awe/radar_system
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor radar system

Browse Source
This commit is contained in:
Ayzen
2026-10-05 16:46:28 +03:00
parent 515049d2ed
commit 8f097dfa2e
179 changed files with 8212 additions and 6676 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.vscode/settings.json
Vendored
+3
View File
@@ -0,0 +1,3 @@
{
"cmake.sourceDirectory": "/home/ayzen/Documents/radar_system/data_acq_and_processing/gpr_cpp"
}
Makefile
+62 -9
View File
@@ -1,9 +1,8 @@
CXX := g++
PYTHON ?= .venv/bin/python
CXXFLAGS := -std=c++20 -O2 -Wall -Wextra -Wpedantic -pthread -MMD -MP
LDFLAGS := -pthread -lrt
ORCH_LDFLAGS := $(LDFLAGS) -lusb-1.0
VISA_CXXFLAGS ?= -I/usr/include/ni-visa
VISA_LDFLAGS := $(LDFLAGS) -lvisa
BUILD_DIR := build
BIN_DIR := $(BUILD_DIR)/bin
@@ -11,9 +10,9 @@ BIN_DIR := $(BUILD_DIR)/bin
# --- CUDA for the sar3d processor (optional). Its kernels are compiled at run
# time by NVRTC and launched through the CUDA driver API, so no nvcc is needed:
# only nvrtc.h, cuda.h, libnvrtc and the NVIDIA driver's libcuda. Headers and
# libnvrtc come from the pip nvidia-cuda-* packages in .venv (requirements.txt,
# libnvrtc come from the pip nvidia-cuda-* packages in .venv (requirements-cuda.txt,
# x86_64) or from a system toolkit in /usr/local/cuda (Jetson). Without them
# data_processor still builds; sar3d then reports that CUDA is missing.
# data_processor still builds; selecting sar3d shows a CUDA warning.
CUDA_ROOTS := $(wildcard $(CURDIR)/.venv/lib/python3*/site-packages/nvidia/*) /usr/local/cuda
CUDA_NVRTC_HEADER := $(firstword $(wildcard $(addsuffix /include/nvrtc.h,$(CUDA_ROOTS))))
CUDA_DRIVER_HEADER := $(firstword $(wildcard $(addsuffix /include/cuda.h,$(CUDA_ROOTS))))
@@ -54,7 +53,6 @@ ORCH_SOURCES := \
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp \
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_transport.cpp \
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_protocol.cpp \
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209/remote_compact_m_k209_driver.cpp \
data_acq_and_processing/sweep_orchestrator/device_drivers/switches/h7992_minimal_driver.cpp \
data_acq_and_processing/sweep_orchestrator/device_drivers/switches/hmc349a_minimal_driver.cpp \
data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp \
@@ -120,17 +118,72 @@ KAMIL_BUILD_DIR := $(BUILD_DIR)/kamil
KAMIL_COLLECTOR_OBJS := $(addprefix $(KAMIL_BUILD_DIR)/,$(notdir $(KAMIL_COLLECTOR_SOURCES:.cpp=.o)))
DEPFILES := $(sort $(SWEEP_ORCH_OBJS:.o=.d) $(PREPROCESSOR_OBJS:.o=.d) $(DATA_PROCESSOR_OBJS:.o=.d) $(KAMIL_COLLECTOR_OBJS:.o=.d))
NATIVE_TEST_SOURCES := $(COMMON_SOURCES) $(LOCATOR_SOURCES) \
data_acq_and_processing/tests/native_regressions.cpp \
data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp \
data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp \
data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp \
$(KAMIL_COLLECTOR_DIR)/src/capture_file_writer.cpp \
$(KAMIL_COLLECTOR_DIR)/src/tty_protocol_writer.cpp
NATIVE_BUILD_DIR := $(BUILD_DIR)/native_tests
NATIVE_TEST_OBJS := $(addprefix $(NATIVE_BUILD_DIR)/,$(NATIVE_TEST_SOURCES:.cpp=.o))
NATIVE_CXXFLAGS := $(CXXFLAGS) -O1 -g $(if $(NATIVE_SANITIZERS),-fsanitize=$(NATIVE_SANITIZERS) -fno-omit-frame-pointer)
NATIVE_FLAGS_STAMP := $(NATIVE_BUILD_DIR)/compiler.flags
$(shell mkdir -p $(NATIVE_BUILD_DIR) && echo '$(NATIVE_CXXFLAGS)' | cmp -s - $(NATIVE_FLAGS_STAMP) || echo '$(NATIVE_CXXFLAGS)' > $(NATIVE_FLAGS_STAMP))
DEPFILES += $(NATIVE_TEST_OBJS:.o=.d)
TARGETS := \
$(BIN_DIR)/sweep_orchestrator \
$(BIN_DIR)/data_preprocessor \
$(BIN_DIR)/data_processor
.PHONY: all clean kamil_adc_collector
.PHONY: all clean kamil_adc_collector check-native check-sar3d
all: $(TARGETS)
kamil_adc_collector: $(BIN_DIR)/kamil_adc_collector
check-native: $(NATIVE_BUILD_DIR)/native_regressions
$(NATIVE_BUILD_DIR)/native_regressions run_config_examples/run_config_simulator.example.json
SAR3D_TEST_SOURCES := data_acq_and_processing/tests/sar3d_fixture.cpp \
data_acq_and_processing/processing/processors/src/sar3d_processor.cpp \
data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp
SAR3D_TEST_OBJS := $(addprefix $(NATIVE_BUILD_DIR)/,$(SAR3D_TEST_SOURCES:.cpp=.o))
SAR3D_EMULATION_OBJ := $(NATIVE_BUILD_DIR)/data_acq_and_processing/tests/sar3d_kernel_emulation.o
SAR3D_KERNEL_TEST_OBJ := $(NATIVE_BUILD_DIR)/data_acq_and_processing/tests/sar3d_kernel_regressions.o
SAR3D_EMULATION_HEADERS := $(NATIVE_BUILD_DIR)/sar3d_kernels.inc $(NATIVE_BUILD_DIR)/sar3d_helpers.inc
DEPFILES += $(SAR3D_TEST_OBJS:.o=.d) $(SAR3D_EMULATION_OBJ:.o=.d) $(SAR3D_KERNEL_TEST_OBJ:.o=.d)
check-sar3d: $(NATIVE_BUILD_DIR)/sar3d_fixture $(NATIVE_BUILD_DIR)/sar3d_math_fixture $(NATIVE_BUILD_DIR)/sar3d_kernel_regressions
$(NATIVE_BUILD_DIR)/sar3d_kernel_regressions
$(PYTHON) data_acq_and_processing/tests/test_sar3d_oracle.py
$(NATIVE_BUILD_DIR)/sar3d_fixture: $(SAR3D_TEST_OBJS)
$(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS) $(SAR3D_LDFLAGS)
$(NATIVE_BUILD_DIR)/sar3d_math_fixture: $(filter-out $(NATIVE_BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o,$(SAR3D_TEST_OBJS)) $(SAR3D_EMULATION_OBJ)
$(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS)
$(NATIVE_BUILD_DIR)/sar3d_kernel_regressions: $(SAR3D_KERNEL_TEST_OBJ) $(SAR3D_EMULATION_OBJ)
$(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS)
$(SAR3D_EMULATION_HEADERS) &: data_acq_and_processing/tests/generate_sar3d_emulation.py data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp
$(PYTHON) data_acq_and_processing/tests/generate_sar3d_emulation.py $(NATIVE_BUILD_DIR)
$(SAR3D_EMULATION_OBJ): NATIVE_CXXFLAGS += -I$(NATIVE_BUILD_DIR)
$(SAR3D_EMULATION_OBJ): $(SAR3D_EMULATION_HEADERS)
$(NATIVE_BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: NATIVE_CXXFLAGS += $(SAR3D_CXXFLAGS)
$(NATIVE_BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: $(SAR3D_CUDA_STAMP)
$(NATIVE_BUILD_DIR)/native_regressions: $(NATIVE_TEST_OBJS)
$(CXX) $(NATIVE_CXXFLAGS) $^ -o $@ $(LDFLAGS) -lutil
$(NATIVE_BUILD_DIR)/%.o: %.cpp $(NATIVE_FLAGS_STAMP)
@mkdir -p $(dir $@)
$(CXX) $(NATIVE_CXXFLAGS) $(INCLUDES) -I$(KAMIL_COLLECTOR_DIR)/include -c $< -o $@
$(BIN_DIR)/sweep_orchestrator: $(SWEEP_ORCH_OBJS)
@mkdir -p $(BIN_DIR)
$(CXX) $(SWEEP_ORCH_OBJS) -o $@ $(ORCH_LDFLAGS)
@@ -142,13 +195,13 @@ $(BIN_DIR)/data_preprocessor: $(PREPROCESSOR_OBJS)
$(BIN_DIR)/data_processor: $(DATA_PROCESSOR_OBJS) $(SAR3D_CUDA_STAMP)
@mkdir -p $(BIN_DIR)
$(CXX) $(DATA_PROCESSOR_OBJS) -o $@ $(LDFLAGS) $(SAR3D_LDFLAGS)
$(if $(SAR3D_LDFLAGS),,@echo "note: CUDA not found, data_processor built without the sar3d GPU backend")
$(if $(SAR3D_LDFLAGS),,@echo "note: CUDA not found, selecting sar3d will show a warning")
$(BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: CXXFLAGS += $(SAR3D_CXXFLAGS)
$(BUILD_DIR)/data_acq_and_processing/processing/processors/src/sar3d_gpu.o: $(SAR3D_CUDA_STAMP)
# The collector compiles into its own build/kamil/ tree with collector-only
# includes (no VISA), so the shared driver/config sources it reuses never collide
# includes, so the shared driver/config sources it reuses never collide
# with the orchestrator's objects of the same name. Basenames are unique, so a
# vpath lets one pattern rule find every source.
vpath %.cpp $(sort $(dir $(KAMIL_COLLECTOR_SOURCES)))
@@ -163,7 +216,7 @@ $(BIN_DIR)/kamil_adc_collector: $(KAMIL_COLLECTOR_OBJS)
$(BUILD_DIR)/%.o: %.cpp
@mkdir -p $(dir $@)
$(CXX) $(CXXFLAGS) $(VISA_CXXFLAGS) $(INCLUDES) -c $< -o $@
$(CXX) $(CXXFLAGS) $(INCLUDES) -c $< -o $@
-include $(DEPFILES)
README.md
+91 -24
View File
@@ -44,8 +44,6 @@ controls plus a pixel-identical live plot through an embedded web UI.
- **librevna** — single LibreVNA.
- **librevna_multi** — synchronized multi-device LibreVNA.
- **compact_m_k209** — Compact-M K209 / S2VNA (including Raspberry Pi remote mode).
- **sn9000** — SNVNA SN9000 over VISA (pure-Python `@py` backend).
- **kamil_adc** — external Kamil ADC acquisition process, with optional laser control.
- **mock / simulator** — driver-less mode for development without hardware.
@@ -56,13 +54,28 @@ controls plus a pixel-identical live plot through an embedded web UI.
- **gpr** — coherent back-projection GPR imaging with object detection.
- **legacy_gpr** — ellipse-intersection MIMO localization (point and extended/region modes).
- **sar3d** — 3D SAR of the reference notebook `Multi_antenna_3D_clean_v1.ipynb`
on an NVIDIA GPU: front (X-Y), side (Z-Y), top (X-Z) and 3D views of the imaged
on an NVIDIA GPU through CUDA: front (X-Y), side (Z-Y), top (X-Z)
and oblique maximum projections of the imaged
volume. Its settings carry the notebook's parameter names; the scene window is
fixed (X −2…2 m, height −0.5…2 m above the ground), its range extent follows
`MIN_DEPTH`/`MAX_DEPTH`, and the voxels are cubes with edge `SAR_VOXEL_M`
(default 2 cm). The radar motion is corrected from each trace's capture
timestamps, so the sweep timing needs no parameters.
For matrix-radar B-scans, **Next combination** advances output first, then input,
and wraps to the first pair. The desktop fields and browser channel selector
redraw the selected cached B-scan immediately without restarting acquisition.
**Save Last 1000** and **Save S21 JSON** use a fixed limit of 1000 recent
measurements from each available stage; older GUI profiles may still contain
`save_count`, which is ignored. **Start + Record** retains its separate count for
recording future measurements to disk.
Tx/Rx antenna positions are edited in tables with switch position and X/Y/Z in
metres, on both the desktop and web interface. Coordinate and legacy-GPR medium
changes apply to the running processor and recompute its retained result. Profiles
preserve all three coordinates. GPR/SAR views wait for a result marked with the
current settings revision, so an older in-flight result cannot overwrite new geometry.
## Repository layout
| Path | Contents |
@@ -72,22 +85,26 @@ controls plus a pixel-identical live plot through an embedded web UI.
| `run_config_examples/` | Ready-made configuration profiles, one per hardware setup. |
| `deploy/` | systemd daemon installer, and the USB update kit for the Pi and Jetson (`pack-usb.sh`, `update.sh`). |
| `build/bin/` | Compiled C++ binaries. |
| `device_firmware/`, `docs/` | Device firmware and vendor manuals. |
| `device_firmware/` | Device firmware and protocol documentation. |
## Requirements
- Linux (POSIX shared memory; GPIO on Raspberry Pi). The deployment target is a
**Raspberry Pi 5**; hardware PWM gating for the generator sweep is Pi 5
specific (GPIO12/13/18/19, `dtoverlay=pwm-2chan`).
- A C++20 compiler and `make` — Eigen and nlohmann/json are vendored, so no extra C++ dependencies are needed.
- A C++20 compiler and `make`; Eigen and nlohmann/json are vendored.
- `libusb-1.0` development headers and `libatomic.so.1` (`libatomic1` on Debian/Ubuntu).
Python ring readers/writers use lock-free atomic loads, stores and cursor CAS
through libatomic, including memory barriers for ARM. Slot sizes must be multiples
of 8 bytes, and the complete ring mapping must not exceed 1 TiB.
- **Python 3.10 or newer**, in a virtual environment.
- For the `sar3d` processor: an NVIDIA GPU with its driver (`libcuda`). The
- For `sar3d`: an NVIDIA GPU with its driver (`libcuda`). The
kernels are compiled at run time by NVRTC, so there is no nvcc step; `make`
needs only `nvrtc.h`, `cuda.h` and `libnvrtc`, taken from the pip
`nvidia-cuda-*` packages in `.venv` (`requirements.txt`, x86_64; CUDA 13 covers
Turing and newer GPUs) or from a system toolkit in `/usr/local/cuda` (Jetson).
Without them everything else still builds, and sar3d reports that CUDA is
missing — a Raspberry Pi needs none of it.
`nvidia-cuda-*` packages in `.venv` (`requirements-cuda.txt`, x86_64) or from
a system toolkit in `/usr/local/cuda` (Jetson).
Without these dependencies or a usable GPU, SAR3D shows a warning above the
plots and does not calculate an image. Other processing modes remain available.
## Build and run
@@ -103,6 +120,8 @@ make # build the C++ binaries into build/bin/
`make` builds the sar3d GPU code when it finds CUDA (see Requirements) and
prints a note when it does not. After installing the CUDA requirements into an
existing venv, run `make` again: it notices the change and rebuilds that part.
For SAR3D on x86_64, install the optional packages with
`.venv/bin/python -m pip install -r requirements-cuda.txt`, then run `make`.
Run the desktop GUI:
@@ -153,17 +172,27 @@ Two ways to run the full pipeline with no radar attached:
**Mock producer (synthetic data).** The default `run_config.json` already sets
`radar.driver_mode: "mock"`, so the producer generates synthetic S-parameters.
The preprocessor still requires S21 calibration/reference sets; create them once:
Mock acquisition supports the LibreVNA models; Kamil ADC uses native acquisition
or recorded-dataset replay without devices.
Both LibreVNA modes share one simulator: a point reflector moves back and forth
along the antenna array and between depths 3.25–4.25 m over an eight-second cycle.
Each S21 uses the actual transmitter–target–receiver distance from the configured
XYZ coordinates; coincident antennas produce identical signals. All combinations
share the same target position and measured capture window, with collections paced
at 50 ms intervals. B-scan's default range is 0–8 m; stored profiles retain their
chosen range. The simulator models a stationary radar, so radar speed should be zero.
The scene uses the antenna positions from its startup profile; live geometry edits
change the reconstruction of captured sweeps, whose samples remain unchanged.
The preprocessor still requires S21 calibration/reference sets. Start the GUI:
```bash
.venv/bin/python -m python_app.scripts.manual_smoke_run --duration 1.0 # creates smoke_cal/smoke_ref
./start.sh # then in the GUI: Preprocessing Panel -> pick the sets -> Start
./start.sh
```
(Alternatively use the GUI's "Create neutral sets" button, which makes
identity calibration + zero reference so preprocessing is a pass-through.)
In **Preprocessing Panel**, use **Create neutral sets**, then **Start**. This makes
identity calibration + zero reference, preserving the simulated S21 samples.
**Dataset replay (recorded scene, meaningful sar3d picture).** When
**Dataset replay (recorded scene).** When
`radar.replay.dataset_dir` is set in the profile, the supervisor launches
`python_app.scripts.replay_raw_producer` instead of the normal producer: it
replays recorded snapshots into the raw ring at the recorded cadence, looping
@@ -179,15 +208,32 @@ A ready profile for the VNA_AlGo tank005 dataset is provided:
./start.sh --profile run_config_examples/run_config_replay_vna.example.json
```
The sar3d processor keeps about 75 MB of buffers on the GPU at the default
2 cm voxel and depth 2…8 m (plus the CUDA context). The volume grows as
1/`SAR_VOXEL_M`³: 1 cm takes about 600 MB and ran about 6 times slower in
testing. If the GPU has no room, it logs the CUDA error once and publishes empty
results until memory frees.
Finer voxels increase SAR3D work as 1/`SAR_VOXEL_M`³. Increase the voxel size when
a faster preview matters more than spatial resolution. Processing failures appear
above the plots and a failed single capture
finishes with its explanation while retaining the acquired data.
The web interface is served in-process at `http://<host>:8080`. It mirrors the live
plot and offers Start / Stop / Capture controls and the active processor settings.
Manual dataset and JSON export keep the last 1000 available collections from each
stage independently, including raw or preprocessed sweeps without a result.
Streaming recording keeps every collection received after arming until the requested
result count or Stop; Stop and closing the window flush accepted data. Acquisition
rings use latest-wins delivery, so disk recording cannot recover sweeps overwritten
before the GUI receives them. NumPy datasets preserve both S-parameters, capture
windows, every supported result payload, and processing time. Failed exports remove
their staging files, leaving existing datasets intact.
Calibration and reference sets keep their trace arrays and metadata together in one
atomically replaced NPZ archive, with a readable JSON sidecar. Earlier NPZ/JSON pairs
remain supported. Invalid trace shapes, duplicate combinations and non-finite
calibration/reference values are rejected before a saved set is replaced.
Median captures also require identical frequency axes and channel coverage.
Dataset replay restores both recorded S-parameters and orders long recordings by
their numeric index; older S21-only datasets still replay with zero S11. Offline
JSON conversion and the GUI share the same sweep-matching code.
## Configuration
The active profile is `run_config.json`. It selects the radar model and sweep, the
@@ -217,13 +263,34 @@ the current version is renamed to `radar_system<N>`, the next free number. Kept
it: `run_config.json`, profiles in `run_config_examples/` changed or added on the
device (the release version of a changed one lands next to it as `.new`),
`python_app/runtime/` without logs, `python_app/data*` and any other top-level
folder that is not part of the release, as hard links. Then it carries the venv over
(or builds a new one with `--python`), builds the C++ part and restarts
file or folder that is not part of the release, as hard links. The venv is copied
independently so package updates cannot modify the rollback environment (or a new
one is built with `--python`). It builds the C++ pipeline and ADC collector and restarts
`radar.service` if it was running. To go back:
`rm -rf radar_system && mv radar_system<N> radar_system`.
## Tests
```bash
.venv/bin/python -m unittest discover -s python_app/tests
QT_QPA_PLATFORM=offscreen .venv/bin/python -m unittest discover -s python_app/tests
make check-native
make check-sar3d PYTHON=.venv/bin/python
ASAN_OPTIONS=detect_leaks=0 make check-native NATIVE_SANITIZERS=address,undefined
```
The tests cover failed writes and settings commits, recordings during manual Save,
repeated Start/Stop/Close, web-action cancellation, malformed IPC frames, hardware
resource cleanup with mock failures, and an isolated deployment update. Native tests
also exercise mutated frames, ring concurrency, locator input and ADC spool recovery.
Firmware host tests check cart timer rollover and SBUS packing; they do not validate
physical timing. Real devices, GPIO and the CUDA backend need their own hardware run.
Invalid device/switch settings and undersized raw/preprocessed slots fail before
acquisition starts. Oversized processed results report the required slot size in
the log; switching to a mode that fits resumes processing without a restart.
GUI profiles, run configs, session state and live processing settings use shared
atomic writers with unique temporary files. Profiles and run/session settings sync
their file and directory; frequent live settings avoid extra disk syncs. A failed
write before replacement leaves the previous file intact. The launcher and supervisor
share acquisition producer selection, with replay taking precedence over the model.
data_acq_and_processing/common_cpp/config/include/run_config.hpp
+2 -4
View File
@@ -32,10 +32,7 @@ struct RadarConfig {
// Radar device identity and runtime mode.
std::string model = "librevna";
std::string serial{};
std::string remote_host = "127.0.0.1";
std::uint32_t remote_port = 50209;
DriverMode driver_mode = DriverMode::Mock;
float mock_signal_hz = 5'000'000.0F;
RadarSweepSettings sweep{};
};
@@ -134,7 +131,7 @@ struct GprRxGeometry {
};
struct GprConfig {
// Stable collection-level GPR processing settings.
// Initial GPR settings; the data_processor can override them through live config.
float relative_permittivity = 1.0F;
std::vector<GprTxGeometry> tx_geometry{};
std::vector<GprRxGeometry> rx_geometry{};
@@ -154,5 +151,6 @@ struct RunConfig {
};
[[nodiscard]] auto load_run_config(const std::string& path) -> RunConfig;
void validate_gpr_config(const GprConfig& config, const RunConfig& run_config);
} // namespace radar::config
data_acq_and_processing/common_cpp/config/include/runtime_wait.hpp
+20
View File
@@ -0,0 +1,20 @@
#pragma once
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdint>
#include <thread>
namespace radar::config {
// Returns true when stop interrupts the delay, checking at least every 50 ms.
inline auto wait_or_stop(std::uint32_t delay_ms, const std::atomic<bool>& stop) -> bool {
std::uint32_t waited = 0;
while (waited < delay_ms && !stop.load(std::memory_order_relaxed)) {
const auto chunk = std::min(50U, delay_ms - waited);
std::this_thread::sleep_for(std::chrono::milliseconds(chunk));
waited += chunk;
}
return stop.load(std::memory_order_relaxed);
}
} // namespace radar::config
data_acq_and_processing/common_cpp/config/src/run_config.cpp
+60 -45
View File
@@ -65,7 +65,18 @@ using Json = nlohmann::json;
if (!value.is_number()) {
throw std::runtime_error("Expected JSON number at " + context);
}
return value.get<double>();
const auto numeric = value.get<double>();
if (!std::isfinite(numeric)) {
throw std::runtime_error("Expected finite number at " + context);
}
return numeric;
}
[[nodiscard]] auto number_to_f32(double value, const std::string& context) -> float {
if (!std::isfinite(value) || std::fabs(value) > std::numeric_limits<float>::max()) {
throw std::runtime_error("Value out of finite float32 range at " + context);
}
return static_cast<float>(value);
}
[[nodiscard]] auto number_to_u32(double value, const std::string& context) -> std::uint32_t {
@@ -74,7 +85,7 @@ using Json = nlohmann::json;
}
const auto rounded = std::round(value);
if (std::fabs(value - rounded) > 0.000001) {
if (value != rounded) {
throw std::runtime_error("Expected integer value at " + context);
}
return static_cast<std::uint32_t>(rounded);
@@ -82,7 +93,7 @@ using Json = nlohmann::json;
[[nodiscard]] auto number_to_i32(double value, const std::string& context) -> std::int32_t {
const auto rounded = std::round(value);
if (std::fabs(value - rounded) > 0.000001) {
if (value != rounded) {
throw std::runtime_error("Expected integer value at " + context);
}
@@ -135,7 +146,7 @@ using Json = nlohmann::json;
[[nodiscard]] auto optional_f32(const Json& object, const std::string& key, float fallback) -> float {
if (const auto* value = optional_field(object, key); value != nullptr) {
return static_cast<float>(as_number(*value, key));
return number_to_f32(as_number(*value, key), key);
}
return fallback;
}
@@ -173,8 +184,8 @@ using Json = nlohmann::json;
}
PreprocessNotchBandConfig band{};
band.low_hz = static_cast<float>(as_number((*band_array)[0], context + ".bands_hz[][0]"));
band.high_hz = static_cast<float>(as_number((*band_array)[1], context + ".bands_hz[][1]"));
band.low_hz = number_to_f32(as_number((*band_array)[0], context + ".bands_hz[][0]"), context);
band.high_hz = number_to_f32(as_number((*band_array)[1], context + ".bands_hz[][1]"), context);
notch.bands_hz.push_back(band);
}
}
@@ -242,6 +253,9 @@ using Json = nlohmann::json;
}
void validate_radar_sweep(const RadarSweepSettings& sweep) {
if (sweep.start_hz < 0.0F || sweep.stop_hz < 0.0F || !(sweep.if_bandwidth_hz > 0.0F)) {
throw std::runtime_error("radar.sweep frequencies must be non-negative and IF bandwidth must be > 0");
}
if (sweep.points == 0U) {
throw std::runtime_error("radar.sweep.points must be > 0");
}
@@ -297,32 +311,6 @@ void validate_combo_key(const ipc::ComboKey& key, const RunConfig& config) {
}
}
void validate_gpr_config(const GprConfig& config, const RunConfig& run_config) {
if (!(config.relative_permittivity > 0.0F)) {
throw std::runtime_error("gpr.relative_permittivity must be > 0");
}
std::unordered_set<std::uint32_t> seen_tx_positions{};
for (const auto& entry : config.tx_geometry) {
if (entry.output_pos >= run_config.output_switch.positions) {
throw std::runtime_error("gpr.tx_geometry output_pos is out of range");
}
if (!seen_tx_positions.insert(entry.output_pos).second) {
throw std::runtime_error("gpr.tx_geometry contains duplicate output_pos");
}
}
std::unordered_set<std::uint32_t> seen_rx_positions{};
for (const auto& entry : config.rx_geometry) {
if (entry.input_pos >= run_config.input_switch.positions) {
throw std::runtime_error("gpr.rx_geometry input_pos is out of range");
}
if (!seen_rx_positions.insert(entry.input_pos).second) {
throw std::runtime_error("gpr.rx_geometry contains duplicate input_pos");
}
}
}
[[nodiscard]] auto parse_gpr_config(const Json& object, const RunConfig& run_config) -> GprConfig {
const auto* gpr_obj = as_object(object, "gpr");
GprConfig config{};
@@ -406,6 +394,9 @@ void validate_gpr_config(const GprConfig& config, const RunConfig& run_config) {
if (endpoint.slot_size_bytes == 0U) {
throw std::runtime_error("Ring slot_size_bytes must be > 0 for " + key);
}
if (endpoint.slot_size_bytes % 8U != 0U) {
throw std::runtime_error("Ring slot_size_bytes must be a multiple of 8 for " + key);
}
if (endpoint.name.empty() || endpoint.name.front() != '/') {
throw std::runtime_error("Ring name must start with '/' for " + key);
}
@@ -415,6 +406,39 @@ void validate_gpr_config(const GprConfig& config, const RunConfig& run_config) {
} // namespace
void validate_gpr_config(const GprConfig& config, const RunConfig& run_config) {
if (!std::isfinite(config.relative_permittivity) || !(config.relative_permittivity > 0.0F)) {
throw std::runtime_error("gpr.relative_permittivity must be > 0");
}
std::unordered_set<std::uint32_t> seen_tx_positions{};
for (const auto& entry : config.tx_geometry) {
if (!std::isfinite(entry.x_m) || !std::isfinite(entry.y_m) || !std::isfinite(entry.z_m)) {
throw std::runtime_error("gpr.tx_geometry coordinates must be finite");
}
if (entry.output_pos >= run_config.output_switch.positions) {
throw std::runtime_error("gpr.tx_geometry output_pos is out of range");
}
if (!seen_tx_positions.insert(entry.output_pos).second) {
throw std::runtime_error("gpr.tx_geometry contains duplicate output_pos");
}
}
std::unordered_set<std::uint32_t> seen_rx_positions{};
for (const auto& entry : config.rx_geometry) {
if (!std::isfinite(entry.x_m) || !std::isfinite(entry.y_m) || !std::isfinite(entry.z_m)) {
throw std::runtime_error("gpr.rx_geometry coordinates must be finite");
}
if (entry.input_pos >= run_config.input_switch.positions) {
throw std::runtime_error("gpr.rx_geometry input_pos is out of range");
}
if (!seen_rx_positions.insert(entry.input_pos).second) {
throw std::runtime_error("gpr.rx_geometry contains duplicate input_pos");
}
}
}
auto load_run_config(const std::string& path) -> RunConfig {
std::ifstream stream(path);
if (!stream.is_open()) {
@@ -440,30 +464,21 @@ auto load_run_config(const std::string& path) -> RunConfig {
const auto* radar_obj = as_object(required_field(*root_obj, "radar"), "radar");
config.radar.model = optional_string(*radar_obj, "model", "librevna");
config.radar.serial = optional_string(*radar_obj, "serial", "");
config.radar.remote_host = optional_string(*radar_obj, "remote_host", "127.0.0.1");
config.radar.remote_port = optional_u32(*radar_obj, "remote_port", 50209);
if (config.radar.remote_host.empty()) {
throw std::runtime_error("radar.remote_host must not be empty");
}
if (config.radar.remote_port == 0U || config.radar.remote_port > 65535U) {
throw std::runtime_error("radar.remote_port must be in 1..65535");
}
config.radar.driver_mode = parse_driver_mode(optional_string(*radar_obj, "driver_mode", "mock"));
config.radar.mock_signal_hz = optional_f32(*radar_obj, "mock_signal_hz", 5'000'000.0F);
const auto* sweep_obj = as_object(required_field(*radar_obj, "sweep"), "radar.sweep");
config.radar.sweep.start_hz =
static_cast<float>(as_number(required_field(*sweep_obj, "start_hz"), "radar.sweep.start_hz"));
number_to_f32(as_number(required_field(*sweep_obj, "start_hz"), "radar.sweep.start_hz"), "radar.sweep.start_hz");
config.radar.sweep.stop_hz =
static_cast<float>(as_number(required_field(*sweep_obj, "stop_hz"), "radar.sweep.stop_hz"));
number_to_f32(as_number(required_field(*sweep_obj, "stop_hz"), "radar.sweep.stop_hz"), "radar.sweep.stop_hz");
if (const auto* points_value = optional_field(*sweep_obj, "points"); points_value != nullptr) {
config.radar.sweep.points =
number_to_u32(as_number(*points_value, "radar.sweep.points"), "radar.sweep.points");
} else if (config.radar.model != "kamil_adc") {
throw std::runtime_error("Missing required config field: points");
}
config.radar.sweep.if_bandwidth_hz = static_cast<float>(
as_number(required_field(*sweep_obj, "if_bandwidth_hz"), "radar.sweep.if_bandwidth_hz")
config.radar.sweep.if_bandwidth_hz = number_to_f32(
as_number(required_field(*sweep_obj, "if_bandwidth_hz"), "radar.sweep.if_bandwidth_hz"), "radar.sweep.if_bandwidth_hz"
);
config.radar.sweep.power_dbm = optional_f32(*sweep_obj, "stimulus_power_dbm", -10.0F);
validate_radar_sweep(config.radar.sweep);
data_acq_and_processing/common_cpp/ipc/include/shared_types.hpp
+3 -1
View File
@@ -2,6 +2,7 @@
#include <cstddef>
#include <cstdint>
#include <functional>
#include <span>
#include <string>
#include <unordered_map>
@@ -26,7 +27,8 @@ struct ComboKey {
struct ComboKeyHash {
[[nodiscard]] auto operator()(const ComboKey& key) const noexcept -> std::size_t {
return (static_cast<std::size_t>(key.input_pos) << 32U) ^ static_cast<std::size_t>(key.output_pos);
const auto packed = (static_cast<std::uint64_t>(key.input_pos) << 32U) | key.output_pos;
return std::hash<std::uint64_t>{}(packed);
}
};
data_acq_and_processing/common_cpp/ipc/include/shm_ring.hpp
+1
View File
@@ -53,6 +53,7 @@ class ShmRing {
[[nodiscard]] auto slot_stride_bytes() const -> std::size_t;
[[nodiscard]] auto slot_header(std::uint64_t sequence) const -> SlotHeader*;
[[nodiscard]] auto slot_payload(SlotHeader* slot) const -> std::uint8_t*;
void advance_read_seq(std::uint64_t target);
void close();
static void validate_name(const std::string& name);
data_acq_and_processing/common_cpp/ipc/src/shared_types.cpp
+18 -8
View File
@@ -95,7 +95,7 @@ class BinaryReader {
private:
void ensure_available(std::size_t size) const {
if (offset_ + size > bytes_.size()) {
if (size > bytes_.size() - offset_) {
throw std::runtime_error("Unexpected end of serialized payload");
}
}
@@ -132,9 +132,8 @@ void write_trace_block(BinaryWriter& writer, const SweepTraceBlock& trace) {
// request gigabytes and crash the process (length_error/bad_alloc/OOM-kill). The
// payload itself is already bounded by the ring slot size, so any count whose
// minimum encoding cannot fit in the bytes still available is structurally invalid.
void require_count_fits(std::uint32_t count, std::size_t min_bytes_each, BinaryReader& reader) {
if (min_bytes_each != 0U
&& static_cast<std::uint64_t>(count) * min_bytes_each > reader.remaining_bytes()) {
void require_count_fits(std::uint64_t count, std::size_t min_bytes_each, BinaryReader& reader) {
if (min_bytes_each != 0U && count > reader.remaining_bytes() / min_bytes_each) {
throw std::runtime_error("Declared element count exceeds remaining payload bytes");
}
}
@@ -257,6 +256,7 @@ void write_trace_result_payload(BinaryWriter& writer, const ResultPayload& paylo
auto read_trace_result_payload(BinaryReader& reader, ResultPayload* payload) -> void {
const auto point_count = reader.read<std::uint32_t>();
require_count_fits(point_count, 12U, reader);
payload->frequency_hz.reserve(point_count);
payload->trace.reserve(point_count);
@@ -296,10 +296,12 @@ void write_image_result_payload(BinaryWriter& writer, const ResultPayload& paylo
auto read_image_result_payload(BinaryReader& reader, ResultPayload* payload) -> void {
const auto x_count = reader.read<std::uint32_t>();
const auto y_count = reader.read<std::uint32_t>();
const auto value_count = static_cast<std::uint64_t>(x_count) * y_count;
require_count_fits(value_count + x_count + y_count, sizeof(float), reader);
payload->image_x_axis.reserve(x_count);
payload->image_y_axis.reserve(y_count);
payload->image_values.reserve(static_cast<std::size_t>(x_count) * static_cast<std::size_t>(y_count));
payload->image_values.reserve(static_cast<std::size_t>(value_count));
for (std::uint32_t index = 0; index < x_count; ++index) {
payload->image_x_axis.push_back(reader.read<float>());
@@ -307,7 +309,7 @@ auto read_image_result_payload(BinaryReader& reader, ResultPayload* payload) ->
for (std::uint32_t index = 0; index < y_count; ++index) {
payload->image_y_axis.push_back(reader.read<float>());
}
for (std::size_t index = 0; index < static_cast<std::size_t>(x_count) * static_cast<std::size_t>(y_count); ++index) {
for (std::size_t index = 0; index < value_count; ++index) {
payload->image_values.push_back(reader.read<float>());
}
}
@@ -334,9 +336,14 @@ void write_table_result_payload(BinaryWriter& writer, const ResultPayload& paylo
auto read_table_result_payload(BinaryReader& reader, ResultPayload* payload) -> void {
const auto column_count = reader.read<std::uint32_t>();
const auto row_count = reader.read<std::uint32_t>();
if (column_count == 0U && row_count != 0U) {
throw std::runtime_error("Result table cannot have rows without columns");
}
const auto value_count = static_cast<std::uint64_t>(column_count) * row_count;
require_count_fits(value_count, sizeof(float), reader);
payload->table_columns = column_count;
payload->table_values.reserve(static_cast<std::size_t>(column_count) * static_cast<std::size_t>(row_count));
for (std::size_t index = 0; index < static_cast<std::size_t>(column_count) * static_cast<std::size_t>(row_count); ++index) {
payload->table_values.reserve(static_cast<std::size_t>(value_count));
for (std::size_t index = 0; index < value_count; ++index) {
payload->table_values.push_back(reader.read<float>());
}
}
@@ -402,6 +409,7 @@ void write_result_block(BinaryWriter& writer, const ResultBlock& block) {
block.combo.output_pos = reader.read<std::uint32_t>();
const auto payload_count = reader.read<std::uint32_t>();
require_count_fits(payload_count, 3U, reader);
block.payloads.reserve(payload_count);
for (std::uint32_t index = 0; index < payload_count; ++index) {
block.payloads.push_back(read_result_payload(reader));
@@ -478,11 +486,13 @@ auto deserialize_result_collection(std::span<const std::uint8_t> bytes) -> Resul
const auto collection_payload_count = reader.read<std::uint32_t>();
const auto block_count = reader.read<std::uint32_t>();
require_count_fits(collection_payload_count, 3U, reader);
collection.collection_payloads.reserve(collection_payload_count);
for (std::uint32_t index = 0; index < collection_payload_count; ++index) {
collection.collection_payloads.push_back(read_result_payload(reader));
}
require_count_fits(block_count, 12U, reader);
collection.blocks.reserve(block_count);
for (std::uint32_t index = 0; index < block_count; ++index) {
collection.blocks.push_back(read_result_block(reader));
data_acq_and_processing/common_cpp/ipc/src/shm_ring.cpp
+64 -25
View File
@@ -30,7 +30,7 @@ struct alignas(64) ShmRing::Header {
std::atomic<std::uint64_t> dropped{};
};
struct alignas(16) ShmRing::SlotHeader {
struct alignas(8) ShmRing::SlotHeader {
std::uint32_t payload_size = 0;
std::uint32_t reserved = 0;
std::uint64_t sequence = 0;
@@ -42,6 +42,7 @@ constexpr std::string_view kRingMagic = "RDRRING2";
constexpr std::uint32_t kRingVersion = 1;
constexpr auto kHeaderInitWaitTimeout = std::chrono::milliseconds(1000);
constexpr auto kHeaderInitPollInterval = std::chrono::milliseconds(2);
constexpr std::uint64_t kMaxMappedSize = 1ULL << 40U;
[[nodiscard]] auto checked_u64_diff(std::uint64_t left, std::uint64_t right) -> std::uint64_t {
return left >= right ? left - right : 0;
@@ -134,6 +135,22 @@ auto ShmRing::open_or_create(
if (slot_size_bytes == 0U) {
throw std::runtime_error("Ring slot size must be > 0");
}
if (slot_size_bytes % alignof(SlotHeader) != 0U) {
throw std::runtime_error("Ring slot size must be a multiple of 8");
}
const auto stride = sizeof(SlotHeader) + static_cast<std::uint64_t>(slot_size_bytes);
if (stride > (kMaxMappedSize - sizeof(Header)) / capacity) {
throw std::runtime_error("Shared memory ring exceeds 1 TiB for " + name);
}
const auto mapped_size_u64 = sizeof(Header) + stride * capacity;
if (mapped_size_u64 > kMaxMappedSize
|| mapped_size_u64 > std::numeric_limits<std::size_t>::max()
|| mapped_size_u64 > static_cast<std::uint64_t>(std::numeric_limits<off_t>::max())) {
throw std::runtime_error("Shared memory ring geometry overflows for " + name);
}
const auto mapped_size = static_cast<std::size_t>(mapped_size_u64);
bool created = false;
int fd = shm_open(name.c_str(), O_RDWR | O_CREAT | O_EXCL, 0660);
@@ -149,13 +166,28 @@ auto ShmRing::open_or_create(
ScopedFd scoped_fd(fd);
const auto slot_stride = sizeof(SlotHeader) + static_cast<std::size_t>(slot_size_bytes);
const auto mapped_size = sizeof(Header) + slot_stride * capacity;
if (created) {
if (ftruncate(fd, static_cast<off_t>(mapped_size)) != 0) {
throw errno_message("Failed to resize shared memory ring", name);
}
} else {
// shm_open publishes the name before the creator calls ftruncate. Mapping
// that zero/short file and reading its header would terminate us with SIGBUS.
const auto deadline = std::chrono::steady_clock::now() + kHeaderInitWaitTimeout;
while (true) {
struct stat info {};
if (fstat(fd, &info) != 0) {
throw errno_message("Failed to stat shared memory ring", name);
}
if (info.st_size >= 0 && static_cast<std::size_t>(info.st_size) >= mapped_size) {
break;
}
if (std::chrono::steady_clock::now() >= deadline) {
throw std::runtime_error("Shared memory ring size is too small for " + name);
}
std::this_thread::sleep_for(kHeaderInitPollInterval);
}
}
void* mapped = mmap(nullptr, mapped_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
@@ -167,7 +199,9 @@ auto ShmRing::open_or_create(
auto* header = static_cast<Header*>(mapped);
if (created) {
std::memset(mapped, 0, mapped_size);
// ftruncate already supplies zero-filled slots; leave them demand-paged
// instead of touching the full ring before any payload is published.
std::memset(mapped, 0, sizeof(Header));
std::memcpy(header->magic, kRingMagic.data(), kRingMagic.size());
header->version = kRingVersion;
header->capacity = capacity;
@@ -319,14 +353,13 @@ auto ShmRing::push(std::span<const std::uint8_t> payload) -> bool {
}
auto* slot = slot_header(write_seq);
// Seqlock publish: write the payload + size FIRST, then store the slot sequence
// LAST (after a release fence). A consumer that observes sequence == write_seq+1
// is therefore guaranteed a fully-written payload. (Publishing the sequence before
// the copy, as before, let a consumer copy a half-written slot.)
// Invalidate the old publication before overwriting its bytes: a slow consumer
// can otherwise accept a mixture of old and new data under the old sequence.
std::atomic_ref(slot->sequence).store(0U, std::memory_order_relaxed);
std::atomic_thread_fence(std::memory_order_seq_cst);
slot->payload_size = static_cast<std::uint32_t>(payload.size());
std::memcpy(slot_payload(slot), payload.data(), payload.size());
std::atomic_thread_fence(std::memory_order_release);
slot->sequence = write_seq + 1U;
std::atomic_ref(slot->sequence).store(write_seq + 1U, std::memory_order_release);
std::atomic_thread_fence(std::memory_order_release);
header_->write_seq.store(write_seq + 1U, std::memory_order_release);
@@ -345,9 +378,9 @@ auto ShmRing::pop(std::vector<std::uint8_t>& payload) -> bool {
}
auto* slot = slot_header(read_seq);
if (slot->sequence != read_seq + 1U) {
if (std::atomic_ref(slot->sequence).load(std::memory_order_acquire) != read_seq + 1U) {
// Slot not published for this seq, or already overwritten (lapped). Resync.
header_->read_seq.store(write_seq, std::memory_order_release);
advance_read_seq(write_seq);
return false;
}
@@ -355,7 +388,7 @@ auto ShmRing::pop(std::vector<std::uint8_t>& payload) -> bool {
if (payload_size > header_->slot_size_bytes) {
// Corrupt/torn slot: skip it (resync) rather than throw — a single bad slot
// must never abort the long-running consumer.
header_->read_seq.store(write_seq, std::memory_order_release);
advance_read_seq(write_seq);
return false;
}
@@ -366,15 +399,21 @@ auto ShmRing::pop(std::vector<std::uint8_t>& payload) -> bool {
// Seqlock verify: if the slot sequence changed during the copy, the producer
// lapped us mid-read and the payload is torn — discard and resync.
std::atomic_thread_fence(std::memory_order_acquire);
if (slot->sequence != read_seq + 1U) {
header_->read_seq.store(write_seq, std::memory_order_release);
if (std::atomic_ref(slot->sequence).load(std::memory_order_acquire) != read_seq + 1U) {
advance_read_seq(write_seq);
return false;
}
header_->read_seq.store(read_seq + 1U, std::memory_order_release);
advance_read_seq(read_seq + 1U);
return true;
}
void ShmRing::advance_read_seq(std::uint64_t target) {
auto current = header_->read_seq.load(std::memory_order_relaxed);
while (current < target && !header_->read_seq.compare_exchange_weak(
current, target, std::memory_order_release, std::memory_order_relaxed)) {}
}
auto ShmRing::slot_stride_bytes() const -> std::size_t {
return sizeof(SlotHeader) + header_->slot_size_bytes;
}
@@ -422,18 +461,18 @@ void ShmRing::validate_geometry(const Header& header, std::size_t mapped_size, c
if (slot_size_bytes == 0U) {
throw std::runtime_error("Shared memory ring slot size is zero for " + name);
}
if (slot_size_bytes % alignof(SlotHeader) != 0U) {
throw std::runtime_error("Shared memory ring slot size must be a multiple of 8 for " + name);
}
constexpr auto kMax = std::numeric_limits<std::size_t>::max();
const auto slot_stride = sizeof(SlotHeader) + static_cast<std::size_t>(slot_size_bytes);
// Guard each multiply/add against std::size_t overflow before computing expected_size.
if (slot_stride > kMax / static_cast<std::size_t>(capacity)) {
const auto stride = sizeof(SlotHeader) + static_cast<std::uint64_t>(slot_size_bytes);
if (stride > (kMaxMappedSize - sizeof(Header)) / capacity) {
throw std::runtime_error("Shared memory ring exceeds 1 TiB for " + name);
}
const auto expected_size = sizeof(Header) + stride * capacity;
if (expected_size > kMaxMappedSize || expected_size > std::numeric_limits<std::size_t>::max()) {
throw std::runtime_error("Shared memory ring geometry overflows for " + name);
}
const auto slots_size = slot_stride * static_cast<std::size_t>(capacity);
if (slots_size > kMax - sizeof(Header)) {
throw std::runtime_error("Shared memory ring geometry overflows for " + name);
}
const auto expected_size = sizeof(Header) + slots_size;
if (mapped_size < expected_size) {
throw std::runtime_error("Shared memory ring size is too small for " + name);
data_acq_and_processing/kamil_adc_collector/include/capture_file_writer.h
+1 -3
View File
@@ -27,9 +27,7 @@ public:
void initialize_live_plot() const;
void initialize_csv(std::size_t channel_count, bool has_di1_trace) const;
void append_csv_packet(const CapturePacket& packet,
double frame_freq_hz,
std::size_t& global_frame_index) const;
void append_csv_packet(const CapturePacket& packet) const;
void update_live_plot(const CapturePacket& packet,
std::size_t packets_seen,
data_acq_and_processing/kamil_adc_collector/src/capture_file_writer.cpp
+41 -13
View File
@@ -234,12 +234,20 @@ std::string csv_spool_path(const std::string& csv_path) {
return replace_extension(csv_path, ".capture_spool.bin");
}
void finish_file(std::ofstream& stream, const std::string& path) {
stream.close();
if (!stream) {
fail_write("Cannot finish writing file: " + path);
}
}
void write_text_file(const std::string& path, const std::string& text) {
std::ofstream file(path, std::ios::binary);
if (!file) {
fail_write("Cannot open file for writing: " + path);
}
file << text;
finish_file(file, path);
}
void write_live_data_script(const std::string& path, const std::string& data_json) {
@@ -584,6 +592,7 @@ void write_live_html_document(const std::string& path, const std::string& data_s
<< " </script>\n"
<< "</body>\n"
<< "</html>\n";
finish_file(html, path);
}
[[noreturn]] void fail_write(const std::string& message) {
@@ -629,11 +638,14 @@ void CaptureFileWriter::initialize_live_plot() const {
}
void CaptureFileWriter::initialize_csv(std::size_t channel_count, bool has_di1_trace) const {
if (channel_count != 1U && channel_count != 2U) {
fail_write("Capture channel count must be 1 or 2");
}
std::ofstream csv_file(csv_path_, std::ios::binary | std::ios::trunc);
if (!csv_file) {
fail_write("Cannot prepare CSV output file: " + csv_path_);
}
csv_file << "";
finish_file(csv_file, csv_path_);
const std::string spool_path = csv_spool_path(csv_path_);
std::ofstream spool(spool_path, std::ios::binary | std::ios::trunc);
@@ -647,16 +659,15 @@ void CaptureFileWriter::initialize_csv(std::size_t channel_count, bool has_di1_t
static_cast<uint32_t>(has_di1_trace ? 1U : 0U)
};
spool.write(reinterpret_cast<const char*>(&header), sizeof(header));
if (!spool) {
fail_write("Cannot initialize CSV spool: " + spool_path);
}
finish_file(spool, spool_path);
}
void CaptureFileWriter::append_csv_packet(const CapturePacket& packet,
double frame_freq_hz,
std::size_t& global_frame_index) const {
(void) frame_freq_hz;
(void) global_frame_index;
void CaptureFileWriter::append_csv_packet(const CapturePacket& packet) const {
if ((packet.channel_count != 1U && packet.channel_count != 2U)
|| (packet.channel_count == 2U && packet.ch1.size() != packet.ch2.size())
|| (packet.has_di1_trace && packet.di1.size() != packet.ch1.size())) {
fail_write("Capture packet channel lengths are inconsistent");
}
const std::string spool_path = csv_spool_path(csv_path_);
std::ofstream spool(spool_path, std::ios::binary | std::ios::app);
@@ -689,9 +700,7 @@ void CaptureFileWriter::append_csv_packet(const CapturePacket& packet,
static_cast<std::streamsize>(frames * sizeof(uint8_t)));
}
}
if (!spool) {
fail_write("Cannot append packet to CSV spool: " + spool_path);
}
finish_file(spool, spool_path);
}
void CaptureFileWriter::update_live_plot(const CapturePacket& packet,
@@ -763,7 +772,7 @@ void CaptureFileWriter::finalize_csv_from_spool(double frame_freq_hz) const {
CsvSpoolPacketHeader packet_header{};
spool.read(reinterpret_cast<char*>(&packet_header), sizeof(packet_header));
if (!spool) {
if (spool.eof()) {
if (spool.eof() && spool.gcount() == 0) {
break;
}
fail_write("Cannot read packet header from CSV spool: " + spool_path);
@@ -773,6 +782,17 @@ void CaptureFileWriter::finalize_csv_from_spool(double frame_freq_hz) const {
const std::size_t channel_count = static_cast<std::size_t>(packet_header.channel_count);
const std::size_t frames = static_cast<std::size_t>(packet_header.frame_count);
const bool has_di1_trace = packet_header.has_di1_trace != 0U;
if (channel_count != 1U && channel_count != 2U) {
fail_write("Invalid channel count in CSV spool: " + spool_path);
}
const auto data_position = spool.tellg();
spool.seekg(0, std::ios::end);
const auto remaining = spool.tellg() - data_position;
spool.seekg(data_position);
const auto bytes_per_frame = channel_count * sizeof(double) + (has_di1_trace ? 1U : 0U);
if (!spool || remaining < 0 || packet_header.frame_count > static_cast<uint64_t>(remaining) / bytes_per_frame) {
fail_write("Truncated or invalid packet data in CSV spool: " + spool_path);
}
if (!header_written) {
csv << "packet_index,frame_index,global_frame_index,time_s,packet_time_s,ch1_v";
@@ -839,6 +859,13 @@ void CaptureFileWriter::finalize_csv_from_spool(double frame_freq_hz) const {
}
csv.flush();
if (!csv) {
fail_write("Cannot flush CSV output; capture spool retained: " + csv_path_);
}
csv.close();
if (!csv) {
fail_write("Cannot close CSV output; capture spool retained: " + csv_path_);
}
std::remove(spool_path.c_str());
}
@@ -1016,4 +1043,5 @@ void CaptureFileWriter::write_svg(const std::vector<CapturePacket>& packets,
<< "\" font-size=\"14\" font-family=\"Segoe UI, Arial, sans-serif\" fill=\"#203040\">DI1</text>\n";
}
file << "</svg>\n";
finish_file(file, svg_path_);
}
data_acq_and_processing/kamil_adc_collector/src/main.cpp
+10 -13
View File
@@ -156,7 +156,8 @@ uint32_t parse_u32(const std::string& text, const std::string& field_name) {
const std::string clean = trim_copy(text);
char* end = nullptr;
const auto value = std::strtoull(clean.c_str(), &end, 0);
if ((end == clean.c_str()) || (*end != '\0') || (value > std::numeric_limits<uint32_t>::max())) {
if ((!clean.empty() && clean.front() == '-') || (end == clean.c_str())
|| (*end != '\0') || (value > std::numeric_limits<uint32_t>::max())) {
fail("Invalid integer for " + field_name + ": " + text);
}
return static_cast<uint32_t>(value);
@@ -191,15 +192,6 @@ uint32_t parse_ipv4(const std::string& text) {
return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3];
}
std::string ipv4_to_string(uint32_t ip_addr) {
std::ostringstream out;
out << ((ip_addr >> 24) & 0xFF) << '.'
<< ((ip_addr >> 16) & 0xFF) << '.'
<< ((ip_addr >> 8) & 0xFF) << '.'
<< (ip_addr & 0xFF);
return out.str();
}
uint32_t parse_range(const std::string& text) {
const double value = parse_double(text, "range");
if (std::fabs(value - 10.0) < 1e-9) {
@@ -2033,8 +2025,14 @@ int run(const Config& cfg) {
}
expect_ok(api, api.StreamsEnable(device.hnd, enabled_streams), "Enable data streams");
const double target_frame_count = (cfg.duration_ms / 1000.0) * actual_frame_freq_hz;
if (!std::isfinite(target_frame_count) || target_frame_count < 0.0
|| target_frame_count >= static_cast<double>(std::numeric_limits<long long>::max())
|| target_frame_count > static_cast<double>(std::numeric_limits<std::size_t>::max())) {
fail("Capture duration and frame clock produce an unsupported frame count");
}
const std::size_t target_frames = std::max<std::size_t>(
1, static_cast<std::size_t>(std::llround((cfg.duration_ms / 1000.0) * actual_frame_freq_hz)));
1, static_cast<std::size_t>(std::llround(target_frame_count)));
std::cout << "Capture settings:\n"
<< " clock source: " << sync_mode_to_string(cfg.sync_mode, false) << "\n"
@@ -2306,7 +2304,6 @@ int run(const Config& cfg) {
if (!fast_tty_avg_stream_mode) {
current_packet.reset(target_frames, cfg.channel_count);
}
std::size_t csv_global_frame_index = 0;
std::size_t packet_avg_steps = 0;
std::size_t fast_packet_frames = 0;
uint64_t packet_clock_count = 0;
@@ -2776,7 +2773,7 @@ int run(const Config& cfg) {
}
std::cout << "\n";
writer->append_csv_packet(packet, actual_frame_freq_hz, csv_global_frame_index);
writer->append_csv_packet(packet);
packets.push_back(std::move(packet));
if ((cfg.svg_history_packets != 0U) && (packets.size() > cfg.svg_history_packets)) {
packets.pop_front();
data_acq_and_processing/kamil_adc_collector/src/tty_protocol_writer.cpp
+22 -3
View File
@@ -196,7 +196,7 @@ TtyProtocolWriter::TtyProtocolWriter(std::string path, std::size_t ring_capacity
: path_(std::move(path)),
impl_(std::make_unique<Impl>(ring_capacity_bytes)) {
if (is_character_device_path(path_)) {
impl_->fd = ::open(path_.c_str(), O_WRONLY | O_NOCTTY);
impl_->fd = ::open(path_.c_str(), O_WRONLY | O_NOCTTY | O_NONBLOCK);
if (impl_->fd < 0) {
throw std::runtime_error(io_error("Cannot open tty output", path_));
}
@@ -233,7 +233,20 @@ TtyProtocolWriter::TtyProtocolWriter(std::string path, std::size_t ring_capacity
}
}
impl_->worker = std::thread([this]() { worker_loop(); });
try {
const auto flags = ::fcntl(impl_->fd, F_GETFL);
if (flags < 0 || ::fcntl(impl_->fd, F_SETFL, flags | O_NONBLOCK) < 0) {
throw std::runtime_error(io_error("Cannot make tty output nonblocking", path_));
}
impl_->worker = std::thread([this]() { worker_loop(); });
} catch (...) {
if (impl_->owns_link) {
::unlink(path_.c_str());
}
close_fd_if_open(impl_->slave_fd);
close_fd_if_open(impl_->fd);
throw;
}
}
TtyProtocolWriter::~TtyProtocolWriter() {
@@ -363,12 +376,12 @@ void TtyProtocolWriter::shutdown() {
impl_->stop_requested = true;
}
close_fd_if_open(impl_->fd);
impl_->data_ready_cv.notify_all();
if (impl_->worker.joinable()) {
impl_->worker.join();
}
close_fd_if_open(impl_->fd);
}
void TtyProtocolWriter::enqueue_frame(uint16_t word0, uint16_t word1, uint16_t word2, uint16_t word3) {
@@ -402,6 +415,12 @@ void TtyProtocolWriter::worker_loop() {
const std::uint8_t* bytes = write_batch.data();
std::size_t remaining = batch_count * kFrameByteCount;
while (remaining != 0U) {
{
std::lock_guard<std::mutex> lock(impl_->mutex);
if (impl_->stop_requested) {
return;
}
}
const ssize_t written = ::write(impl_->fd, bytes, remaining);
if (written < 0) {
if (errno == EINTR) {
data_acq_and_processing/preprocessing/calibration_master/include/channel_bundle_support.hpp
+3
View File
@@ -16,6 +16,9 @@ struct ChannelTrace {
std::vector<ipc::Complex32> samples{};
};
void validate_channel_trace_layout(const ChannelTrace& trace, const std::string& label, const ipc::ComboKey& combo);
[[nodiscard]] auto frequency_axes_match(std::span<const float> left, std::span<const float> right) -> bool;
class S21CalibrationBundle {
public:
void load(const std::string& path);
data_acq_and_processing/preprocessing/calibration_master/src/channel_bundle_support.cpp
+23 -10
View File
@@ -12,6 +12,7 @@
#include <stdexcept>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include "calibrator_interface.hpp"
@@ -59,6 +60,8 @@ void validate_raw_trace_layout(const ipc::SweepTraceBlock& trace, const std::str
}
}
} // namespace
void validate_channel_trace_layout(
const ChannelTrace& trace,
const std::string& trace_label,
@@ -69,6 +72,12 @@ void validate_channel_trace_layout(
trace_label + " frequency/channel vector size mismatch for combo " + combo_to_string(combo)
);
}
for (std::size_t i = 0; i < trace.samples.size(); ++i) {
if (!std::isfinite(trace.frequency_hz[i]) || !std::isfinite(trace.samples[i].re)
|| !std::isfinite(trace.samples[i].im)) {
throw std::runtime_error(trace_label + " contains non-finite values for combo " + combo_to_string(combo));
}
}
}
[[nodiscard]] auto frequency_axes_match(std::span<const float> left, std::span<const float> right) -> bool {
@@ -77,6 +86,9 @@ void validate_channel_trace_layout(
}
for (std::size_t index = 0; index < left.size(); ++index) {
if (!std::isfinite(left[index]) || !std::isfinite(right[index])) {
return false;
}
const auto delta = std::fabs(left[index] - right[index]);
const auto scale = std::max(std::fabs(left[index]), std::fabs(right[index]));
const auto tolerance = std::max(kFrequencyAbsoluteTolerance, scale * kFrequencyRelativeTolerance);
@@ -87,6 +99,8 @@ void validate_channel_trace_layout(
return true;
}
namespace {
void ensure_frequency_axis_match(
std::span<const float> expected,
std::span<const float> actual,
@@ -122,6 +136,9 @@ void ensure_frequency_axis_match(
const std::string& bundle_label
) -> ChannelTrace {
validate_raw_trace_layout(trace, bundle_label + " trace");
if (trace.frequency_hz.empty()) {
throw std::runtime_error(bundle_label + " trace contains no frequency samples");
}
ChannelTrace channel_trace{};
channel_trace.frequency_hz = trace.frequency_hz;
@@ -275,8 +292,6 @@ void ensure_combo_coverage(
} // namespace
void S21CalibrationBundle::load(const std::string& path) {
traces_by_combo_.clear();
if (path.empty()) {
throw std::runtime_error("S21 calibration bundle path must not be empty");
}
@@ -311,8 +326,6 @@ auto S21CalibrationBundle::resolve(const ipc::ComboKey& combo) const -> const Ch
}
void S21ReferenceBundle::load(const std::string& path) {
traces_by_combo_.clear();
if (path.empty()) {
throw std::runtime_error("S21 reference bundle path must not be empty");
}
@@ -337,9 +350,8 @@ void S11CalibrationBundle::load(
const std::string& short_path,
const std::string& load_path
) {
coefficients_by_combo_.clear();
if (open_path.empty() && short_path.empty() && load_path.empty()) {
coefficients_by_combo_.clear();
// #38: S11 calibration disabled because no OSL paths were provided; surface it instead of silently skipping.
std::cerr << "S11 calibration: no open/short/load bundle paths provided; reflection correction disabled\n";
return;
@@ -357,13 +369,15 @@ void S11CalibrationBundle::load(
ensure_combo_coverage(short_traces, open_traces, "S11 short calibration", "S11 open calibration");
ensure_combo_coverage(load_traces, open_traces, "S11 load calibration", "S11 open calibration");
coefficients_by_combo_.reserve(open_traces.size());
decltype(coefficients_by_combo_) coefficients;
coefficients.reserve(open_traces.size());
for (const auto& [combo, open_trace] : open_traces) {
coefficients_by_combo_.insert_or_assign(
coefficients.insert_or_assign(
combo,
solve_osl_coefficients(combo, open_trace, short_traces.at(combo), load_traces.at(combo))
);
}
coefficients_by_combo_ = std::move(coefficients);
}
auto S11CalibrationBundle::is_enabled() const noexcept -> bool {
@@ -422,9 +436,8 @@ auto S11CalibrationBundle::resolve(const ipc::ComboKey& combo) const -> const Co
}
void S11ReferenceBundle::load(const std::string& path) {
traces_by_combo_.clear();
if (path.empty()) {
traces_by_combo_.clear();
// #38: S11 reference disabled because no path was provided; surface it instead of silently skipping.
std::cerr << "S11 reference: no bundle path provided; reference normalization disabled\n";
return;
data_acq_and_processing/preprocessing/data_preprocessor/src/data_preprocessor.cpp
+3 -4
View File
@@ -1,14 +1,13 @@
#include "data_preprocessor.hpp"
#include "runtime_wait.hpp"
#include <algorithm>
#include <chrono>
#include <cmath>
#include <cstdint>
#include <iostream>
#include <span>
#include <stdexcept>
#include <string_view>
#include <thread>
#include <vector>
namespace radar::preprocessing {
@@ -117,7 +116,6 @@ auto DataPreprocessor::try_pop_raw_collection(
ipc::RawSweepCollection* out_collection
) -> bool {
if (!raw_ring_.pop(serialized_raw)) {
std::this_thread::sleep_for(std::chrono::milliseconds(config_.runtime.idle_sleep_ms));
return false;
}
@@ -145,6 +143,7 @@ void DataPreprocessor::run(const std::atomic<bool>& stop_requested) {
while (!stop_requested.load(std::memory_order_relaxed)) {
try {
if (!try_pop_raw_collection(serialized_raw, &raw_collection)) {
(void)config::wait_or_stop(config_.runtime.idle_sleep_ms, stop_requested);
continue;
}
const auto preprocessed_collection = preprocess_collection(raw_collection);
@@ -174,7 +173,7 @@ void DataPreprocessor::run(const std::atomic<bool>& stop_requested) {
// Back-off floored at 1ms (the empty-ring sleep is never reached while a
// persistent error keeps the ring non-empty) so the catch cannot busy-spin
// a CPU core even when idle_sleep_ms is configured to 0.
std::this_thread::sleep_for(std::chrono::milliseconds(std::max<int>(1, config_.runtime.idle_sleep_ms)));
(void)config::wait_or_stop(std::max(1U, config_.runtime.idle_sleep_ms), stop_requested);
}
}
}
data_acq_and_processing/preprocessing/reference_master/src/reference_master.cpp
-33
View File
@@ -1,7 +1,5 @@
#include "reference_master.hpp"
#include <algorithm>
#include <cmath>
#include <span>
#include <stdexcept>
#include <string>
@@ -15,41 +13,10 @@
namespace radar::preprocessing {
namespace {
constexpr float kFrequencyRelativeTolerance = 1e-5F;
constexpr float kFrequencyAbsoluteTolerance = 1e-3F;
[[nodiscard]] auto combo_to_string(const ipc::ComboKey& combo) -> std::string {
return "input=" + std::to_string(combo.input_pos) + " output=" + std::to_string(combo.output_pos);
}
void validate_channel_trace_layout(
const ChannelTrace& trace,
const std::string& trace_label,
const ipc::ComboKey& combo
) {
if (trace.frequency_hz.size() != trace.samples.size()) {
throw std::runtime_error(
trace_label + " frequency/channel vector size mismatch for combo " + combo_to_string(combo)
);
}
}
[[nodiscard]] auto frequency_axes_match(std::span<const float> left, std::span<const float> right) -> bool {
if (left.size() != right.size()) {
return false;
}
for (std::size_t index = 0; index < left.size(); ++index) {
const auto delta = std::fabs(left[index] - right[index]);
const auto scale = std::max(std::fabs(left[index]), std::fabs(right[index]));
const auto tolerance = std::max(kFrequencyAbsoluteTolerance, scale * kFrequencyRelativeTolerance);
if (delta > tolerance) {
return false;
}
}
return true;
}
} // namespace
void ReferenceMaster::load_s21_reference_bundle(const std::string& path) {
data_acq_and_processing/processing/data_processor/include/data_processor.hpp
+1
View File
@@ -54,6 +54,7 @@ class DataProcessor {
void publish_locator(const ipc::ResultCollection& collection, const ProcessingLiveConfig& live_config);
const config::RunConfig& config_;
config::RunConfig processing_config_;
ipc::ShmRing& preprocessed_ring_;
ipc::ShmRing& results_ring_;
ProcessorRegistry processors_{};
data_acq_and_processing/processing/data_processor/include/processing_live_config.hpp
+10 -3
View File
@@ -2,9 +2,12 @@
#include <chrono>
#include <cstdint>
#include <optional>
#include <string>
#include <vector>
#include "run_config.hpp"
namespace radar::processing {
enum class HistoryCommand {
@@ -34,12 +37,14 @@ struct ProcessingLiveConfig {
float pass_through_y_max_db = 0.0F;
std::string bscan_axis = "abs";
std::string bscan_channel = "s21";
float bscan_cut_m = 0.824F;
float bscan_max_depth_m = 1.0F;
float bscan_cut_m = 0.0F;
float bscan_max_depth_m = 8.0F;
float bscan_gain = 1.0F;
float bscan_start_freq_mhz = 100.0F;
float bscan_stop_freq_mhz = 8800.0F;
LegacyGprMode legacy_gpr_mode = LegacyGprMode::Point;
// Absent in legacy live files: use the initial run_config geometry and medium.
std::optional<config::GprConfig> gpr{};
std::vector<std::uint32_t> gpr_input_positions{};
std::vector<std::uint32_t> gpr_output_positions{};
float gpr_min_depth_m = 2.0F;
@@ -108,11 +113,12 @@ struct ProcessingLiveConfig {
bool reprocess_current_result = true;
std::uint64_t history_command_seq = 0;
HistoryCommand history_command = HistoryCommand::None;
std::uint64_t settings_revision = 0;
};
class ProcessingLiveConfigLoader {
public:
explicit ProcessingLiveConfigLoader(std::string path);
ProcessingLiveConfigLoader(std::string path, const config::RunConfig& run_config);
[[nodiscard]] auto current() const -> ProcessingLiveConfig;
[[nodiscard]] auto refresh_if_needed() -> ProcessingLiveConfig;
@@ -122,6 +128,7 @@ class ProcessingLiveConfigLoader {
[[nodiscard]] auto read_file_text() const -> std::string;
std::string path_{};
const config::RunConfig& run_config_;
ProcessingLiveConfig current_{};
std::string last_json_text_{};
bool has_last_json_text_ = false;
data_acq_and_processing/processing/data_processor/src/data_processor.cpp
+30 -17
View File
@@ -1,4 +1,5 @@
#include "data_processor.hpp"
#include "runtime_wait.hpp"
#include <algorithm>
#include <chrono>
@@ -7,7 +8,6 @@
#include <optional>
#include <span>
#include <stdexcept>
#include <thread>
#include <utility>
#include "bscan_processor.hpp"
@@ -27,10 +27,15 @@ constexpr std::size_t kBscanReplayWindow = 50U;
return std::min({preprocessed_capacity, results_capacity, kBscanReplayWindow});
}
void publish_result_collection(const ipc::ResultCollection& collection, ipc::ShmRing& results_ring) {
void publish_result_collection(
const ipc::ResultCollection& collection, ipc::ShmRing& results_ring, std::uint64_t& oversized_results
) {
const auto out_bytes = ipc::serialize_result_collection(collection);
if (!results_ring.push(out_bytes)) {
throw std::runtime_error("Results ring slot is too small for serialized collection");
if (!results_ring.push(out_bytes) && ++oversized_results % 100U == 1U) {
std::cerr << "data_processor: result " << collection.collection_id << " requires " << out_bytes.size()
<< " bytes, results ring slot holds " << results_ring.slot_size_bytes()
<< "; increase rings.results.slot_size_bytes or reduce processing output"
<< " (dropped total=" << oversized_results << ")\n";
}
}
@@ -44,11 +49,12 @@ DataProcessor::DataProcessor(
radar::locator::TcpServer* locator_server
)
: config_(config),
processing_config_(config),
preprocessed_ring_(preprocessed_ring),
results_ring_(results_ring),
processors_(std::move(processors)),
default_processor_mode_(kDefaultProcessorMode),
live_config_loader_(config.runtime.processing_live_config_path),
live_config_loader_(config.runtime.processing_live_config_path, config),
locator_server_(locator_server) {
if (processors_.empty()) {
throw std::runtime_error("DataProcessor requires at least one processor");
@@ -66,6 +72,7 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
std::uint64_t last_applied_history_command_seq = 0;
std::uint64_t error_count = 0;
std::uint64_t consecutive_errors = 0;
std::uint64_t oversized_results = 0;
// track the socket-fed speed used for the last reprocess so a change
// arriving without a live-config revision bump still triggers a reprocess of
// the current result (gated below by reprocess_current_result).
@@ -82,6 +89,9 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
live_config.gpr_speed_m_s = static_cast<float>(*current_socket_speed);
}
const auto live_revision = live_config_loader_.revision();
if (live_revision != last_replayed_revision) {
processing_config_.gpr = live_config.gpr.value_or(config_.gpr);
}
auto& processor = resolve_processor(live_config);
const bool socket_speed_dirty = current_socket_speed != last_reprocessed_socket_speed;
@@ -108,7 +118,7 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
processor,
live_config
);
publish_result_collection(replay_result, results_ring_);
publish_result_collection(replay_result, results_ring_, oversized_results);
publish_locator(replay_result, live_config);
}
} else if (!preprocessed_history.empty()) {
@@ -118,7 +128,7 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
processor,
live_config
);
publish_result_collection(replay_result, results_ring_);
publish_result_collection(replay_result, results_ring_, oversized_results);
publish_locator(replay_result, live_config);
}
last_replayed_revision = live_revision;
@@ -156,7 +166,7 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
processor,
live_config
);
publish_result_collection(result_collection, results_ring_);
publish_result_collection(result_collection, results_ring_, oversized_results);
if (!fed_back) {
publish_locator(result_collection, live_config);
}
@@ -164,14 +174,8 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
continue;
}
std::this_thread::sleep_for(std::chrono::milliseconds(config_.runtime.idle_sleep_ms));
(void)config::wait_or_stop(config_.runtime.idle_sleep_ms, stop_requested);
} catch (const std::exception& exc) {
// A ring-slot-too-small failure is a permanent config error (the slot
// cannot hold a serialized collection): swallowing it would silently
// drop every collection forever, so fail fast and let the operator fix it.
if (std::string(exc.what()).find("slot is too small") != std::string::npos) {
throw;
}
// A single bad collection (torn ring slot, decode/processing error) must
// not kill the long-running processor: drop it and keep going.
++error_count;
@@ -189,7 +193,7 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
}
// Back-off (floored at 1ms even if idle_sleep_ms==0) so a persistent error
// cannot busy-spin a core or flood the log.
std::this_thread::sleep_for(std::chrono::milliseconds(std::max<int>(1, config_.runtime.idle_sleep_ms)));
(void)config::wait_or_stop(std::max(1U, config_.runtime.idle_sleep_ms), stop_requested);
}
}
}
@@ -201,11 +205,20 @@ auto DataProcessor::process_collection(
const ProcessingLiveConfig& live_config
) -> ipc::ResultCollection {
const auto started_at = std::chrono::steady_clock::now();
auto result = processor.process_collection(config_, preprocessed, previous_collections, live_config);
auto result = processor.process_collection(processing_config_, preprocessed, previous_collections, live_config);
const auto finished_at = std::chrono::steady_clock::now();
result.processing_duration_ns = static_cast<std::uint64_t>(
std::chrono::duration_cast<std::chrono::nanoseconds>(finished_at - started_at).count()
);
// Four 16-bit words retain the uint64 exactly in the existing float32 table format.
ipc::ResultPayload revision{};
revision.processing_name = "processing_revision";
revision.kind = ipc::ResultKind::TableF32;
revision.table_columns = 4U;
for (unsigned shift = 0; shift < 64; shift += 16) {
revision.table_values.push_back(static_cast<float>((live_config.settings_revision >> shift) & 0xffffU));
}
result.collection_payloads.push_back(std::move(revision));
return result;
}
data_acq_and_processing/processing/data_processor/src/processing_live_config.cpp
+117 -196
View File
@@ -9,6 +9,7 @@
#include <sstream>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <utility>
#include <nlohmann/json.hpp>
@@ -92,16 +93,55 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s
throw std::runtime_error(field_name + " must be one of: s21, s11");
}
[[nodiscard]] auto parse_f32(const Json& value, const std::string& field_name) -> float {
if (!value.is_number()) {
throw std::runtime_error(field_name + " must be number");
}
const auto number = value.get<double>();
if (!std::isfinite(number) || std::fabs(number) > std::numeric_limits<float>::max()) {
throw std::runtime_error(field_name + " must fit finite float32");
}
return static_cast<float>(number);
}
template <typename T>
void assign_optional(const Json& root, const char* key, T& target) {
const auto found = root.find(key);
if (found == root.end()) {
return;
}
const auto field_name = std::string("processing.") + key;
if constexpr (std::is_same_v<T, float>) {
target = parse_f32(*found, field_name);
} else {
const bool valid = std::is_same_v<T, bool> ? found->is_boolean() : found->is_string();
if (!valid) {
throw std::runtime_error(field_name + " has invalid type");
}
target = found->get<T>();
}
}
[[nodiscard]] auto parse_u64_number(const Json& value, const std::string& field_name) -> std::uint64_t {
if (!value.is_number()) {
throw std::runtime_error(field_name + " must be number");
}
if (value.is_number_unsigned()) {
return value.get<std::uint64_t>();
}
if (value.is_number_integer()) {
const auto integer = value.get<std::int64_t>();
if (integer < 0) {
throw std::runtime_error(field_name + " is out of uint64 range");
}
return static_cast<std::uint64_t>(integer);
}
const double numeric = value.get<double>();
if (numeric < 0.0 || numeric > static_cast<double>(std::numeric_limits<std::uint64_t>::max())) {
if (!std::isfinite(numeric) || numeric < 0.0 || numeric >= static_cast<double>(std::numeric_limits<std::uint64_t>::max())) {
throw std::runtime_error(field_name + " is out of uint64 range");
}
const double rounded = std::round(numeric);
if (std::fabs(numeric - rounded) > 1e-6) {
if (numeric != rounded) {
throw std::runtime_error(field_name + " must be integer");
}
return static_cast<std::uint64_t>(rounded);
@@ -128,7 +168,31 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s
return result;
}
[[nodiscard]] auto parse_live_config(const std::string& json_text, const std::string& path) -> ProcessingLiveConfig {
template <typename Entry>
auto parse_geometry(const Json& value, const char* position_field, std::uint32_t Entry::*position_member)
-> std::vector<Entry> {
if (!value.is_array()) {
throw std::runtime_error("processing geometry must be an array");
}
std::vector<Entry> entries{};
entries.reserve(value.size());
for (const auto& record : value) {
if (!record.is_object()) {
throw std::runtime_error("processing geometry entry must be an object");
}
Entry entry{};
entry.*position_member = parse_u32_number(record.at(position_field), position_field);
entry.x_m = parse_f32(record.at("x_m"), "processing.x_m");
assign_optional(record, "y_m", entry.y_m);
assign_optional(record, "z_m", entry.z_m);
entries.push_back(entry);
}
return entries;
}
[[nodiscard]] auto parse_live_config(
const std::string& json_text, const std::string& path, const config::RunConfig& run_config
) -> ProcessingLiveConfig {
if (json_text.empty()) {
throw std::runtime_error("Processing live config is empty: " + path);
}
@@ -145,12 +209,23 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s
}
ProcessingLiveConfig config{};
if (const auto found = root.find("processor_mode"); found != root.end()) {
if (!found->is_string()) {
throw std::runtime_error("processing.processor_mode must be string");
}
config.processor_mode = found->get<std::string>();
if (const auto found = root.find("settings_revision"); found != root.end()) {
config.settings_revision = parse_u64_number(*found, "processing.settings_revision");
}
if (root.contains("relative_permittivity") || root.contains("tx_geometry") || root.contains("rx_geometry")) {
config.gpr = run_config.gpr;
if (const auto found = root.find("relative_permittivity"); found != root.end()) {
config.gpr->relative_permittivity = parse_f32(*found, "processing.relative_permittivity");
}
if (const auto found = root.find("tx_geometry"); found != root.end()) {
config.gpr->tx_geometry = parse_geometry(*found, "output_pos", &config::GprTxGeometry::output_pos);
}
if (const auto found = root.find("rx_geometry"); found != root.end()) {
config.gpr->rx_geometry = parse_geometry(*found, "input_pos", &config::GprRxGeometry::input_pos);
}
config::validate_gpr_config(*config.gpr, run_config);
}
assign_optional(root, "processor_mode", config.processor_mode);
if (const auto found = root.find("pass_through_channel"); found != root.end()) {
if (!found->is_string()) {
throw std::runtime_error("processing.pass_through_channel must be string");
@@ -158,66 +233,21 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s
config.pass_through_channel =
parse_s_parameter_channel(found->get<std::string>(), "processing.pass_through_channel");
}
if (const auto found = root.find("pass_through_fixed_y_enabled"); found != root.end()) {
if (!found->is_boolean()) {
throw std::runtime_error("processing.pass_through_fixed_y_enabled must be bool");
}
config.pass_through_fixed_y_enabled = found->get<bool>();
}
if (const auto found = root.find("pass_through_y_min_db"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.pass_through_y_min_db must be number");
}
config.pass_through_y_min_db = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("pass_through_y_max_db"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.pass_through_y_max_db must be number");
}
config.pass_through_y_max_db = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("bscan_axis"); found != root.end()) {
if (!found->is_string()) {
throw std::runtime_error("processing.bscan_axis must be string");
}
config.bscan_axis = found->get<std::string>();
}
assign_optional(root, "pass_through_fixed_y_enabled", config.pass_through_fixed_y_enabled);
assign_optional(root, "pass_through_y_min_db", config.pass_through_y_min_db);
assign_optional(root, "pass_through_y_max_db", config.pass_through_y_max_db);
assign_optional(root, "bscan_axis", config.bscan_axis);
if (const auto found = root.find("bscan_channel"); found != root.end()) {
if (!found->is_string()) {
throw std::runtime_error("processing.bscan_channel must be string");
}
config.bscan_channel = parse_s_parameter_channel(found->get<std::string>(), "processing.bscan_channel");
}
if (const auto found = root.find("bscan_cut_m"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.bscan_cut_m must be number");
}
config.bscan_cut_m = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("bscan_max_depth_m"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.bscan_max_depth_m must be number");
}
config.bscan_max_depth_m = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("bscan_gain"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.bscan_gain must be number");
}
config.bscan_gain = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("bscan_start_freq_mhz"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.bscan_start_freq_mhz must be number");
}
config.bscan_start_freq_mhz = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("bscan_stop_freq_mhz"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.bscan_stop_freq_mhz must be number");
}
config.bscan_stop_freq_mhz = static_cast<float>(found->get<double>());
}
assign_optional(root, "bscan_cut_m", config.bscan_cut_m);
assign_optional(root, "bscan_max_depth_m", config.bscan_max_depth_m);
assign_optional(root, "bscan_gain", config.bscan_gain);
assign_optional(root, "bscan_start_freq_mhz", config.bscan_start_freq_mhz);
assign_optional(root, "bscan_stop_freq_mhz", config.bscan_stop_freq_mhz);
if (const auto found = root.find("legacy_gpr_mode"); found != root.end()) {
if (!found->is_string()) {
throw std::runtime_error("processing.legacy_gpr_mode must be string");
@@ -237,36 +267,11 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s
if (const auto found = root.find("gpr_output_positions"); found != root.end()) {
config.gpr_output_positions = parse_u32_array(*found, "processing.gpr_output_positions");
}
if (const auto found = root.find("gpr_min_depth_m"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_min_depth_m must be number");
}
config.gpr_min_depth_m = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_max_depth_m"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_max_depth_m must be number");
}
config.gpr_max_depth_m = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_range_comp_power"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_range_comp_power must be number");
}
config.gpr_range_comp_power = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_object_min_frac"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_object_min_frac must be number");
}
config.gpr_object_min_frac = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_comp_power"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_comp_power must be number");
}
config.gpr_comp_power = static_cast<float>(found->get<double>());
}
assign_optional(root, "gpr_min_depth_m", config.gpr_min_depth_m);
assign_optional(root, "gpr_max_depth_m", config.gpr_max_depth_m);
assign_optional(root, "gpr_range_comp_power", config.gpr_range_comp_power);
assign_optional(root, "gpr_object_min_frac", config.gpr_object_min_frac);
assign_optional(root, "gpr_comp_power", config.gpr_comp_power);
if (const auto found = root.find("gpr_score_mode"); found != root.end()) {
if (!found->is_string()) {
throw std::runtime_error("processing.gpr_score_mode must be string");
@@ -279,30 +284,10 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s
}
config.gpr_motion_mode = parse_gpr_motion_mode(found->get<std::string>(), "processing.gpr_motion_mode");
}
if (const auto found = root.find("gpr_speed_m_s"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_speed_m_s must be number");
}
config.gpr_speed_m_s = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_look_angle_deg"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_look_angle_deg must be number");
}
config.gpr_look_angle_deg = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_direction_sign"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_direction_sign must be number");
}
config.gpr_direction_sign = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_apply_freq_phase_correction"); found != root.end()) {
if (!found->is_boolean()) {
throw std::runtime_error("processing.gpr_apply_freq_phase_correction must be bool");
}
config.gpr_apply_freq_phase_correction = found->get<bool>();
}
assign_optional(root, "gpr_speed_m_s", config.gpr_speed_m_s);
assign_optional(root, "gpr_look_angle_deg", config.gpr_look_angle_deg);
assign_optional(root, "gpr_direction_sign", config.gpr_direction_sign);
assign_optional(root, "gpr_apply_freq_phase_correction", config.gpr_apply_freq_phase_correction);
if (const auto found = root.find("gpr_reference_mode"); found != root.end()) {
if (!found->is_string()) {
throw std::runtime_error("processing.gpr_reference_mode must be string");
@@ -310,70 +295,25 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s
config.gpr_reference_mode =
parse_legacy_gpr_reference_mode(found->get<std::string>(), "processing.gpr_reference_mode");
}
if (const auto found = root.find("gpr_snr_thresh"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_snr_thresh must be number");
}
config.gpr_snr_thresh = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_snr_comp_max"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_snr_comp_max must be number");
}
config.gpr_snr_comp_max = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_start_freq_mhz"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_start_freq_mhz must be number");
}
config.gpr_start_freq_mhz = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_stop_freq_mhz"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_stop_freq_mhz must be number");
}
config.gpr_stop_freq_mhz = static_cast<float>(found->get<double>());
}
if (const auto found = root.find("gpr_background_subtract_enabled"); found != root.end()) {
if (!found->is_boolean()) {
throw std::runtime_error("processing.gpr_background_subtract_enabled must be bool");
}
config.gpr_background_subtract_enabled = found->get<bool>();
}
assign_optional(root, "gpr_snr_thresh", config.gpr_snr_thresh);
assign_optional(root, "gpr_snr_comp_max", config.gpr_snr_comp_max);
assign_optional(root, "gpr_start_freq_mhz", config.gpr_start_freq_mhz);
assign_optional(root, "gpr_stop_freq_mhz", config.gpr_stop_freq_mhz);
assign_optional(root, "gpr_background_subtract_enabled", config.gpr_background_subtract_enabled);
if (const auto found = root.find("gpr_background_mean_count"); found != root.end()) {
config.gpr_background_mean_count = parse_u32_number(*found, "processing.gpr_background_mean_count");
}
if (const auto found = root.find("gpr_remove_sidelobe_objects_enabled"); found != root.end()) {
if (!found->is_boolean()) {
throw std::runtime_error("processing.gpr_remove_sidelobe_objects_enabled must be bool");
}
config.gpr_remove_sidelobe_objects_enabled = found->get<bool>();
}
if (const auto found = root.find("gpr_imaging_plane_y_m"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.gpr_imaging_plane_y_m must be number");
}
config.gpr_imaging_plane_y_m = static_cast<float>(found->get<double>());
}
assign_optional(root, "gpr_remove_sidelobe_objects_enabled", config.gpr_remove_sidelobe_objects_enabled);
assign_optional(root, "gpr_imaging_plane_y_m", config.gpr_imaging_plane_y_m);
for (const auto& [key, target] : {
std::pair{"gpr_visible_x_min_m", &config.gpr_visible_x_min_m},
std::pair{"gpr_visible_x_max_m", &config.gpr_visible_x_max_m},
std::pair{"gpr_visible_z_min_m", &config.gpr_visible_z_min_m},
std::pair{"gpr_visible_z_max_m", &config.gpr_visible_z_max_m},
}) {
if (const auto found = root.find(key); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error(std::string("processing.") + key + " must be number");
}
*target = static_cast<float>(found->get<double>());
}
}
if (const auto found = root.find("legacy_gpr_min_visible_pair_count"); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error("processing.legacy_gpr_min_visible_pair_count must be number");
}
config.legacy_gpr_min_visible_pair_count = static_cast<float>(found->get<double>());
assign_optional(root, key, *target);
}
assign_optional(root, "legacy_gpr_min_visible_pair_count", config.legacy_gpr_min_visible_pair_count);
if (const auto found = root.find("gpr_max_detected_objects_to_draw"); found != root.end()) {
config.gpr_max_detected_objects_to_draw =
parse_u32_number(*found, "processing.gpr_max_detected_objects_to_draw");
@@ -394,36 +334,16 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s
std::pair{"sar3d_bp_ground_depth_m", &config.sar3d_bp_ground_depth_m},
std::pair{"sar3d_comp_range_power", &config.sar3d_comp_range_power},
}) {
if (const auto found = root.find(key); found != root.end()) {
if (!found->is_number()) {
throw std::runtime_error(std::string("processing.") + key + " must be number");
}
*target = static_cast<float>(found->get<double>());
}
assign_optional(root, key, *target);
}
for (const auto& [key, target] : {
std::pair{"sar3d_use_compensation", &config.sar3d_use_compensation},
std::pair{"sar3d_normalize_each_frame", &config.sar3d_normalize_each_frame},
}) {
if (const auto found = root.find(key); found != root.end()) {
if (!found->is_boolean()) {
throw std::runtime_error(std::string("processing.") + key + " must be bool");
}
*target = found->get<bool>();
}
}
if (const auto found = root.find("ignore_socket_speed"); found != root.end()) {
if (!found->is_boolean()) {
throw std::runtime_error("processing.ignore_socket_speed must be bool");
}
config.ignore_socket_speed = found->get<bool>();
}
if (const auto found = root.find("reprocess_current_result"); found != root.end()) {
if (!found->is_boolean()) {
throw std::runtime_error("processing.reprocess_current_result must be bool");
}
config.reprocess_current_result = found->get<bool>();
assign_optional(root, key, *target);
}
assign_optional(root, "ignore_socket_speed", config.ignore_socket_speed);
assign_optional(root, "reprocess_current_result", config.reprocess_current_result);
if (const auto found = root.find("history_command_seq"); found != root.end()) {
config.history_command_seq = parse_u64_number(*found, "processing.history_command_seq");
}
@@ -438,8 +358,9 @@ void apply_legacy_gpr_algorithm_alias(ProcessingLiveConfig& config, const std::s
} // namespace
ProcessingLiveConfigLoader::ProcessingLiveConfigLoader(std::string path)
ProcessingLiveConfigLoader::ProcessingLiveConfigLoader(std::string path, const config::RunConfig& run_config)
: path_(std::move(path)),
run_config_(run_config),
next_check_at_(std::chrono::steady_clock::now()) {
if (path_.empty()) {
throw std::runtime_error("Processing live config path must not be empty");
@@ -472,7 +393,7 @@ auto ProcessingLiveConfigLoader::refresh_if_needed() -> ProcessingLiveConfig {
return current_;
}
current_ = parse_live_config(json_text, path_);
current_ = parse_live_config(json_text, path_, run_config_);
last_json_text_ = json_text;
has_last_json_text_ = true;
++revision_;
data_acq_and_processing/processing/locator/include/locator/tcp_server.hpp
+2 -5
View File
@@ -88,6 +88,7 @@ class ClientSession {
private:
void writer_loop();
void reader_loop(std::atomic<double>& shared_vlc_slot, std::atomic<std::int64_t>& shared_vlc_at_ns);
void finish_loop();
int socket_fd_;
std::string peer_name_;
@@ -148,11 +149,7 @@ class TcpServer {
void acceptor_loop();
void enroll_client(std::unique_ptr<ClientSession> session);
void broadcast_packet(const std::vector<std::uint8_t>& packet);
void reap_finished_clients();
// Best-effort reap from the publish path: acquires clients_mutex_ via try-lock
// (skips if contended) and joins exited sessions outside the lock. Keeps
// clients_ bounded even when no new connections arrive to trigger a reap.
void try_reap_finished_clients();
void reap_finished_clients(bool wait_for_lock = true);
void cache_latest_packet(std::vector<std::uint8_t> packet);
[[nodiscard]] auto latest_packet_copy() const -> std::optional<std::vector<std::uint8_t>>;
data_acq_and_processing/processing/locator/src/payload_builder.cpp
+5 -20
View File
@@ -6,7 +6,9 @@
#include <cstring>
#include <ctime>
#include <iomanip>
#include <limits>
#include <sstream>
#include <stdexcept>
#include <string>
#include <nlohmann/json.hpp>
@@ -96,17 +98,6 @@ constexpr std::uint32_t kMinTableColumns = 3U; // [x_m, z_m, score, ...]
return stream.str();
}
// Wall-clock epoch milliseconds at packet-build time. Unlike the human-readable
// `tim` field (which has no date and rolls over at midnight), this is a
// monotonically increasing generation stamp consumers can use to compute the
// true age of a snapshot and detect a stalled pipeline.
[[nodiscard]] auto epoch_millis_now() -> std::int64_t {
using Clock = std::chrono::system_clock;
return std::chrono::duration_cast<std::chrono::milliseconds>(
Clock::now().time_since_epoch()
).count();
}
void append_u32_little_endian(std::vector<std::uint8_t>& buffer, std::uint32_t value) {
buffer.push_back(static_cast<std::uint8_t>(value & 0xFFU));
buffer.push_back(static_cast<std::uint8_t>((value >> 8U) & 0xFFU));
@@ -200,15 +191,6 @@ auto build_payload_json(
{"obs", std::move(obs_array)},
};
// `gen` is a build-time generation stamp (epoch ms) that lets a consumer compute
// a packet's true age and flag a cached/stalled snapshot — something `tim` cannot
// express, having no date and rolling over at midnight. It is currently NOT sent;
// set this flag to true to re-enable it (epoch_millis_now() is kept ready for that).
constexpr bool kEmitGenerationStamp = false;
if (kEmitGenerationStamp) {
root["gen"] = epoch_millis_now();
}
return root.dump();
}
@@ -216,6 +198,9 @@ auto encode_packet(
const std::string& payload_json,
std::uint32_t device_id
) -> std::vector<std::uint8_t> {
if (payload_json.size() > std::numeric_limits<std::uint32_t>::max()) {
throw std::runtime_error("Locator packet payload exceeds uint32 wire limit");
}
const auto payload_size = static_cast<std::uint32_t>(payload_json.size());
std::vector<std::uint8_t> packet;
packet.reserve(static_cast<std::size_t>(8U) + payload_json.size());
data_acq_and_processing/processing/locator/src/tcp_server.cpp
+91 -92
View File
@@ -225,10 +225,16 @@ void ClientSession::start(
std::atomic<double>& shared_vlc_slot,
std::atomic<std::int64_t>& shared_vlc_at_ns
) {
writer_thread_ = std::thread([this]() { writer_loop(); });
reader_thread_ = std::thread([this, &shared_vlc_slot, &shared_vlc_at_ns]() {
reader_loop(shared_vlc_slot, shared_vlc_at_ns);
});
try {
writer_thread_ = std::thread([this]() { writer_loop(); });
reader_thread_ = std::thread([this, &shared_vlc_slot, &shared_vlc_at_ns]() {
reader_loop(shared_vlc_slot, shared_vlc_at_ns);
});
} catch (...) {
request_stop();
join();
throw;
}
}
void ClientSession::enqueue(std::vector<std::uint8_t> packet) {
@@ -278,14 +284,7 @@ void ClientSession::writer_loop() {
break;
}
}
request_stop();
// Mark exited only once BOTH loops have finished; otherwise a writer-only
// death (e.g. a send timeout while the reader still blocks on recv) would
// never be reaped. request_stop() above shuts the socket so the reader's
// recv() returns promptly.
if (loops_finished_.fetch_add(1, std::memory_order_acq_rel) + 1U == 2U) {
exited_.store(true, std::memory_order_release);
}
finish_loop();
}
void ClientSession::reader_loop(
@@ -295,51 +294,58 @@ void ClientSession::reader_loop(
std::array<std::uint8_t, kPacketHeaderSize> header_buffer{};
std::vector<std::uint8_t> payload_buffer;
while (!stop_requested_.load(std::memory_order_acquire)) {
if (!read_exact(socket_fd_, header_buffer.data(), header_buffer.size(), stop_requested_)) {
break;
}
const auto payload_len = decode_u32_little_endian(header_buffer.data() + 4U);
if (payload_len > max_payload_bytes_) {
log_warning(
"closing client " + peer_name_ + " after payload size "
+ std::to_string(payload_len) + " exceeded the configured limit"
);
break;
}
try {
while (!stop_requested_.load(std::memory_order_acquire)) {
if (!read_exact(socket_fd_, header_buffer.data(), header_buffer.size(), stop_requested_)) {
break;
}
const auto payload_len = decode_u32_little_endian(header_buffer.data() + 4U);
if (payload_len > max_payload_bytes_) {
log_warning(
"closing client " + peer_name_ + " after payload size "
+ std::to_string(payload_len) + " exceeded the configured limit"
);
break;
}
payload_buffer.assign(payload_len, std::uint8_t{0});
if (payload_len > 0U
&& !read_exact(socket_fd_, payload_buffer.data(), payload_len, stop_requested_)) {
break;
}
payload_buffer.assign(payload_len, std::uint8_t{0});
if (payload_len > 0U
&& !read_exact(socket_fd_, payload_buffer.data(), payload_len, stop_requested_)) {
break;
}
try {
const auto json = Json::parse(payload_buffer.begin(), payload_buffer.end());
if (json.is_object()) {
const auto found = json.find("vlc");
if (found != json.end() && found->is_number()) {
const double value = found->get<double>();
if (std::isfinite(value)) {
// Stamp the value first, then the time, so a reader that
// observes a fresh timestamp also observes the matching value.
shared_vlc_slot.store(value, std::memory_order_release);
const auto now_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch()
).count();
shared_vlc_at_ns.store(now_ns, std::memory_order_release);
try {
const auto json = Json::parse(payload_buffer.begin(), payload_buffer.end());
if (json.is_object()) {
const auto found = json.find("vlc");
if (found != json.end() && found->is_number()) {
const double value = found->get<double>();
if (std::isfinite(value) && std::fabs(value) <= std::numeric_limits<float>::max()) {
// Stamp the value first, then the time, so a reader that
// observes a fresh timestamp also observes the matching value.
shared_vlc_slot.store(value, std::memory_order_release);
const auto now_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch()
).count();
shared_vlc_at_ns.store(now_ns, std::memory_order_release);
}
}
}
} catch (const Json::exception& error) {
log_warning(
"ignoring malformed packet from " + peer_name_ + ": " + error.what()
);
}
} catch (const Json::parse_error& error) {
log_warning(
"ignoring malformed packet from " + peer_name_ + ": " + error.what()
);
}
} catch (const std::exception& error) {
log_warning("closing client " + peer_name_ + ": " + error.what());
}
finish_loop();
}
void ClientSession::finish_loop() {
request_stop();
// Mark exited only once BOTH loops have finished (see writer_loop), so a
// reader-only death still waits for the writer before this session is reaped.
// A session is reapable after both threads have stopped using its socket.
if (loops_finished_.fetch_add(1, std::memory_order_acq_rel) + 1U == 2U) {
exited_.store(true, std::memory_order_release);
}
@@ -417,7 +423,13 @@ void TcpServer::start() {
}
listen_fd_ = fd;
acceptor_thread_ = std::thread([this]() { acceptor_loop(); });
try {
acceptor_thread_ = std::thread([this]() { acceptor_loop(); });
} catch (...) {
shutdown_and_close(listen_fd_);
running_.store(false, std::memory_order_release);
throw;
}
}
void TcpServer::stop() {
@@ -427,13 +439,14 @@ void TcpServer::stop() {
if (listen_fd_ >= 0) {
(void)::shutdown(listen_fd_, SHUT_RDWR);
(void)::close(listen_fd_);
listen_fd_ = -1;
}
if (acceptor_thread_.joinable()) {
acceptor_thread_.join();
}
// Keep the descriptor stable until accept() has returned: closing it earlier
// permits fd reuse and races the acceptor's read of listen_fd_.
shutdown_and_close(listen_fd_);
std::vector<std::unique_ptr<ClientSession>> sessions;
{
@@ -466,7 +479,7 @@ void TcpServer::publish(const ipc::ResultCollection& collection, const FilterPar
// Reap exited sessions on the publish path too, so clients_ stays bounded
// even when no new connections arrive to trigger an accept-time reap. Uses a
// try-lock and joins outside the mutex, so publish() never blocks on it.
try_reap_finished_clients();
reap_finished_clients(false);
}
auto TcpServer::latest_socket_speed() const -> std::optional<double> {
@@ -525,19 +538,27 @@ void TcpServer::acceptor_loop() {
reap_finished_clients();
apply_socket_keepalive(client_fd);
auto session = std::make_unique<ClientSession>(
client_fd,
format_peer(peer_addr),
config_.client_queue_size,
config_.max_payload_bytes
);
const auto snapshot = latest_packet_copy();
if (snapshot.has_value()) {
session->enqueue(*snapshot);
std::unique_ptr<ClientSession> session;
bool socket_owned = false;
try {
apply_socket_keepalive(client_fd);
session = std::make_unique<ClientSession>(
client_fd, format_peer(peer_addr), config_.client_queue_size,
config_.max_payload_bytes
);
socket_owned = true;
const auto snapshot = latest_packet_copy();
if (snapshot.has_value()) {
session->enqueue(*snapshot);
}
session->start(latest_socket_speed_, latest_socket_speed_at_ns_);
enroll_client(std::move(session));
} catch (const std::exception& error) {
if (!socket_owned) {
::close(client_fd);
}
log_warning(std::string("cannot start client: ") + error.what());
}
session->start(latest_socket_speed_, latest_socket_speed_at_ns_);
enroll_client(std::move(session));
}
}
@@ -553,35 +574,13 @@ void TcpServer::broadcast_packet(const std::vector<std::uint8_t>& packet) {
}
}
void TcpServer::reap_finished_clients() {
void TcpServer::reap_finished_clients(bool wait_for_lock) {
std::vector<std::unique_ptr<ClientSession>> to_join;
{
std::lock_guard<std::mutex> guard(clients_mutex_);
auto first_dead = std::partition(
clients_.begin(),
clients_.end(),
[](const std::unique_ptr<ClientSession>& session) {
return !session->has_exited();
}
);
for (auto it = first_dead; it != clients_.end(); ++it) {
to_join.push_back(std::move(*it));
}
clients_.erase(first_dead, clients_.end());
}
for (auto& session : to_join) {
session->request_stop();
session->join();
}
}
void TcpServer::try_reap_finished_clients() {
std::vector<std::unique_ptr<ClientSession>> to_join;
{
// try_to_lock: if the acceptor already holds the mutex (it reaps too),
// skip this round rather than block the publish path.
std::unique_lock<std::mutex> guard(clients_mutex_, std::try_to_lock);
if (!guard.owns_lock()) {
std::unique_lock<std::mutex> guard(clients_mutex_, std::defer_lock);
if (wait_for_lock) {
guard.lock();
} else if (!guard.try_lock()) {
return;
}
auto first_dead = std::partition(
data_acq_and_processing/processing/processors/src/gpr_legacy_processor.ipp
+3 -3
View File
@@ -20,10 +20,10 @@
// 5. Optional extended-mode region detection for diffuse reflectors.
//
// Sweep-event model — radar topology matters:
// * Matrix radars (`librevna_multi`, `sn9000`) fire one Tx at a time and
// * Matrix radars (`librevna_multi`) fire one Tx at a time and
// receive on all Rx channels in parallel. A run of 8 traces is just 2
// Tx-events; all (tx_k, *) pairs share one timestamp.
// * Sequential radars (single librevna with switches, kamil_adc, k209…)
// * Sequential radars (single librevna with switches, kamil_adc)
// measure each pair separately. A run of 8 traces is 8 separate sweeps;
// every pair has its own timestamp.
// Both cases reduce to: there are N events in the frame, each event lasts
@@ -283,7 +283,7 @@ enum class LegacyPeakDomain {
}
[[nodiscard]] auto is_matrix_radar_model(const std::string& model) -> bool {
return model == "librevna_multi" || model == "sn9000";
return model == "librevna_multi";
}
// Assign an `event_index` to every selected pair. The mapping depends on the
data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp
+225 -87
View File
@@ -1,19 +1,76 @@
#include "sar3d_gpu.hpp"
#include <algorithm>
#include <bit>
#include <climits>
#include <cmath>
#include <limits>
#include <stdexcept>
#include <string>
#ifdef SAR3D_WITH_CUDA
#include <cuda.h>
#include <nvrtc.h>
#include <algorithm>
#include <climits>
#include <cmath>
#include <string>
#include <type_traits>
#endif
namespace radar::processing::sar3d {
namespace {
constexpr std::size_t kMaxIndexedItems = static_cast<std::size_t>(INT_MAX);
auto checked_product(std::size_t a, std::size_t b, std::size_t limit, const char* what) -> std::size_t {
if (a == 0U || b == 0U || a > limit / b) {
throw std::runtime_error(std::string("sar3d ") + what + " is too large or empty");
}
return a * b;
}
#ifdef SAR3D_WITH_CUDA
auto gaussian_taps(double sigma) -> std::vector<float> {
if (!(sigma > 0.0) || !std::isfinite(sigma) || sigma > 100.0) {
throw std::runtime_error("sar3d smoothing sigma must be between 0 and 100");
}
const int radius = static_cast<int>(4.0 * sigma + 0.5);
std::vector<double> weights;
double total = 0.0;
for (int k = -radius; k <= radius; ++k) {
weights.push_back(std::exp(-0.5 * k * k / (sigma * sigma)));
total += weights.back();
}
std::vector<float> taps;
for (const double weight : weights) {
taps.push_back(static_cast<float>(weight / total));
}
return taps;
}
auto reference_plane(const VolumeRequest& request, std::size_t iy) -> bool {
const double margin = request.norm_edge_margin_m;
if (request.y.size() == 2U || request.y.back() - request.y.front() <= 2.5 * margin) {
return true;
}
return request.y[iy] >= request.y.front() + margin && request.y[iy] <= request.y.back() - margin;
}
void choose_peak(Projections& result, const VolumeRequest& request,
std::span<const float> levels, std::span<const unsigned int> indices = {}) {
for (std::size_t iy = 0U; iy < request.y.size(); ++iy) {
if (!std::isfinite(levels[iy])) {
throw std::runtime_error("sar3d volume contains a non-finite value");
}
if (reference_plane(request, iy) && levels[iy] > result.level) {
result.level = levels[iy];
if (!indices.empty()) {
result.peak = {iy, indices[iy] / request.x.size(), indices[iy] % request.x.size()};
}
}
}
}
#endif
} // namespace
#ifdef SAR3D_WITH_CUDA
namespace {
@@ -39,8 +96,9 @@ extern "C" __global__ void inverse_dft(
const float2* __restrict__ bands, const int points, const int first_bin,
const int size, const int items, float2* __restrict__ ascans)
{
const int item = blockIdx.x * blockDim.x + threadIdx.x;
if (item >= items) return;
const unsigned index = blockIdx.x * blockDim.x + threadIdx.x;
if (index >= (unsigned)items) return;
const int item = (int)index;
const unsigned k = (unsigned)(item % size);
const unsigned mask = (unsigned)size - 1u;
const float2* band = bands + (item / size) * points;
@@ -72,8 +130,9 @@ extern "C" __global__ void backproject(
const float comp_power, const float comp_max,
float* __restrict__ volume)
{
const int voxel = blockIdx.x * blockDim.x + threadIdx.x;
if (voxel >= voxels) return;
const unsigned index = blockIdx.x * blockDim.x + threadIdx.x;
if (index >= (unsigned)voxels) return;
const int voxel = (int)index;
const int row = voxel / nx;
// Scene -> radar axes (sar_scene_to_radar_yz). The depth gate is decided in
// double so voxels on its edge are classified like in the reference.
@@ -87,7 +146,7 @@ extern "C" __global__ void backproject(
const float px = (float)gx[voxel - row * nx];
const float py = (float)(y0 * cos_p + zs * sin_p);
const float pz = (float)depth;
const float last = (float)(samples - 1);
const double last = (double)(samples - 1);
float re = 0.0f, im = 0.0f, weights = 0.0f;
for (int c = 0; c < count; ++c) {
const Contribution k = contributions[c];
@@ -96,7 +155,7 @@ extern "C" __global__ void backproject(
const float rt = sqrtf(dxt * dxt + dyt * dyt + dzt * dzt);
const float rr = sqrtf(dxr * dxr + dyr * dyr + dzr * dzr);
const float position = (rt + rr) * samples_per_metre;
if (position > last) continue;
if (!isfinite(position) || position < 0.0f || (double)position > last) continue;
const int i = min((int)position, samples - 2);
const float frac = position - (float)i;
const float2* row_ascan = ascans + (long long)c * samples;
@@ -111,7 +170,7 @@ extern "C" __global__ void backproject(
im += w * (a.y + frac * (b.y - a.y));
weights += k.weight;
}
volume[voxel] = sqrtf(re * re + im * im) / (weights + 1e-12f);
volume[voxel] = hypotf(re, im) / (weights + 1e-12f);
}
// One pass of scipy.ndimage.gaussian_filter1d (mode 'reflect') along Z or X.
@@ -120,8 +179,9 @@ extern "C" __global__ void smooth(
const float* __restrict__ taps, const int radius,
const int nx, const int nz, const int voxels, const int along_z)
{
const int voxel = blockIdx.x * blockDim.x + threadIdx.x;
if (voxel >= voxels) return;
const unsigned thread = blockIdx.x * blockDim.x + threadIdx.x;
if (thread >= (unsigned)voxels) return;
const int voxel = (int)thread;
const int length = along_z ? nz : nx;
const int index = along_z ? (voxel / nx) % nz : voxel % nx;
const int stride = along_z ? nx : 1;
@@ -134,17 +194,36 @@ extern "C" __global__ void smooth(
dst[voxel] = sum;
}
// dst[o] = max over r of src[(o / inner) * outer_stride + (o % inner) * inner_stride + r * reduce_stride].
// Strides select the displayed axes and the axis reduced to its maximum.
extern "C" __global__ void max_project(
const float* __restrict__ src, float* __restrict__ dst, const int outputs, const int inner,
const int outer_stride, const int inner_stride, const int reduce, const int reduce_stride)
{
const int o = blockIdx.x * blockDim.x + threadIdx.x;
if (o >= outputs) return;
const float* p = src + (o / inner) * outer_stride + (o % inner) * inner_stride;
float m = 0.0f;
for (int r = 0; r < reduce; ++r) m = fmaxf(m, p[r * reduce_stride]);
dst[o] = m;
const unsigned thread = blockIdx.x * blockDim.x + threadIdx.x;
if (thread >= (unsigned)outputs) return;
const int o = (int)thread;
const float* row = src + (o / inner) * outer_stride + (o % inner) * inner_stride;
float maximum = 0.0f;
for (int index = 0; index < reduce; ++index) maximum = fmaxf(maximum, row[index * reduce_stride]);
dst[o] = maximum;
}
// Keep the first joint (y,z,x) maximum. Separate projections cannot locate it.
extern "C" __global__ void peak_planes(
const float* volume, const int plane, const int ny, float* levels, unsigned int* indices)
{
const unsigned thread = blockIdx.x * blockDim.x + threadIdx.x;
if (thread >= (unsigned)ny) return;
const int y = (int)thread;
const float* row = volume + y * plane;
float level = 0.0f;
unsigned int index = 0u;
for (int i = 0; i < plane; ++i) {
if (!isfinite(row[i])) { levels[y] = row[i]; indices[y] = (unsigned int)i; return; }
if (row[i] > level) { level = row[i]; index = (unsigned int)i; }
}
levels[y] = level;
indices[y] = index;
}
// Orthographic max projection. The volume is non-negative, so float bit
@@ -158,8 +237,9 @@ extern "C" __global__ void project_oblique(
const double u0, const double v0, const double inv_pixel, const int nu, const int nv,
unsigned int* __restrict__ image)
{
const int voxel = blockIdx.x * blockDim.x + threadIdx.x;
if (voxel >= voxels) return;
const unsigned index = blockIdx.x * blockDim.x + threadIdx.x;
if (index >= (unsigned)voxels) return;
const int voxel = (int)index;
const float value = volume[voxel];
if (!(value > 0.0f)) return;
const int row = voxel / nx;
@@ -175,11 +255,13 @@ extern "C" __global__ void project_oblique(
constexpr int kBlockSize = 256;
void check(CUresult result, const char* what) {
void check(CUresult result, const char* what, bool device_required = false) {
if (result != CUDA_SUCCESS) {
const char* text = nullptr;
cuGetErrorString(result, &text);
throw std::runtime_error(std::string{"CUDA "} + what + ": " + (text != nullptr ? text : "unknown error"));
const auto message = std::string{"CUDA "} + what + ": " + (text != nullptr ? text : "unknown error");
if (device_required) { throw CudaUnavailable(message); }
throw std::runtime_error(message);
}
}
@@ -299,20 +381,20 @@ template <class T>
template <class... Args>
void launch(CUfunction kernel, int items, Args... args) {
void* parameters[] = {static_cast<void*>(&args)...};
const auto blocks = static_cast<unsigned>((items + kBlockSize - 1) / kBlockSize);
const auto blocks = (static_cast<unsigned>(items) + kBlockSize - 1U) / kBlockSize;
check(cuLaunchKernel(kernel, blocks, 1U, 1U, kBlockSize, 1U, 1U, 0U, nullptr, parameters, nullptr), "launch");
}
} // namespace
struct Gpu::Impl {
struct CudaBackend {
// Declared first, destroyed last: buffers and the module need the context.
struct Context {
CUdevice device = 0;
CUcontext handle = nullptr;
Context() {
check(cuInit(0U), "init");
check(cuDeviceGet(&device, 0), "device");
check(cuInit(0U), "init", true);
check(cuDeviceGet(&device, 0), "device", true);
check(cuDevicePrimaryCtxRetain(&handle, device), "context");
if (const CUresult result = cuCtxSetCurrent(handle); result != CUDA_SUCCESS) {
cuDevicePrimaryCtxRelease(device);
@@ -331,10 +413,11 @@ struct Gpu::Impl {
return kernel;
}
} module{compile_kernels(context.device)};
CUfunction inverse_dft = module.function("inverse_dft");
CUfunction inverse_dft_kernel = module.function("inverse_dft");
CUfunction backproject = module.function("backproject");
CUfunction smooth = module.function("smooth");
CUfunction max_project = module.function("max_project");
CUfunction peak_planes = module.function("peak_planes");
CUfunction project_oblique = module.function("project_oblique");
int smooth_radius = 0;
DeviceBuffer taps;
@@ -345,50 +428,36 @@ struct Gpu::Impl {
DeviceBuffer volume;
DeviceBuffer scratch;
DeviceBuffer images;
DeviceBuffer peak_stats;
~Impl() { cuCtxSetCurrent(context.handle); }
~CudaBackend() { cuCtxSetCurrent(context.handle); }
explicit CudaBackend(std::span<const float> weights) {
smooth_radius = static_cast<int>(weights.size() / 2U);
taps.upload(weights);
}
auto inverse_dft(std::span<const std::complex<float>>, std::size_t, std::size_t, std::size_t) -> std::vector<std::complex<float>>;
auto project(const VolumeRequest&) -> Projections;
};
Gpu::Gpu(double smooth_sigma_voxels) : impl_(std::make_unique<Impl>()) {
// scipy.ndimage.gaussian_filter1d taps: radius int(4 sigma + 0.5), normalized.
const int radius = static_cast<int>(4.0 * smooth_sigma_voxels + 0.5);
std::vector<double> weights{};
for (int k = -radius; k <= radius; ++k) {
weights.push_back(std::exp(-0.5 * k * k / (smooth_sigma_voxels * smooth_sigma_voxels)));
}
double total = 0.0;
for (const double weight : weights) {
total += weight;
}
std::vector<float> taps{};
for (const double weight : weights) {
taps.push_back(static_cast<float>(weight / total));
}
impl_->smooth_radius = radius;
impl_->taps.upload(std::span<const float>(taps));
}
Gpu::~Gpu() = default;
auto Gpu::inverse_dft(
auto CudaBackend::inverse_dft(
std::span<const std::complex<float>> bands,
std::size_t points,
std::size_t first_bin,
std::size_t size
) -> std::vector<std::complex<float>> {
Impl& gpu = *impl_;
CudaBackend& gpu = *this;
check(cuCtxSetCurrent(gpu.context.handle), "context");
const std::size_t samples = bands.size() / points * size;
const int items = checked_int(samples, "A-scan stack");
const CUdeviceptr input = gpu.bands.upload(bands);
const CUdeviceptr output = gpu.ascans.reserve(sizeof(std::complex<float>) * samples);
launch(gpu.inverse_dft, items, input, checked_int(points, "band"), checked_int(first_bin, "first bin"),
launch(gpu.inverse_dft_kernel, items, input, checked_int(points, "band"), checked_int(first_bin, "first bin"),
checked_int(size, "A-scan"), items, output);
return download<std::complex<float>>(output, samples);
}
auto Gpu::project(const VolumeRequest& request) -> Projections {
Impl& gpu = *impl_;
auto CudaBackend::project(const VolumeRequest& request) -> Projections {
CudaBackend& gpu = *this;
check(cuCtxSetCurrent(gpu.context.handle), "context");
const int nx = checked_int(request.x.size(), "grid");
@@ -413,6 +482,12 @@ auto Gpu::project(const VolumeRequest& request) -> Projections {
std::cos(request.pitch_rad), std::sin(request.pitch_rad), request.ground_depth_m,
request.min_depth_m, request.max_depth_m, static_cast<float>(request.samples_per_metre),
static_cast<int>(request.use_compensation), request.comp_range_power, request.comp_weight_max, volume);
const int plane = nz * nx;
const auto stats = gpu.peak_stats.reserve(2U * sizeof(float) * static_cast<std::size_t>(ny));
launch(gpu.peak_planes, ny, volume, plane, ny, stats, stats + sizeof(float) * static_cast<std::size_t>(ny));
Projections raw_peak;
const auto raw_levels = download<float>(stats, static_cast<std::size_t>(ny));
choose_peak(raw_peak, request, raw_levels);
// Separable Gaussian with sigma (0, s, s) over [y][z][x]: Z first, then X, like scipy.
const CUdeviceptr taps = gpu.taps.pointer();
launch(gpu.smooth, voxels, volume, scratch, taps, gpu.smooth_radius, nx, nz, voxels, 1);
@@ -423,49 +498,112 @@ auto Gpu::project(const VolumeRequest& request) -> Projections {
const int side_size = ny * nz;
const int top_size = nz * nx;
const int oblique_size = checked_int(static_cast<std::size_t>(view.nv) * static_cast<std::size_t>(view.nu), "view");
const auto image_floats = static_cast<std::size_t>(front_size + side_size + top_size + oblique_size);
const auto image_floats = static_cast<std::size_t>(front_size) + side_size + top_size + oblique_size + 2U * ny;
const CUdeviceptr front = gpu.images.reserve(sizeof(float) * image_floats);
const CUdeviceptr side = front + sizeof(float) * static_cast<std::size_t>(front_size);
const CUdeviceptr top = side + sizeof(float) * static_cast<std::size_t>(side_size);
const CUdeviceptr oblique = top + sizeof(float) * static_cast<std::size_t>(top_size);
const int plane = nz * nx;
const CUdeviceptr levels = oblique + sizeof(float) * static_cast<std::size_t>(oblique_size);
const CUdeviceptr indices = levels + sizeof(float) * static_cast<std::size_t>(ny);
launch(gpu.peak_planes, ny, volume, plane, ny, levels, indices);
Projections result;
result.raw_level = raw_peak.level;
const auto plane_levels = download<float>(levels, static_cast<std::size_t>(ny));
const auto plane_indices = download<unsigned int>(indices, static_cast<std::size_t>(ny));
choose_peak(result, request, plane_levels, plane_indices);
launch(gpu.max_project, front_size, volume, front, front_size, nx, plane, 1, nz, nx);
launch(gpu.max_project, side_size, volume, side, side_size, nz, plane, nx, nx, 1);
launch(gpu.max_project, top_size, volume, top, top_size, nx, nx, 1, ny, plane);
check(cuMemsetD32(oblique, 0U, static_cast<std::size_t>(oblique_size)), "clear");
launch(gpu.project_oblique, voxels, volume, gx, gy, gz, nx, nz, voxels, view.u[0], view.u[1], view.u[2],
view.v[0], view.v[1], view.v[2], view.u0, view.v0, 1.0 / view.pixel_m, view.nu, view.nv, oblique);
return Projections{
.front = download<float>(front, static_cast<std::size_t>(front_size)),
.side = download<float>(side, static_cast<std::size_t>(side_size)),
.top = download<float>(top, static_cast<std::size_t>(top_size)),
.oblique = download<float>(oblique, static_cast<std::size_t>(oblique_size)),
};
}
#else // !SAR3D_WITH_CUDA
struct Gpu::Impl {};
Gpu::Gpu(double /*smooth_sigma_voxels*/) {
throw std::runtime_error(
"data_processor was built without CUDA (nvrtc.h, cuda.h, libnvrtc or libcuda not found), "
"so sar3d cannot run; install the CUDA requirements and rebuild"
);
}
Gpu::~Gpu() = default;
auto Gpu::inverse_dft(std::span<const std::complex<float>>, std::size_t, std::size_t, std::size_t)
-> std::vector<std::complex<float>> {
return {};
}
auto Gpu::project(const VolumeRequest& /*request*/) -> Projections {
return {};
result.front = download<float>(front, static_cast<std::size_t>(front_size));
result.side = download<float>(side, static_cast<std::size_t>(side_size));
result.top = download<float>(top, static_cast<std::size_t>(top_size));
result.oblique = download<float>(oblique, static_cast<std::size_t>(oblique_size));
return result;
}
#endif
struct Engine::Impl {
#ifdef SAR3D_WITH_CUDA
std::unique_ptr<CudaBackend> cuda;
#endif
};
Engine::Engine(double smooth_sigma_voxels) : impl_(std::make_unique<Impl>()) {
#ifdef SAR3D_WITH_CUDA
impl_->cuda = std::make_unique<CudaBackend>(gaussian_taps(smooth_sigma_voxels));
#else
(void)smooth_sigma_voxels;
throw CudaUnavailable("SAR3D requires an NVIDIA GPU with CUDA; this program was built without "
"nvrtc.h, cuda.h, libnvrtc or libcuda. Install CUDA and rebuild.");
#endif
}
Engine::~Engine() = default;
auto Engine::inverse_dft(std::span<const std::complex<float>> bands, std::size_t points,
std::size_t first_bin, std::size_t size) -> std::vector<std::complex<float>> {
if (points == 0U || bands.empty() || bands.size() % points != 0U || !std::has_single_bit(size) ||
size > kMaxIndexedItems || first_bin >= size || points > size - first_bin) {
throw std::runtime_error("sar3d invalid inverse FFT dimensions");
}
(void)checked_product(bands.size() / points, size, kMaxIndexedItems, "A-scan stack");
#ifdef SAR3D_WITH_CUDA
return impl_->cuda->inverse_dft(bands, points, first_bin, size);
#else
throw CudaUnavailable("SAR3D requires an NVIDIA GPU with CUDA");
#endif
}
auto Engine::project(const VolumeRequest& request) -> Projections {
const auto nx = request.x.size(), ny = request.y.size(), nz = request.z.size();
const auto plane = checked_product(nx, nz, kMaxIndexedItems, "grid plane");
(void)checked_product(plane, ny, kMaxIndexedItems, "grid (32-bit indexing; increase SAR_VOXEL_M)");
const auto& view = request.oblique;
if (nx < 2U || ny < 2U || nz < 2U || request.samples_per_ascan < 2U ||
request.contributions.empty() || request.samples_per_metre <= 0.0 ||
!std::isfinite(request.samples_per_metre) || request.samples_per_metre > std::numeric_limits<float>::max() ||
view.nu <= 0 || view.nv <= 0 || !(view.pixel_m > 0.0) || !std::isfinite(view.pixel_m)) {
throw std::runtime_error("sar3d invalid grid, aperture, A-scan or oblique dimensions");
}
const auto scan_count = checked_product(request.contributions.size(), request.samples_per_ascan,
kMaxIndexedItems, "A-scan stack");
if (request.ascans.size() != scan_count) {
throw std::runtime_error("sar3d A-scan stack does not match the aperture");
}
for (const auto axis : {request.x, request.y, request.z}) {
for (const double coordinate : axis) {
if (!std::isfinite(coordinate) || std::abs(coordinate) > std::numeric_limits<float>::max()) {
throw std::runtime_error("sar3d grid contains an unsupported coordinate");
}
}
}
for (const auto& sample : request.ascans) {
if (!std::isfinite(sample.real()) || !std::isfinite(sample.imag())) {
throw std::runtime_error("sar3d A-scan contains a non-finite sample");
}
}
for (const auto& term : request.contributions) {
if (!std::isfinite(term.weight) || term.weight < 0.0F ||
!std::isfinite(term.inv_reference_ranges) || term.inv_reference_ranges <= 0.0F) {
throw std::runtime_error("sar3d aperture contains an invalid weight or reference range");
}
for (const auto point : {term.tx, term.rx}) {
if (!std::ranges::all_of(point, [](float value) { return std::isfinite(value); })) {
throw std::runtime_error("sar3d aperture contains a non-finite antenna position");
}
}
}
(void)checked_product(static_cast<std::size_t>(view.nu), static_cast<std::size_t>(view.nv),
kMaxIndexedItems, "oblique image");
#ifdef SAR3D_WITH_CUDA
return impl_->cuda->project(request);
#else
throw CudaUnavailable("SAR3D requires an NVIDIA GPU with CUDA");
#endif
}
} // namespace radar::processing::sar3d
data_acq_and_processing/processing/processors/src/sar3d_gpu.hpp
+20 -10
View File
@@ -5,13 +5,19 @@
#include <cstddef>
#include <memory>
#include <span>
#include <stdexcept>
#include <vector>
// GPU half of the sar3d processor: A-scans of the band spectra, and the
// Numerical backend of the sar3d processor: A-scans of the band spectra, and the
// backprojection of an aperture into the 3D volume, smoothed and projected.
// Only the projections of the volume leave the GPU.
// Only the projections and peak leave the backend.
namespace radar::processing::sar3d {
class CudaUnavailable : public std::runtime_error {
public:
using std::runtime_error::runtime_error;
};
// One (aperture frame, Tx/Rx pair) term of the backprojection sum. The layout
// is shared with the CUDA kernel.
struct Contribution {
@@ -51,25 +57,29 @@ struct VolumeRequest {
bool use_compensation = true;
float comp_range_power = 0.0F;
float comp_weight_max = 0.0F;
double norm_edge_margin_m = 0.0;
ObliqueView oblique{};
};
// Max-projections of the smoothed, not yet normalized volume (row-major).
// Maximum projections of the smoothed volume (row-major), before normalization.
struct Projections {
std::vector<float> front; // [y][x], maximum along Z
std::vector<float> side; // [y][z], maximum along X
std::vector<float> top; // [z][x], maximum along Y
std::vector<float> oblique; // [v][u]
float level = 0.0F;
float raw_level = 0.0F;
std::array<std::size_t, 3> peak{}; // joint voxel maximum, [y, z, x]
};
// CUDA context with the kernels compiled for the installed GPU. Construction
// throws when no usable GPU is present or the processor was built without CUDA.
class Gpu {
// CUDA context and runtime-compiled kernels. A missing device raises
// CudaUnavailable; computation and compilation failures remain ordinary errors.
class Engine {
public:
explicit Gpu(double smooth_sigma_voxels);
~Gpu();
Gpu(const Gpu&) = delete;
auto operator=(const Gpu&) -> Gpu& = delete;
explicit Engine(double smooth_sigma_voxels);
~Engine();
Engine(const Engine&) = delete;
auto operator=(const Engine&) -> Engine& = delete;
// numpy.fft.ifft of each band spectrum placed at bins [first_bin, first_bin +
// points) of a `size`-point spectrum; `bands` holds the bands back to back.
data_acq_and_processing/processing/processors/src/sar3d_processor.cpp
+142 -85
View File
@@ -7,11 +7,13 @@
#include <complex>
#include <cstddef>
#include <cstdint>
#include <exception>
#include <iostream>
#include <limits>
#include <map>
#include <memory>
#include <numbers>
#include <set>
#include <span>
#include <stdexcept>
#include <string>
@@ -158,22 +160,26 @@ void select_antennas(
struct TraceClock {
double start_s = 0.0; // sweep start
double event_s = 0.0; // sweep duration / (stages * points)
long long stage = 0;
long long stages = 1;
std::size_t stage = 0U;
std::size_t stages = 1U;
[[nodiscard]] auto sample_time_s(std::size_t point) const -> double {
return start_s + static_cast<double>(stages * static_cast<long long>(point) + stage) * event_s;
return start_s + (static_cast<double>(stages) * point + stage) * event_s;
}
};
struct CollectionTiming {
double center_s = 0.0; // absolute acquisition centre, positions the frame in the aperture
std::uint64_t center_ns = 0U; // integer centre avoids overflow and loss of relative timing
double center_fraction_ns = 0.0;
std::vector<TraceClock> clocks; // per collection trace
};
// Timing from the per-trace capture windows (snapshot_acquisition_timing); every
// trace takes part, so the stages do not depend on the INPUT_IDX/OUTPUT_IDX choice.
[[nodiscard]] auto collection_timing(const ipc::PreprocessedCollection& collection) -> CollectionTiming {
if (collection.traces.empty()) {
throw std::runtime_error("collection has no traces");
}
std::map<std::pair<std::uint64_t, std::uint64_t>, std::vector<std::uint32_t>> sweep_outputs{};
std::uint64_t first = std::numeric_limits<std::uint64_t>::max();
std::uint64_t last = 0U;
@@ -181,6 +187,9 @@ struct CollectionTiming {
if (trace.capture_end_ns <= trace.capture_start_ns) {
throw std::runtime_error("collection has traces without a capture window");
}
if (trace.frequency_hz.empty()) {
throw std::runtime_error("collection has a trace without frequency points");
}
sweep_outputs[{trace.capture_start_ns, trace.capture_end_ns}].push_back(trace.combo.output_pos);
first = std::min(first, trace.capture_start_ns);
last = std::max(last, trace.capture_end_ns);
@@ -191,18 +200,19 @@ struct CollectionTiming {
}
CollectionTiming timing{};
const auto center_x2_ns = static_cast<std::int64_t>(first + last);
timing.center_s = 0.5e-9 * static_cast<double>(center_x2_ns);
const auto duration_ns = last - first;
timing.center_ns = first + duration_ns / 2U;
timing.center_fraction_ns = 0.5 * static_cast<double>(duration_ns % 2U);
for (const auto& trace : collection.traces) {
const auto& stages = sweep_outputs.at({trace.capture_start_ns, trace.capture_end_ns});
const auto stage = std::find(stages.begin(), stages.end(), trace.combo.output_pos) - stages.begin();
TraceClock clock{};
clock.start_s =
0.5e-9 * static_cast<double>(2 * static_cast<std::int64_t>(trace.capture_start_ns) - center_x2_ns);
clock.start_s = 1e-9 * (static_cast<double>(trace.capture_start_ns - first) -
0.5 * static_cast<double>(duration_ns));
clock.stage = stage;
clock.stages = static_cast<long long>(stages.size());
clock.stages = stages.size();
clock.event_s = 1e-9 * static_cast<double>(trace.capture_end_ns - trace.capture_start_ns) /
static_cast<double>(clock.stages * static_cast<long long>(trace.frequency_hz.size()));
static_cast<double>(clock.stages) / trace.frequency_hz.size();
timing.clocks.push_back(clock);
}
return timing;
@@ -226,6 +236,10 @@ struct Band {
}
std::vector<std::size_t> points{};
for (std::size_t point = 0U; point < frequency.size(); ++point) {
if (!std::isfinite(frequency[point]) || frequency[point] < 0.0F ||
(point > 0U && frequency[point] <= frequency[point - 1U])) {
throw std::runtime_error("trace frequencies must be finite, non-negative and increasing");
}
if (frequency[point] >= settings.f_start_hz && frequency[point] <= settings.f_stop_hz) {
points.push_back(point);
}
@@ -242,7 +256,15 @@ struct Band {
if (!(band.df_hz > 0.0)) {
throw std::runtime_error("trace frequencies do not increase");
}
band.first_bin = static_cast<std::size_t>(std::nearbyint(frequency[points.front()] / band.df_hz));
// Bound before converting to an integer or calling bit_ceil. CUDA uses
// 32-bit indexing; pathological sub-Hz steps must not wrap CUDA indices.
constexpr std::size_t max_fft = 1U << 30U; // largest power of two inside CUDA's signed index
const double first_bin = std::nearbyint(frequency[points.front()] / band.df_hz);
if (!std::isfinite(first_bin) || first_bin < 0.0 ||
first_bin + points.size() - 1U > max_fft / (2U * kOversample)) {
throw std::runtime_error("frequency band exceeds the 32-bit CUDA A-scan indexing");
}
band.first_bin = static_cast<std::size_t>(first_bin);
band.size = std::bit_ceil(2U * (band.first_bin + points.size() - 1U)) * kOversample;
// The radar moves SPEED_M_S * cos(LOOK_ANGLE_DEG) along range; its offset
@@ -255,6 +277,9 @@ struct Band {
const std::size_t point = points[index];
const double hann = 0.5 - 0.5 * std::cos(2.0 * pi * static_cast<double>(index) / last);
const std::complex<double> sample(trace.s21[point].re, trace.s21[point].im);
if (!std::isfinite(sample.real()) || !std::isfinite(sample.imag())) {
throw std::runtime_error("S21 contains a non-finite sample");
}
band.spectrum.emplace_back(
sample * std::polar(hann, phase_per_hz_s * frequency[point] * clock.sample_time_s(point))
);
@@ -289,7 +314,8 @@ void normalize_ascan(std::span<std::complex<float>> ascan, double dt_s, const Se
// A-scans of every selected Tx/Rx pair of one collection, (Tx, Rx) order.
struct Frame {
double center_s = 0.0;
std::uint64_t center_ns = 0U;
double center_fraction_ns = 0.0;
double dt_s = 0.0;
std::size_t samples = 0U; // per A-scan
std::vector<std::complex<float>> ascans{}; // [pair][sample]
@@ -298,7 +324,7 @@ struct Frame {
[[nodiscard]] auto compute_frame(
const ipc::PreprocessedCollection& collection,
const Settings& settings,
sar3d::Gpu& gpu
sar3d::Engine& backend
) -> Frame {
const auto timing = collection_timing(collection);
std::vector<Band> bands{};
@@ -327,10 +353,11 @@ struct Frame {
spectra.insert(spectra.end(), band.spectrum.begin(), band.spectrum.end());
}
Frame frame{};
frame.center_s = timing.center_s;
frame.center_ns = timing.center_ns;
frame.center_fraction_ns = timing.center_fraction_ns;
frame.samples = first.size;
frame.dt_s = 1.0 / (static_cast<double>(first.size) * first.df_hz);
frame.ascans = gpu.inverse_dft(spectra, first.spectrum.size(), first.first_bin, first.size);
frame.ascans = backend.inverse_dft(spectra, first.spectrum.size(), first.first_bin, first.size);
if (settings.normalize_each_frame) {
for (std::size_t offset = 0U; offset < frame.ascans.size(); offset += frame.samples) {
normalize_ascan(std::span(frame.ascans).subspan(offset, frame.samples), frame.dt_s, settings);
@@ -350,9 +377,11 @@ struct Frame {
return axis;
}
// Axis with about the given pitch (sar_build_full_region_grid: round(span/pitch)+1 points).
[[nodiscard]] auto grid_axis(double start, double stop, double pitch) -> std::vector<double> {
return linspace(start, stop, static_cast<std::size_t>(std::max(std::nearbyint((stop - start) / pitch) + 1.0, 2.0)));
[[nodiscard]] auto checked_axis_size(double count) -> std::size_t {
if (!std::isfinite(count) || count < 1.0 || count > std::numeric_limits<int>::max()) {
throw std::runtime_error("SAR axis exceeds 32-bit kernel indexing; increase SAR_VOXEL_M");
}
return static_cast<std::size_t>(count);
}
struct Grid {
@@ -367,6 +396,9 @@ struct Grid {
[[nodiscard]] auto scene_grid(const Settings& settings) -> Grid {
const double cos_p = std::cos(settings.look_angle_rad);
const double sin_p = std::sin(settings.look_angle_rad);
if (std::abs(cos_p) < 1e-6) {
throw std::runtime_error("SAR grid is undefined at a radar look angle of 90 degrees");
}
double z_min = std::numeric_limits<double>::infinity();
double z_max = -std::numeric_limits<double>::infinity();
for (const double height : {kSceneYMin, kSceneYMax}) {
@@ -377,11 +409,19 @@ struct Grid {
z_max = std::max(z_max, z);
}
}
return {
grid_axis(kSceneXMin, kSceneXMax, settings.voxel_m),
grid_axis(kSceneYMin, kSceneYMax, settings.voxel_m),
grid_axis(z_min, z_max, settings.voxel_m),
// Match sar_build_full_region_grid and reject oversized products before
// allocating the axes. Every downstream kernel uses signed 32-bit indices.
const auto axis_size = [&](double span) {
return checked_axis_size(std::max(std::nearbyint(span / settings.voxel_m) + 1.0, 2.0));
};
const auto nx = axis_size(kSceneXMax - kSceneXMin);
const auto ny = axis_size(kSceneYMax - kSceneYMin);
const auto nz = axis_size(z_max - z_min);
if (nx > static_cast<std::size_t>(std::numeric_limits<int>::max()) / ny / nz) {
throw std::runtime_error("SAR grid exceeds 32-bit kernel indexing; increase SAR_VOXEL_M");
}
return {linspace(kSceneXMin, kSceneXMax, nx), linspace(kSceneYMin, kSceneYMax, ny),
linspace(z_min, z_max, nz)};
}
// Cosine taper over the aperture: 1 at the centre frame, SAR_EDGE_FRAME_WEIGHT at
@@ -463,8 +503,11 @@ struct Oblique {
}
}
}
view.nu = static_cast<int>(std::lround((u_max - view.u0) / view.pixel_m)) + 1;
view.nv = static_cast<int>(std::lround((v_max - view.v0) / view.pixel_m)) + 1;
view.nu = static_cast<int>(checked_axis_size(std::round((u_max - view.u0) / view.pixel_m) + 1.0));
view.nv = static_cast<int>(checked_axis_size(std::round((v_max - view.v0) / view.pixel_m) + 1.0));
if (view.nu > std::numeric_limits<int>::max() / view.nv) {
throw std::runtime_error("SAR oblique image exceeds 32-bit kernel indexing; increase SAR_VOXEL_M");
}
for (int index = 0; index < view.nu; ++index) {
oblique.u.push_back(view.u0 + index * view.pixel_m);
}
@@ -479,6 +522,11 @@ struct Oblique {
}
[[nodiscard]] auto to_float(const std::array<double, 3>& point) -> std::array<float, 3> {
for (const double coordinate : point) {
if (!std::isfinite(coordinate) || std::abs(coordinate) > std::numeric_limits<float>::max()) {
throw std::runtime_error("SAR antenna motion exceeds the supported coordinate range");
}
}
return {static_cast<float>(point[0]), static_cast<float>(point[1]), static_cast<float>(point[2])};
}
@@ -511,6 +559,15 @@ struct Oblique {
return payload;
}
[[nodiscard]] auto status_payload(float code, const std::string& message) -> ipc::ResultPayload {
std::vector<float> values{code};
for (const unsigned char byte : message) {
values.push_back(byte);
}
const auto columns = static_cast<std::uint32_t>(values.size());
return table_payload("sar3d_status", columns, std::move(values));
}
} // namespace
struct Sar3dProcessor::State {
@@ -521,44 +578,42 @@ struct Sar3dProcessor::State {
return {collection.collection_id, collection.monotonic_ns};
}
// A-scans by sweep for `frames_key`; null marks a sweep that could not be
// processed, so its error is reported once and not retried.
std::map<SweepKey, std::shared_ptr<const Frame>> frames{};
struct CachedFrame {
std::shared_ptr<const Frame> value;
std::string error;
};
std::map<SweepKey, CachedFrame> frames{};
AscanKey frames_key{};
std::unique_ptr<sar3d::Gpu> gpu{};
std::string gpu_error{};
std::unique_ptr<sar3d::Engine> engine{};
std::exception_ptr engine_error{};
std::string last_error{};
std::uint64_t repeated_errors = 0U;
auto frame(const ipc::PreprocessedCollection& collection, const Settings& settings, sar3d::Gpu& gpu)
auto frame(const ipc::PreprocessedCollection& collection, const Settings& settings, sar3d::Engine& backend)
-> std::shared_ptr<const Frame> {
if (const auto found = frames.find(sweep_key(collection)); found != frames.end()) {
return found->second;
if (found->second.value == nullptr) { report_error(found->second.error); }
return found->second.value;
}
std::shared_ptr<const Frame> frame{};
std::string failure;
try {
frame = std::make_shared<const Frame>(compute_frame(collection, settings, gpu));
frame = std::make_shared<const Frame>(compute_frame(collection, settings, backend));
} catch (const std::exception& error) {
report_error(error.what());
failure = error.what();
report_error(failure);
}
frames.emplace(sweep_key(collection), frame);
frames.emplace(sweep_key(collection), CachedFrame{frame, std::move(failure)});
return frame;
}
// Set up once: compiling the kernels takes seconds, so a failure is kept
// instead of being retried on every frame.
auto gpu_engine() -> sar3d::Gpu& {
if (gpu == nullptr && gpu_error.empty()) {
try {
gpu = std::make_unique<sar3d::Gpu>(kSmoothSigma);
} catch (const std::exception& error) {
gpu_error = error.what();
}
auto numerical_engine() -> sar3d::Engine& {
if (engine == nullptr && engine_error == nullptr) {
try { engine = std::make_unique<sar3d::Engine>(kSmoothSigma); }
catch (...) { engine_error = std::current_exception(); }
}
if (gpu == nullptr) {
throw std::runtime_error(gpu_error);
}
return *gpu;
if (engine == nullptr) { std::rethrow_exception(engine_error); }
return *engine;
}
auto render(
@@ -566,23 +621,29 @@ struct Sar3dProcessor::State {
std::span<const ipc::PreprocessedCollection> previous,
const Settings& settings
) -> std::vector<ipc::ResultPayload> {
auto& gpu = gpu_engine();
auto& backend = numerical_engine();
if (settings.ascan_key() != frames_key) {
frames.clear();
frames_key = settings.ascan_key();
}
const auto newest = frame(collection, settings, gpu);
const auto history_count = settings.frame_radius >= previous.size() / 2U + previous.size() % 2U
? previous.size() : 2U * settings.frame_radius;
const auto recent = previous.last(history_count);
std::set<SweepKey> retained{sweep_key(collection)};
for (const auto& member : recent) { retained.insert(sweep_key(member)); }
// Prune even when the newest frame is invalid, so a bad input stream
// cannot grow the failed-frame cache forever.
std::erase_if(frames, [&](const auto& entry) { return !retained.contains(entry.first); });
const auto newest = frame(collection, settings, backend);
if (newest == nullptr) {
return {};
return {status_payload(2.0F, last_error)};
}
// Aperture: the newest 2r+1 collections, centred r collections before
// the newest one, so the image trails the input by r frames. Collections
// that left the aperture are dropped from the cache.
std::vector<std::shared_ptr<const Frame>> aperture{};
std::map<SweepKey, std::shared_ptr<const Frame>> kept{{sweep_key(collection), newest}};
for (const auto& member : previous.last(std::min(previous.size(), 2U * settings.frame_radius))) {
auto member_frame = frame(member, settings, gpu);
kept.emplace(sweep_key(member), member_frame);
for (const auto& member : recent) {
auto member_frame = frame(member, settings, backend);
const bool same_axis = member_frame != nullptr && member_frame->samples == newest->samples &&
member_frame->dt_s == newest->dt_s;
if (same_axis) {
@@ -590,14 +651,16 @@ struct Sar3dProcessor::State {
}
}
aperture.push_back(newest);
frames = std::move(kept);
const std::size_t centre_index =
aperture.size() > settings.frame_radius ? aperture.size() - 1U - settings.frame_radius : 0U;
const Frame& centre = *aperture[centre_index];
std::vector<double> offsets_s{};
for (const auto& member : aperture) {
offsets_s.push_back(member->center_s - centre.center_s);
const double delta_ns = member->center_ns >= centre.center_ns
? static_cast<double>(member->center_ns - centre.center_ns)
: -static_cast<double>(centre.center_ns - member->center_ns);
offsets_s.push_back(1e-9 * (delta_ns + member->center_fraction_ns - centre.center_fraction_ns));
}
const auto weights = frame_weights(offsets_s);
const Grid grid = scene_grid(settings);
@@ -637,8 +700,9 @@ struct Sar3dProcessor::State {
// A Tx+Rx path is convex, so the box corners bound it: samples beyond the
// longest path are never read and are not sent to the GPU.
const double samples_per_metre = 1.0 / (kWaveSpeed * newest->dt_s);
const std::size_t samples =
std::min(newest->samples, static_cast<std::size_t>(std::ceil(longest_path * samples_per_metre)) + 2U);
const double needed = std::ceil(longest_path * samples_per_metre) + 2.0;
const std::size_t samples = needed >= newest->samples
? newest->samples : static_cast<std::size_t>(needed);
std::vector<std::complex<float>> ascans{};
for (const auto& member : aperture) {
const std::span<const std::complex<float>> rows = member->ascans;
@@ -649,7 +713,7 @@ struct Sar3dProcessor::State {
}
const Oblique oblique = oblique_view(grid, settings.voxel_m);
auto images = gpu.project({
auto images = backend.project({
.x = grid.x,
.y = grid.y,
.z = grid.z,
@@ -664,6 +728,7 @@ struct Sar3dProcessor::State {
.use_compensation = settings.use_compensation,
.comp_range_power = settings.comp_range_power,
.comp_weight_max = kCompRangeWeightMax,
.norm_edge_margin_m = kNormEdgeMargin,
.oblique = oblique.view,
});
@@ -671,30 +736,17 @@ struct Sar3dProcessor::State {
// edges, where a constant-range ridge leaves the grid and would set a
// false maximum (sar_norm_reference_mask, sar_finish_volume,
// sar_global_max_index). The level is the volume maximum, so the
// published projections peak at 1.
const std::size_t nx = grid.x.size();
const std::size_t nz = grid.z.size();
float level = 0.0F;
std::size_t peak_x = 0U;
std::size_t peak_y = 0U;
for (std::size_t iy = 0U; iy < grid.y.size(); ++iy) {
if (grid.y[iy] < grid.y.front() + kNormEdgeMargin || grid.y[iy] > grid.y.back() - kNormEdgeMargin) {
continue;
}
for (std::size_t ix = 0U; ix < nx; ++ix) {
if (images.front[iy * nx + ix] > level) {
level = images.front[iy * nx + ix];
peak_x = ix;
peak_y = iy;
}
}
}
const auto side_row = std::span<const float>(images.side).subspan(peak_y * nz, nz);
const auto peak_z = static_cast<std::size_t>(std::ranges::max_element(side_row) - side_row.begin());
const float scale = level > 0.0F ? 1.0F / level : 0.0F;
// Values at excluded Y edges can exceed 1, as in sar_finish_volume.
const float level = images.level;
const auto [peak_y, peak_z, peak_x] = images.peak;
// The notebook normalizes before and after the linear Gaussian pass.
// Combining both factors in double preserves its epsilons and avoids
// float32 reciprocal overflow for a weak, nonzero signal.
const double normalization = static_cast<double>(level) + (images.raw_level + 1e-12) * 1e-12;
for (auto* image : {&images.front, &images.side, &images.top, &images.oblique}) {
for (auto& value : *image) {
value *= scale;
value = static_cast<float>(static_cast<double>(value) / normalization);
if (!std::isfinite(value)) { throw std::runtime_error("SAR normalization exceeds the float32 range"); }
}
}
@@ -707,10 +759,12 @@ struct Sar3dProcessor::State {
basis.insert(basis.end(), oblique.view.v.begin(), oblique.view.v.end());
payloads.push_back(table_payload("sar3d_oblique_basis", 3U, std::move(basis)));
if (level > 0.0F) {
payloads.push_back(table_payload("sar3d_peak", 4U, {level, static_cast<float>(grid.x[peak_x]),
payloads.push_back(table_payload("sar3d_peak", 4U, {static_cast<float>(normalization), static_cast<float>(grid.x[peak_x]),
static_cast<float>(grid.y[peak_y]),
static_cast<float>(grid.z[peak_z])}));
}
payloads.push_back(status_payload(level > 0.0F ? 0.0F : 1.0F,
level > 0.0F ? "CUDA" : "No signal inside the selected depth range"));
return payloads;
}
@@ -748,10 +802,13 @@ auto Sar3dProcessor::process_collection(
try {
results.collection_payloads =
state_->render(collection, previous_collections, read_settings(run_config, live_config));
} catch (const std::exception& error) {
// A data or CUDA error must not stop the data_processor loop: publish an
// empty result and keep going.
} catch (const sar3d::CudaUnavailable& error) {
state_->report_error(error.what());
results.collection_payloads = {status_payload(3.0F, error.what())};
} catch (const std::exception& error) {
// Keep the processing loop alive and make the reason visible in the GUI.
state_->report_error(error.what());
results.collection_payloads = {status_payload(2.0F, error.what())};
}
return results;
}
data_acq_and_processing/sweep_orchestrator/device_drivers/interfaces/radar_driver.hpp
-12
View File
@@ -35,18 +35,6 @@ class RadarDriver {
/** @brief Acquire one sweep containing the forward traces exposed by the driver. */
[[nodiscard]] virtual auto acquire_sweep() -> SweepTrace = 0;
/**
* @brief Announce which switch combo the next `acquire_sweep` belongs to.
*
* Real radars are agnostic to this because the switch state itself decides
* what they see. Mock drivers use it to synthesise per-combo variation so
* downstream plots show eight distinct traces for an eight-combo run
* instead of eight identical curves stacked on top of each other.
*
* Default implementation is a no-op so production drivers do not need to
* override.
*/
virtual void set_active_combo(const ipc::ComboKey& /*combo*/) {}
};
} // namespace radar::drivers
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/compact_m_k209/compact_m_k209_driver.cpp
-414
View File
@@ -1,414 +0,0 @@
#include "../compact_m_k209_driver.hpp"
#include <algorithm>
#include <array>
#include <cctype>
#include <cmath>
#include <cstring>
#include <limits>
#include <sstream>
#include <stdexcept>
#include <utility>
namespace radar::drivers {
namespace {
constexpr std::uint32_t kFloatBytes = 4U;
constexpr std::uint32_t kComplexScalarCount = 2U;
constexpr std::uint64_t kMaxBinaryBlockBytes = 512ULL * 1024ULL * 1024ULL;
[[nodiscard]] auto visa_resource(const std::string& resource) -> ViRsrc {
return reinterpret_cast<ViRsrc>(const_cast<char*>(resource.c_str()));
}
[[nodiscard]] auto visa_buffer(std::uint8_t* data) -> ViBuf {
return reinterpret_cast<ViBuf>(data);
}
[[nodiscard]] auto visa_const_buffer(const char* data) -> ViBuf {
return reinterpret_cast<ViBuf>(const_cast<char*>(data));
}
[[nodiscard]] auto is_line_ending(std::uint8_t value) -> bool {
return value == static_cast<std::uint8_t>('\n') || value == static_cast<std::uint8_t>('\r');
}
[[nodiscard]] auto is_response_separator(std::uint8_t value) -> bool {
return is_line_ending(value) || value == static_cast<std::uint8_t>(';');
}
[[nodiscard]] auto checked_vi_count(std::uint64_t value) -> ViUInt32 {
const auto max_count = static_cast<std::uint64_t>(std::numeric_limits<ViUInt32>::max());
return static_cast<ViUInt32>(std::min(value, max_count));
}
} // namespace
CompactMK209Driver::CompactMK209Driver(CompactMK209DriverSettings settings) : settings_(std::move(settings)) {}
CompactMK209Driver::~CompactMK209Driver() {
try {
close();
} catch (...) {
}
}
void CompactMK209Driver::open() {
if (is_open_) {
return;
}
if (settings_.resource.empty()) {
throw std::runtime_error("K209 VISA resource must not be empty");
}
if (settings_.sweep.points < 2U) {
throw std::runtime_error("K209 sweep points must be >= 2");
}
if (settings_.sweep.stop_hz < settings_.sweep.start_hz) {
throw std::runtime_error("K209 sweep stop_hz must be >= start_hz");
}
if (!(settings_.sweep.if_bandwidth_hz > 0.0F)) {
throw std::runtime_error("K209 IF bandwidth must be > 0");
}
ViStatus status = viOpenDefaultRM(&resource_manager_);
if (status < VI_SUCCESS) {
throw std::runtime_error("Failed to open VISA resource manager: status=" + std::to_string(status));
}
try {
check_status(
viOpen(resource_manager_, visa_resource(settings_.resource), VI_NULL, VI_NULL, &instrument_),
"open K209 VISA resource"
);
check_status(
viSetAttribute(instrument_, VI_ATTR_TMO_VALUE, static_cast<ViAttrState>(settings_.timeout_ms)),
"set K209 VISA timeout"
);
check_status(
viSetAttribute(instrument_, VI_ATTR_TERMCHAR_EN, static_cast<ViAttrState>(VI_FALSE)),
"disable K209 VISA termchar"
);
write_command("*CLS");
configure_device();
} catch (...) {
close();
throw;
}
is_open_ = true;
}
void CompactMK209Driver::close() {
if (instrument_ != VI_NULL) {
viClose(instrument_);
instrument_ = VI_NULL;
}
if (resource_manager_ != VI_NULL) {
viClose(resource_manager_);
resource_manager_ = VI_NULL;
}
is_open_ = false;
}
auto CompactMK209Driver::acquire_sweep() -> SweepTrace {
require_open();
auto traces = query_sweep_trace_pair(settings_.sweep.points);
SweepTrace trace{};
trace.frequency_hz = frequency_hz_;
trace.s11 = complex_trace_from_interleaved(traces.first);
trace.s21 = complex_trace_from_interleaved(traces.second);
return trace;
}
auto CompactMK209Driver::query_identity() -> std::string {
require_open();
return trim_ascii(query_string("*IDN?"));
}
auto CompactMK209Driver::query_system_error() -> std::string {
require_open();
return trim_ascii(query_string("SYST:ERR?"));
}
void CompactMK209Driver::configure_device() {
if (settings_.preset_on_open) {
write_command("SYST:PRES");
expect_operation_complete("*OPC?", "wait for K209 preset");
}
write_command("SENS:FREQ:STAR " + format_frequency(settings_.sweep.start_hz));
write_command("SENS:FREQ:STOP " + format_frequency(settings_.sweep.stop_hz));
write_command("SENS:SWE:POIN " + std::to_string(settings_.sweep.points));
write_command("SENS:SWE:POIN:TIME 0");
write_command("SENS:BAND " + format_frequency(settings_.sweep.if_bandwidth_hz));
write_command("SOUR:POW " + format_power(settings_.sweep.power_dbm));
write_command("SENS:AVER OFF");
write_command("CALC:PAR1:DEF S21");
write_command("CALC:PAR1:SEL");
write_command("FORM:DATA REAL32");
write_command("FORM:BORD SWAP");
write_command("INIT:CONT ON");
write_command("TRIG:SOUR BUS");
expect_operation_complete("*OPC?", "wait for K209 setup");
frequency_hz_ = query_float_array("SENS:FREQ:DATA?", settings_.sweep.points);
}
void CompactMK209Driver::expect_operation_complete(const std::string& command, const std::string& context) {
const auto response = trim_ascii(query_string(command));
if (response != "1") {
throw std::runtime_error(context + " returned unexpected *OPC? response: " + response);
}
}
void CompactMK209Driver::write_command(const std::string& command) {
require_open();
const std::string message = command + "\n";
ViUInt32 written = 0;
check_status(
viWrite(instrument_, visa_const_buffer(message.data()), static_cast<ViUInt32>(message.size()), &written),
"write K209 command: " + command
);
if (written != message.size()) {
throw std::runtime_error("Incomplete K209 command write: " + command);
}
}
auto CompactMK209Driver::query_string(const std::string& command) -> std::string {
write_command(command);
return read_line();
}
auto CompactMK209Driver::query_float_array(
const std::string& command,
std::uint32_t expected_values
) -> std::vector<float> {
write_command(command);
return read_float_array_response(command, expected_values);
}
auto CompactMK209Driver::query_sweep_trace_pair(std::uint32_t points)
-> std::pair<std::vector<float>, std::vector<float>> {
write_command("TRIG:SING;*OPC?;:SENS:DATA:CORR? S11;:SENS:DATA:CORR? S21");
const auto opc_response = read_ascii_token();
if (opc_response != "1") {
throw std::runtime_error("K209 sweep returned unexpected *OPC? response: " + opc_response);
}
return {
read_float_array_response("SENS:DATA:CORR? S11", points * kComplexScalarCount),
read_float_array_response("SENS:DATA:CORR? S21", points * kComplexScalarCount),
};
}
auto CompactMK209Driver::read_float_array_response(
const std::string& context,
std::uint32_t expected_values
) -> std::vector<float> {
auto values = float_array_from_little_endian(read_ieee_block());
if (values.size() != expected_values) {
throw std::runtime_error(
"K209 response for " + context + " returned " + std::to_string(values.size()) +
" float32 values, expected " + std::to_string(expected_values)
);
}
return values;
}
auto CompactMK209Driver::read_ascii_token() -> std::string {
require_open();
std::string token{};
token.reserve(32);
while (true) {
const auto byte = read_byte();
if (is_response_separator(byte)) {
if (!token.empty()) {
return token;
}
continue;
}
token.push_back(static_cast<char>(byte));
if (token.size() > 64U * 1024U) {
throw std::runtime_error("K209 ASCII response token is too long");
}
}
}
auto CompactMK209Driver::read_line() -> std::string {
require_open();
std::string line{};
line.reserve(128);
bool has_content = false;
while (true) {
const auto byte = read_byte();
if (!has_content && is_line_ending(byte)) {
continue;
}
if (byte == static_cast<std::uint8_t>('\n')) {
break;
}
if (byte != static_cast<std::uint8_t>('\r')) {
line.push_back(static_cast<char>(byte));
has_content = true;
}
if (line.size() > 64U * 1024U) {
throw std::runtime_error("K209 ASCII response line is too long");
}
}
return line;
}
auto CompactMK209Driver::read_byte() -> std::uint8_t {
auto bytes = read_exact(1);
return bytes[0];
}
auto CompactMK209Driver::read_exact(std::uint64_t size) -> std::vector<std::uint8_t> {
require_open();
std::vector<std::uint8_t> bytes(size);
std::uint64_t offset = 0;
while (offset < size) {
ViUInt32 transferred = 0;
const auto count = checked_vi_count(size - offset);
check_status(
viRead(instrument_, visa_buffer(bytes.data() + offset), count, &transferred),
"read K209 response bytes"
);
if (transferred == 0U) {
throw std::runtime_error("K209 VISA read returned zero bytes before expected payload was complete");
}
offset += transferred;
}
return bytes;
}
auto CompactMK209Driver::read_ieee_block() -> std::vector<std::uint8_t> {
std::uint8_t marker = read_byte();
while (is_response_separator(marker)) {
marker = read_byte();
}
if (marker != static_cast<std::uint8_t>('#')) {
throw std::runtime_error("K209 binary response does not start with IEEE block marker '#'");
}
const auto digit = read_byte();
if (digit != static_cast<std::uint8_t>('8')) {
throw std::runtime_error("K209 binary response uses unsupported IEEE block header width");
}
const auto size_text_bytes = read_exact(8);
std::uint64_t payload_size = 0;
for (const auto byte : size_text_bytes) {
if (!std::isdigit(static_cast<unsigned char>(byte))) {
throw std::runtime_error("K209 binary response has non-numeric payload size");
}
payload_size = (payload_size * 10ULL) + static_cast<std::uint64_t>(byte - static_cast<std::uint8_t>('0'));
}
if (payload_size == 0U || payload_size > kMaxBinaryBlockBytes) {
throw std::runtime_error("K209 binary response payload size is out of allowed range");
}
if ((payload_size % kFloatBytes) != 0U) {
throw std::runtime_error("K209 binary response payload size is not aligned to float32 values");
}
return read_exact(payload_size);
}
void CompactMK209Driver::check_status(ViStatus status, const std::string& context) const {
if (status >= VI_SUCCESS) {
return;
}
throw std::runtime_error(context + ": " + status_message(status));
}
auto CompactMK209Driver::status_message(ViStatus status) const -> std::string {
std::array<ViChar, 256> description{};
const auto session = instrument_ != VI_NULL ? instrument_ : resource_manager_;
if (session != VI_NULL && viStatusDesc(session, status, description.data()) >= VI_SUCCESS) {
return std::string(description.data()) + " (status=" + std::to_string(status) + ")";
}
return "VISA status=" + std::to_string(status);
}
void CompactMK209Driver::require_open() const {
if (instrument_ == VI_NULL) {
throw std::runtime_error("K209 VISA instrument is not open");
}
}
auto CompactMK209Driver::float_array_from_little_endian(std::vector<std::uint8_t> bytes) -> std::vector<float> {
if ((bytes.size() % kFloatBytes) != 0U) {
throw std::runtime_error("K209 binary float payload is not aligned to 4 bytes");
}
std::vector<float> values(bytes.size() / kFloatBytes, 0.0F);
for (std::size_t index = 0; index < values.size(); ++index) {
const auto offset = index * kFloatBytes;
const std::uint32_t bits = static_cast<std::uint32_t>(bytes[offset]) |
(static_cast<std::uint32_t>(bytes[offset + 1U]) << 8U) |
(static_cast<std::uint32_t>(bytes[offset + 2U]) << 16U) |
(static_cast<std::uint32_t>(bytes[offset + 3U]) << 24U);
std::memcpy(&values[index], &bits, sizeof(float));
}
return values;
}
auto CompactMK209Driver::complex_trace_from_interleaved(const std::vector<float>& values)
-> std::vector<ipc::Complex32> {
if ((values.size() % kComplexScalarCount) != 0U) {
throw std::runtime_error("K209 complex trace payload has odd scalar count");
}
std::vector<ipc::Complex32> trace{};
trace.reserve(values.size() / kComplexScalarCount);
for (std::size_t index = 0; index < values.size(); index += kComplexScalarCount) {
trace.push_back(ipc::Complex32{.re = values[index], .im = values[index + 1U]});
}
return trace;
}
auto CompactMK209Driver::format_frequency(float value_hz) -> std::string {
if (!std::isfinite(value_hz)) {
throw std::runtime_error("K209 frequency value is not finite");
}
std::ostringstream stream;
stream.precision(9);
stream << std::fixed << value_hz;
return stream.str();
}
auto CompactMK209Driver::format_power(float value_dbm) -> std::string {
if (!std::isfinite(value_dbm)) {
throw std::runtime_error("K209 power value is not finite");
}
std::ostringstream stream;
stream.precision(3);
stream << std::fixed << value_dbm;
return stream.str();
}
auto CompactMK209Driver::trim_ascii(std::string value) -> std::string {
const auto first = std::find_if_not(value.begin(), value.end(), [](unsigned char ch) {
return std::isspace(ch) != 0;
});
const auto last = std::find_if_not(value.rbegin(), value.rend(), [](unsigned char ch) {
return std::isspace(ch) != 0;
}).base();
if (first >= last) {
return {};
}
return std::string(first, last);
}
} // namespace radar::drivers
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/compact_m_k209_driver.hpp
-71
View File
@@ -1,71 +0,0 @@
#pragma once
#include <cstdint>
#include <string>
#include <utility>
#include <vector>
#include <visa.h>
#include "radar_driver.hpp"
#include "run_config.hpp"
namespace radar::drivers {
struct CompactMK209DriverSettings {
std::string resource{};
config::RadarSweepSettings sweep{};
std::uint32_t timeout_ms = 20'000;
bool preset_on_open = true;
};
class CompactMK209Driver final : public RadarDriver {
public:
explicit CompactMK209Driver(CompactMK209DriverSettings settings);
~CompactMK209Driver() override;
void open() override;
void close() override;
[[nodiscard]] auto acquire_sweep() -> SweepTrace override;
[[nodiscard]] auto query_identity() -> std::string;
[[nodiscard]] auto query_system_error() -> std::string;
private:
void configure_device();
void expect_operation_complete(const std::string& command, const std::string& context);
void write_command(const std::string& command);
[[nodiscard]] auto query_string(const std::string& command) -> std::string;
[[nodiscard]] auto query_float_array(const std::string& command, std::uint32_t expected_values)
-> std::vector<float>;
[[nodiscard]] auto query_sweep_trace_pair(std::uint32_t points)
-> std::pair<std::vector<float>, std::vector<float>>;
[[nodiscard]] auto read_float_array_response(const std::string& context, std::uint32_t expected_values)
-> std::vector<float>;
[[nodiscard]] auto read_ascii_token() -> std::string;
[[nodiscard]] auto read_line() -> std::string;
[[nodiscard]] auto read_byte() -> std::uint8_t;
[[nodiscard]] auto read_exact(std::uint64_t size) -> std::vector<std::uint8_t>;
[[nodiscard]] auto read_ieee_block() -> std::vector<std::uint8_t>;
void check_status(ViStatus status, const std::string& context) const;
[[nodiscard]] auto status_message(ViStatus status) const -> std::string;
void require_open() const;
[[nodiscard]] static auto float_array_from_little_endian(std::vector<std::uint8_t> bytes) -> std::vector<float>;
[[nodiscard]] static auto complex_trace_from_interleaved(const std::vector<float>& values)
-> std::vector<ipc::Complex32>;
[[nodiscard]] static auto format_frequency(float value_hz) -> std::string;
[[nodiscard]] static auto format_power(float value_dbm) -> std::string;
[[nodiscard]] static auto trim_ascii(std::string value) -> std::string;
CompactMK209DriverSettings settings_{};
std::vector<float> frequency_hz_{};
bool is_open_ = false;
ViSession resource_manager_ = VI_NULL;
ViSession instrument_ = VI_NULL;
};
} // namespace radar::drivers
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna/librevna_minimal_driver_lifecycle.cpp
+35 -203
View File
@@ -3,15 +3,11 @@
#include <algorithm>
#include <array>
#include <chrono>
#include <cmath>
#include <complex>
#include <cstdint>
#include <exception>
#include <random>
#include <string>
#include <string_view>
#include <stdexcept>
#include <thread>
#include <utility>
#include <vector>
@@ -43,60 +39,6 @@ constexpr std::uint32_t kNativeAcquireMaxAttempts = 3U;
);
}
// One synthetic GPR reflector. `range_m` is its physical depth, `reflection`
// is the dimensionless complex reflection coefficient (|Γ| ≤ 1).
struct MockTarget {
float range_m;
float reflection_magnitude;
};
// Three reflectors at GPR-typical depths: a strong near-surface scatterer,
// a mid-depth target, and a weak deeper one. The magnitudes are tuned so the
// summed S21 stays within unit modulus across the sweep band.
constexpr std::array<MockTarget, 3> kMockTargets = {{
{0.45F, 0.55F},
{1.30F, 0.30F},
{2.90F, 0.18F},
}};
// Group velocity in moderately wet soil (≈ c / sqrt(εr), εr ≈ 4). The choice
// is what maps reflector depth to round-trip phase delay; it is held constant
// to keep the simulator deterministic.
constexpr float kGroundVelocityMps = 1.5e8F;
// Soil attenuation grows with frequency. Calibrated so a target at 3 m sees
// roughly −20 dB extra loss at 6 GHz on top of geometric spreading.
constexpr float kAttenuationCoeffPerMeterAtRefHz = 0.22F;
constexpr float kAttenuationReferenceHz = 6e9F;
// Antenna mismatch dominates S11: simulate one shallow reflection right at
// the connector, plus a small amount of cross-coupling from S21 targets.
constexpr float kS11ConnectorReflection = 0.55F;
constexpr float kS11ConnectorRangeM = 0.02F;
constexpr float kS11CrossCouplingFactor = 0.06F;
// Noise floor in linear voltage units. Real LibreVNA hits ~ −90 dB at 1 kHz
// IFBW; pick something a touch noisier so the GPR processor has to work.
constexpr float kNoiseAmplitudeLinear = 0.004F;
// Minimum simulated dwell so 0-point or pathological configs don't busy-loop.
constexpr auto kMockMinimumSweepDuration = std::chrono::microseconds(50);
// Compute the dwell time the mock pretends to spend on the device. Mirrors
// the real LibreVNA contract: per-point dwell ≈ 1 / IFBW. Capped so absurd
// configs (IFBW ≈ 0 or huge sweeps) cannot freeze the producer for hours.
[[nodiscard]] auto mock_target_sweep_duration(const config::RadarSweepSettings& sweep)
-> std::chrono::nanoseconds {
const float points = std::max(1.0F, static_cast<float>(sweep.points));
const float if_bw = std::max(1.0F, sweep.if_bandwidth_hz);
const double seconds = static_cast<double>(points) / static_cast<double>(if_bw);
const auto duration_ns = std::chrono::nanoseconds(
static_cast<std::chrono::nanoseconds::rep>(seconds * 1e9)
);
constexpr auto kHardCap = std::chrono::seconds(5);
return std::clamp<std::chrono::nanoseconds>(duration_ns, kMockMinimumSweepDuration, kHardCap);
}
[[nodiscard]] auto is_retryable_native_acquire_error(std::string_view message) -> bool {
// A stop request must not be retried: drain to the caller immediately (#26).
if (message.find("aborted by stop request") != std::string_view::npos) {
@@ -139,9 +81,7 @@ void LibreVnaMinimalDriver::open() {
throw std::runtime_error("Radar sweep points must be > 0");
}
if (settings_.mode == config::DriverMode::Native) {
open_native();
}
open_native();
is_open_ = true;
}
@@ -151,166 +91,58 @@ void LibreVnaMinimalDriver::close() {
return;
}
if (settings_.mode == config::DriverMode::Native) {
close_native();
}
close_native();
is_open_ = false;
}
void LibreVnaMinimalDriver::set_active_combo(const ipc::ComboKey& combo) {
active_combo_ = combo;
}
auto LibreVnaMinimalDriver::acquire_sweep() -> SweepTrace {
if (!is_open_) {
throw std::runtime_error("Radar driver is not open");
}
++sweep_index_;
switch (settings_.mode) {
case config::DriverMode::Mock:
return acquire_mock();
case config::DriverMode::Native: {
std::exception_ptr last_exception{};
std::string last_message{};
std::exception_ptr last_exception{};
std::string last_message{};
for (std::uint32_t attempt = 1; attempt <= kNativeAcquireMaxAttempts; ++attempt) {
try {
return acquire_native();
} catch (const std::exception& exception) {
last_exception = std::current_exception();
last_message = exception.what();
for (std::uint32_t attempt = 1; attempt <= kNativeAcquireMaxAttempts; ++attempt) {
try {
return acquire_native();
} catch (const std::exception& exception) {
last_exception = std::current_exception();
last_message = exception.what();
if (!is_retryable_native_acquire_error(last_message) || attempt == kNativeAcquireMaxAttempts) {
break;
}
if (detail::native_stop_requested()) {
break; // #26: do not waste reconnect attempts while shutting down.
}
// #31: recover from transient USB/protocol stalls without
// destroying the libusb_context. Release only the interface
// and handle, then reopen against the surviving context so
// we ride out a brief device re-enumeration instead of
// re-initialising libusb from scratch on every retry.
rx_buffer_.clear();
packet_queue_.clear();
if (usb_handle_ != nullptr && interface_claimed_) {
libusb_release_interface(usb_handle_, detail::kUsbInterface);
interface_claimed_ = false;
}
if (usb_handle_ != nullptr) {
libusb_close(usb_handle_);
usb_handle_ = nullptr;
}
protocol_version_ = 0;
device_num_ports_ = 0;
open_native(); // Reuses the surviving usb_context_ and retries discovery.
}
if (!is_retryable_native_acquire_error(last_message) || attempt == kNativeAcquireMaxAttempts) {
break;
}
if (detail::native_stop_requested()) {
break; // #26: do not waste reconnect attempts while shutting down.
}
if (last_exception != nullptr) {
std::rethrow_exception(last_exception);
// #31: recover from transient USB/protocol stalls without
// destroying the libusb_context. Release only the interface
// and handle, then reopen against the surviving context so
// we ride out a brief device re-enumeration instead of
// re-initialising libusb from scratch on every retry.
rx_buffer_.clear();
packet_queue_.clear();
if (usb_handle_ != nullptr && interface_claimed_) {
libusb_release_interface(usb_handle_, detail::kUsbInterface);
interface_claimed_ = false;
}
throw std::runtime_error("Native acquisition failed without exception detail");
if (usb_handle_ != nullptr) {
libusb_close(usb_handle_);
usb_handle_ = nullptr;
}
protocol_version_ = 0;
device_num_ports_ = 0;
open_native(); // Reuses the surviving usb_context_ and retries discovery.
}
default:
throw std::runtime_error("Unsupported radar driver mode");
}
}
auto LibreVnaMinimalDriver::acquire_mock() -> SweepTrace {
// Synthesise a frequency-domain VNA response containing several discrete
// reflectors at known depths, plus light Gaussian noise. The sweep dwells
// for a realistic duration derived from the configured IF bandwidth so
// downstream stages cannot be flooded faster than a real device would
// produce data.
const auto started_at = std::chrono::steady_clock::now();
const auto target_duration = mock_target_sweep_duration(settings_.sweep);
SweepTrace trace{};
trace.frequency_hz.reserve(settings_.sweep.points);
trace.s11.reserve(settings_.sweep.points);
trace.s21.reserve(settings_.sweep.points);
const float span_hz = settings_.sweep.stop_hz - settings_.sweep.start_hz;
const float denominator =
settings_.sweep.points > 1U ? static_cast<float>(settings_.sweep.points - 1U) : 1.0F;
// A tiny per-sweep range drift gives the rendered B-scan a visible motion
// signature so the simulator does not look frozen.
const float range_drift_m =
0.01F * std::sin(0.07F * static_cast<float>(sweep_index_));
// Combo-dependent variation. Without this the mock returns near-identical
// S21 for every (input, output) combo and an eight-combo pass-through plot
// collapses into a single visible trace. The factors below are arbitrary
// but chosen small enough that the overall response stays in a reasonable
// band and large enough that each pair is visually distinct.
const auto input_pos = static_cast<float>(active_combo_.input_pos);
const auto output_pos = static_cast<float>(active_combo_.output_pos);
const float combo_amplitude_gain = 0.55F + 0.08F * input_pos + 0.05F * output_pos;
const float combo_phase_offset = 0.4F * input_pos + 0.9F * output_pos;
const float combo_range_offset_m = 0.05F * input_pos + 0.12F * output_pos;
// Deterministic-per-sweep noise so two consecutive frames look distinct
// but the test stays reproducible for any given sweep index.
std::mt19937 noise_engine(
static_cast<std::uint32_t>(0x9E3779B9ULL ^ sweep_index_)
);
std::normal_distribution<float> noise_dist(0.0F, kNoiseAmplitudeLinear);
for (std::uint32_t point = 0; point < settings_.sweep.points; ++point) {
const float ratio = static_cast<float>(point) / denominator;
const float frequency_hz = settings_.sweep.start_hz + span_hz * ratio;
const float frequency_scale = frequency_hz / kAttenuationReferenceHz;
std::complex<float> s21_total{0.0F, 0.0F};
std::complex<float> s11_total{0.0F, 0.0F};
for (const auto& target : kMockTargets) {
const float range_m = target.range_m + range_drift_m + combo_range_offset_m;
// Round-trip phase: 2π·f·(2R/v).
const float round_trip_phase =
2.0F * detail::kPi * frequency_hz * (2.0F * range_m / kGroundVelocityMps);
// Geometric spreading: 1/(1 + R) keeps near-zero ranges finite.
const float spreading = 1.0F / (1.0F + range_m);
// Frequency-dependent soil attenuation in linear amplitude.
const float attenuation =
std::exp(-kAttenuationCoeffPerMeterAtRefHz * range_m * frequency_scale);
const std::complex<float> contribution = std::polar<float>(
target.reflection_magnitude * spreading * attenuation * combo_amplitude_gain,
-round_trip_phase + combo_phase_offset
);
s21_total += contribution;
s11_total += kS11CrossCouplingFactor * contribution;
}
// Antenna mismatch dominates the near-field S11 response.
const float antenna_phase =
2.0F * detail::kPi * frequency_hz * (2.0F * kS11ConnectorRangeM / kGroundVelocityMps);
s11_total += std::polar<float>(kS11ConnectorReflection, -antenna_phase);
// Independent noise per channel; complex variance ≈ kNoiseAmplitudeLinear².
s21_total += std::complex<float>(noise_dist(noise_engine), noise_dist(noise_engine));
s11_total += std::complex<float>(noise_dist(noise_engine), noise_dist(noise_engine));
trace.frequency_hz.push_back(frequency_hz);
trace.s11.push_back({.re = s11_total.real(), .im = s11_total.imag()});
trace.s21.push_back({.re = s21_total.real(), .im = s21_total.imag()});
}
// Honour the IFBW-derived dwell time. If generation already took longer
// than the simulated device would have needed (huge `points` × CPU jitter)
// we skip the sleep so the producer does not fall further behind.
const auto elapsed = std::chrono::steady_clock::now() - started_at;
if (elapsed < target_duration) {
std::this_thread::sleep_for(target_duration - elapsed);
if (last_exception != nullptr) {
std::rethrow_exception(last_exception);
}
return trace;
throw std::runtime_error("Native acquisition failed without exception detail");
}
auto LibreVnaMinimalDriver::acquire_native() -> SweepTrace {
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/librevna_minimal_driver.hpp
-9
View File
@@ -23,10 +23,8 @@ namespace radar::drivers {
* @brief Runtime settings for minimal LibreVNA acquisition.
*/
struct LibreVnaMinimalDriverSettings {
config::DriverMode mode = config::DriverMode::Mock;
std::string serial{};
config::RadarSweepSettings sweep{};
float mock_signal_hz = 5'000'000.0F;
};
/**
@@ -43,7 +41,6 @@ class LibreVnaMinimalDriver final : public RadarDriver {
void open() override;
void close() override;
[[nodiscard]] auto acquire_sweep() -> SweepTrace override;
void set_active_combo(const ipc::ComboKey& combo) override;
private:
/**
@@ -66,7 +63,6 @@ class LibreVnaMinimalDriver final : public RadarDriver {
bool has_s21 = false;
};
[[nodiscard]] auto acquire_mock() -> SweepTrace;
[[nodiscard]] auto acquire_native() -> SweepTrace;
void open_native();
@@ -92,11 +88,6 @@ class LibreVnaMinimalDriver final : public RadarDriver {
LibreVnaMinimalDriverSettings settings_{};
bool is_open_ = false;
std::uint64_t sweep_index_ = 0;
// Latest combo announced by the orchestrator. Used by the mock backend to
// give each (input, output) pair a slightly different reflectivity profile
// so a multi-combo run does not render as eight identical traces.
ipc::ComboKey active_combo_{};
libusb_context* usb_context_ = nullptr;
libusb_device_handle* usb_handle_ = nullptr;
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209/remote_compact_m_k209_driver.cpp
-271
View File
@@ -1,271 +0,0 @@
#include "../remote_compact_m_k209_driver.hpp"
#include <arpa/inet.h>
#include <netdb.h>
#include <netinet/tcp.h>
#include <sys/socket.h>
#include <unistd.h>
#include <cerrno>
#include <cstring>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
namespace radar::drivers {
namespace {
constexpr std::uint8_t kCommandConfigure = static_cast<std::uint8_t>('C');
constexpr std::uint8_t kCommandAcquire = static_cast<std::uint8_t>('A');
constexpr std::uint8_t kStatusOk = static_cast<std::uint8_t>('O');
constexpr std::uint8_t kStatusError = static_cast<std::uint8_t>('E');
[[nodiscard]] auto system_error_message(const std::string& context) -> std::string {
return context + ": " + std::strerror(errno);
}
void append_u32_be(std::vector<std::uint8_t>& buffer, std::uint32_t value) {
buffer.push_back(static_cast<std::uint8_t>((value >> 24U) & 0xFFU));
buffer.push_back(static_cast<std::uint8_t>((value >> 16U) & 0xFFU));
buffer.push_back(static_cast<std::uint8_t>((value >> 8U) & 0xFFU));
buffer.push_back(static_cast<std::uint8_t>(value & 0xFFU));
}
void append_u64_be(std::vector<std::uint8_t>& buffer, std::uint64_t value) {
for (int shift = 56; shift >= 0; shift -= 8) {
buffer.push_back(static_cast<std::uint8_t>((value >> static_cast<unsigned>(shift)) & 0xFFU));
}
}
void append_double_be(std::vector<std::uint8_t>& buffer, double value) {
std::uint64_t bits = 0;
static_assert(sizeof(bits) == sizeof(value));
std::memcpy(&bits, &value, sizeof(bits));
append_u64_be(buffer, bits);
}
[[nodiscard]] auto parse_u32_be(const std::uint8_t* data) -> std::uint32_t {
return (static_cast<std::uint32_t>(data[0]) << 24U) |
(static_cast<std::uint32_t>(data[1]) << 16U) |
(static_cast<std::uint32_t>(data[2]) << 8U) |
static_cast<std::uint32_t>(data[3]);
}
void send_all(int socket_fd, const void* data, std::size_t size) {
const auto* cursor = static_cast<const std::uint8_t*>(data);
auto remaining = size;
while (remaining > 0U) {
const auto sent = ::send(socket_fd, cursor, remaining, MSG_NOSIGNAL);
if (sent < 0) {
throw std::runtime_error(system_error_message("send to K209 remote server failed"));
}
if (sent == 0) {
throw std::runtime_error("send to K209 remote server returned zero bytes");
}
cursor += sent;
remaining -= static_cast<std::size_t>(sent);
}
}
void recv_exact(int socket_fd, void* data, std::size_t size) {
auto* cursor = static_cast<std::uint8_t*>(data);
auto remaining = size;
while (remaining > 0U) {
const auto received = ::recv(socket_fd, cursor, remaining, 0);
if (received < 0) {
throw std::runtime_error(system_error_message("read from K209 remote server failed"));
}
if (received == 0) {
throw std::runtime_error("K209 remote server closed the connection");
}
cursor += received;
remaining -= static_cast<std::size_t>(received);
}
}
[[nodiscard]] auto recv_u32(int socket_fd) -> std::uint32_t {
std::uint8_t bytes[4]{};
recv_exact(socket_fd, bytes, sizeof(bytes));
return parse_u32_be(bytes);
}
void read_status(int socket_fd) {
std::uint8_t status = 0;
recv_exact(socket_fd, &status, 1);
if (status == kStatusOk) {
return;
}
if (status == kStatusError) {
const auto message_size = recv_u32(socket_fd);
std::string message(message_size, '\0');
if (message_size > 0U) {
recv_exact(socket_fd, message.data(), message.size());
}
throw std::runtime_error("K209 remote server error: " + message);
}
throw std::runtime_error("K209 remote server returned invalid status byte");
}
[[nodiscard]] auto recv_float32_array(int socket_fd, std::uint32_t expected_values, const std::string& context)
-> std::vector<float> {
const auto payload_size = recv_u32(socket_fd);
const auto expected_size = expected_values * static_cast<std::uint32_t>(sizeof(float));
if (payload_size != expected_size) {
throw std::runtime_error(
"K209 remote " + context + " payload has " + std::to_string(payload_size) +
" bytes, expected " + std::to_string(expected_size)
);
}
std::vector<float> values(expected_values);
if (payload_size > 0U) {
recv_exact(socket_fd, values.data(), payload_size);
}
return values;
}
[[nodiscard]] auto connect_socket(const std::string& host, std::uint32_t port, std::uint32_t timeout_ms) -> int {
addrinfo hints{};
hints.ai_family = AF_UNSPEC;
hints.ai_socktype = SOCK_STREAM;
hints.ai_protocol = IPPROTO_TCP;
addrinfo* result = nullptr;
const auto port_text = std::to_string(port);
const auto rc = ::getaddrinfo(host.c_str(), port_text.c_str(), &hints, &result);
if (rc != 0) {
throw std::runtime_error("getaddrinfo failed for K209 remote host " + host + ": " + ::gai_strerror(rc));
}
int connected_fd = -1;
std::string last_error;
for (auto* item = result; item != nullptr; item = item->ai_next) {
const int fd = ::socket(item->ai_family, item->ai_socktype, item->ai_protocol);
if (fd < 0) {
last_error = system_error_message("socket");
continue;
}
const int one = 1;
static_cast<void>(::setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof(one)));
timeval timeout{};
timeout.tv_sec = static_cast<long>(timeout_ms / 1000U);
timeout.tv_usec = static_cast<long>((timeout_ms % 1000U) * 1000U);
static_cast<void>(::setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)));
static_cast<void>(::setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)));
if (::connect(fd, item->ai_addr, item->ai_addrlen) == 0) {
connected_fd = fd;
break;
}
last_error = system_error_message("connect");
::close(fd);
}
::freeaddrinfo(result);
if (connected_fd < 0) {
throw std::runtime_error("Failed to connect to K209 remote server " + host + ":" + port_text + ": " + last_error);
}
return connected_fd;
}
[[nodiscard]] auto complex_from_interleaved(const std::vector<float>& values, std::uint32_t points, const std::string& context)
-> std::vector<ipc::Complex32> {
if (values.size() != static_cast<std::size_t>(points) * 2U) {
throw std::runtime_error("K209 remote " + context + " returned unexpected scalar count");
}
std::vector<ipc::Complex32> output(points);
for (std::uint32_t index = 0; index < points; ++index) {
output[index] = ipc::Complex32{
.re = values[static_cast<std::size_t>(index) * 2U],
.im = values[static_cast<std::size_t>(index) * 2U + 1U],
};
}
return output;
}
} // namespace
RemoteCompactMK209Driver::RemoteCompactMK209Driver(RemoteCompactMK209DriverSettings settings)
: settings_(std::move(settings)) {
if (settings_.host.empty()) {
throw std::runtime_error("K209 remote host must not be empty");
}
if (settings_.port == 0U || settings_.port > 65535U) {
throw std::runtime_error("K209 remote port must be in 1..65535");
}
if (settings_.timeout_ms == 0U) {
throw std::runtime_error("K209 remote timeout_ms must be > 0");
}
}
RemoteCompactMK209Driver::~RemoteCompactMK209Driver() {
try {
close();
} catch (...) {
}
}
void RemoteCompactMK209Driver::open() {
if (socket_fd_ >= 0) {
return;
}
socket_fd_ = connect_socket(settings_.host, settings_.port, settings_.timeout_ms);
try {
std::vector<std::uint8_t> payload{};
payload.reserve(1U + 8U + 8U + 4U + 8U + 8U);
payload.push_back(kCommandConfigure);
append_double_be(payload, static_cast<double>(settings_.sweep.start_hz));
append_double_be(payload, static_cast<double>(settings_.sweep.stop_hz));
append_u32_be(payload, settings_.sweep.points);
append_double_be(payload, static_cast<double>(settings_.sweep.if_bandwidth_hz));
append_double_be(payload, static_cast<double>(settings_.sweep.power_dbm));
send_all(socket_fd_, payload.data(), payload.size());
read_status(socket_fd_);
const auto points = recv_u32(socket_fd_);
if (points != settings_.sweep.points) {
throw std::runtime_error("K209 remote server returned unexpected configured point count");
}
frequency_hz_ = recv_float32_array(socket_fd_, points, "frequency");
} catch (...) {
close();
throw;
}
}
void RemoteCompactMK209Driver::close() {
if (socket_fd_ >= 0) {
::close(socket_fd_);
socket_fd_ = -1;
}
frequency_hz_.clear();
}
auto RemoteCompactMK209Driver::acquire_sweep() -> SweepTrace {
if (socket_fd_ < 0) {
throw std::runtime_error("K209 remote driver is not open");
}
const std::uint8_t command = kCommandAcquire;
send_all(socket_fd_, &command, 1);
read_status(socket_fd_);
const auto points = recv_u32(socket_fd_);
if (points != settings_.sweep.points) {
throw std::runtime_error("K209 remote server returned unexpected sweep point count");
}
const auto s11_values = recv_float32_array(socket_fd_, points * 2U, "S11");
const auto s21_values = recv_float32_array(socket_fd_, points * 2U, "S21");
return SweepTrace{
.frequency_hz = frequency_hz_,
.s11 = complex_from_interleaved(s11_values, points, "S11"),
.s21 = complex_from_interleaved(s21_values, points, "S21"),
};
}
} // namespace radar::drivers
data_acq_and_processing/sweep_orchestrator/device_drivers/radar/remote_compact_m_k209_driver.hpp
-34
View File
@@ -1,34 +0,0 @@
#pragma once
#include <cstdint>
#include <string>
#include <vector>
#include "radar_driver.hpp"
#include "run_config.hpp"
namespace radar::drivers {
struct RemoteCompactMK209DriverSettings {
std::string host = "127.0.0.1";
std::uint32_t port = 50209;
config::RadarSweepSettings sweep{};
std::uint32_t timeout_ms = 20'000;
};
class RemoteCompactMK209Driver final : public RadarDriver {
public:
explicit RemoteCompactMK209Driver(RemoteCompactMK209DriverSettings settings);
~RemoteCompactMK209Driver() override;
void open() override;
void close() override;
[[nodiscard]] auto acquire_sweep() -> SweepTrace override;
private:
RemoteCompactMK209DriverSettings settings_{};
int socket_fd_ = -1;
std::vector<float> frequency_hz_{};
};
} // namespace radar::drivers
data_acq_and_processing/sweep_orchestrator/src/main.cpp
+3 -22
View File
@@ -10,7 +10,6 @@
#include "hmc349a_minimal_driver.hpp"
#include "librevna_minimal_driver.hpp"
#include "radar_driver.hpp"
#include "remote_compact_m_k209_driver.hpp"
#include "run_config.hpp"
#include "shm_ring.hpp"
#include "sweep_orchestrator.hpp"
@@ -20,7 +19,6 @@ namespace {
constexpr const char* kDefaultConfigPath = "run_config.json";
constexpr const char* kLibreVnaModel = "librevna";
constexpr const char* kLibreVnaMultiModel = "librevna_multi";
constexpr const char* kCompactMK209Model = "compact_m_k209";
std::atomic<bool> g_stop_requested{false};
void signal_handler(int /*signal*/) {
@@ -48,26 +46,13 @@ void install_signal_handlers() {
[[nodiscard]] auto make_radar_driver(const radar::config::RadarConfig& config)
-> std::unique_ptr<radar::drivers::RadarDriver> {
if (config.model == kLibreVnaModel || config.model.empty()) {
if (config.driver_mode != radar::config::DriverMode::Native) {
throw std::runtime_error("LibreVNA simulation uses python_app.scripts.matrix_raw_producer; launch through start.sh");
}
return std::make_unique<radar::drivers::LibreVnaMinimalDriver>(
radar::drivers::LibreVnaMinimalDriverSettings{
.mode = config.driver_mode,
.serial = config.serial,
.sweep = config.sweep,
.mock_signal_hz = config.mock_signal_hz,
}
);
}
if (config.model == kCompactMK209Model) {
if (config.driver_mode != radar::config::DriverMode::Native) {
throw std::runtime_error("compact_m_k209 requires radar.driver_mode='native'");
}
return std::make_unique<radar::drivers::RemoteCompactMK209Driver>(
radar::drivers::RemoteCompactMK209DriverSettings{
.host = config.remote_host,
.port = config.remote_port,
.sweep = config.sweep,
.timeout_ms = 20'000,
}
);
}
@@ -76,10 +61,6 @@ void install_signal_handlers() {
throw std::runtime_error("librevna_multi is handled by python_app.scripts.matrix_raw_producer");
}
if (config.model == "sn9000") {
throw std::runtime_error("sn9000 is handled by python_app.scripts.matrix_raw_producer");
}
throw std::runtime_error("Unsupported radar.model for sweep_orchestrator: " + config.model);
}
data_acq_and_processing/sweep_orchestrator/src/sweep_orchestrator.cpp
+22 -52
View File
@@ -1,12 +1,11 @@
#include "sweep_orchestrator.hpp"
#include "runtime_wait.hpp"
#include <algorithm>
#include <chrono>
#include <cstdint>
#include <exception>
#include <iostream>
#include <stdexcept>
#include <thread>
#include <vector>
namespace radar::acq {
@@ -26,28 +25,6 @@ constexpr std::uint64_t kOpenRetryLogEvery = 30U;
return stop_requested.load(std::memory_order_relaxed);
}
void sleep_if_needed_ms(std::uint32_t delay_ms) {
if (delay_ms == 0U) {
return;
}
std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms));
}
// Interruptible sleep: returns true if a stop was requested while waiting.
[[nodiscard]] auto interruptible_sleep_ms(std::uint32_t delay_ms, const std::atomic<bool>& stop_requested) -> bool {
constexpr std::uint32_t kPollMs = 50U;
std::uint32_t waited = 0U;
while (waited < delay_ms) {
if (should_stop(stop_requested)) {
return true;
}
const std::uint32_t chunk = std::min(kPollMs, delay_ms - waited);
std::this_thread::sleep_for(std::chrono::milliseconds(chunk));
waited += chunk;
}
return should_stop(stop_requested);
}
void validate_sweep(const drivers::SweepTrace& sweep) {
if (sweep.frequency_hz.size() != sweep.s11.size() || sweep.frequency_hz.size() != sweep.s21.size()) {
throw std::runtime_error("Radar driver returned inconsistent sweep vectors");
@@ -66,30 +43,14 @@ class DriverLifecycleGuard {
output_switch_driver_(output_switch_driver) {}
~DriverLifecycleGuard() noexcept {
if (!active_) {
return;
}
// Shutdown should never hide the primary error path.
try {
output_switch_driver_.close();
} catch (...) {
}
try {
input_switch_driver_.close();
} catch (...) {
}
try {
radar_driver_.close();
} catch (...) {
}
close_all();
}
void open_all() {
active_ = true; // A later open failure still owns partially opened devices.
radar_driver_.open();
input_switch_driver_.open();
output_switch_driver_.open();
active_ = true;
}
// Open all devices, retrying forever with capped exponential backoff until success or
@@ -118,7 +79,7 @@ class DriverLifecycleGuard {
<< "); retrying up to every " << (kOpenRetryMaxMs / 1'000U)
<< "s until present: " << exc.what() << '\n';
}
if (interruptible_sleep_ms(delay_ms, stop_requested)) {
if (config::wait_or_stop(delay_ms, stop_requested)) {
return false;
}
delay_ms = std::min(delay_ms * 2U, kOpenRetryMaxMs);
@@ -127,14 +88,23 @@ class DriverLifecycleGuard {
return false;
}
void close_all() {
void close_all() noexcept {
if (!active_) {
return;
}
output_switch_driver_.close();
input_switch_driver_.close();
radar_driver_.close();
active_ = false;
const auto close = [](auto& driver) {
try {
driver.close();
} catch (const std::exception& error) {
std::cerr << "sweep_orchestrator: driver cleanup failed: " << error.what() << '\n';
} catch (...) {
std::cerr << "sweep_orchestrator: driver cleanup failed with an unknown error\n";
}
};
close(output_switch_driver_);
close(input_switch_driver_);
close(radar_driver_);
}
private:
@@ -238,7 +208,7 @@ void SweepOrchestrator::run(const std::atomic<bool>& stop_requested) {
if (!config_.runtime.continuous) {
break;
}
sleep_if_needed_ms(config_.runtime.idle_sleep_ms);
(void)config::wait_or_stop(config_.runtime.idle_sleep_ms, stop_requested);
continue;
}
@@ -289,11 +259,11 @@ auto SweepOrchestrator::acquire_one_collection(
// 1) switch output path, 2) switch input path, 3) optional settle.
output_switch_driver_.switch_to(combo.output_pos);
input_switch_driver_.switch_to(combo.input_pos);
sleep_if_needed_ms(config_.runtime.settling_ms);
if (config::wait_or_stop(config_.runtime.settling_ms, stop_requested)) {
interrupted = true;
break;
}
// Production drivers ignore this; mock drivers use it to give every
// (input, output) pair its own synthetic response.
radar_driver_.set_active_combo(combo);
// Stamp around the sweep only: the switch drive and settling above belong to
// neither the previous combo nor this one, so excluding them keeps the window
// an honest "when was this combo actually measured".
data_acq_and_processing/tests/generate_sar3d_emulation.py
+18
View File
@@ -0,0 +1,18 @@
"""Extract production CUDA kernels and host helpers for the test-only adapter."""
from pathlib import Path
import re
import sys
source = (Path(__file__).resolve().parents[1] / "processing/processors/src/sar3d_gpu.cpp").read_text()
kernel = re.search(r'kKernelSource\s*=\s*R"CUDA\((.*?)\)CUDA";', source, re.DOTALL)
if kernel is None:
raise RuntimeError("SAR CUDA kernel literal was not found")
first = source.index("constexpr std::size_t kMaxIndexedItems")
last = source.index("} // namespace", first)
destination = Path(sys.argv[1])
destination.mkdir(parents=True, exist_ok=True)
(destination / "sar3d_kernels.inc").write_text(kernel.group(1))
helpers = re.sub(r"^#(?:ifdef SAR3D_WITH_CUDA|endif)$", "", source[first:last], flags=re.MULTILINE)
(destination / "sar3d_helpers.inc").write_text(helpers)
data_acq_and_processing/tests/native_regressions.cpp
+398
View File
@@ -0,0 +1,398 @@
#include "capture_file_writer.h"
#include "channel_bundle_support.hpp"
#include "locator/tcp_server.hpp"
#include "processing_live_config.hpp"
#include "runtime_wait.hpp"
#include "shm_ring.hpp"
#include "sweep_orchestrator.hpp"
#include "tty_protocol_writer.h"
#include <sys/mman.h>
#include <sys/socket.h>
#include <sys/wait.h>
#include <unistd.h>
#include <array>
#include <atomic>
#include <chrono>
#include <cstring>
#include <filesystem>
#include <fstream>
#include <iostream>
#include <limits>
#include <random>
#include <stdexcept>
#include <thread>
#include <nlohmann/json.hpp>
namespace {
using namespace radar;
using Json = nlohmann::json;
void require(bool condition, const char* message) {
if (!condition) {
throw std::runtime_error(message);
}
}
template <typename Callable>
void expect_error(Callable operation) {
try {
operation();
} catch (const std::bad_alloc&) {
throw; // A malformed count must be rejected before allocating.
} catch (const std::exception&) {
return;
}
throw std::runtime_error("Expected operation to reject malformed input");
}
struct Workspace {
std::filesystem::path path;
Workspace() {
char pattern[] = "/tmp/radar-native-XXXXXX";
const auto* created = ::mkdtemp(pattern);
require(created != nullptr, "mkdtemp failed");
path = created;
}
~Workspace() { std::filesystem::remove_all(path); }
};
struct RingName {
std::string name = "/radar_native_test_" + std::to_string(::getpid());
RingName() { ipc::ShmRing::unlink_ring(name); }
~RingName() { ipc::ShmRing::unlink_ring(name); }
};
void write_u32(std::vector<std::uint8_t>& bytes, std::size_t offset, std::uint32_t value) {
require(offset + sizeof(value) <= bytes.size(), "invalid test offset");
std::memcpy(bytes.data() + offset, &value, sizeof(value));
}
void test_wire_decoders() {
ipc::RawSweepCollection raw{.collection_id = 7, .monotonic_ns = 42,
.capture_start_ns = 21, .capture_end_ns = 40,
.traces = {{.combo = {1, 2}, .frequency_hz = {1, 2},
.s11 = {{1, 2}, {3, 4}}, .s21 = {{5, 6}, {7, 8}},
.capture_start_ns = 25, .capture_end_ns = 39}}};
ipc::ResultCollection results{.collection_id = 7, .processing_duration_ns = 17,
.collection_payloads = {
{.kind = ipc::ResultKind::TraceComplex, .frequency_hz = {1}, .trace = {{2, 3}}},
{.kind = ipc::ResultKind::ScalarF32, .scalar_value = 4},
{.kind = ipc::ResultKind::ImageF32, .image_x_axis = {1, 2},
.image_y_axis = {3}, .image_values = {4, 5}},
{.kind = ipc::ResultKind::TableF32, .table_columns = 2, .table_values = {6, 7}},
{.kind = ipc::ResultKind::ImageF32, .image_y_axis = {1}},
{.kind = ipc::ResultKind::TableF32, .table_columns = 3}},
.blocks = {{.combo = {1, 2}, .payloads = {{.kind = ipc::ResultKind::ScalarF32}}}}};
const auto raw_bytes = ipc::serialize_raw_collection(raw);
const auto pre_bytes = ipc::serialize_preprocessed_collection(raw);
const auto result_bytes = ipc::serialize_result_collection(results);
require(ipc::serialize_raw_collection(ipc::deserialize_raw_collection(raw_bytes)) == raw_bytes,
"raw roundtrip changed data");
require(ipc::serialize_preprocessed_collection(ipc::deserialize_preprocessed_collection(pre_bytes)) == pre_bytes,
"preprocessed roundtrip changed data");
require(ipc::serialize_result_collection(ipc::deserialize_result_collection(result_bytes)) == result_bytes,
"result roundtrip changed data or empty shape");
for (std::size_t size = 0; size < result_bytes.size(); ++size) {
expect_error([&] { (void)ipc::deserialize_result_collection(std::span(result_bytes).first(size)); });
}
for (std::size_t count_offset : {28U, 32U, 39U}) {
auto corrupt = result_bytes;
write_u32(corrupt, count_offset, std::numeric_limits<std::uint32_t>::max());
expect_error([&] { (void)ipc::deserialize_result_collection(corrupt); });
}
for (auto kind : {ipc::ResultKind::ImageF32, ipc::ResultKind::TableF32}) {
ipc::ResultCollection empty{.collection_payloads = {{.kind = kind}}};
auto corrupt = ipc::serialize_result_collection(empty);
write_u32(corrupt, 39, std::numeric_limits<std::uint32_t>::max());
write_u32(corrupt, 43, std::numeric_limits<std::uint32_t>::max());
expect_error([&] { (void)ipc::deserialize_result_collection(corrupt); });
}
// Deterministic random mutations include wire-compatible valid frames as seeds.
std::mt19937 random(0x52414452U);
for (unsigned iteration = 0; iteration < 10'000U; ++iteration) {
auto bytes = iteration % 3U == 0U ? raw_bytes : iteration % 3U == 1U ? pre_bytes : result_bytes;
for (unsigned edit = 0; edit < 1U + random() % 5U; ++edit) {
bytes[random() % bytes.size()] = static_cast<std::uint8_t>(random());
}
if (iteration % 4U == 0U) {
bytes.resize(random() % bytes.size());
}
try {
if (iteration % 3U == 0U) { (void)ipc::deserialize_raw_collection(bytes); }
else if (iteration % 3U == 1U) { (void)ipc::deserialize_preprocessed_collection(bytes); }
else { (void)ipc::deserialize_result_collection(bytes); }
} catch (const std::bad_alloc&) { throw; }
catch (const std::runtime_error&) {}
}
}
void test_configuration(const std::string& seed_path, const Workspace& workspace) {
std::ifstream input(seed_path);
const auto seed = Json::parse(input);
const auto config_path = workspace.path / "config.json";
const auto save = [&](const Json& config) { std::ofstream(config_path) << config; };
save(seed);
const auto config = config::load_run_config(config_path.string());
for (const auto& [pointer, value] : std::array<std::pair<const char*, double>, 8>{{
{"/radar/sweep/start_hz", -1}, {"/radar/sweep/stop_hz", 1e100},
{"/radar/sweep/if_bandwidth_hz", 0}, {"/radar/sweep/points", 0},
{"/radar/sweep/points", 4294967296.0}, {"/run/combos/0/input", 0.0000001},
{"/rings/raw/slot_size_bytes", 65}, {"/gpr/rx_geometry/0/x_m", 1e100}}}) {
auto invalid = seed;
invalid[Json::json_pointer(pointer)] = value;
save(invalid);
expect_error([&] { (void)config::load_run_config(config_path.string()); });
}
const auto live_path = workspace.path / "live.json";
std::ofstream(live_path) << Json{{"bscan_gain", 2}, {"settings_revision", 11}};
processing::ProcessingLiveConfigLoader loader(live_path.string(), config);
require(loader.refresh_if_needed().bscan_gain == 2, "valid live config did not load");
const auto revision = loader.revision();
for (const auto& invalid : {Json{{"bscan_gain", 1e100}},
Json{{"gpr_input_positions", {0.0000001}}}, Json{{"bscan_gain", true}}}) {
std::ofstream(live_path) << invalid;
std::this_thread::sleep_for(std::chrono::milliseconds(110));
const auto current = loader.refresh_if_needed();
require(current.bscan_gain == 2 && loader.revision() == revision,
"invalid live config replaced the last valid settings");
}
const std::array<float, 1> nan_axis{std::numeric_limits<float>::quiet_NaN()};
require(!preprocessing::frequency_axes_match(nan_axis, nan_axis), "NaN frequency axes matched");
preprocessing::ChannelTrace invalid_trace{.frequency_hz = {1},
.samples = {{std::numeric_limits<float>::infinity(), 0}}};
expect_error([&] { preprocessing::validate_channel_trace_layout(invalid_trace, "test", {}); });
}
void test_ring() {
RingName name;
expect_error([&] { (void)ipc::ShmRing::open_or_create(name.name, 2, 65); });
expect_error([&] { (void)ipc::ShmRing::open_or_create(name.name, 0xffffffffU, 0xfffffff8U); });
auto ring = ipc::ShmRing::open_or_create(name.name, 2, 4096);
const auto child = ::fork();
require(child >= 0, "fork failed");
if (child == 0) {
std::array<std::uint64_t, 512> words{};
for (std::uint64_t sequence = 1; sequence <= 20'000U; ++sequence) {
words.fill(sequence);
if (!ring.push({reinterpret_cast<const std::uint8_t*>(words.data()), sizeof(words)})) {
::_exit(2);
}
}
::_exit(0);
}
std::uint64_t previous = 0;
std::vector<std::uint8_t> payload;
int status = 0;
bool finished = false;
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10);
try {
while (true) {
if (ring.pop(payload)) {
require(payload.size() == 4096, "stress payload changed size");
std::uint64_t sequence = 0;
std::memcpy(&sequence, payload.data(), sizeof(sequence));
require(sequence > previous, "stress ring yielded duplicate or older data");
for (std::size_t offset = 0; offset < payload.size(); offset += sizeof(sequence)) {
std::uint64_t word = 0;
std::memcpy(&word, payload.data() + offset, sizeof(word));
require(word == sequence, "stress ring accepted a torn payload");
}
previous = sequence;
} else if (finished) { break; }
finished = finished || ::waitpid(child, &status, WNOHANG) == child;
require(std::chrono::steady_clock::now() < deadline, "ring stress timed out");
}
require(WIFEXITED(status) && WEXITSTATUS(status) == 0 && previous != 0,
"ring stress writer failed");
} catch (...) {
::kill(child, SIGKILL);
if (!finished) { (void)::waitpid(child, &status, 0); }
throw;
}
}
void test_partial_driver_open() {
struct Radar final : drivers::RadarDriver {
unsigned closes = 0;
void open() override {}
void close() override { ++closes; }
auto acquire_sweep() -> drivers::SweepTrace override { return {}; }
} radar;
struct Switch final : drivers::SwitchDriver {
std::atomic<bool>& stop;
bool fail_open = false;
bool fail_close = false;
unsigned closes = 0;
explicit Switch(std::atomic<bool>& stop_flag) : stop(stop_flag) {}
void open() override {
if (fail_open) { stop.store(true); throw std::runtime_error("open failure"); }
}
void close() override { ++closes; if (fail_close) { throw std::runtime_error("close failure"); } }
auto position_count() const -> std::uint32_t override { return 1; }
void switch_to(std::uint32_t) override {}
};
std::atomic<bool> stop{false};
Switch input(stop), output(stop);
input.fail_open = true;
output.fail_close = true;
config::RunConfig config;
config.radar.sweep.points = 1;
config.run_combos = {{0, 0}};
RingName name;
auto ring = ipc::ShmRing::open_or_create(name.name, 2, 256);
acq::SweepOrchestrator(config, radar, input, output, ring).run(stop);
require(radar.closes == 1 && input.closes == 1 && output.closes == 1,
"partial driver startup did not close each driver exactly once");
}
void test_interruptible_wait() {
std::atomic<bool> stop{false};
std::jthread request_stop([&] {
std::this_thread::sleep_for(std::chrono::milliseconds(10));
stop.store(true);
});
const auto start = std::chrono::steady_clock::now();
require(config::wait_or_stop(0xffffffffU, stop), "large wait ignored stop");
require(std::chrono::steady_clock::now() - start < std::chrono::seconds(1),
"large wait delayed shutdown");
}
void test_csv_failures(const Workspace& workspace) {
const auto csv = workspace.path / "capture.csv";
const auto spool = workspace.path / "capture.capture_spool.bin";
CaptureFileWriter writer(csv.string(), (workspace.path / "capture.svg").string(), "", "");
writer.initialize_csv(2, false);
CapturePacket packet{.ch1 = {1, 2}, .ch2 = {3, 4}, .di1 = {}};
auto invalid_packet = packet;
invalid_packet.ch2.pop_back();
expect_error([&] { writer.append_csv_packet(invalid_packet); });
require(std::filesystem::file_size(spool) == 16, "invalid packet partially changed the spool");
writer.append_csv_packet(packet);
std::filesystem::remove(csv);
std::filesystem::create_symlink("/dev/full", csv);
expect_error([&] { writer.write({packet}, 1000, 10); });
require(std::filesystem::file_size(spool) > 16, "failed CSV write deleted the recovery spool");
std::filesystem::remove(csv);
writer.write({packet}, 1000, 10);
require(!std::filesystem::exists(spool), "successful CSV export retained the spool");
std::ifstream csv_input(csv);
const std::string csv_text((std::istreambuf_iterator<char>(csv_input)), {});
require(csv_text.find("1.000000000,3.000000000") != std::string::npos,
"CSV retry lost channel samples");
writer.initialize_csv(2, false);
std::ofstream(spool, std::ios::binary | std::ios::app).put('x');
expect_error([&] { writer.write({}, 1000, 10); });
require(std::filesystem::exists(spool), "truncated packet header deleted the spool");
writer.initialize_csv(2, false);
const std::array<std::uint64_t, 4> corrupt_packet{0, 2, 0xffffffffffffffffULL, 0};
std::ofstream(spool, std::ios::binary | std::ios::app).write(
reinterpret_cast<const char*>(corrupt_packet.data()), sizeof(corrupt_packet));
expect_error([&] { writer.write({}, 1000, 10); });
require(std::filesystem::exists(spool), "oversized packet count deleted the spool");
}
void test_asset_reload_preserves_data(const Workspace& workspace) {
const auto save = [&](const char* name, float sample) {
const auto path = workspace.path / name;
const auto bytes = ipc::serialize_raw_collection({.traces = {
{.frequency_hz = {1}, .s11 = {{sample, 0}}, .s21 = {{sample, 0}}}}});
std::ofstream(path, std::ios::binary).write(
reinterpret_cast<const char*>(bytes.data()), static_cast<std::streamsize>(bytes.size()));
return path.string();
};
const auto open = save("open.bin", 1);
const auto short_standard = save("short.bin", -1);
const auto load = save("load.bin", 0);
const auto nan = save("nan.bin", std::numeric_limits<float>::quiet_NaN());
preprocessing::S21ReferenceBundle s21_reference;
s21_reference.load(open);
expect_error([&] { s21_reference.load(nan); });
require(s21_reference.resolve({}).samples.front().re == 1, "failed S21 reload lost previous reference");
preprocessing::S21CalibrationBundle s21_calibration;
s21_calibration.load(open);
expect_error([&] { s21_calibration.load(nan); });
require(s21_calibration.is_enabled(), "failed S21 reload disabled previous calibration");
preprocessing::S11ReferenceBundle s11_reference;
s11_reference.load(open);
expect_error([&] { s11_reference.load(nan); });
require(s11_reference.resolve({}).samples.front().re == 1, "failed S11 reload lost previous reference");
preprocessing::S11CalibrationBundle s11_calibration;
s11_calibration.load(open, short_standard, load);
expect_error([&] { s11_calibration.load(open, open, load); });
const std::array<float, 1> axis{1};
require(s11_calibration.apply({}, axis, {{0.5, 0}}).front().re == 0.5F,
"failed OSL reload lost previous calibration");
}
void test_locator_packet_errors() {
int sockets[2]{};
require(::socketpair(AF_UNIX, SOCK_STREAM, 0, sockets) == 0, "socketpair failed");
std::atomic<double> speed{0};
std::atomic<std::int64_t> at_ns{0};
locator::ClientSession client(sockets[0], "regression", 2, 1024);
client.start(speed, at_ns);
const auto send_json = [&](const std::string& json) {
std::vector<std::uint8_t> bytes(8 + json.size());
write_u32(bytes, 4, static_cast<std::uint32_t>(json.size()));
std::memcpy(bytes.data() + 8, json.data(), json.size());
const auto sent = ::send(sockets[1], bytes.data(), bytes.size(), MSG_NOSIGNAL);
if (sent != static_cast<ssize_t>(bytes.size())) {
throw std::runtime_error(std::string("send test packet failed: ") + std::strerror(errno));
}
};
send_json("{\"vlc\":1e400}");
send_json("{\"vlc\":1e308}");
std::this_thread::sleep_for(std::chrono::milliseconds(50));
require(at_ns.load() == 0 && speed.load() == 0, "invalid socket speed reached the processor");
send_json("{\"vlc\":3.5}");
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);
while (at_ns.load() == 0 && std::chrono::steady_clock::now() < deadline) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
require(speed.load() == 3.5 && at_ns.load() > 0, "locator stopped after oversized JSON number");
client.request_stop();
client.join();
::close(sockets[1]);
require(client.has_exited(), "locator session did not finish both loops");
locator::LocatorServerConfig config{.host = "127.0.0.1", .port = 0};
locator::TcpServer server(config);
for (unsigned iteration = 0; iteration < 30; ++iteration) { server.start(); server.stop(); }
}
void test_unread_tty_shutdown(const Workspace& workspace) {
TtyProtocolWriter writer((workspace.path / "tty").string(), 160'000);
const std::vector<std::uint16_t> words(80'000, 42);
writer.enqueue_encoded_frames(words.data(), words.size() / 4U);
std::this_thread::sleep_for(std::chrono::milliseconds(50));
const auto start = std::chrono::steady_clock::now();
writer.shutdown();
require(std::chrono::steady_clock::now() - start < std::chrono::seconds(1),
"unread tty blocked shutdown");
writer.shutdown();
}
} // namespace
int main(int argc, char** argv) {
try {
require(argc == 2, "Pass the simulator config fixture path");
Workspace workspace;
test_wire_decoders();
test_configuration(argv[1], workspace);
test_asset_reload_preserves_data(workspace);
test_ring();
test_partial_driver_open();
test_interruptible_wait();
test_csv_failures(workspace);
test_locator_packet_errors();
test_unread_tty_shutdown(workspace);
std::cout << "native regressions passed (10,000 decoder mutations; 20,000 concurrent ring frames)\n";
} catch (const std::exception& error) {
std::cerr << "native regression failed: " << error.what() << '\n';
return 1;
}
}
data_acq_and_processing/tests/sar3d_fixture.cpp
+90
View File
@@ -0,0 +1,90 @@
#include "sar3d_processor.hpp"
#include "../processing/processors/src/sar3d_gpu.hpp"
#include <iostream>
#include <nlohmann/json.hpp>
using Json = nlohmann::json;
using namespace radar;
namespace {
auto complex_samples(const Json& samples) -> std::vector<std::complex<float>> {
std::vector<std::complex<float>> values;
for (const auto& sample : samples) {
values.emplace_back(sample.at(0).get<float>(), sample.at(1).get<float>());
}
return values;
}
auto collection(const Json& source) -> ipc::PreprocessedCollection {
ipc::PreprocessedCollection result;
result.collection_id = source.at("id");
result.monotonic_ns = source.at("timestamp");
for (const auto& entry : source.at("traces")) {
ipc::SweepTraceBlock trace;
trace.combo = {entry.at("input"), entry.at("output")};
trace.capture_start_ns = entry.at("start");
trace.capture_end_ns = entry.at("end");
trace.frequency_hz = entry.at("frequency").get<std::vector<float>>();
for (const auto& sample : complex_samples(entry.at("s21"))) {
trace.s21.push_back({sample.real(), sample.imag()});
}
result.traces.push_back(std::move(trace));
}
return result;
}
}
int main() {
try {
const Json input = Json::parse(std::cin);
Json output;
if (input.at("operation") == "processor") {
config::RunConfig config;
for (const auto& row : input.at("tx")) {
config.gpr.tx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)});
}
for (const auto& row : input.at("rx")) {
config.gpr.rx_geometry.push_back({row.at(0), row.at(1), row.at(2), row.at(3)});
}
processing::ProcessingLiveConfig live;
live.gpr_start_freq_mhz = input.at("f_start").get<float>() / 1e6F;
live.gpr_stop_freq_mhz = input.at("f_stop").get<float>() / 1e6F;
live.gpr_min_depth_m = input.at("min_depth");
live.gpr_max_depth_m = input.at("max_depth");
live.gpr_speed_m_s = input.at("speed");
live.gpr_look_angle_deg = input.at("pitch");
live.sar3d_voxel_m = input.at("voxel");
live.sar3d_bp_ground_depth_m = input.at("ground");
live.sar3d_frame_radius = input.at("radius");
live.sar3d_comp_range_power = input.at("power");
live.sar3d_use_compensation = input.at("compensation");
live.sar3d_normalize_each_frame = input.at("normalize");
std::vector<ipc::PreprocessedCollection> previous;
for (const auto& member : input.at("previous")) {
previous.push_back(collection(member));
}
processing::Sar3dProcessor processor;
const auto result = processor.process_collection(config, collection(input.at("latest")), previous, live);
output["payloads"] = Json::array();
for (const auto& payload : result.collection_payloads) {
output["payloads"].push_back({{"name", payload.processing_name},
{"x", payload.image_x_axis}, {"y", payload.image_y_axis},
{"image", payload.image_values}, {"table", payload.table_values},
{"columns", payload.table_columns}});
}
} else {
processing::sar3d::Engine engine(1.5);
const auto samples = engine.inverse_dft(complex_samples(input.at("bands")),
input.at("points"), input.at("first_bin"), input.at("size"));
output["ascans"] = Json::array();
for (const auto& sample : samples) {
output["ascans"].push_back({sample.real(), sample.imag()});
}
}
std::cout << output << '\n';
} catch (const std::exception& error) {
std::cout << Json{{"error", error.what()}} << '\n';
return 1;
}
}
data_acq_and_processing/tests/sar3d_kernel_emulation.cpp
+136
View File
@@ -0,0 +1,136 @@
// Test-only execution of the exact CUDA kernel source, one logical thread at a time.
// Linked solely into sar3d_math_fixture; never used as a production backend.
#include "../processing/processors/src/sar3d_gpu.hpp"
#include <algorithm>
#include <bit>
#include <climits>
#include <cmath>
#include <limits>
#include <stdexcept>
#include <string>
#define __global__
using std::isfinite;
using std::min;
struct Index { unsigned x; };
Index blockIdx{0}, blockDim{1}, threadIdx{0};
struct float2 { float x, y; };
auto make_float2(float x, float y) -> float2 { return {x, y}; }
void sincospif(float x, float* sine, float* cosine) {
*sine = std::sin(x * 3.14159265358979323846F);
*cosine = std::cos(x * 3.14159265358979323846F);
}
auto __powf(float x, float y) -> float { return std::pow(x, y); }
auto __float_as_uint(float x) -> unsigned int { return std::bit_cast<unsigned int>(x); }
auto __double2int_rn(double x) -> int { return static_cast<int>(std::nearbyint(x)); }
void atomicMax(unsigned int* destination, unsigned int value) { *destination = std::max(*destination, value); }
#include "sar3d_kernels.inc"
auto probe_ascan_upper_bound() -> float {
blockIdx.x = 0;
const double x[]{1073741824.0}, zero[]{0.0};
const ::Contribution contribution{{}, {}, 1.0F, 1.0F};
float volume = -1.0F;
::backproject(nullptr, INT_MAX, &contribution, 1, x, zero, zero, 1, 1, 1,
1.0, 0.0, 0.0, 0.0, 1.0, 1.0F, 0, 0.0F, 0.0F, &volume);
return volume;
}
namespace {
template <class Operation>
void each(int count, Operation operation) {
for (blockIdx.x = 0; blockIdx.x < static_cast<unsigned>(count); ++blockIdx.x) { operation(); }
}
auto float_pairs(std::span<const std::complex<float>> values) -> std::vector<float2> {
std::vector<float2> pairs;
pairs.reserve(values.size());
for (const auto value : values) { pairs.push_back({value.real(), value.imag()}); }
return pairs;
}
}
namespace radar::processing::sar3d {
namespace {
#include "sar3d_helpers.inc"
}
struct Engine::Impl { std::vector<float> taps; };
Engine::Engine(double sigma) : impl_(std::make_unique<Impl>()) { impl_->taps = gaussian_taps(sigma); }
Engine::~Engine() = default;
auto Engine::inverse_dft(std::span<const std::complex<float>> bands, std::size_t points,
std::size_t first_bin, std::size_t size) -> std::vector<std::complex<float>> {
if (points == 0U || bands.empty() || bands.size() % points != 0U || !std::has_single_bit(size) ||
size > kMaxIndexedItems || first_bin >= size || points > size - first_bin) {
throw std::runtime_error("sar3d invalid inverse FFT dimensions");
}
const auto count = checked_product(bands.size() / points, size, kMaxIndexedItems, "A-scan stack");
const auto input = float_pairs(bands);
std::vector<float2> output(count);
each(static_cast<int>(count), [&] {
::inverse_dft(input.data(), points, first_bin, size, count, output.data());
});
std::vector<std::complex<float>> result;
result.reserve(count);
for (const auto value : output) { result.emplace_back(value.x, value.y); }
return result;
}
auto Engine::project(const VolumeRequest& request) -> Projections {
const int nx = request.x.size(), ny = request.y.size(), nz = request.z.size();
const int plane = checked_product(nx, nz, kMaxIndexedItems, "grid plane");
const int voxels = checked_product(plane, ny, kMaxIndexedItems, "grid");
std::vector<::Contribution> contributions;
for (const auto& term : request.contributions) {
contributions.push_back(std::bit_cast<::Contribution>(term));
}
const auto ascans = float_pairs(request.ascans);
std::vector<float> volume(voxels), scratch(voxels), levels(ny);
std::vector<unsigned int> indices(ny);
each(voxels, [&] {
::backproject(ascans.data(), request.samples_per_ascan, contributions.data(), contributions.size(),
request.x.data(), request.y.data(), request.z.data(), nx, nz, voxels,
std::cos(request.pitch_rad), std::sin(request.pitch_rad), request.ground_depth_m,
request.min_depth_m, request.max_depth_m, request.samples_per_metre,
request.use_compensation, request.comp_range_power, request.comp_weight_max, volume.data());
});
each(ny, [&] { ::peak_planes(volume.data(), plane, ny, levels.data(), indices.data()); });
Projections raw;
choose_peak(raw, request, levels);
const int radius = impl_->taps.size() / 2;
each(voxels, [&] {
::smooth(volume.data(), scratch.data(), impl_->taps.data(), radius, nx, nz, voxels, 1);
});
each(voxels, [&] {
::smooth(scratch.data(), volume.data(), impl_->taps.data(), radius, nx, nz, voxels, 0);
});
each(ny, [&] { ::peak_planes(volume.data(), plane, ny, levels.data(), indices.data()); });
Projections result;
result.raw_level = raw.level;
choose_peak(result, request, levels, indices);
result.front.resize(ny * nx);
result.side.resize(ny * nz);
result.top.resize(plane);
const auto& view = request.oblique;
std::vector<unsigned int> oblique(checked_product(view.nu, view.nv, kMaxIndexedItems, "oblique image"));
each(result.front.size(), [&] {
::max_project(volume.data(), result.front.data(), result.front.size(), nx, plane, 1, nz, nx);
});
each(result.side.size(), [&] {
::max_project(volume.data(), result.side.data(), result.side.size(), nz, plane, nx, nx, 1);
});
each(result.top.size(), [&] {
::max_project(volume.data(), result.top.data(), result.top.size(), nx, nx, 1, ny, plane);
});
each(voxels, [&] {
::project_oblique(volume.data(), request.x.data(), request.y.data(), request.z.data(), nx, nz, voxels,
view.u[0], view.u[1], view.u[2], view.v[0], view.v[1], view.v[2], view.u0, view.v0,
1 / view.pixel_m, view.nu, view.nv, oblique.data());
});
result.oblique.reserve(oblique.size());
for (const auto value : oblique) { result.oblique.push_back(std::bit_cast<float>(value)); }
return result;
}
}
data_acq_and_processing/tests/sar3d_kernel_regressions.cpp
+85
View File
@@ -0,0 +1,85 @@
// Kernel arithmetic checks linked only with the host emulation fixture.
#include "../processing/processors/src/sar3d_gpu.hpp"
#include <cmath>
#include <iostream>
#include <limits>
#include <stdexcept>
#include <string_view>
extern "C" void peak_planes(const float*, int, int, float*, unsigned int*);
struct Index { unsigned x; };
extern Index blockIdx;
auto probe_ascan_upper_bound() -> float;
namespace {
void rejects_unsigned_padding() {
blockIdx.x = 0x80000000U;
peak_planes(nullptr, 1, 1, nullptr, nullptr);
}
void rejects_rounded_sample_bound() {
if (probe_ascan_upper_bound() != 0.0F) {
throw std::runtime_error("out-of-range float A-scan position was accepted");
}
}
void rejects_nan_peak() {
blockIdx.x = 0;
const float volume[]{std::numeric_limits<float>::quiet_NaN(), 0.0F};
float level = -1.0F;
unsigned int index = 0U;
peak_planes(volume, 2, 1, &level, &index);
if (std::isfinite(level)) { throw std::runtime_error("peak kernel silently ignored NaN"); }
}
void keeps_first_joint_peak() {
blockIdx.x = 0;
const float volume[]{0, 1, 0, 1, 0, 0};
float level = 0;
unsigned int index = 0U;
peak_planes(volume, 6, 1, &level, &index);
if (level != 1.0F || index != 1U) { throw std::runtime_error("joint peak tie chose a nonexistent voxel"); }
}
void rejects_finite_coherent_overflow() {
using namespace radar::processing::sar3d;
Engine engine(1.5);
const std::vector<double> x{-1, 1}, y{-.5, .5}, z{2, 3};
std::vector<Contribution> terms(3);
for (auto& term : terms) { term.weight = 1; term.inv_reference_ranges = 1; }
std::vector<std::complex<float>> scans(3 * 64);
for (int i = 0; i < 64; ++i) {
scans[i] = {1e38F, 0};
scans[64 + i] = {-1e38F, 0};
scans[128 + i] = {1, 0};
}
const VolumeRequest request{
.x = x, .y = y, .z = z, .min_depth_m = 1, .max_depth_m = 4,
.contributions = terms, .ascans = scans, .samples_per_ascan = 64, .samples_per_metre = 1,
.use_compensation = true, .comp_range_power = 1, .comp_weight_max = 5, .norm_edge_margin_m = .3,
.oblique = {.u = {1, 0, 0}, .v = {0, 0, 1}, .u0 = -1, .v0 = 2, .pixel_m = 1, .nu = 3, .nv = 2},
};
try {
(void)engine.project(request);
} catch (const std::runtime_error& error) {
if (std::string_view(error.what()).find("non-finite") == std::string_view::npos) { throw; }
return;
}
throw std::runtime_error("finite coherent overflow was silently accepted as no signal");
}
}
int main() {
try {
rejects_unsigned_padding();
rejects_rounded_sample_bound();
rejects_nan_peak();
keeps_first_joint_peak();
rejects_finite_coherent_overflow();
std::cout << "SAR CUDA kernel arithmetic regressions passed\n";
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}
data_acq_and_processing/tests/test_sar3d_oracle.py
+297
View File
@@ -0,0 +1,297 @@
"""Compare the native SAR port with notebook cells 0/4, without dataset or GPU fixtures."""
from __future__ import annotations
import json
from pathlib import Path
import re
import subprocess
import tempfile
import unittest
import numpy as np
ROOT = Path(__file__).resolve().parents[2]
FIXTURE = ROOT / "build/native_tests/sar3d_math_fixture"
CUDA_FIXTURE = ROOT / "build/native_tests/sar3d_fixture"
C = 3e8
def smooth(volume):
# scipy gaussian_filter(sigma=(0,1.5,1.5), mode='reflect', truncate=4).
taps = np.exp(-.5 * (np.arange(-6, 7) / 1.5) ** 2)
taps /= taps.sum()
for axis in (1, 2):
volume = np.apply_along_axis(lambda row: np.convolve(
np.pad(row, 6, mode="symmetric"), taps, mode="valid"), axis, volume)
return volume
def native(request, executable=None):
run = subprocess.run([str(executable or FIXTURE)], input=json.dumps(request), text=True,
capture_output=True, timeout=30, check=False)
if not run.stdout:
raise AssertionError(run.stderr)
return json.loads(run.stdout)
def fixture(base=2**64 - 3_000_000_000):
request = dict(operation="processor", tx=[[0, -.4, 0, 0], [1, .1, .2, 0], [2, .45, .1, .05]],
rx=[[0, -.2, .1, 0], [1, .3, .35, .02]], f_start=1e9, f_stop=4e9,
min_depth=1., max_depth=3., speed=.4, pitch=13., ground=.3,
voxel=.4, radius=1, power=1.5, compensation=True, normalize=True)
members = []
frequency = np.linspace(1e9, 4e9, 64, dtype=np.float32)
for frame, offset in enumerate([0, 180_000_000, 510_000_000]):
traces = []
for tx in range(3):
for rx in range(2):
# Two transmitters interleave in one real capture window; the
# third has a longer independent segment, matching meta timing.
start = base + offset + (0 if tx < 2 else 70_000_000)
end = base + offset + (60_000_000 if tx < 2 else 165_000_001)
samples = ((1 + .1 * rx) * np.exp(-2j * np.pi * frequency *
(4.8 + .08 * tx - .03 * frame) / C) +
.35 * np.exp(-2j * np.pi * frequency * (5.6 - .05 * rx) / C)).astype(np.complex64)
traces.append(dict(input=rx, output=tx, start=start, end=end,
frequency=frequency.tolist(), s21=np.column_stack((samples.real, samples.imag)).tolist()))
members.append(dict(id=frame + 1, timestamp=base + offset, traces=traces))
request.update(previous=members[:-1], latest=members[-1])
return request
def oracle(request):
"""Notebook ground/frequency-domain mode, with actual per-trace capture windows."""
pitch = np.deg2rad(float(np.float32(request["pitch"])))
cos_p, sin_p = np.cos(pitch), np.sin(pitch)
speed, voxel, ground = (float(np.float32(request[key])) for key in ("speed", "voxel", "ground"))
axis = lambda lo, hi: np.linspace(lo, hi, max(round((hi - lo) / voxel) + 1, 2))
x, y = axis(-2, 2), axis(-.5, 2)
limits = [(depth + (height - ground) * sin_p) / cos_p
for height in (y[0], y[-1]) for depth in (request["min_depth"], request["max_depth"])]
z = axis(min(limits), max(limits))
xx, zz = np.meshgrid(x, z)
frames, centres = [], []
for member in request["previous"] + [request["latest"]]:
traces = member["traces"]
first, last = min(t["start"] for t in traces), max(t["end"] for t in traces)
centres.append((first + last) / 2 if first < 2**53 else (first, last))
ascans = []
for trace in traces:
frequency = np.asarray(trace["frequency"], dtype=np.float32)
samples = np.asarray(trace["s21"])
samples = samples[:, 0] + 1j * samples[:, 1]
mask = (frequency >= request["f_start"]) & (frequency <= request["f_stop"])
points = np.flatnonzero(mask)
stages = sorted({t["output"] for t in traces
if (t["start"], t["end"]) == (trace["start"], trace["end"])})
# Relative integer subtraction retains precision at uint64 limits.
t = ((trace["start"] - first) - .5 * (last - first)) * 1e-9
t += (len(stages) * points + stages.index(trace["output"])) * (
(trace["end"] - trace["start"]) * 1e-9 / len(stages) / len(frequency))
band = samples[mask] * np.exp(-4j * np.pi * speed * cos_p * frequency[mask] * t / C)
df = float(np.median(np.diff(frequency[mask].astype(float))))
k0, n = round(float(frequency[mask][0]) / df), len(band)
size = 2 ** (2 * (k0 + n - 1) - 1).bit_length() * 8
spectrum = np.zeros(size, dtype=complex)
spectrum[k0:k0+n] = band * np.hanning(n)
h = np.fft.ifft(spectrum)
dt = 1 / (size * df)
if request["normalize"]:
depth = np.arange(size) * dt * C / 2
gate = (depth >= request["min_depth"]) & (depth <= request["max_depth"])
scale = np.percentile(np.abs(h)[gate], 50)
h /= (scale if np.isfinite(scale) and scale > 1e-15 else 1) + 1e-15
ascans.append(h)
frames.append(ascans)
# All members use identical grids. Native centre trails latest by radius.
centre_index = max(len(frames) - 1 - request["radius"], 0)
def relative(a, b):
if isinstance(a, tuple):
return .5e-9 * ((a[0] - b[0]) + (a[1] - b[1]))
return 1e-9 * (a - b)
offsets = np.array([relative(t, centres[centre_index]) for t in centres])
farthest = np.max(np.abs(offsets))
weights = np.ones(len(frames)) if farthest <= 1e-12 else .8 + .1 * (1 + np.cos(np.pi * abs(offsets) / farthest))
acc = np.zeros((len(y), len(z), len(x)), dtype=complex)
count = np.zeros(acc.shape)
tx = np.asarray(request["tx"], dtype=np.float32)[:, 1:].astype(float)
rx = np.asarray(request["rx"], dtype=np.float32)[:, 1:].astype(float)
for scans, offset, weight in zip(frames, offsets, weights):
shift = np.array([0, speed * offset * sin_p, speed * offset * cos_p])
for iy, height in enumerate(y):
yr, zr = (height - ground) * cos_p + zz * sin_p, -(height - ground) * sin_p + zz * cos_p
depth_ok = (zr >= request["min_depth"]) & (zr <= request["max_depth"])
for it, antenna_t in enumerate(tx):
for ir, antenna_r in enumerate(rx):
def distance(antenna):
return np.sqrt((xx-antenna[0])**2 + (yr-antenna[1])**2 + (zr-antenna[2])**2)
rt, rr = distance(antenna_t + shift), distance(antenna_r + shift)
position = (rt + rr) / (C * dt)
h = scans[it * len(rx) + ir]
valid = depth_ok & (position <= len(h) - 1)
sample = np.interp(position, np.arange(len(h)), h)
if request["compensation"]:
point = np.array([.5 * (antenna_t[0] + antenna_r[0]),
(-ground + 5 * sin_p) / cos_p, 5])
ref = 1 / (np.linalg.norm(point-antenna_t) * np.linalg.norm(point-antenna_r) + 1e-12) + 1e-30
ratio = (1 / (rt * rr + 1e-12) + 1e-30) / ref
sample *= np.clip(1 / (ratio ** request["power"] + 1e-12), 0, 5)
acc[iy] += weight * np.where(valid, sample, 0)
count[iy] += weight * valid
eligible = (y >= y[0]+.3) & (y <= y[-1]-.3)
raw_volume = np.abs(acc / (count + 1e-12))
raw_level = raw_volume[eligible].max() + 1e-12
volume = smooth(raw_volume / raw_level)
smooth_level = volume[eligible].max() + 1e-12
level = raw_level * smooth_level
peak = np.unravel_index(np.argmax(np.where(eligible[:, None, None], volume, -1)), volume.shape)
volume /= smooth_level
iy, iz, ix = peak
eye = np.array([1.7, -2.2, .85])
forward = -eye / np.linalg.norm(eye)
right = np.cross(forward, [0, 0, 1])
right /= np.linalg.norm(right)
up = np.cross(right, forward)
u, v = right[[0, 2, 1]], up[[0, 2, 1]]
points = np.array(np.meshgrid(x, y, z, indexing="ij")).reshape(3, -1).T
us, vs = points @ u, points @ v
u0, v0 = us.min(), vs.min()
nu, nv = round((us.max() - u0) / voxel) + 1, round((vs.max() - v0) / voxel) + 1
oblique = np.zeros((nv, nu))
pu, pv = np.rint((us-u0)/voxel).astype(int), np.rint((vs-v0)/voxel).astype(int)
valid = (pu >= 0) & (pu < nu) & (pv >= 0) & (pv < nv)
np.maximum.at(oblique, (pv[valid], pu[valid]), volume.transpose(2, 0, 1).ravel()[valid])
return dict(x=x, y=y, z=z, volume=volume, level=level, peak=[x[ix], y[iy], z[iz]],
peak_index=peak, oblique=oblique, u=u0 + np.arange(nu)*voxel, v=v0 + np.arange(nv)*voxel)
class Sar3dOracleTest(unittest.TestCase):
def test_cuda_branch_cpp_syntax(self):
# Hardware-free name/type structure check. Variadic API placeholders do
# not validate the NVIDIA ABI or replace testing NVRTC on an actual GPU.
source_path = ROOT / "data_acq_and_processing/processing/processors/src/sar3d_gpu.cpp"
source = source_path.read_text()
with tempfile.TemporaryDirectory(prefix="radar_cuda_syntax_") as directory:
path = Path(directory)
functions = set(re.findall(r"\b(cu[A-Z]\w*)\s*\(", source))
constants = set(re.findall(r"\bCU_DEVICE_ATTRIBUTE_\w+", source))
(path / "cuda.h").write_text(
"#pragma once\n#include <cstdint>\nenum CUresult {CUDA_SUCCESS=0};\n"
"using CUdevice=int;using CUdeviceptr=std::uintptr_t;using CUcontext=struct Context*;"
"using CUmodule=struct Module*;using CUfunction=struct Function*;\n" +
"".join(f"constexpr int {name}=0;\n" for name in constants) +
"".join(f"inline CUresult {name}(...) {{return CUDA_SUCCESS;}}\n" for name in functions))
functions = set(re.findall(r"\b(nvrtc[A-Z]\w*)\s*\(", source)) | {
"nvrtcGetCUBINSize", "nvrtcGetPTXSize", "nvrtcGetCUBIN", "nvrtcGetPTX"}
(path / "nvrtc.h").write_text(
"#pragma once\nenum nvrtcResult {NVRTC_SUCCESS=0};using nvrtcProgram=struct Program*;\n"
'inline const char* nvrtcGetErrorString(...) {return "stub";}\n' +
"".join(f"inline nvrtcResult {name}(...) {{return NVRTC_SUCCESS;}}\n"
for name in functions if name != "nvrtcGetErrorString"))
result = subprocess.run(["g++", "-std=c++20", "-pthread", "-fsyntax-only", "-Wall", "-Wextra",
"-Wpedantic", "-Werror", "-DSAR3D_WITH_CUDA", f"-I{path}", str(source_path)],
text=True, capture_output=True, timeout=30, check=False)
self.assertEqual(result.returncode, 0, result.stderr)
def ready(self, result):
payloads = {p["name"]: p for p in result["payloads"]}
status = payloads["sar3d_status"]["table"]
self.assertEqual(payloads["sar3d_status"]["columns"], len(status))
self.assertEqual(status[0], 0)
return payloads
def test_inverse_fft_matches_numpy(self):
random = np.random.default_rng(209)
bands = random.normal(size=(2, 48)) + 1j * random.normal(size=(2, 48))
result = native(dict(operation="ifft", bands=np.column_stack((bands.ravel().real, bands.ravel().imag)).tolist(),
points=48, first_bin=17, size=512))
actual = np.asarray(result["ascans"])
spectrum = np.zeros((2, 512), dtype=complex)
spectrum[:, 17:65] = bands.astype(np.complex64)
expected = np.fft.ifft(spectrum).ravel()
np.testing.assert_allclose(actual[:, 0] + 1j * actual[:, 1], expected, rtol=3e-4, atol=2e-7)
def test_production_backend_reports_cuda_warning(self):
result = native(fixture(), CUDA_FIXTURE)
payloads = {p["name"]: p for p in result["payloads"]}
status = payloads["sar3d_status"]["table"]
if status[0] == 3:
self.assertIn("CUDA", bytes(map(int, status[1:])).decode("utf-8"))
self.assertFalse(any(item["image"] for item in payloads.values()))
else:
# On a CUDA-equipped test host, also run the real compiled backend.
self.ready(result)
def test_motion_volume_matches_notebook(self):
for compensation, normalize in ((True, True), (True, False), (False, True), (False, False)):
with self.subTest(compensation=compensation, normalize=normalize):
request = fixture()
request.update(compensation=compensation, normalize=normalize)
payloads = self.ready(native(request))
expected = oracle(request)
for name, image, x, y in (
("sar3d_front", expected["volume"].max(axis=1), expected["x"], expected["y"]),
("sar3d_side", expected["volume"].max(axis=2), expected["z"], expected["y"]),
("sar3d_top", expected["volume"].max(axis=0), expected["x"], expected["z"]),
("sar3d_oblique", expected["oblique"], expected["u"], expected["v"])):
np.testing.assert_allclose(payloads[name]["x"], x, rtol=2e-6, atol=1e-6)
np.testing.assert_allclose(payloads[name]["y"], y, rtol=2e-6, atol=1e-6)
np.testing.assert_allclose(np.asarray(payloads[name]["image"]).reshape(image.shape), image,
rtol=2e-3, atol=3e-4)
np.testing.assert_allclose(payloads["sar3d_peak"]["table"][1:], expected["peak"], atol=2e-6)
np.testing.assert_allclose(payloads["sar3d_peak"]["table"][0], expected["level"], rtol=2e-3)
def test_errors_are_published(self):
for malformed in ("empty", "capture", "frequency", "axis", "angle"):
with self.subTest(malformed=malformed):
request = fixture()
if malformed == "empty":
request["latest"]["traces"] = []
elif malformed == "capture":
request["latest"]["traces"][0]["end"] = request["latest"]["traces"][0]["start"]
elif malformed == "frequency":
request["latest"]["traces"][0]["frequency"].reverse()
elif malformed == "axis":
request["voxel"] = 1e-8
else:
request["pitch"] = 90
result = native(request)
payloads = {p["name"]: p for p in result["payloads"]}
status = payloads["sar3d_status"]["table"]
self.assertEqual(payloads["sar3d_status"]["columns"], len(status))
self.assertEqual(status[0], 2)
self.assertTrue(bytes(map(int, status[1:])).decode())
def test_empty_signal_is_reported(self):
request = fixture()
request.update(previous=[], radius=0)
for trace in request["latest"]["traces"]:
trace["s21"] = [[0, 0]] * len(trace["frequency"])
payloads = {p["name"]: p for p in native(request)["payloads"]}
self.assertEqual(payloads["sar3d_status"]["table"][0], 1)
self.assertNotIn("sar3d_peak", payloads)
for name in ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique"):
self.assertTrue(np.all(np.asarray(payloads[name]["image"]) == 0))
def test_invalid_fft_dimensions_are_rejected(self):
for points, first_bin, size in ((0, 0, 8), (1, 0, 7), (1, 8, 8), (2, 7, 8), (1, 0, 2**31)):
with self.subTest(points=points, first_bin=first_bin, size=size):
result = native(dict(operation="ifft", bands=[[1, 0]],
points=points, first_bin=first_bin, size=size))
self.assertIn("error", result)
def test_weak_signal_stays_finite(self):
request = fixture()
request.update(previous=[], radius=0, normalize=False)
for trace in request["latest"]["traces"]:
trace["s21"] = (np.asarray(trace["s21"]) * 1e-38).tolist()
payloads = self.ready(native(request))
for name in ("sar3d_front", "sar3d_side", "sar3d_top", "sar3d_oblique"):
values = np.asarray(payloads[name]["image"])
self.assertTrue(np.isfinite(values).all())
self.assertGreater(values.max(), 0)
self.assertLess(values.max(), 1e-10)
if __name__ == "__main__":
unittest.main()
deploy/pack-usb.sh
+2 -2
View File
@@ -9,9 +9,9 @@
set -euo pipefail
PROJECT_ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
# What runs on the Pi or Jetson; device_firmware/ and docs/ stay on this machine.
# What runs on the Pi or Jetson; device_firmware/ stays on this machine.
HOST_PATHS=(data_acq_and_processing deploy python_app run_config_examples Makefile
requirements.txt run_config.json start.sh locator_test_client.py)
"requirements*.txt" run_config.json start.sh locator_test_client.py)
drive="${1:-}"
if [[ -z "${drive}" ]]; then
deploy/update.sh
+7 -4
View File
@@ -35,6 +35,7 @@ while (( $# > 0 )); do
*) archive="$1"; shift ;;
esac
done
if [[ -n "${archive}" ]]; then archive="$(realpath -- "${archive}")"; fi
cd "${HOME}" # never stand in the directory that is about to be renamed
# --- check everything before touching the project ------------------------------
@@ -149,8 +150,8 @@ if [[ -n "${backup}" ]]; then
# writes new files instead of rewriting old ones.
data=()
for entry in "${backup}"/python_app/data*; do data+=("python_app/${entry##*/}"); done
for dir in "${backup}"/*/; do
name="$(basename -- "${dir}")"
for entry in "${backup}"/*; do
name="${entry##*/}"
case "${name}" in build|graphify-out|device_firmware|docs) continue ;; esac
[[ -e "${target}/${name}" ]] || data+=("${name}")
done
@@ -168,7 +169,9 @@ fi
venv="${target}/.venv"
if [[ -z "${python}" && -n "${backup}" && -x "${backup}/.venv/bin/python" ]]; then
say "carrying the venv over"
cp -al -- "${backup}/.venv" "${venv}"
# pip and generated caches may rewrite existing files: the rollback copy
# must own its files independently of the environment being upgraded.
cp -a -- "${backup}/.venv" "${venv}"
"${venv}/bin/python" -m pip install --quiet -r "${target}/requirements.txt"
else
python="${python:-$(command -v python3)}"
@@ -178,7 +181,7 @@ else
"${venv}/bin/python" -m pip install -r "${target}/requirements.txt"
fi
say "building the C++ part"
make -C "${target}" -j"$(nproc)" all
make -C "${target}" -j"$(nproc)" all kamil_adc_collector
echo "${archive##*/}" > "${target}/RELEASE"
trap - ERR
device_firmware/cart_firmware/src/main.cpp
+3 -20
View File
@@ -8,15 +8,7 @@ static const int X_FWD = 650, X_MID = 2014, X_BACK = 3378; // PA0 (плечи
static const int Y_RIGHT = 479, Y_MID = 1981, Y_LEFT = 3586; // PA1
static const int DEADZONE = 15; // отсечка дребезга вокруг нейтрали
// ---- SBUS-значения старого пульта ----
static const uint16_t SBUS_MID = 1024;
static const uint16_t CH2_MIN = 433, CH2_MAX = 1643; // назад..вперёд
static const uint16_t CH4_MIN = 446, CH4_MAX = 1654; // влево..вправо
// ---- expo: 0 = линейно, 1 = максимально мягкая нейтраль ----
static const float EXPO_K = 0.0f;
// ---- общий масштаб выхода: 1.0 = диапазон старого пульта ----
static const float RANGE_SCALE = 0.75f;
// симметричные плечи: вперёд и назад дают одинаковый максимум
static const uint16_t CH2_SPAN = 591; // min(1643-1024, 1024-433)
@@ -75,15 +67,6 @@ static float axisNorm(int adc, int lowEnd, int mid, int highEnd) {
return constrain(x, -1.0f, 1.0f);
}
// expo-кривая: гасит чувствительность у нейтрали, сохраняет края
static float expo(float x) {
return EXPO_K * x * x * x + (1.0f - EXPO_K) * x;
}
static uint16_t toSbus(float x, uint16_t span) {
return (uint16_t)(SBUS_MID + lroundf(x * span));
}
// стик [-1..1] -> SBUS-значение канала: ниже MOVE_START — нейтраль, выше —
// линейно от порога срабатывания приёмника (db) до POWER_FRAC*span на полном стике
static uint16_t driveToSbus(float x, uint16_t span, uint16_t db,
@@ -129,11 +112,11 @@ void setup() {
}
void loop() {
static uint32_t next = 0;
static uint32_t last = millis() - FRAME_PERIOD_MS;
uint32_t now = millis();
if (now < next)
if (uint32_t(now - last) < FRAME_PERIOD_MS)
return;
next = now + FRAME_PERIOD_MS;
last = now;
int adcX = readAvg(PIN_X);
int adcY = readAvg(PIN_Y);
docs/S2VNA Operating and Programing Manual ver 26.1 2.pdf
BIN
View File
Binary file not shown.
docs/Руководство_по_эксплуатации_SNVNA_часть_2_ver_26.1.pdf
BIN
View File
Binary file not shown.
locator_test_client.py
+23 -16
View File
@@ -3,8 +3,8 @@ import random
import socket
import struct
import threading
import time
from typing import Any, Dict, Tuple
from contextlib import suppress
from typing import Any
HOST = "127.0.0.1"
PORT = 8888
@@ -18,7 +18,7 @@ MAX_PAYLOAD_BYTES = 64 * 1024
HEADER_STRUCT = struct.Struct("<II")
def encode_packet(payload: Dict[str, Any], device_id: int) -> bytes:
def encode_packet(payload: dict[str, Any], device_id: int) -> bytes:
"""Encode a JSON payload using the protocol binary header."""
payload_bytes = json.dumps(
payload,
@@ -28,12 +28,17 @@ def encode_packet(payload: Dict[str, Any], device_id: int) -> bytes:
return HEADER_STRUCT.pack(device_id, len(payload_bytes)) + payload_bytes
def recv_exactly(sock: socket.socket, size: int) -> bytes:
def recv_exactly(sock: socket.socket, size: int, stop_event: threading.Event | None = None) -> bytes:
"""Receive an exact number of bytes from the socket."""
chunks = bytearray()
while len(chunks) < size:
chunk = sock.recv(size - len(chunks))
if stop_event is not None and stop_event.is_set():
raise ConnectionError("Client stopped.")
try:
chunk = sock.recv(size - len(chunks))
except socket.timeout:
continue # Preserve a partial header/payload across timeout polls.
if not chunk:
raise ConnectionError("Connection closed by peer.")
chunks.extend(chunk)
@@ -41,9 +46,9 @@ def recv_exactly(sock: socket.socket, size: int) -> bytes:
return bytes(chunks)
def read_packet(sock: socket.socket) -> Tuple[int, Any]:
def read_packet(sock: socket.socket, stop_event: threading.Event | None = None) -> tuple[int, Any]:
"""Read and decode a single packet from the server."""
header_bytes = recv_exactly(sock, HEADER_STRUCT.size)
header_bytes = recv_exactly(sock, HEADER_STRUCT.size, stop_event)
device_id, payload_length = HEADER_STRUCT.unpack(header_bytes)
if payload_length > MAX_PAYLOAD_BYTES:
@@ -52,10 +57,11 @@ def read_packet(sock: socket.socket) -> Tuple[int, Any]:
% (payload_length, MAX_PAYLOAD_BYTES)
)
payload_bytes = recv_exactly(sock, payload_length)
payload_bytes = recv_exactly(sock, payload_length, stop_event)
payload = json.loads(payload_bytes.decode("utf-8"))
return device_id, payload
def send_vlc_message(sock: socket.socket, device_id: int, vlc: float) -> None:
"""Send one test client payload to the server."""
payload = {"vlc": vlc}
@@ -67,10 +73,8 @@ def receive_loop(sock: socket.socket, stop_event: threading.Event) -> None:
"""Print all packets received from the locator server."""
while not stop_event.is_set():
try:
device_id, payload = read_packet(sock)
except socket.timeout:
continue
except (ConnectionError, OSError, ValueError, json.JSONDecodeError) as error:
device_id, payload = read_packet(sock, stop_event)
except (OSError, ValueError) as error:
if not stop_event.is_set():
print("Receiver stopped:", error)
stop_event.set()
@@ -89,12 +93,10 @@ def send_loop(sock: socket.socket, stop_event: threading.Event) -> None:
)
if SEND_INTERVAL_SECONDS <= 0:
while not stop_event.is_set():
time.sleep(0.1)
stop_event.wait()
return
while not stop_event.is_set():
time.sleep(SEND_INTERVAL_SECONDS)
while not stop_event.wait(SEND_INTERVAL_SECONDS):
send_vlc_message(
sock,
CLIENT_DEVICE_ID,
@@ -124,8 +126,13 @@ def main() -> None:
send_loop(sock, stop_event)
except KeyboardInterrupt:
print("Stopping client.")
except OSError as error:
print("Sender stopped:", error)
finally:
stop_event.set()
with suppress(OSError):
sock.shutdown(socket.SHUT_RDWR)
receiver.join(RECV_TIMEOUT_SECONDS + 1.0)
if __name__ == "__main__":
mimo_calibration.py
+544
View File
@@ -0,0 +1,544 @@
#!/usr/bin/env python3
"""Фазовая калибровка MIMO-радара по точечному отражателю (шару) и применение её к данным.
Расчёт тот же, что в MIMO_calibration.ipynb, без графиков и диагностики. Формат файлов
калибровки (cal_*.npy, cal_meta.json) и папки postprocessed — тот же.
Запуск целиком: python mimo_calibration.py (работает по параметрам из блока ниже)
Из другого кода:
import mimo_calibration as mc
cal = mc.compute_calibration(cal_sets, empty_scene_path, mc.Settings(...))
mc.save_calibration(cal, cal_dir)
cal = mc.load_calibration(cal_dir)
s21_corr = mc.apply_calibration(s21, freq, cal) # массив (TX, RX, Nf) в памяти
mc.apply_to_measurement(meas_dir, cal) # preprocessed -> postprocessed
"""
import datetime
import json
import re
import shutil
from dataclasses import dataclass
from pathlib import Path
import numpy as np
from scipy.optimize import minimize
# ╔══════════════════════════════════════════════════════════════════════════════════════╗
# ║ ЗАДАВАЕМЫЕ ПАРАМЕТРЫ ║
# ╚══════════════════════════════════════════════════════════════════════════════════════╝
# --- что делать -------------------------------------------------------------------------
RUN_CALIBRATION = True # True -> посчитать калибровку по CAL_SETS и записать в CAL_DIR;
# False -> взять готовую калибровку из CAL_DIR
APPLY_TO_DATA = False # True -> записать исправленные данные в <измерение>/postprocessed
# --- калибровочные измерения ------------------------------------------------------------
# Каждая запись — одно положение отражателя. snapshots: список префиксов снимков ('0003')
# или None (все снимки папки). Несколько снимков одной записи усредняются.
# target_scene_m = (X, Y, Z) в осях сцены, метры; для шара — координаты его ЦЕНТРА,
# измеренные физически (а не взятые с некалиброванного изображения):
# X — поперёк, вправо, от начала координат антенн (config_profile, x = 0);
# Y — высота центра отражателя над землёй;
# Z — расстояние вдоль земли от точки под началом координат антенн до отражателя.
CAL_ROOT = Path('/Users/ivan_root/Downloads/Telegram_dwnld/2026-09-31 balls calib/'
'librevna_multi_0.5-6GHz_1031pts_-3dBm_big_ball_h_0cm_z_5m_x_0cm/preprocessed')
CAL_SETS = [
dict(path=CAL_ROOT, snapshots=['0000'], target_scene_m=(0.0, 0.16, 5.20)), # Здесь нужно указывать координаты, измернные
dict(path=CAL_ROOT, snapshots=['0001'], target_scene_m=(0.0, 0.16, 5.20)), # в системе координат земли (x,y,z)
dict(path=CAL_ROOT, snapshots=['0002'], target_scene_m=(0.0, 0.16, 5.20)), # X - поеперчная координата от центра радара
dict(path=CAL_ROOT, snapshots=['0003'], target_scene_m=(0.0, 0.16, 5.20)), # y - выота от поверхности земли
dict(path=CAL_ROOT, snapshots=['0004'], target_scene_m=(0.0, 0.16, 5.20)), # z - длина вдоль земли от вертикальной плоскости, про
] # проходящей через нижние антенны до центра шара (до переднего края + радиус)
SPHERE_RADIUS_M = 0.16 # радиус шара-отражателя [м]; 0 -> точечная цель
EMPTY_SCENE_PATH = Path('/Users/ivan_root/Downloads/Telegram_dwnld/2026-09-31 balls calib/'
'librevna_multi_0.5-6GHz_1031pts_-3dBm_bkgd_h_0cm/preprocessed')
# папка снимков той же сцены БЕЗ отражателя (вычитается их среднее);
# None -> без вычитания
# --- геометрия (как в софте построения изображений) ---------------------------------------
INPUT_IDX = [0, 1, 2, 3] # физические входы (Rx), input_pos
OUTPUT_IDX = [0, 1, 2, 3] # физические выходы (Tx), output_pos
ANTENNA_HEIGHT_M = 0.92 # высота начала координат антенн (нижний ряд) над землёй [м]
PITCH_DEG = 10.0 # наклон оси радара ниже горизонта [град]
# --- оценка -----------------------------------------------------------------------------
CAL_F_MIN = 2.0e9 # полоса, по которой оценивается калибровка [Гц]
CAL_F_MAX = 6.0e9
GATE_HALF_M = 0.15 # окно вокруг эха цели, ± по бистатическому пути [м]
GATE_TAPER_M = 0.03 # плавные края окна [м]
COARSE_RANGE_SEARCH_M = 0.6 # грубый поиск эха по дальности перед точной подгонкой, ± по пути [м];
# 0 -> выключен
REFINE_AXES = ('x', 'y', 'z') # какие координаты цели уточнять по данным; если X и Y измерены
# точно — ('z',)
REFINE_RANGE_M = 0.10 # уточнение не дальше этого от найденной точки [м]
APPLY_AMPLITUDE = False # True -> поправка выравнивает и амплитуды антенн
PHASE_MODEL = 'linear' # модель фазы антенны по частоте: 'const' | 'linear' | 'poly3' | 'raw'
# --- куда писать калибровку ---------------------------------------------------------------
CAL_NAME = 'cal_20260931_big_ball_h0_script'
CAL_DIR = Path(__file__).resolve().parent / 'results' / 'calibration' / CAL_NAME
OVERWRITE = True # True -> перезаписать файлы калибровки в CAL_DIR
# --- к каким измерениям применять ---------------------------------------------------------
# Список папок измерений (тех, где лежат preprocessed, raw, results).
APPLY_TO_MEASUREMENTS = [
# '/путь/к/измерению',
]
OVERWRITE_POSTPROCESSED = False # True -> пересоздать postprocessed (только созданный этой калибровкой:
# папка без calibration_applied.json не удаляется никогда)
# ╔══════════════════════════════════════════════════════════════════════════════════════╗
# ║ КОНЕЦ ЗАДАВАЕМЫХ ПАРАМЕТРОВ ║
# ╚══════════════════════════════════════════════════════════════════════════════════════╝
C_LIGHT = 299_792_458.0
@dataclass
class Settings:
"""Настройки расчёта; значения по умолчанию — из блока параметров выше."""
sphere_radius_m: float = SPHERE_RADIUS_M
input_idx: tuple = tuple(INPUT_IDX)
output_idx: tuple = tuple(OUTPUT_IDX)
antenna_height_m: float = ANTENNA_HEIGHT_M
pitch_deg: float = PITCH_DEG
f_min: float = CAL_F_MIN
f_max: float = CAL_F_MAX
gate_half_m: float = GATE_HALF_M
gate_taper_m: float = GATE_TAPER_M
coarse_range_search_m: float = COARSE_RANGE_SEARCH_M
refine_axes: tuple = tuple(REFINE_AXES)
refine_range_m: float = REFINE_RANGE_M
apply_amplitude: bool = APPLY_AMPLITUDE
phase_model: str = PHASE_MODEL
# ══════════════════════════════════════════════════════
# ЗАГРУЗКА
# ══════════════════════════════════════════════════════
def find_config_profile(start, max_levels=8):
p = Path(start)
for _ in range(max_levels):
if (p / 'config_profile.json').exists():
return p / 'config_profile.json'
p = p.parent
raise FileNotFoundError(f'config_profile.json не найден выше {start}')
def load_geometry(config_path, input_idx, output_idx):
"""Координаты антенн в осях радара [м], массивы (N,3) в порядке сортировки физических портов."""
gpr = json.load(open(config_path, encoding='utf-8'))['gpr']
tx = {int(e['output_pos']): (e['x_m'], e.get('y_m', 0.0), e.get('z_m', 0.0)) for e in gpr['tx_geometry']}
rx = {int(e['input_pos']): (e['x_m'], e.get('y_m', 0.0), e.get('z_m', 0.0)) for e in gpr['rx_geometry']}
return (np.array([tx[k] for k in sorted(output_idx)], float),
np.array([rx[k] for k in sorted(input_idx)], float))
def list_snapshots(root, prefixes=None):
dirs = sorted(p for p in Path(root).iterdir() if p.is_dir() and (p / 'meta.json').exists())
if prefixes is None:
return dirs
out = [d for d in dirs if any(d.name.startswith(str(pr)) for pr in prefixes)]
if len(out) != len(prefixes):
raise FileNotFoundError(f'В {root} найдены не все снимки {prefixes}')
return out
def load_s21(snapshot_dir, input_idx, output_idx):
"""S21 (TX, RX, Nf) и частоты. Индексы — порядок сортировки физических портов."""
s21, freq = None, None
for a, o in enumerate(sorted(output_idx)):
for b, i in enumerate(sorted(input_idx)):
s = np.load(Path(snapshot_dir) / f'i{i}_o{o}_s21.npy')
f = np.load(Path(snapshot_dir) / f'i{i}_o{o}_freq.npy')
if s21 is None:
s21 = np.zeros((len(output_idx), len(input_idx), s.size), complex); freq = f
elif not np.allclose(f, freq):
raise ValueError(f'Разные частотные сетки у пар в {snapshot_dir}')
s21[a, b] = s
return s21, freq
# ══════════════════════════════════════════════════════
# РАСЧЁТ КАЛИБРОВКИ
# ══════════════════════════════════════════════════════
def best_phases(Z):
"""max |Σ Z_ij·e^{-j(t_i+r_j)}| по фазам антенн, t_0 = r_0 = 0."""
t = np.zeros(Z.shape[0]); r = np.zeros(Z.shape[1])
for _ in range(100):
t = np.angle((Z * np.exp(-1j * r)[None, :]).sum(axis=1))
r = np.angle((Z * np.exp(-1j * t)[:, None]).sum(axis=0))
t = t - t[0]; r = r - r[0]
return t, r
def coherence(Z, t=None, r=None):
"""Сфазированность вкладов пар: |Σ| / Σ|·| (1 — все в фазе)."""
if t is not None:
Z = Z * np.exp(-1j * (t[:, None] + r[None, :]))
return float(np.abs(Z.sum()) / (np.abs(Z).sum() + 1e-30))
def circular_median(phases):
"""Круговая медиана по первой оси: значение с минимальной суммой угловых расстояний до остальных."""
d = np.abs(np.angle(np.exp(1j * (phases[:, None] - phases[None, :]))))
best = np.argmin(d.sum(axis=1), axis=0)
return np.take_along_axis(phases, best[None], axis=0)[0]
def combine(sets):
"""Добавки антенн по всем калибровочным точкам: круговая медиана фаз и медиана усиления."""
out = []
for key in ('T', 'R'):
Z = np.array([s[key] for s in sets]) # (точки, антенны, частоты)
out.append(np.exp(np.median(np.log(np.abs(Z)), axis=0)) * np.exp(1j * circular_median(np.angle(Z))))
return tuple(out)
class _Estimator:
"""Геометрия, полоса и окно одной калибровки."""
def __init__(self, st, tx, rx, freq):
self.st, self.tx, self.rx, self.freq = st, tx, rx, freq
self.n_tx, self.n_rx = len(tx), len(rx)
self.band = (freq >= st.f_min) & (freq <= st.f_max)
self.fb = freq[self.band]
self.win = np.hanning(self.fb.size)
self.ok = self.win > 0.2 # края полосы после деления на окно ненадёжны
self.nfft = 1 << int(np.ceil(np.log2(self.fb.size * 8)))
self.dpath = np.fft.fftfreq(self.nfft, self.fb[1] - self.fb[0]) * C_LIGHT # ось «лишний путь», м
self.gate = np.clip(np.minimum((self.dpath + st.gate_half_m) / st.gate_taper_m,
(st.gate_half_m - self.dpath) / st.gate_taper_m) + 0.5, 0, 1)
def scene_to_radar(self, q):
"""(X, Y над землёй, Z вдоль земли) -> оси радара (x, y, z)."""
p = np.radians(self.st.pitch_deg); y0 = q[1] - self.st.antenna_height_m
return np.array([q[0], y0 * np.cos(p) + q[2] * np.sin(p), -y0 * np.sin(p) + q[2] * np.cos(p)])
def pair_paths(self, q_scene):
"""Бистатический путь каждой пары до цели (TX, RX). Для шара радиуса a (q_scene — центр)
путь короче на 2a·cos(beta/2), beta — угол между направлениями из центра на Tx и на Rx."""
a = self.st.sphere_radius_m
pr = self.scene_to_radar(np.asarray(q_scene, float))
vt = self.tx - pr; vr = self.rx - pr
dt = np.linalg.norm(vt, axis=1); dr = np.linalg.norm(vr, axis=1)
paths = dt[:, None] + dr[None, :]
if a:
cos_beta = (vt @ vr.T) / (dt[:, None] * dr[None, :])
paths = paths - 2 * a * np.sqrt((1 + cos_beta) / 2)
return paths
def _unwound(self, s21_band, paths):
return s21_band * np.exp(2j * np.pi * self.fb[None, None, :] * paths[:, :, None] / C_LIGHT)
def target_echo(self, s21_band, paths):
"""Эхо цели E (TX, RX, Nb) со снятым путём: окно ±gate_half_m вокруг заданной точки."""
h = np.fft.ifft(self._unwound(s21_band, paths) * self.win, self.nfft, axis=2)
return np.fft.fft(h * self.gate, axis=2)[:, :, :self.fb.size] / np.maximum(self.win, 1e-3)
def broadband(self, E):
"""Вклад пары в изображение в точке: сумма эха по надёжной части полосы."""
return E[:, :, self.ok].sum(axis=2)
def coarse_range(self, s21, q_nominal):
"""Грубая привязка по дальности: максимум суммарной огибающей всех пар в пределах
±coarse_range_search_m по пути; заданная дальность Z сдвигается под него."""
q = q_nominal.copy()
if not self.st.coarse_range_search_m:
return q
s = np.mean(s21, axis=0)[:, :, self.band]; sel = np.abs(self.dpath) <= self.st.coarse_range_search_m
for _ in range(3):
env = np.abs(np.fft.ifft(self._unwound(s, self.pair_paths(q)) * self.win, self.nfft, axis=2)).sum(axis=(0, 1))
q[2] += self.dpath[sel][np.argmax(env[sel])] / 2
return q
def refine_target(self, s21, q0):
"""Положение цели и широкополосные фазы антенн, максимизирующие сфазированность."""
P0 = self.pair_paths(q0)
E0 = np.mean([self.target_echo(s[:, :, self.band], P0) for s in s21], axis=0)
axes = [('xyz').index(a) for a in self.st.refine_axes]
def E_at(v):
q = q0.copy(); q[axes] += v
dP = self.pair_paths(q) - P0
return E0 * np.exp(2j * np.pi * self.fb[None, None, :] * dP[:, :, None] / C_LIGHT), q
def score(v, t, r):
if np.any(np.abs(v) > self.st.refine_range_m):
return 0.0
return coherence(self.broadband(E_at(v)[0]), t, r)
v = np.zeros(len(axes))
if axes:
for _ in range(6):
t, r = best_phases(self.broadband(E_at(v)[0]))
simplex = np.vstack([v] + [v + 0.03 * np.eye(len(axes))[k] for k in range(len(axes))])
v = minimize(lambda w: -score(w, t, r), v, method='Nelder-Mead',
options=dict(xatol=5e-4, fatol=1e-7, initial_simplex=simplex)).x
return E_at(v)
def factorize(self, E):
"""E_ij(f) ≈ g(f)·T_i(f)·R_j(f) на каждой частоте: фазы — best_phases, амплитуды — МНК по log|E|."""
nb = E.shape[2]
T = np.ones((self.n_tx, nb), complex); R = np.ones((self.n_rx, nb), complex)
L = np.log(np.abs(E) + 1e-30)
for k in range(nb):
t, r = best_phases(E[:, :, k])
tt = L[:, :, k].mean(axis=1) - L[:, :, k].mean(); rr = L[:, :, k].mean(axis=0) - L[:, :, k].mean()
T[:, k] = np.exp(tt + 1j * t); R[:, k] = np.exp(rr + 1j * r)
return T, R
def antenna_params(self, Z):
"""Фаза, продолженная к 0 Гц, задержка (эквивалентный лишний путь) и усиление антенны."""
ph = np.unwrap(np.angle(Z[self.ok])); f = self.fb[self.ok]
slope, ph0 = np.polyfit(f, ph, 1)
return dict(phase0_deg=float(np.degrees(ph0)), delay_path_m=float(-slope * C_LIGHT / (2 * np.pi)),
gain_db=float(20 * np.log10(np.mean(np.abs(Z[self.ok])))),
phase_rms_after_linear_deg=float(np.degrees(np.std(ph - (slope * f + ph0)))))
def phase_model(self, Z):
"""Фаза добавки антенны по модели phase_model (подгонка по надёжной части полосы)."""
model = self.st.phase_model
if model == 'raw':
return Z
ph = np.unwrap(np.angle(Z)); f = (self.fb - self.fb.mean()) / 1e9
if model == 'const':
u = np.exp(1j * np.angle(Z[self.ok])).mean()
return np.abs(Z) * np.exp(1j * np.angle(u))
deg = {'linear': 1, 'poly3': 3}.get(model)
if deg is None:
raise ValueError(f"phase_model должен быть 'const', 'linear', 'poly3' или 'raw', а не {model!r}")
return np.abs(Z) * np.exp(1j * np.polyval(np.polyfit(f[self.ok], ph[self.ok], deg), f))
def pair_correction(self, T, R):
"""Поправка пары C_ij(f) = 1/(T_i·R_j); без apply_amplitude — только фаза."""
Z = T[:, None, :] * R[None, :, :]
return np.conj(Z) / np.abs(Z) if not self.st.apply_amplitude else 1.0 / Z
def to_full_grid(self, Z_band, params):
"""На полную сетку частот: внутри надёжной полосы — оценка, вне её — продолжение с наклоном
линейной модели фазы антенны, пристыкованное к краю полосы без скачка."""
idx = np.flatnonzero(self.band)[self.ok]
ph_in = np.unwrap(np.angle(Z_band[self.ok])); mag_in = np.abs(Z_band[self.ok])
slope = -2 * np.pi * params['delay_path_m'] / C_LIGHT # рад/Гц
ph = np.empty(self.freq.size); mag = np.empty(self.freq.size)
ph[idx] = ph_in; mag[idx] = mag_in
lo, hi = idx[0], idx[-1]
ph[:lo] = ph_in[0] + slope * (self.freq[:lo] - self.freq[lo]); mag[:lo] = mag_in[0]
ph[hi + 1:] = ph_in[-1] + slope * (self.freq[hi + 1:] - self.freq[hi]); mag[hi + 1:] = mag_in[-1]
return mag * np.exp(1j * ph)
def compute_calibration(cal_sets, empty_scene_path=None, settings=None):
"""Калибровка по набору измерений отражателя. Возвращает словарь в том же виде, что load_calibration."""
st = settings or Settings()
cal_sets = list(cal_sets)
config_path = find_config_profile(cal_sets[0]['path'])
tx, rx = load_geometry(config_path, st.input_idx, st.output_idx)
freq = None
empty = None
if empty_scene_path is not None:
loaded = [load_s21(d, st.input_idx, st.output_idx) for d in list_snapshots(empty_scene_path)]
empty = np.mean([s for s, _ in loaded], axis=0); freq = loaded[0][1]
sets = []
for n, cs in enumerate(cal_sets):
snaps = list_snapshots(cs['path'], cs.get('snapshots'))
data = []
for d in snaps:
s21, f = load_s21(d, st.input_idx, st.output_idx)
if freq is None:
freq = f
elif f.shape != freq.shape or not np.allclose(f, freq):
raise ValueError(f'Частотная сетка в {d} отличается от остальных калибровочных данных')
cfg = find_config_profile(d)
if cfg != config_path:
tx_d, rx_d = load_geometry(cfg, st.input_idx, st.output_idx)
if not (np.allclose(tx_d, tx) and np.allclose(rx_d, rx)):
raise ValueError(f'Геометрия в {d} отличается от {config_path}')
data.append(s21 - empty if empty is not None else s21)
sets.append(dict(n=n, snaps=[d.name for d in snaps], s21=np.array(data),
q_nominal=np.array(cs['target_scene_m'], float)))
est = _Estimator(st, tx, rx, freq)
for s in sets:
s['E'], s['q'] = est.refine_target(s['s21'], est.coarse_range(s['s21'], s['q_nominal']))
s['T'], s['R'] = est.factorize(s['E'])
T_all, R_all = combine(sets)
T_all = np.array([est.phase_model(z) for z in T_all]); R_all = np.array([est.phase_model(z) for z in R_all])
ant = {f'Tx{k}': est.antenna_params(T_all[k]) for k in range(est.n_tx)}
ant.update({f'Rx{k}': est.antenna_params(R_all[k]) for k in range(est.n_rx)})
# качество: сфазированность пар в точке цели без поправки и с ней
coh_with = lambda s, C: coherence(est.broadband(s['E'] * C))
quality = dict(coherence_before=[], coherence_own=[], coherence_leave_one_out=[], coherence_final=[])
for s in sets:
others = [o for o in sets if o is not s]
quality['coherence_before'].append(coh_with(s, 1.0))
quality['coherence_own'].append(coh_with(s, est.pair_correction(s['T'], s['R'])))
quality['coherence_leave_one_out'].append(coh_with(s, est.pair_correction(*combine(others))) if others else float('nan'))
quality['coherence_final'].append(coh_with(s, est.pair_correction(T_all, R_all)))
T_full = np.array([est.to_full_grid(T_all[k], ant[f'Tx{k}']) for k in range(est.n_tx)])
R_full = np.array([est.to_full_grid(R_all[k], ant[f'Rx{k}']) for k in range(est.n_rx)])
Z_full = T_full[:, None, :] * R_full[None, :, :]
unit = (lambda Z: np.conj(Z) / np.abs(Z)) if not st.apply_amplitude else (lambda Z: 1.0 / Z)
valid = np.zeros(freq.size, bool); valid[np.flatnonzero(est.band)[est.ok]] = True
meta = dict(
created=datetime.datetime.now().isoformat(timespec='seconds'), name=None,
convention=dict(apply='S21_corrected[tx, rx, f] = S21[tx, rx, f] * cal_pair[tx, rx, f]',
index_order='tx = номер в cal_tx_output_pos (output_pos по возрастанию), rx = cal_rx_input_pos',
files='S21 пары в данных: i{input_pos}_o{output_pos}_s21.npy',
phase_reference='фазы относительно Tx0 и Rx0; общая фаза/задержка не входит',
amplitude='выравнивается' if st.apply_amplitude else 'не меняется (только фаза)',
outside_band='вне cal_valid: продолжение линейной моделью фазы антенны'),
geometry=dict(config=str(config_path), tx_xyz_m=tx.tolist(), rx_xyz_m=rx.tolist(),
antenna_height_m=st.antenna_height_m, pitch_deg=st.pitch_deg),
estimation=dict(band_hz=[st.f_min, st.f_max], gate_half_m=st.gate_half_m, refine_axes=list(st.refine_axes),
phase_model=st.phase_model, sphere_radius_m=st.sphere_radius_m, empty_scene=str(empty_scene_path)),
targets=[dict(path=str(cal_sets[s['n']]['path']), snapshots=s['snaps'], nominal_scene_m=s['q_nominal'].tolist(),
refined_scene_m=s['q'].tolist()) for s in sets],
antennas=ant, quality=quality)
return dict(
freq_hz=freq.astype(np.float64),
pair=unit(Z_full).astype(np.complex64), # (TX, RX, Nf): S21_corr = S21 * pair
tx=unit(T_full).astype(np.complex64), # (TX, Nf): pair = tx[:,None] * rx[None,:]
rx=unit(R_full).astype(np.complex64), # (RX, Nf)
valid=valid, # (Nf,) True — внутри полосы калибровки
tx_output_pos=np.array(sorted(st.output_idx), np.int32), # индекс -> физический выход
rx_input_pos=np.array(sorted(st.input_idx), np.int32), # индекс -> физический вход
antenna_params=np.array([[np.radians(a['phase0_deg']), a['delay_path_m'], a['gain_db']] for a in ant.values()],
np.float64), # (TX+RX, 3): фаза к 0 Гц [рад], путь [м], усиление [дБ]
meta=meta, dir=None)
# ══════════════════════════════════════════════════════
# ФАЙЛЫ КАЛИБРОВКИ
# ══════════════════════════════════════════════════════
_CAL_ARRAYS = ('freq_hz', 'pair', 'tx', 'rx', 'valid', 'tx_output_pos', 'rx_input_pos', 'antenna_params')
def save_calibration(cal, cal_dir, overwrite=False):
cal_dir = Path(cal_dir)
cal_dir.mkdir(parents=True, exist_ok=True)
existing = [p.name for p in cal_dir.glob('cal_*')]
if existing and not overwrite:
raise FileExistsError(f'В {cal_dir} уже есть файлы калибровки ({existing[:3]}...). '
f'Задайте новое имя или overwrite=True.')
for k in _CAL_ARRAYS:
np.save(cal_dir / f'cal_{k}.npy', cal[k])
cal['meta']['name'] = cal_dir.name; cal['dir'] = cal_dir
json.dump(cal['meta'], open(cal_dir / 'cal_meta.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=2)
return cal_dir
def load_calibration(cal_dir):
cal_dir = Path(cal_dir)
cal = {k: np.load(cal_dir / f'cal_{k}.npy') for k in _CAL_ARRAYS}
cal['meta'] = json.load(open(cal_dir / 'cal_meta.json', encoding='utf-8'))
cal['dir'] = cal_dir
return cal
# ══════════════════════════════════════════════════════
# ПРИМЕНЕНИЕ
# ══════════════════════════════════════════════════════
def calibration_on_grid(cal, freq):
"""Поправка пар (TX, RX, Nf) на частотной сетке freq. Совпадает с сеткой калибровки — берётся как есть;
иначе фаза и модуль поправок антенн интерполируются, вне сетки калибровки — линейная модель фазы."""
fc = cal['freq_hz']
if freq.shape == fc.shape and np.allclose(freq, fc):
return cal['pair']
def interp(Z, par):
ph = np.unwrap(np.angle(Z)); mag = np.abs(Z)
out_ph = np.interp(freq, fc, ph); out_mag = np.interp(freq, fc, mag)
lin = -(par[0] - 2 * np.pi * freq * par[1] / C_LIGHT) # поправка = −добавка
outside = (freq < fc[0]) | (freq > fc[-1])
out_ph[outside] = lin[outside] + np.round((out_ph[outside] - lin[outside]) / (2 * np.pi)) * 2 * np.pi
return out_mag * np.exp(1j * out_ph)
n_tx = cal['tx'].shape[0]
tx = np.array([interp(cal['tx'][k], cal['antenna_params'][k]) for k in range(n_tx)])
rx = np.array([interp(cal['rx'][k], cal['antenna_params'][n_tx + k]) for k in range(cal['rx'].shape[0])])
return (tx[:, None, :] * rx[None, :, :]).astype(cal['pair'].dtype)
def apply_calibration(s21, freq, cal):
"""s21 (TX, RX, Nf) в порядке cal['tx_output_pos'] x cal['rx_input_pos'] -> исправленный массив."""
return s21 * calibration_on_grid(cal, np.asarray(freq))
def apply_to_measurement(meas_dir, cal, overwrite=False):
"""<измерение>/preprocessed -> <измерение>/postprocessed: S21 пар умножаются на поправку,
остальные файлы копируются. Возвращает (папка, снимков, исправлено S21, скопировано файлов)."""
meas_dir = Path(meas_dir); pre = meas_dir / 'preprocessed'; post = meas_dir / 'postprocessed'
if not pre.is_dir():
raise FileNotFoundError(f'Нет {pre}')
# калибровка годится только для той же сборки антенн
tx_m, rx_m = load_geometry(find_config_profile(pre), cal['rx_input_pos'].tolist(), cal['tx_output_pos'].tolist())
if not (np.allclose(tx_m, cal['meta']['geometry']['tx_xyz_m']) and np.allclose(rx_m, cal['meta']['geometry']['rx_xyz_m'])):
raise ValueError(f'Геометрия антенн в {meas_dir.name} отличается от геометрии калибровки — применять нельзя')
if post.exists():
if not overwrite:
raise FileExistsError(f'{post} уже существует. overwrite=True, чтобы пересоздать.')
if not (post / 'calibration_applied.json').exists():
raise FileExistsError(f'{post} существует, но создан не калибровкой (нет calibration_applied.json) — '
f'удалять не буду, разберитесь вручную.')
shutil.rmtree(post)
tx_of = {int(p): k for k, p in enumerate(cal['tx_output_pos'])}
rx_of = {int(p): k for k, p in enumerate(cal['rx_input_pos'])}
pat = re.compile(r'^i(\d+)_o(\d+)_s21\.npy$')
n_snap = n_fix = n_copy = 0
grid_cache = {}
post.mkdir()
for item in sorted(pre.iterdir()):
if item.is_file():
shutil.copy2(item, post / item.name); n_copy += 1
continue
out = post / item.name; out.mkdir(); n_snap += 1
for f in sorted(item.iterdir()):
m = pat.match(f.name)
if f.is_dir() or m is None or int(m.group(2)) not in tx_of or int(m.group(1)) not in rx_of:
if f.is_dir():
shutil.copytree(f, out / f.name)
else:
shutil.copy2(f, out / f.name)
n_copy += 1
continue
s21 = np.load(f); freq = np.load(item / f'i{m.group(1)}_o{m.group(2)}_freq.npy')
key = (freq.size, float(freq[0]), float(freq[-1]))
if key not in grid_cache:
grid_cache[key] = calibration_on_grid(cal, freq)
corr = grid_cache[key][tx_of[int(m.group(2))], rx_of[int(m.group(1))]]
np.save(out / f.name, (s21 * corr).astype(s21.dtype)); n_fix += 1
json.dump(dict(calibration=cal['meta']['name'], calibration_dir=str(Path(cal['dir']).resolve()),
created=datetime.datetime.now().isoformat(timespec='seconds'), source=str(pre),
rule='postprocessed i*_o*_s21 = preprocessed i*_o*_s21 * cal_pair[tx(o), rx(i)]; остальные файлы скопированы',
frequency_grids=[list(k) for k in grid_cache]),
open(post / 'calibration_applied.json', 'w', encoding='utf-8'), ensure_ascii=False, indent=2)
return post, n_snap, n_fix, n_copy
def main():
if RUN_CALIBRATION:
cal = compute_calibration(CAL_SETS, EMPTY_SCENE_PATH, Settings())
save_calibration(cal, CAL_DIR, overwrite=OVERWRITE)
q = cal['meta']['quality']
print(f'Калибровка записана в {CAL_DIR}: сфазированность пар на отражателе '
f'{np.mean(q["coherence_before"]):.2f} -> {np.mean(q["coherence_final"]):.2f}')
if APPLY_TO_DATA:
cal = load_calibration(CAL_DIR)
targets = APPLY_TO_MEASUREMENTS
if isinstance(targets, (str, Path)): # один путь вместо списка
targets = [targets]
for meas in targets:
post, n_snap, n_fix, _ = apply_to_measurement(meas, cal, OVERWRITE_POSTPROCESSED)
print(f'{Path(meas).name}: {n_snap} снимков, исправлено S21 {n_fix} -> {post}')
if __name__ == '__main__':
main()
python_app/file_io.py
+47
View File
@@ -0,0 +1,47 @@
"""Atomic writes for JSON settings shared by GUI and pipeline processes."""
from __future__ import annotations
import json
import os
from contextlib import suppress
from pathlib import Path
import tempfile
def atomic_write_json(path: Path, payload: object, *, durable: bool = False) -> Path:
"""Serialize strict JSON before opening any files, then atomically commit it."""
return atomic_write_text(path, json.dumps(payload, indent=2, allow_nan=False), durable=durable)
def atomic_write_text(path: Path, text: str, *, durable: bool = False) -> Path:
"""Replace a text file only after a complete write, cleaning up failed writes.
Each call owns its temporary file, so concurrent writers cannot truncate or
rename one another's work. Durable writes also sync the file and directory;
frequent live settings can omit those syncs to avoid unnecessary disk I/O.
"""
path.parent.mkdir(parents=True, exist_ok=True)
temporary = None
try:
with tempfile.NamedTemporaryFile(
mode="w", encoding="utf-8", dir=path.parent,
prefix=f".{path.name}.", suffix=".tmp", delete=False,
) as handle:
temporary = Path(handle.name)
handle.write(text)
if durable:
handle.flush()
os.fsync(handle.fileno())
os.replace(temporary, path)
if durable:
directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY)
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
finally:
if temporary is not None:
with suppress(OSError):
temporary.unlink(missing_ok=True)
return path
python_app/generator_sweep/config.py
+15 -8
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
from dataclasses import dataclass
import logging
import math
logger = logging.getLogger(__name__)
@@ -121,17 +122,13 @@ def _build_frequency_grid(model: GeneratorSweepConfig) -> tuple[int, ...]:
if has_step:
_validate_positive_int(step_hz, "generator_sweep.step_hz")
frequencies: list[int] = []
current_hz = start_hz
while current_hz < stop_hz:
frequencies.append(current_hz)
current_hz += step_hz
if not frequencies or frequencies[-1] != stop_hz:
frequencies.append(stop_hz)
return tuple(frequencies)
_require((stop_hz - start_hz + step_hz - 1) // step_hz + 1 <= 1_000_000,
"generator sweep grid must not exceed 1000000 frequencies")
return (*range(start_hz, stop_hz, step_hz), stop_hz)
_validate_positive_int(points, "generator_sweep.points")
_require(points >= 2, "generator_sweep.points must be >= 2 when start_hz != stop_hz")
_require(points <= 1_000_000, "generator sweep grid must not exceed 1000000 frequencies")
span_hz = stop_hz - start_hz
frequencies = tuple(
int(round(start_hz + (span_hz * index) / float(points - 1)))
@@ -148,6 +145,16 @@ def _build_frequency_grid(model: GeneratorSweepConfig) -> tuple[int, ...]:
def resolve_generator_sweep_config(model: GeneratorSweepConfig) -> ResolvedGeneratorSweepConfig:
"""Validate and resolve runtime configuration for generator sweep mode."""
for name in ("points", "settle_timeout_ms", "post_lock_delay_us", "pwm_frequency_hz",
"sweep_start_pin", "curr_step_pin", "pwm_pin", "curr_step_initial_level",
"strict_protocol_version", "port"):
value = getattr(model, name)
_require(type(value) is int, f"generator_sweep.{name} must be an integer")
for name in ("start_hz", "stop_hz", "step_hz", "hold_time_ms", "status_poll_ms", "power_dbm", "pwm_duty_cycle"):
value = getattr(model, name)
_require(type(value) in (int, float) and math.isfinite(value),
f"generator_sweep.{name} must be a finite number")
_require(model.points >= 0 and model.step_hz >= 0, "generator_sweep points and step_hz must be >= 0")
log_level = _resolve_log_level(model.log_level)
frequencies_hz = _build_frequency_grid(model)
hold_time_ms = float(model.hold_time_ms)
python_app/generator_sweep/runner.py
+11 -16
View File
@@ -142,24 +142,19 @@ class GeneratorSweepRunner:
def _close(self) -> None:
"""Best-effort cleanup of PWM, GPIO, and device state."""
self._pwm.disable()
device = self._device
device, self._device = self._device, None
cleanup = [self._pwm.disable]
if device is not None:
cleanup.extend([
lambda: device.generator.set_idle(timeout_s=1.0) if device.is_connected else None,
device.disconnect,
])
cleanup.extend([self._markers.reset_to_idle, self._markers.close, self._pwm.close])
for action in cleanup:
try:
if device.is_connected:
device.generator.set_idle(timeout_s=1.0)
except Exception as exc: # noqa: BLE001
logger.warning("Failed to set LibreVNA idle mode during shutdown: %s", exc)
finally:
device.disconnect()
self._device = None
try:
self._markers.reset_to_idle()
finally:
self._markers.close()
self._pwm.close()
action()
except Exception as exc:
logger.warning("Generator sweep cleanup failed: %s", exc)
def _run_single_sweep(self) -> None:
"""Execute one pass from the first configured frequency to the last."""
python_app/gui/antenna_geometry_editor.py
+133
View File
@@ -0,0 +1,133 @@
"""Table editor for antenna switch positions and coordinates in metres."""
from __future__ import annotations
from dataclasses import asdict, is_dataclass
import math
from PyQt6.QtCore import QLocale, QSignalBlocker, pyqtSignal
from PyQt6.QtGui import QDoubleValidator, QIntValidator
from PyQt6.QtWidgets import (
QAbstractItemView, QHeaderView, QHBoxLayout, QLineEdit, QMessageBox, QPushButton,
QStyledItemDelegate, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget,
)
class _CoordinateDelegate(QStyledItemDelegate):
def createEditor(self, parent, option, index):
editor = QLineEdit(parent)
validator = (
QIntValidator(0, self.parent().property("position_count") - 1, editor)
if index.column() == 0 else QDoubleValidator(editor)
)
validator.setLocale(QLocale.c())
editor.setValidator(validator)
return editor
class AntennaGeometryEditor(QWidget):
"""Keep geometry explicit and preserve all three coordinates on profile round trips."""
changed = pyqtSignal()
def __init__(self, position_field: str, entries, *, position_count: int, parent=None) -> None:
super().__init__(parent)
self.position_field = position_field
self._table = QTableWidget(0, 4, self)
self.set_position_count(position_count)
self._table.setHorizontalHeaderLabels([
"Output" if position_field == "output_pos" else "Input", "X (m)", "Y (m)", "Z (m)",
])
self._table.horizontalHeader().setSectionResizeMode(QHeaderView.ResizeMode.Stretch)
self._table.verticalHeader().hide()
self._table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows)
self._table.setItemDelegate(_CoordinateDelegate(self._table))
self._table.setMinimumHeight(130)
self._table.setMaximumHeight(220)
layout = QVBoxLayout(self)
layout.setContentsMargins(0, 0, 0, 0)
layout.addWidget(self._table)
buttons = QHBoxLayout()
self._add_button = QPushButton("Add antenna", self)
self._remove_button = QPushButton("Remove selected", self)
buttons.addWidget(self._add_button)
buttons.addWidget(self._remove_button)
layout.addLayout(buttons)
self.set_records(entries)
self._table.itemChanged.connect(self._on_item_changed)
self._add_button.clicked.connect(self._add_row)
self._remove_button.clicked.connect(self._remove_rows)
def records(self) -> list[dict]:
records = []
for row in range(self._table.rowCount()):
records.append(dict(zip(
(self.position_field, "x_m", "y_m", "z_m"),
(self._table.item(row, column).text() for column in range(4)),
strict=True,
)))
return self._normalize(records)
def _normalize(self, entries) -> list[dict]:
records = []
positions = set()
for entry in entries:
entry = asdict(entry) if is_dataclass(entry) else entry
if any(isinstance(entry.get(key), bool) for key in (self.position_field, "x_m", "y_m", "z_m")):
raise ValueError("Antenna positions and coordinates must be numbers, not booleans")
position = int(entry[self.position_field])
if float(entry[self.position_field]) != position or not 0 <= position < self._table.property("position_count"):
raise ValueError("Antenna position is outside the configured switch positions")
if position in positions:
raise ValueError(f"Duplicate antenna position: {position}")
positions.add(position)
coordinates = {key: float(entry.get(key, 0.0)) for key in ("x_m", "y_m", "z_m")}
if any(not math.isfinite(value) or abs(value) > 3.4e38 for value in coordinates.values()):
raise ValueError("Antenna coordinates must be finite numbers")
records.append({self.position_field: position, **coordinates})
return records
def set_position_count(self, count: int) -> None:
self.position_count = int(count)
self._table.setProperty("position_count", int(count))
def set_records(self, entries) -> None:
records = self._normalize(entries)
with QSignalBlocker(self._table):
self._table.setRowCount(len(records))
for row, record in enumerate(records):
for column, value in enumerate(record.values()):
self._table.setItem(row, column, QTableWidgetItem(str(value)))
self._valid_records = records
self._add_button.setEnabled(len(records) < self._table.property("position_count"))
self.changed.emit()
def _on_item_changed(self, _item) -> None:
try:
self._valid_records = self.records()
except ValueError as exc:
with QSignalBlocker(self):
self.set_records(self._valid_records)
QMessageBox.warning(self, "Invalid antenna geometry", str(exc))
return
self.changed.emit()
def _add_row(self) -> None:
records = self.records()
if len(records) >= self._table.property("position_count"):
return
positions = {record[self.position_field] for record in records}
position = next(index for index in range(len(positions) + 1) if index not in positions)
self.set_records([*records, {self.position_field: position}])
self._table.setCurrentCell(len(records), 1)
self._table.editItem(self._table.item(len(records), 1))
def _remove_rows(self) -> None:
rows = {index.row() for index in self._table.selectionModel().selectedRows()}
if rows:
with QSignalBlocker(self._table):
for row in sorted(rows, reverse=True):
self._table.removeRow(row)
self._valid_records = self.records()
self._add_button.setEnabled(True)
self.changed.emit()
python_app/gui/app_window.py
+22 -46
View File
@@ -23,6 +23,7 @@ from PyQt6.QtGui import QTextCursor
from PyQt6.QtWidgets import QApplication, QMainWindow, QMessageBox
from python_app.gui.controllers.app_window_config_mixin import AppWindowConfigMixin
from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT
from python_app.gui.controllers.app_window_control_button_mixin import AppWindowControlButtonMixin
from python_app.gui.controllers.app_window_pipeline_mixin import AppWindowPipelineMixin
from python_app.gui.controllers.app_window_plot_mixin import AppWindowPlotMixin
@@ -160,6 +161,9 @@ class AppWindow(
self._pending_startup_log_entries: list[tuple[str, str, str | None]] = []
# Guards closeEvent against re-entrant teardown (e.g. a second signal).
self._closing = False
self._close_after_stop = False
self._stopping_run = False
self._stop_request_seq = 0
def _init_logging(self) -> None:
"""Configure the application logger and the bridge that feeds the GUI panel.
@@ -336,11 +340,10 @@ class AppWindow(
def _init_history_state(self) -> None:
"""Initialize runtime history buffers and render-cache state."""
bscan_cpp_replay_window = self._cpp_bscan_replay_window_from_config()
save_history_limit = self._save_history_limit_from_config()
self._raw_history: deque[SweepCollection] = deque(maxlen=save_history_limit)
self._pre_history: deque[SweepCollection] = deque(maxlen=save_history_limit)
self._result_history: deque[ResultCollection] = deque(maxlen=save_history_limit)
bscan_cpp_replay_window = self._cpp_bscan_replay_window_for_config(self._defaults_config)
self._raw_history: deque[SweepCollection] = deque(maxlen=GUI_SAVE_HISTORY_LIMIT)
self._pre_history: deque[SweepCollection] = deque(maxlen=GUI_SAVE_HISTORY_LIMIT)
self._result_history: deque[ResultCollection] = deque(maxlen=GUI_SAVE_HISTORY_LIMIT)
# Sequence id must survive GUI restarts so history commands stay monotonic.
self._history_command_seq = self._load_history_command_seq(self._live_config_writer.path)
@@ -371,20 +374,11 @@ class AppWindow(
self._gpr_region_contours = []
self._gpr_geometry_signature = None
self._gpr_selected_geometry = None
self._phase_viewbox = None
self._history_run_signature = None
self._processor_run_signature = None
self._active_processing_mode = "pass_through"
self._radar_limits: dict[str, float | int] | None = None
def _cpp_bscan_replay_window_from_config(self) -> int:
"""Return the C++ B-scan replay window for the active config."""
return self._cpp_bscan_replay_window_for_config(self._defaults_config)
def _save_history_limit_from_config(self) -> int:
"""Return maxlen for GUI snapshot-save deques (independent of ring capacities)."""
return self._save_history_limit_for_config(self._defaults_config)
def _init_runtime_limits(self) -> None:
"""Initialize read/drain loop limits used by polling and snapshot code."""
self._max_pop_per_poll = 256
@@ -638,30 +632,6 @@ class AppWindow(
)
return "<div style='margin:0 0 6px 0;'>" + "".join(body_parts) + "</div>"
def _append_log_entry(
self,
level: str,
text: str,
*,
details: str | None = None,
once_key: str | None = None,
) -> None:
"""Append formatted log entry with timestamp and optional details."""
if once_key is not None:
if once_key in self._logged_once_keys:
return
self._logged_once_keys.add(once_key)
cursor = self._log_box.textCursor()
cursor.movePosition(QTextCursor.MoveOperation.End)
self._log_box.setTextCursor(cursor)
self._log_box.insertHtml(self._render_log_entry_html(level, text, details))
self._log_box.insertPlainText("\n")
self._log_box.ensureCursorVisible()
if level.upper() == "ERROR" and hasattr(self, "_status_label"):
self._status_label.setText("Status: error")
def _on_log_level_selected(self, level_text: str) -> None:
"""Apply the chosen log level immediately (sub-level logs stop being generated).
@@ -746,12 +716,8 @@ class AppWindow(
# Missing or malformed file should not block startup.
return 0
raw_value = payload.get("history_command_seq", 0)
if isinstance(raw_value, bool):
return 0
if isinstance(raw_value, (int, float)):
return max(0, int(raw_value))
return 0
raw_value = payload.get("history_command_seq", 0) if isinstance(payload, dict) else 0
return raw_value if type(raw_value) is int and 0 <= raw_value < (1 << 64) - 1 else 0
def _capture_web_action_error(self, message: str) -> bool:
"""Record the first error of an in-flight web action so the browser can show it.
@@ -804,18 +770,24 @@ class AppWindow(
Guarded against re-entrancy so a second close signal (or a ``window.close()``
after the event loop has already returned) does not run teardown twice.
"""
if self._stopping_run:
# A drain pumps Qt events. Finish ingesting/finalizing before teardown
# can close the same readers and recording writer beneath that drain.
self._close_after_stop = True
self._stop_all_requested = True
event.ignore()
return
if self._closing:
# Re-entrant close (second signal, or window.close() after the event loop
# already returned): teardown is in progress or done — do nothing more.
super().closeEvent(event)
return
self._closing = True
self._timer.stop()
self._log("Window closing; shutting down runtime.")
try:
# 0) Stop the web server first so a late request cannot start work.
self._shutdown_web_ui()
# 0) Stop the disk-recording writer thread so it is not orphaned.
self._shutdown_recording()
# 0) Stop the GPIO button watcher so a late press cannot start work.
self._stop_control_button_watcher()
self._resume_pipeline_after_capture = False
@@ -825,11 +797,15 @@ class AppWindow(
self._close_replay_ring_writer()
# 1) Abort active capture first (releases exclusive hardware resources).
self._abort_capture_sequence(resume_pipeline=False)
# Drain acquisition stages before finalizing the last recording chunk.
if self._recording_active:
self._stop_run()
# 2) Stop all managed processes/readers.
self._stop_all_processes()
# 3) Close auxiliary dialog windows.
if self._preprocess_dialog is not None:
self._preprocess_dialog.close()
finally:
self._shutdown_recording()
self._log_debug("Window teardown finished.")
super().closeEvent(event)
python_app/gui/controllers/app_window_config/live_processing_mixin.py
+79 -52
View File
@@ -2,17 +2,21 @@
from __future__ import annotations
from contextlib import ExitStack
import math
import time
from PyQt6.QtCore import QSignalBlocker
from PyQt6.QtWidgets import (
QCheckBox,
QComboBox,
QDoubleSpinBox,
QLineEdit,
QPlainTextEdit,
QSpinBox,
)
from python_app.gui.runtime.constraints import validate_processing_mode_constraints
from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor
from python_app.orchestration.live_processing_config import ProcessingLiveConfig
_GPR_MODES = ("gpr", "legacy_gpr")
@@ -56,6 +60,9 @@ _WEB_LIVE_SCHEMA = [
("bscan_start_freq_mhz", "B-scan", ("bscan",), _attr("_bscan_start_freq_mhz")),
("bscan_stop_freq_mhz", "B-scan", ("bscan",), _attr("_bscan_stop_freq_mhz")),
("legacy_gpr_mode", "Mode", ("legacy_gpr",), _attr("_legacy_gpr_config_mode")),
("relative_permittivity", "Geometry & medium", ("legacy_gpr",), _attr("_gpr_relative_permittivity")),
("tx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_tx_geometry_input")),
("rx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_rx_geometry_input")),
("gpr_input_positions", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_input_positions_input", {"legacy_gpr": "_legacy_gpr_input_positions_input", "sar3d": "_sar3d_input_positions_input"})),
("gpr_output_positions", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_output_positions_input", {"legacy_gpr": "_legacy_gpr_output_positions_input", "sar3d": "_sar3d_output_positions_input"})),
("gpr_min_depth_m", "Geometry & depth", _GPR_LIKE_MODES, _dual("_gpr_min_depth_m", {"legacy_gpr": "_legacy_gpr_min_depth_m", "sar3d": "_sar3d_min_depth_m"})),
@@ -107,6 +114,7 @@ _NON_WIDGET_LIVE_FIELDS = frozenset({
"reprocess_current_result",
"history_command",
"history_command_seq",
"settings_revision",
})
# Display toggles: GUI-only rendering choices (kept in the GUI profile, not the live
@@ -118,21 +126,10 @@ _WEB_DISPLAY_SCHEMA = [
("combo_filter", "Display", ("pass_through",), _attr("_pass_through_combo_filter_input")),
]
# Stable run_config fields (GPR medium and antenna geometry). They live in run_config and
# take effect only when the pipeline (re)starts — not hot-reloaded — so the web marks them
# "applies on Start" and editing them just updates the widget for the next start.
_WEB_STABLE_SCHEMA = [
# Coherent BP fixes the medium to eps_r = 1 (Horns_motion_3libre.py), so relative
# permittivity is a legacy-GPR-only knob; BP ignores it.
("relative_permittivity", "Geometry & medium", ("legacy_gpr",), _attr("_gpr_relative_permittivity")),
("tx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_tx_geometry_input")),
("rx_geometry", "Geometry & medium", _GPR_LIKE_MODES, _attr("_gpr_rx_geometry_input")),
]
# Every field the web form renders and may write back, across all three categories.
# Every field the web form renders and may write back.
_WEB_FORM_GETTERS = {
field: getter
for source in (_WEB_LIVE_SCHEMA, _WEB_DISPLAY_SCHEMA, _WEB_STABLE_SCHEMA)
for source in (_WEB_LIVE_SCHEMA, _WEB_DISPLAY_SCHEMA)
for field, _group, _modes, getter in source
}
@@ -157,8 +154,9 @@ def _web_field_schema(field: str, group: str, widget) -> dict | None:
return {**base, "kind": "float", "value": float(widget.value()),
"min": float(widget.minimum()), "max": float(widget.maximum()),
"step": float(widget.singleStep()), "decimals": int(widget.decimals())}
if isinstance(widget, QPlainTextEdit):
return {**base, "kind": "textarea", "value": widget.toPlainText()}
if isinstance(widget, AntennaGeometryEditor):
return {**base, "kind": "geometry", "value": widget.records(),
"position_field": widget.position_field, "position_count": widget.position_count}
if isinstance(widget, QLineEdit):
return {**base, "kind": "text", "value": widget.text()}
return None
@@ -180,48 +178,47 @@ def _read_live_widget_value(widget):
return float(widget.value())
if isinstance(widget, QLineEdit):
return widget.text()
if isinstance(widget, AntennaGeometryEditor):
return widget.records()
raise TypeError(f"Unsupported live-settings widget: {type(widget).__name__}")
def _build_web_settings_schema(window) -> list[dict]:
"""Build the settings schema for the active mode straight from the Qt widgets.
Covers all three widget categories — live settings, GUI display toggles, and stable
run_config fields — so the web form shows exactly what the desktop processing panel
does. Stable fields are tagged ``applies_on_start`` (they take effect on the next run).
Covers live settings and GUI display toggles, matching the desktop panel.
"""
mode = window._processing_mode.currentText()
schema: list[dict] = []
for source, extra in (
(_WEB_LIVE_SCHEMA, {}),
(_WEB_DISPLAY_SCHEMA, {}),
(_WEB_STABLE_SCHEMA, {"applies_on_start": True}),
):
for source in (_WEB_LIVE_SCHEMA, _WEB_DISPLAY_SCHEMA):
for field, group, modes, getter in source:
if modes is not None and mode not in modes:
continue
entry = _web_field_schema(field, group, getter(window))
if entry is not None:
schema.append({**entry, **extra})
schema.append(entry)
return schema
def _set_web_live_field(window, field: str, value) -> None:
"""Write one web value into the desktop widget that feeds ``field`` (any category)."""
getter = _WEB_FORM_GETTERS.get(field)
if getter is None:
return
widget = getter(window)
def _set_widget_value(window, widget, value) -> None:
"""Validate and write a web value, also used when rolling back a rejected batch."""
if isinstance(widget, QComboBox):
if widget.findText(str(value)) < 0:
raise ValueError(f"Unknown setting option: {value}")
window._set_combo_current_text(widget, str(value))
elif isinstance(widget, QCheckBox):
widget.setChecked(bool(value))
elif isinstance(widget, QSpinBox):
widget.setValue(int(float(value)))
elif isinstance(widget, QDoubleSpinBox):
widget.setValue(float(value))
elif isinstance(widget, QPlainTextEdit):
widget.setPlainText(str(value))
if not isinstance(value, bool):
raise ValueError("Checkbox value must be true or false")
widget.setChecked(value)
elif isinstance(widget, (QSpinBox, QDoubleSpinBox)):
numeric = float(value)
if not math.isfinite(numeric) or not widget.minimum() <= numeric <= widget.maximum():
raise ValueError("Numeric setting is outside its allowed range")
if isinstance(widget, QSpinBox) and numeric != int(numeric):
raise ValueError("Setting requires an integer")
widget.setValue(int(numeric) if isinstance(widget, QSpinBox) else numeric)
elif isinstance(widget, AntennaGeometryEditor):
widget.set_records(value)
elif isinstance(widget, QLineEdit):
widget.setText(",".join(str(int(v)) for v in value) if isinstance(value, list) else str(value))
@@ -297,6 +294,7 @@ class AppWindowLiveProcessingMixin:
reprocess_current_result=bool(reprocess_current_result),
history_command=str(history_command),
history_command_seq=int(self._history_command_seq),
settings_revision=getattr(self, "_processing_settings_revision", 0),
**values,
)
@@ -315,12 +313,15 @@ class AppWindowLiveProcessingMixin:
"""
if bump_history_seq:
self._history_command_seq += 1
self._processing_settings_revision = time.monotonic_ns()
self._live_config_writer.write(
self._live_processing_config(
history_command=history_command,
reprocess_current_result=not self._can_reprocess_history(),
)
)
if getattr(self, "_await_processing_revision", False):
self._required_processing_revision = self._processing_settings_revision
def _apply_web_live_settings(self, fields: dict) -> None:
"""Apply web-requested live settings through the SAME path as a desktop edit.
@@ -339,6 +340,10 @@ class AppWindowLiveProcessingMixin:
"""
try:
history_command = str(fields.get("history_command", "none"))
if history_command not in {"none", "clear_all", "remove_last"}:
raise ValueError("Unknown history command")
if history_command != "none" and set(fields) != {"history_command"}:
raise ValueError("History commands must be sent separately from settings")
if history_command == "clear_all":
self._clear_all_runtime_history()
return
@@ -356,15 +361,30 @@ class AppWindowLiveProcessingMixin:
self._log_debug(f"Applying web live settings: fields={sorted(settings)}.")
self._suppress_live_settings_handler = True
try:
# processor_mode first: dual-sourced gpr_* fields route to the gpr or
# legacy_gpr widget based on the active mode.
if "processor_mode" in settings:
_set_web_live_field(self, "processor_mode", settings.pop("processor_mode"))
for name, value in settings.items():
_set_web_live_field(self, name, value)
with ExitStack() as blockers:
snapshots = []
try:
# Set the mode under a blocker before resolving per-mode fields.
ordered = sorted(settings, key=lambda name: name != "processor_mode")
for name in ordered:
widget = _WEB_FORM_GETTERS[name](self)
blockers.enter_context(QSignalBlocker(widget))
snapshots.append((widget, _read_live_widget_value(widget)))
_set_widget_value(self, widget, settings[name])
self._validate_processing_mode_selection(self._processing_mode.currentText())
except Exception:
for widget, value in reversed(snapshots):
_set_widget_value(self, widget, value)
raise
finally:
self._suppress_live_settings_handler = False
self._on_processing_live_settings_changed()
self._sync_socket_speed_controls()
if {"tx_geometry", "rx_geometry", "relative_permittivity"} & settings.keys():
self._await_processing_revision = True
if "processor_mode" in settings:
self._on_processing_mode_changed(self._processing_mode.currentText())
else:
self._on_processing_live_settings_changed()
except Exception as exc: # noqa: BLE001
self._show_exception("Failed to apply web live settings", exc)
@@ -372,6 +392,12 @@ class AppWindowLiveProcessingMixin:
"""Return the live-settings schema for the active mode (read from widgets)."""
return _build_web_settings_schema(self)
def _on_gpr_geometry_changed(self) -> None:
if getattr(self, "_suppress_live_settings_handler", False):
return
self._await_processing_revision = True
self._on_processing_live_settings_changed()
def _on_processing_live_settings_changed(self, *_args) -> None:
"""Handle live-processing setting changes and trigger redraw when needed."""
# Web-apply sets many widgets at once; their individual change signals are
@@ -399,11 +425,12 @@ class AppWindowLiveProcessingMixin:
collection = latest
if collection is None and self._result_history:
collection = self._result_history[-1]
if collection is None or not self._draw_results(collection):
if current_mode == "sar3d":
self._clear_sar3d_plot()
else:
self._clear_gpr_plot()
if current_mode == "sar3d":
self._clear_sar3d_plot()
if collection is not None:
self._draw_results(collection)
elif collection is None or not self._draw_results(collection):
self._clear_gpr_plot()
elif self._result_history:
self._draw_results(self._result_history[-1])
else:
@@ -589,9 +616,9 @@ class AppWindowLiveProcessingMixin:
self._clear_gpr_plot()
return
if self._processing_mode.currentText() == "sar3d":
if self._result_history and self._draw_results(self._result_history[-1]):
return
self._clear_sar3d_plot()
if self._result_history:
self._draw_results(self._result_history[-1])
return
if self._result_history:
self._draw_results(self._result_history[-1])
python_app/gui/controllers/app_window_config/profile_io_mixin.py
+38 -20
View File
@@ -10,6 +10,8 @@ from pathlib import Path
from PyQt6.QtCore import QSignalBlocker
from PyQt6.QtWidgets import QFileDialog
from python_app.file_io import atomic_write_json
from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT
from python_app.models.gui_profile_model import GuiProfileModel
from python_app.models.run_config_model import RunConfigModel
from python_app.orchestration.preprocess_assets import (
@@ -27,8 +29,7 @@ class AppWindowConfigProfileIOMixin:
raise FileExistsError(f"Config profile output already exists: {output_path}")
profile = self._build_gui_profile()
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_text(json.dumps(profile.to_dict(), indent=2), encoding="utf-8")
atomic_write_json(output_path, profile.to_dict(), durable=True)
return profile
def _set_combo_selection_mode(self, mode: str) -> None:
@@ -42,6 +43,7 @@ class AppWindowConfigProfileIOMixin:
self._single_combo_input.setEnabled(True)
self._run_combos_select_button.setEnabled(False)
self._single_combo_select_button.setEnabled(False)
self._next_combo_button.setEnabled(True)
return
text_selected = mode != "single"
@@ -52,6 +54,32 @@ class AppWindowConfigProfileIOMixin:
self._single_combo_input.setEnabled(not text_selected)
self._run_combos_select_button.setEnabled(True)
self._single_combo_select_button.setEnabled(True)
self._next_combo_button.setEnabled(not text_selected)
def _select_display_combo(self, input_pos: int, output_pos: int) -> None:
"""Select an available pair and redraw its cached B-scan immediately."""
key = (int(input_pos), int(output_pos))
if key not in self._available_display_combo_keys():
raise ValueError(f"Unknown channel combination: {key[0]}:{key[1]}")
self._single_combo_input.setText(str(key[0]))
self._single_combo_output.setText(str(key[1]))
self._on_bscan_display_combo_changed()
def _select_next_combo(self) -> None:
"""Advance output first, then input, wrapping after the last configured pair."""
available = self._available_display_combo_keys()
if not available:
return
try:
current = (int(self._single_combo_input.text()), int(self._single_combo_output.text()))
index = available.index(current) + 1
except ValueError:
index = 0
self._select_display_combo(*available[index % len(available)])
def _available_display_combo_keys(self) -> list[tuple[int, int]]:
"""Return configured pairs ordered by input, then output for display navigation."""
return sorted((combo.input, combo.output) for combo in self._defaults_config.build_runtime_combos())
def _is_matrix_radar_model_selected(self) -> bool:
"""Return whether the loaded config targets a matrix-mode radar acquisition."""
@@ -124,11 +152,10 @@ class AppWindowConfigProfileIOMixin:
C++ replay window still follows ring capacities, since it bounds how much
of the history the processor can re-publish coherently.
"""
save_history_limit = self._save_history_limit_for_config(config)
bscan_cpp_replay_window = self._cpp_bscan_replay_window_for_config(config)
self._raw_history = deque(self._raw_history, maxlen=save_history_limit)
self._pre_history = deque(self._pre_history, maxlen=save_history_limit)
self._result_history = deque(self._result_history, maxlen=save_history_limit)
self._raw_history = deque(self._raw_history, maxlen=GUI_SAVE_HISTORY_LIMIT)
self._pre_history = deque(self._pre_history, maxlen=GUI_SAVE_HISTORY_LIMIT)
self._result_history = deque(self._result_history, maxlen=GUI_SAVE_HISTORY_LIMIT)
self._bscan_cpp_replay_window = bscan_cpp_replay_window
self._clear_bscan_plot_history()
@@ -351,7 +378,6 @@ class AppWindowConfigProfileIOMixin:
self._sar3d_comp_range_power,
self._sar3d_use_compensation,
self._sar3d_normalize_each_frame,
self._save_count,
self._record_count,
self._save_path_input,
self._save_name_input,
@@ -457,18 +483,10 @@ class AppWindowConfigProfileIOMixin:
self._bscan_history_window.setValue(int(gui_state.processing.bscan.history_window_scans))
self._gpr_relative_permittivity.setValue(float(config.gpr.relative_permittivity))
self._gpr_tx_geometry_input.setPlainText(
"\n".join(
f"{int(entry.output_pos)} {float(entry.x_m):g}"
for entry in config.gpr.tx_geometry
)
)
self._gpr_rx_geometry_input.setPlainText(
"\n".join(
f"{int(entry.input_pos)} {float(entry.x_m):g}"
for entry in config.gpr.rx_geometry
)
)
self._gpr_tx_geometry_input.set_position_count(config.output_switch.positions)
self._gpr_rx_geometry_input.set_position_count(config.input_switch.positions)
self._gpr_tx_geometry_input.set_records(config.gpr.tx_geometry)
self._gpr_rx_geometry_input.set_records(config.gpr.rx_geometry)
self._gpr_input_positions_input.setText(str(gui_state.processing.gpr.input_positions))
self._gpr_output_positions_input.setText(str(gui_state.processing.gpr.output_positions))
self._gpr_min_depth_m.setValue(float(gui_state.processing.gpr.min_depth_m))
@@ -555,7 +573,6 @@ class AppWindowConfigProfileIOMixin:
self._sar3d_normalize_each_frame.setChecked(bool(sar3d_state.normalize_each_frame))
self._sync_socket_speed_controls()
self._save_count.setValue(int(gui_state.data_actions.save_count))
self._record_count.setValue(int(gui_state.data_actions.record_count))
self._save_path_input.setText(str(gui_state.data_actions.save_path))
self._save_name_input.setText(str(gui_state.data_actions.save_name))
@@ -590,6 +607,7 @@ class AppWindowConfigProfileIOMixin:
self._apply_initial_radar_limits()
if self._preprocess_dialog is not None:
self._refresh_sets()
self._await_processing_revision = True
self._on_processing_mode_changed(gui_state.processing.selected_mode)
self._update_history_indicator()
self._remember_active_profile_path(profile_path)
python_app/gui/controllers/app_window_config/radar_limits_mixin.py
-3
View File
@@ -28,9 +28,6 @@ class AppWindowRadarLimitsMixin:
else:
radar_service = create_single_radar_service(config)
if isinstance(radar_service, LibreVnaService) and not radar_service.driver_available:
raise RuntimeError("LibreVNA Python driver is not available for device limits query")
limits = radar_service.read_device_limits()
return self._apply_radar_limits_to_ui(limits)
python_app/gui/controllers/app_window_config/state_builders.py
+2 -126
View File
@@ -81,52 +81,6 @@ class AppWindowConfigStateBuildersMixin:
env[key] = value
return env
@staticmethod
def _parse_geometry_line_coordinates(parts: list[str]) -> tuple[float, float, float]:
"""Parse 1/2/3 trailing coordinate fields into (x, y, z); missing axes default to 0."""
x_m = float(parts[0])
y_m = float(parts[1]) if len(parts) >= 2 else 0.0
z_m = float(parts[2]) if len(parts) >= 3 else 0.0
return x_m, y_m, z_m
@staticmethod
def _parse_gpr_tx_geometry_text(text: str) -> list[GprTxGeometryModel]:
"""Parse line-based Tx geometry editor text: `output_pos x_m [y_m] [z_m]`."""
entries: list[GprTxGeometryModel] = []
for line_number, raw_line in enumerate(text.splitlines(), start=1):
line = raw_line.strip()
if not line:
continue
parts = line.split()
if len(parts) < 2 or len(parts) > 4:
raise ValueError(
f"Invalid Tx geometry line {line_number}: expected `output_pos x_m [y_m] [z_m]`"
)
x_m, y_m, z_m = AppWindowConfigStateBuildersMixin._parse_geometry_line_coordinates(parts[1:])
entries.append(
GprTxGeometryModel(output_pos=int(parts[0]), x_m=x_m, y_m=y_m, z_m=z_m)
)
return entries
@staticmethod
def _parse_gpr_rx_geometry_text(text: str) -> list[GprRxGeometryModel]:
"""Parse line-based Rx geometry editor text: `input_pos x_m [y_m] [z_m]`."""
entries: list[GprRxGeometryModel] = []
for line_number, raw_line in enumerate(text.splitlines(), start=1):
line = raw_line.strip()
if not line:
continue
parts = line.split()
if len(parts) < 2 or len(parts) > 4:
raise ValueError(
f"Invalid Rx geometry line {line_number}: expected `input_pos x_m [y_m] [z_m]`"
)
x_m, y_m, z_m = AppWindowConfigStateBuildersMixin._parse_geometry_line_coordinates(parts[1:])
entries.append(
GprRxGeometryModel(input_pos=int(parts[0]), x_m=x_m, y_m=y_m, z_m=z_m)
)
return entries
@staticmethod
def _format_combos_text_from_config(config: RunConfigModel) -> str:
"""Render configured combos for UI text editor, keeping full matrix as empty."""
@@ -192,18 +146,6 @@ class AppWindowConfigStateBuildersMixin:
),
)
@staticmethod
def _save_history_limit_for_config(config: RunConfigModel) -> int:
"""Return maxlen for GUI snapshot-save deques.
Independent of ring capacities — see :data:`GUI_SAVE_HISTORY_LIMIT`.
The `config` argument is kept for symmetry with
:meth:`_cpp_bscan_replay_window_for_config` and possible future per-profile
overrides.
"""
del config
return GUI_SAVE_HISTORY_LIMIT
def _default_gui_state_for_config(self, config: RunConfigModel) -> GuiStateModel:
"""Build fallback GUI-only defaults for a stable run config."""
default_combo = config.combos[0] if config.combos else ComboModel(input=0, output=0)
@@ -217,74 +159,17 @@ class AppWindowConfigStateBuildersMixin:
single_output=str(int(default_combo.output)),
),
processing=GuiProcessingStateModel(
selected_mode="pass_through",
pass_through=GuiPassThroughStateModel(
show_magnitude=True,
show_phase=True,
unwrap_phase=False,
combo_filter="",
fixed_y_enabled=False,
y_min_db=-100.0,
y_max_db=0.0,
),
bscan=GuiBscanStateModel(
axis="abs",
cut_m=0.824,
max_depth_m=1.0,
gain=1.0,
start_freq_mhz=100.0,
stop_freq_mhz=8800.0,
subtract_mean_ascan=False,
),
gpr=GuiGprStateModel(
input_positions=self._default_gpr_input_positions_from_config(config),
output_positions=self._default_gpr_output_positions_from_config(config),
min_depth_m=2.0,
max_depth_m=14.0,
range_comp_power=0.1,
object_min_frac=0.7,
score_mode="combined",
motion_mode="int_minus",
look_angle_deg=0.0,
direction_sign=1.0,
speed_m_s=0.0,
ignore_socket_speed_enabled=False,
max_detected_objects_to_draw=5,
draw_top_m_objects=2,
object_approach_min_frames=3,
start_freq_mhz=3000.0,
stop_freq_mhz=6000.0,
background_subtract_enabled=True,
background_mean_count=10,
remove_sidelobe_objects_enabled=True,
imaging_plane_y_m=0.0,
render_mode="heatmap",
visible_x_min_m=default_gpr_x_min_m,
visible_x_max_m=default_gpr_x_max_m,
visible_z_min_m=0.0,
visible_z_max_m=14.0,
),
legacy_gpr=GuiLegacyGprStateModel(
input_positions=self._default_gpr_input_positions_from_config(config),
output_positions=self._default_gpr_output_positions_from_config(config),
min_depth_m=2.0,
max_depth_m=14.0,
comp_power=0.2,
start_freq_mhz=3000.0,
stop_freq_mhz=6000.0,
speed_m_s=0.0,
ignore_socket_speed_enabled=False,
look_angle_deg=0.0,
snr_thresh=4.5,
snr_comp_max=25.0,
background_subtract_enabled=True,
background_mean_count=10,
render_mode="heatmap",
min_visible_pair_count=1,
visible_x_min_m=default_gpr_x_min_m,
visible_x_max_m=default_gpr_x_max_m,
visible_z_min_m=0.0,
visible_z_max_m=14.0,
),
sar3d=GuiSar3dStateModel(
input_positions=self._default_gpr_input_positions_from_config(config),
@@ -292,15 +177,7 @@ class AppWindowConfigStateBuildersMixin:
),
),
data_actions=GuiDataActionsStateModel(
save_count=10,
save_path=str(self._project_root / "python_app/data/snapshots"),
save_name="snapshot_manual",
record_count=100,
),
preprocess_dialog=GuiPreprocessDialogStateModel(
set_name="set_001",
radar_config_dir="",
use_all_radar_configs=False,
),
)
@@ -442,7 +319,6 @@ class AppWindowConfigStateBuildersMixin:
),
),
data_actions=GuiDataActionsStateModel(
save_count=int(self._save_count.value()),
save_path=self._save_path_input.text().strip(),
save_name=self._save_name_input.text().strip(),
record_count=int(self._record_count.value()),
@@ -502,8 +378,8 @@ class AppWindowConfigStateBuildersMixin:
for key in VISIBLE_PREPROCESS_ASSET_KEYS:
preprocess_asset_model(config, key).set_name = self._selected_preprocess_sets.get(key, "")
config.gpr.relative_permittivity = float(self._gpr_relative_permittivity.value())
config.gpr.tx_geometry = self._parse_gpr_tx_geometry_text(self._gpr_tx_geometry_input.toPlainText())
config.gpr.rx_geometry = self._parse_gpr_rx_geometry_text(self._gpr_rx_geometry_input.toPlainText())
config.gpr.tx_geometry = [GprTxGeometryModel(**entry) for entry in self._gpr_tx_geometry_input.records()]
config.gpr.rx_geometry = [GprRxGeometryModel(**entry) for entry in self._gpr_rx_geometry_input.records()]
validate_gpr_model(
config.gpr,
input_switch_positions=config.input_switch.positions,
python_app/gui/controllers/app_window_pipeline_mixin.py
+123 -84
View File
@@ -7,6 +7,7 @@ import time
from PyQt6.QtCore import QCoreApplication, QEventLoop
from python_app.gui.runtime.constraints import validate_processing_mode_constraints
from python_app.gui.controllers.app_window_plot.sar3d_plot_mixin import sar3d_status
from python_app.gui.runtime.history import (
build_processor_run_signature,
build_run_history_signature,
@@ -16,6 +17,7 @@ from python_app.hardware_full.kamil_adc import apply_kamil_adc_laser_control
from python_app.hardware_full.single_radar_service import create_single_radar_service
from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection
from python_app.models.run_config_model import RunConfigModel
from python_app.models.run_config_validation import validate_acquisition_config
from python_app.orchestration.preprocess_assets import (
PREPROCESS_ASSET_SPECS,
REQUIRED_PREPROCESS_ASSET_KEYS,
@@ -57,6 +59,11 @@ class AppWindowPipelineMixin:
def _start_run(self, *, single_capture: bool = False) -> None:
"""Start pipeline processes and ring readers."""
if getattr(self, "_closing", False) or getattr(self, "_close_after_stop", False):
return
if getattr(self, "_stopping_run", False):
self._show_error("Wait for pipeline shutdown before starting again")
return
if self._capture_session is not None:
self._show_error(
"Cannot start pipeline during active capture sequence",
@@ -76,15 +83,20 @@ class AppWindowPipelineMixin:
return
self._log("Stopping continuous pipeline before single capture")
self._stop_run()
if getattr(self, "_closing", False):
return
try:
processor_was_running = self._supervisor.is_processor_running()
config = self._build_config()
validate_acquisition_config(config)
run_signature = self._build_run_history_signature(config)
processor_signature = self._build_processor_run_signature(config)
if self._processor_requires_restart(processor_signature):
self._log("Restarting data_processor because stable run settings changed")
self._stop_all_processes()
if getattr(self, "_closing", False):
return
processor_was_running = False
if not processor_was_running:
self._reset_runtime_history()
@@ -207,13 +219,17 @@ class AppWindowPipelineMixin:
return
was_running = self._supervisor.is_running()
resume_seq = getattr(self, "_stop_request_seq", 0) + 1
processor_only_running = self._supervisor.is_processor_running() and not was_running
if was_running:
self._stop_run()
if not self._pipeline_resume_allowed(resume_seq):
return
try:
config = self._build_config()
if config.radar.driver_mode == "native":
validate_acquisition_config(config)
if config.radar.driver_mode == "native" and not config.radar.replay.dataset_dir:
self._refresh_radar_limits_from_device()
processor_signature = self._build_processor_run_signature(config)
if processor_only_running and self._processor_requires_restart(processor_signature):
@@ -241,13 +257,14 @@ class AppWindowPipelineMixin:
except Exception as exc: # noqa: BLE001
self._show_exception("Failed to apply radar settings", exc)
finally:
if was_running:
if was_running and self._pipeline_resume_allowed(resume_seq):
self._start_run()
def _prepare_radar_for_native_acquisition(self, config: RunConfigModel) -> None:
"""Preconfigure native single-radar hardware using current sweep settings."""
if config.radar.model == RunConfigModel.COMPACT_M_K209_MODEL and config.radar.driver_mode != "native":
raise RuntimeError("Compact-M K209 requires radar.driver_mode='native'")
if config.radar.replay.dataset_dir:
self._log("Radar pre-configuration skipped (dataset replay)")
return
if config.radar.driver_mode != "native":
self._log("Radar pre-configuration skipped (mock mode)")
return
@@ -265,9 +282,6 @@ class AppWindowPipelineMixin:
return
radar_service = create_single_radar_service(config)
if not getattr(radar_service, "driver_available", True):
raise RuntimeError("LibreVNA Python driver is not available for native pre-configuration")
try:
radar_service.open()
radar_service.configure(config.radar.sweep)
@@ -278,49 +292,77 @@ class AppWindowPipelineMixin:
def _stop_run(self) -> None:
"""Stop acquisition-side processes and close readers as needed."""
self._pipeline_should_run = False # an explicit stop disables crash auto-restart
was_running = self._supervisor.is_running()
if was_running:
self._supervisor.stop_orchestrator()
self._drain_rings_until_quiet(timeout_s=0.35, poll_s=0.02)
self._supervisor.stop_preprocessor()
self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02)
else:
self._supervisor.stop()
self._drain_rings_once_for_history()
# Flush any in-progress disk recording so its buffered (not-yet-full) chunk is
# written rather than lost; the drains above already captured the last data.
self._finalize_recording_on_stop()
keep_results_reader = self._supervisor.is_processor_running()
self._close_readers(keep_results=keep_results_reader)
self._single_capture_active = False
self._single_capture_start_ns = None
self._single_capture_seen_raw = False
self._single_capture_target_collection_id = None
self._update_history_indicator()
self._status_label.setText("Status: idle")
if was_running:
if keep_results_reader:
self._log("Acquisition stopped (data_processor kept running)")
else:
self._log("Pipeline stopped")
self._stop_request_seq = getattr(self, "_stop_request_seq", 0) + 1
self._resume_pipeline_after_capture = False
self._stop_pipeline(keep_processor=True)
def _stop_all_processes(self) -> None:
"""Stop all managed pipeline processes and close all readers."""
self._pipeline_should_run = False # an explicit stop disables crash auto-restart
was_running = self._supervisor.is_running() or self._supervisor.is_processor_running()
self._supervisor.stop_all()
self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02)
self._close_readers(keep_results=False)
self._single_capture_active = False
self._single_capture_start_ns = None
self._single_capture_seen_raw = False
self._single_capture_target_collection_id = None
self._processor_run_signature = None
self._update_history_indicator()
self._status_label.setText("Status: idle")
if was_running:
self._log("All pipeline processes stopped")
self._stop_pipeline(keep_processor=False)
def _stop_pipeline(self, *, keep_processor: bool) -> None:
"""Share shutdown/drain/finalization, protecting event-loop re-entry."""
self._pipeline_should_run = False
if getattr(self, "_stopping_run", False):
self._stop_all_requested |= not keep_processor
return
self._stopping_run = True
self._stop_all_requested = not keep_processor
try:
drain_error = None
try:
was_running = self._supervisor.is_running()
if keep_processor and was_running:
self._supervisor.stop_orchestrator()
self._drain_rings_until_quiet(timeout_s=0.35, poll_s=0.02)
self._supervisor.stop_preprocessor()
self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02)
elif keep_processor:
self._supervisor.stop()
self._drain_rings_once_for_history()
if self._stop_all_requested:
self._supervisor.stop_all()
self._drain_rings_until_quiet(timeout_s=0.25, poll_s=0.02)
except Exception as exc: # noqa: BLE001 - corrupt data must not prevent Stop
drain_error = exc
self._stop_request_seq = getattr(self, "_stop_request_seq", 0) + 1
if self._stop_all_requested:
self._supervisor.stop_all()
else:
self._supervisor.stop()
if self._stop_all_requested:
self._processor_run_signature = None
self._finalize_recording_on_stop()
keep_results = not self._stop_all_requested and self._supervisor.is_processor_running()
result_ring_name = getattr(self._result_reader, "_ring_name", None) if drain_error else None
self._close_readers(keep_results=keep_results and drain_error is None)
self._single_capture_active = False
self._single_capture_start_ns = None
self._single_capture_seen_raw = False
self._single_capture_target_collection_id = None
self._update_history_indicator()
self._status_label.setText("Status: idle")
self._log("Acquisition stopped (data_processor kept running)" if keep_results else "Pipeline stopped")
if drain_error is not None:
self._status_label.setText("Status: error")
self._show_exception("Pipeline stopped, but final ring data could not be read", drain_error)
if keep_results and result_ring_name:
self._result_reader = ShmRingReader(result_ring_name)
except Exception as exc: # noqa: BLE001 - this is also a direct Qt button slot
self._stop_request_seq = getattr(self, "_stop_request_seq", 0) + 1
self._status_label.setText("Status: error")
self._show_exception("Failed to stop pipeline", exc)
finally:
self._stopping_run = False
if getattr(self, "_close_after_stop", False):
self._close_after_stop = False
self.close()
def _pipeline_resume_allowed(self, stop_seq: int) -> bool:
"""A Stop or Close received during a temporary pause cancels its restart."""
return not getattr(self, "_closing", False) and not getattr(self, "_close_after_stop", False) and (
getattr(self, "_stop_request_seq", 0) == stop_seq
)
def _close_readers(self, *, keep_results: bool = False) -> None:
"""Close active ring readers; keep the results reader when ``keep_results``."""
@@ -342,6 +384,8 @@ class AppWindowPipelineMixin:
def _poll_rings(self) -> None:
"""Poll readers, ingest history, and trigger rendering."""
if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False):
return
self._web_update_snapshot() # guarded internally; never raises
self._handle_process_exit_reports()
@@ -366,12 +410,10 @@ class AppWindowPipelineMixin:
# crash-storm budget (it only caps consecutive crash-restarts).
self._pipeline_restart_count = 0
render_started_ns = time.monotonic_ns()
self._draw_preferred_collection(result_latest=result_latest)
self._draw_results(result_latest)
self._pipeline_metrics.record(
"rendering", time.monotonic_ns() - render_started_ns
)
else:
self._draw_preferred_collection(result_latest=None)
except Exception as exc: # noqa: BLE001
self._handle_reader_poll_error(exc)
@@ -395,7 +437,12 @@ class AppWindowPipelineMixin:
self._log_exception("Failed to collect process exit reports", exc, level="ERROR")
return
if unexpected and getattr(self, "_pipeline_should_run", False):
if getattr(self, "_pipeline_should_run", False) and (
unexpected
or not {"data_processor", "data_preprocessor", "sweep_orchestrator"}.issubset(self._supervisor.pids())
):
# Exit reports are consumed once. A missing child must keep triggering
# recovery after a back-off delay or a failed start with no reports left.
self._recover_pipeline_after_crash()
def _recover_pipeline_after_crash(self) -> None:
@@ -441,7 +488,7 @@ class AppWindowPipelineMixin:
self._drop_pending_ring_payloads(include_results=True)
self._status_label.setText("Status: running")
self._log("Pipeline auto-restarted after crash.")
except Exception as exc: # noqa: BLE001 - retry on the next crash signal
except Exception as exc: # noqa: BLE001 - retry on subsequent polls
self._log_exception("Pipeline auto-restart failed; will retry", exc, level="ERROR")
def _check_single_capture_deadline(self) -> None:
@@ -529,8 +576,15 @@ class AppWindowPipelineMixin:
return False
self._draw_results(target_result)
self._log("Single capture completed")
status = sar3d_status(target_result) if self._processing_mode.currentText() == "sar3d" else None
self._stop_run()
if status is not None and status[0] in (1, 2, 3):
state = {1: "no signal in selected range", 2: "error", 3: "warning"}[status[0]]
self._status_label.setText(f"Status: {state}")
log = self._log_error if status[0] == 2 else self._log_warning
log(f"Single capture: SAR3D {state}: {status[1]}")
else:
self._log("Single capture completed")
return True
def _find_single_capture_target_result(self) -> ResultCollection | None:
@@ -551,12 +605,12 @@ class AppWindowPipelineMixin:
return None
def _read_all_raw(self) -> SweepCollection | None:
def _read_all_raw(self, *, max_collections: int | None = None) -> SweepCollection | None:
"""Read available raw collections from raw ring."""
if self._raw_reader is None:
raise RuntimeError("Raw ring reader is not initialised")
latest: SweepCollection | None = None
for _ in range(self._max_pop_per_poll):
for _ in range(self._max_pop_per_poll if max_collections is None else max_collections):
collection = self._raw_reader.pop_raw_collection()
if collection is None:
break
@@ -575,24 +629,27 @@ class AppWindowPipelineMixin:
self._single_capture_target_collection_id = collection.collection_id
return latest
def _read_all_preprocessed(self) -> None:
def _read_all_preprocessed(self, *, max_collections: int | None = None) -> SweepCollection | None:
"""Read available preprocessed collections from preprocessed ring."""
if self._pre_reader is None:
return
for _ in range(self._max_pop_per_poll):
latest: SweepCollection | None = None
for _ in range(self._max_pop_per_poll if max_collections is None else max_collections):
collection = self._pre_reader.pop_preprocessed_collection()
if collection is None:
break
self._pre_history.append(collection)
self._record_collection("preprocessed", collection)
latest = collection
return latest
def _read_all_results(self) -> ResultCollection | None:
def _read_all_results(self, *, max_collections: int | None = None) -> ResultCollection | None:
"""Read available result collections from results ring."""
if self._result_reader is None:
raise RuntimeError("Result ring reader is not initialised")
latest: ResultCollection | None = None
for _ in range(self._max_pop_per_poll):
for _ in range(self._max_pop_per_poll if max_collections is None else max_collections):
collection = self._result_reader.pop_result_collection()
if collection is None:
break
@@ -627,36 +684,19 @@ class AppWindowPipelineMixin:
# residual in tiny slices so we neither busy-spin nor block too long.
time.sleep(min(0.002, max(0.0, deadline - time.monotonic())))
def _drain_rings_once_for_history(self) -> None:
def _drain_rings_once_for_history(self) -> bool:
"""Perform one non-blocking read pass to extend histories."""
if self._raw_reader is not None:
self._read_all_raw()
self._read_all_preprocessed()
if self._result_reader is not None:
self._read_all_results()
raw = self._read_all_raw() if self._raw_reader is not None else None
preprocessed = self._read_all_preprocessed()
result = self._read_all_results() if self._result_reader is not None else None
return raw is not None or preprocessed is not None or result is not None
def _drain_rings_until_quiet(self, *, timeout_s: float, poll_s: float) -> None:
"""Drain rings until history sizes stabilize or timeout expires."""
"""Drain rings until readers are quiet or timeout expires, even at full history capacity."""
deadline = time.monotonic() + timeout_s
stable_rounds = 0
previous = (
len(self._raw_history),
len(self._pre_history),
len(self._result_history),
)
while time.monotonic() < deadline and stable_rounds < 2:
self._drain_rings_once_for_history()
current = (
len(self._raw_history),
len(self._pre_history),
len(self._result_history),
)
if current == previous:
stable_rounds += 1
else:
stable_rounds = 0
previous = current
stable_rounds = 0 if self._drain_rings_once_for_history() else stable_rounds + 1
# Event-loop-friendly wait so timers/signals keep firing during drain.
self._pump_events_during_drain(poll_s)
@@ -734,4 +774,3 @@ class AppWindowPipelineMixin:
config,
self._live_processing_config(),
)
python_app/gui/controllers/app_window_plot/bscan_plot_mixin.py
+3 -48
View File
@@ -198,7 +198,7 @@ class AppWindowBscanPlotMixin:
def _draw_bscan_heatmap(self, _collection: ResultCollection) -> bool:
"""Draw B-scan image rebuilt from processed result history."""
self._disable_phase_axis()
self._trace_phase_plot.clear()
self._sync_bscan_history_from_results()
return self._draw_bscan_heatmap_from_history()
@@ -256,8 +256,8 @@ class AppWindowBscanPlotMixin:
image_item.setRect(QRectF(x_min, depth_min, sweep_width, depth_span))
axis_mode = self._bscan_axis.currentText()
image_item.setLookupTable(self._bscan_lookup_table(axis_mode))
image_item.setLevels(self._bscan_levels(sweeps, axis_mode))
image_item.setLookupTable(bscan_lookup_table(axis_mode))
image_item.setLevels(bscan_levels(sweeps, axis_mode))
self._bscan_plot.clear()
self._configure_bscan_plot_axes()
@@ -463,53 +463,8 @@ class AppWindowBscanPlotMixin:
except ValueError:
return None
def _bscan_lookup_table(self, axis_mode: str) -> np.ndarray:
"""Return lookup table for current B-scan axis mode."""
return bscan_lookup_table(axis_mode)
@staticmethod
def _build_lut(stops: list[str], *, size: int = 256) -> np.ndarray:
"""Backward-compatible wrapper around LUT builder."""
return build_lut(stops, size=size)
@staticmethod
def _bscan_levels(sweeps: np.ndarray, axis_mode: str) -> tuple[float, float]:
"""Return display levels for B-scan image."""
return bscan_levels(sweeps, axis_mode)
def _clear_bscan_plot_history(self) -> None:
"""Drop cached B-scan history and invalidate cache signature."""
self._bscan_history_by_combo.clear()
self._bscan_depth_axis_by_combo.clear()
self._bscan_render_signature = None
def _ensure_phase_view_box(self) -> pg.ViewBox:
"""Create or return secondary right-axis ViewBox for phase curves."""
plot_item = self._bscan_plot.getPlotItem()
phase_view_box = self._phase_viewbox
if phase_view_box is None:
phase_view_box = pg.ViewBox()
self._phase_viewbox = phase_view_box
plot_item.scene().addItem(phase_view_box)
plot_item.getAxis("right").linkToView(phase_view_box)
phase_view_box.setXLink(plot_item.vb)
plot_item.vb.sigResized.connect(self._update_phase_view_box_geometry)
self._update_phase_view_box_geometry()
return phase_view_box
def _update_phase_view_box_geometry(self) -> None:
"""Keep right-axis ViewBox geometry in sync with main plot ViewBox."""
phase_view_box = self._phase_viewbox
if phase_view_box is None:
return
plot_item = self._bscan_plot.getPlotItem()
phase_view_box.setGeometry(plot_item.vb.sceneBoundingRect())
phase_view_box.linkedViewChanged(plot_item.vb, phase_view_box.XAxis)
def _clear_phase_overlay(self) -> None:
"""Remove all phase curves from secondary ViewBox."""
self._trace_phase_plot.clear()
def _disable_phase_axis(self) -> None:
"""Hide right axis and clear phase overlay when phase is not rendered."""
self._clear_phase_overlay()
python_app/gui/controllers/app_window_plot/gpr_plot_mixin.py
+12 -11
View File
@@ -6,6 +6,7 @@ from PyQt6.QtCore import QRectF
import numpy as np
import pyqtgraph as pg
from python_app.gui.controllers.app_window_plot.bscan_plot_mixin import build_lut
from python_app.models.dataset_model import ResultCollection
from python_app.orchestration.gpr_locator import (
collection_payload_by_name as gpr_collection_payload_by_name,
@@ -95,7 +96,7 @@ class AppWindowGprPlotMixin:
self._configure_gpr_plot_axes()
if self._gpr_lookup_table is None:
self._gpr_lookup_table = self._build_lut(["#081c15", "#1b4332", "#ffd166", "#f94144"])
self._gpr_lookup_table = build_lut(["#081c15", "#1b4332", "#ffd166", "#f94144"])
self._gpr_image_item = pg.ImageItem(axisOrder="row-major")
self._gpr_image_item.setZValue(0)
@@ -179,29 +180,29 @@ class AppWindowGprPlotMixin:
output_positions_text = self._gpr_output_positions_input.text()
requested_inputs = tuple(self._parse_csv_int_list(input_positions_text))
requested_outputs = tuple(self._parse_csv_int_list(output_positions_text))
tx_entries = self._gpr_tx_geometry_input.records()
rx_entries = self._gpr_rx_geometry_input.records()
signature = (
self._gpr_tx_geometry_input.toPlainText(),
self._gpr_rx_geometry_input.toPlainText(),
tuple(tuple(entry.values()) for entry in tx_entries),
tuple(tuple(entry.values()) for entry in rx_entries),
requested_inputs,
requested_outputs,
)
if signature == self._gpr_geometry_signature and self._gpr_selected_geometry is not None:
return self._gpr_selected_geometry
tx_entries = self._parse_gpr_tx_geometry_text(signature[0])
rx_entries = self._parse_gpr_rx_geometry_text(signature[1])
requested_input_set = set(requested_inputs)
requested_output_set = set(requested_outputs)
rx_entries = sorted(rx_entries, key=lambda entry: int(entry.input_pos))
tx_entries = sorted(tx_entries, key=lambda entry: int(entry.output_pos))
rx_entries = sorted(rx_entries, key=lambda entry: entry["input_pos"])
tx_entries = sorted(tx_entries, key=lambda entry: entry["output_pos"])
if requested_input_set:
rx_entries = [entry for entry in rx_entries if int(entry.input_pos) in requested_input_set]
rx_entries = [entry for entry in rx_entries if entry["input_pos"] in requested_input_set]
if requested_output_set:
tx_entries = [entry for entry in tx_entries if int(entry.output_pos) in requested_output_set]
tx_entries = [entry for entry in tx_entries if entry["output_pos"] in requested_output_set]
x_tx = np.asarray([float(entry.x_m) for entry in tx_entries], dtype=np.float32)
x_rx = np.asarray([float(entry.x_m) for entry in rx_entries], dtype=np.float32)
x_tx = np.asarray([entry["x_m"] for entry in tx_entries], dtype=np.float32)
x_rx = np.asarray([entry["x_m"] for entry in rx_entries], dtype=np.float32)
self._gpr_geometry_signature = signature
self._gpr_selected_geometry = (x_tx, x_rx)
return self._gpr_selected_geometry
python_app/gui/controllers/app_window_plot/sar3d_plot_mixin.py
+37 -8
View File
@@ -1,14 +1,17 @@
"""3D SAR views of the volume published by the sar3d processor.
Scene axes: X across, Y height above the ground (Y = 0 is the ground), Z along
the ground away from the radar. Payloads, the volume normalized to a peak of 1:
the ground away from the radar. Payloads share normalization by the maximum
away from the Y boundaries:
- ``sar3d_front``, ``sar3d_side``, ``sar3d_top``: maximum projections onto X-Y
(looking ahead from the radar), Z-Y (from the side) and X-Z (from above);
- ``sar3d_oblique``: orthographic projection along the camera of the notebook's
3D scene; ``sar3d_oblique_basis`` (2x3) gives its image axes u and v as scene
directions, which places the scene box, the ground and the maximum on it;
- ``sar3d_peak``: [level, X, Y, Z] of the volume maximum before normalization.
- ``sar3d_peak``: [level, X, Y, Z] of that maximum before normalization.
- ``sar3d_status``: [code, UTF-8 bytes...] with ready CUDA (0), no signal (1),
processing error (2), or CUDA unavailable (3), including the explanation.
"""
from __future__ import annotations
@@ -22,7 +25,7 @@ from PyQt6.QtCore import QRectF, Qt
from python_app.gui.controllers.app_window_plot.bscan_plot_mixin import build_lut
from python_app.models.dataset_model import ResultCollection
# Colours start at this fraction of the maximum; weaker voxels stay transparent,
# Colours start at this fraction of the normalization reference; weaker voxels stay transparent,
# like those under the threshold of the notebook's point cloud.
DISPLAY_THRESHOLD = 0.75
@@ -31,7 +34,7 @@ SAR3D_VIEWS: dict[str, tuple[str, str, str, tuple[int, int] | None]] = {
"sar3d_front": ("Front (X-Y)", "X", "Y", (0, 1)),
"sar3d_side": ("Side (Z-Y)", "Z", "Y", (2, 1)),
"sar3d_top": ("Top (X-Z)", "X", "Z", (0, 2)),
"sar3d_oblique": ("3D", "", "", None),
"sar3d_oblique": ("3D projection", "", "", None),
}
# Corners of the scene box indexed by bits (x, y, z); an edge joins corners one bit apart.
@@ -40,6 +43,22 @@ _BOX_EDGES = [(a, b) for a, b in itertools.combinations(range(8), 2) if bin(a ^
_AXIS_EDGES = {"X": (0, 1), "Y": (1, 3), "Z": (1, 5)}
def sar3d_status(collection: ResultCollection) -> tuple[int, str] | None:
"""Decode processor status without letting malformed telemetry break a Qt slot."""
payload = next((item for item in collection.collection_payloads
if item.processing_name == "sar3d_status" and item.kind == 4), None)
if payload is None:
return None
table = payload.table
if table.ndim != 2 or table.shape[0] != 1 or table.shape[1] == 0:
return 2, "Invalid SAR3D status payload"
row = table[0]
if (not np.isfinite(row).all() or not np.equal(row, np.floor(row)).all()
or row[0] not in (0, 1, 2, 3) or np.any((row[1:] < 0) | (row[1:] > 255))):
return 2, "Invalid SAR3D status payload"
return int(row[0]), bytes(row[1:].astype(np.uint8)).decode("utf-8", errors="replace")
def _image_rect(x_axis: np.ndarray, y_axis: np.ndarray) -> QRectF:
"""Rectangle covering the pixels centred on the axis samples."""
dx = (x_axis[-1] - x_axis[0]) / max(len(x_axis) - 1, 1)
@@ -97,7 +116,7 @@ class AppWindowSar3dPlotMixin:
oblique.addItem(label)
self._sar3d_axis_labels[axis_name] = label
def _clear_sar3d_plot(self) -> None:
def _clear_sar3d_plot(self, message: str = "SAR3D: waiting for measurements") -> None:
"""Clear all four 3D SAR views."""
for name, (title, *_rest) in SAR3D_VIEWS.items():
self._sar3d_images[name].hide()
@@ -105,13 +124,21 @@ class AppWindowSar3dPlotMixin:
self._sar3d_plots[name].setTitle(title)
self._sar3d_box.setData([], [])
self._sar3d_ground.setData([], [])
for label in self._sar3d_axis_labels.values():
label.hide()
self._sar3d_status_label.setText(message)
def _draw_sar3d_projections(self, collection: ResultCollection) -> bool:
"""Draw the 3D SAR views of a result collection; return success."""
"""Draw four maximum projections of the volume; return success."""
status = sar3d_status(collection)
if status is not None and status[0] in (1, 2, 3):
state = {1: "no signal", 2: "error", 3: "warning"}[status[0]]
self._clear_sar3d_plot(f"SAR3D: {state} — {status[1]}")
return False
payloads = {name: self._collection_payload_by_name(collection, name, kind=3) for name in SAR3D_VIEWS}
basis = self._collection_payload_by_name(collection, "sar3d_oblique_basis", kind=4)
if basis is None or any(payload is None or payload.image.size == 0 for payload in payloads.values()):
self._clear_sar3d_plot()
self._clear_sar3d_plot("SAR3D: waiting for projection data")
return False
peak = self._collection_payload_by_name(collection, "sar3d_peak", kind=4)
level, *peak_xyz = peak.table[0] if peak is not None else (None,)
@@ -147,12 +174,14 @@ class AppWindowSar3dPlotMixin:
for axis_name, (start, end) in _AXIS_EDGES.items():
middle = (projected[start] + projected[end]) / 2
self._sar3d_axis_labels[axis_name].setPos(*(middle + 0.08 * (middle - centre))) # just outside
self._sar3d_axis_labels[axis_name].show()
(x0, x1), _, (z0, z1) = bounds
ground = (np.array([[x0, 0.0, z0], [x1, 0.0, z0], [x1, 0.0, z1], [x0, 0.0, z1], [x0, 0.0, z0]]) @ basis.table.T).T
self._sar3d_ground.setData(ground[0], ground[1])
if level is not None:
x, y, z = peak_xyz
self._sar3d_plots["sar3d_oblique"].setTitle(
f"3D | max X {x:+.2f}, Y {y:+.2f}, Z {z:.2f} m | level {level:.3g}"
f"3D projection | max X {x:+.2f}, Y {y:+.2f}, Z {z:.2f} m | level {level:.3g}"
)
self._sar3d_status_label.setText(f"SAR3D: ready — {status[1]}" if status else "SAR3D: ready")
return True
python_app/gui/controllers/app_window_plot_mixin.py
+14 -13
View File
@@ -21,18 +21,19 @@ class AppWindowPlotMixin(
):
"""Routes plotting to trace, B-scan, GPR, or 3D SAR-specific mixins."""
def _draw_preferred_collection(
self,
*,
result_latest: ResultCollection | None,
) -> None:
"""Draw latest available result collection if present."""
if result_latest is None:
return
self._draw_results(result_latest)
def _draw_results(self, collection: ResultCollection) -> bool:
"""Draw collection based on currently selected processing mode."""
expected_revision = getattr(self, "_required_processing_revision", None)
if expected_revision is not None and self._is_gpr_like_processing_mode(self._processing_mode.currentText()):
revision = next((payload for payload in collection.collection_payloads
if payload.processing_name == "processing_revision" and payload.kind == 4), None)
if revision is None or revision.table.shape != (1, 4):
return False
if any(not 0 <= word <= 0xffff or word != int(word) for word in revision.table[0]):
return False
actual_revision = sum(int(word) << (index * 16) for index, word in enumerate(revision.table[0]))
if actual_revision != expected_revision:
return False
if self._processing_mode.currentText() == "bscan":
return self._draw_bscan_heatmap(collection)
if self._processing_mode.currentText() in {"gpr", "legacy_gpr"}:
@@ -42,11 +43,11 @@ class AppWindowPlotMixin(
return self._draw_trace_lines(collection)
def _result_collection_has_trace(self, collection: ResultCollection) -> bool:
"""Return `True` when collection contains at least one trace payload."""
if collection.collection_payloads:
"""Recognize processed output, including an empty trace after a valid depth/frequency cut."""
if any(payload.processing_name != "processing_revision" for payload in collection.collection_payloads):
return True
for block in collection.blocks:
for payload in block.payloads:
if payload.kind == 1 and payload.trace.size > 0:
if payload.kind == 1:
return True
return False
python_app/gui/controllers/app_window_preprocess_mixin.py
+28 -7
View File
@@ -122,6 +122,8 @@ class AppWindowPreprocessMixin:
def _capture_tmp_reference(self) -> None:
"""Capture, save, and select a temporary S21 reference with current sweep settings."""
if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False):
return
if self._capture_session is not None:
self._show_error(
"Cannot capture tmp reference during active capture sequence",
@@ -138,12 +140,15 @@ class AppWindowPreprocessMixin:
pipeline_was_running = self._supervisor.is_running()
pipeline_was_paused = False
resume_seq = getattr(self, "_stop_request_seq", 0) + 1
try:
if pipeline_was_running:
self._log("Pipeline paused for tmp reference capture")
self._stop_run()
pipeline_was_paused = True
if not self._pipeline_resume_allowed(resume_seq):
return
config = self._build_config()
radar_key = self._radar_key(config)
@@ -194,7 +199,7 @@ class AppWindowPreprocessMixin:
except Exception as exc: # noqa: BLE001
self._show_exception("Failed to capture tmp reference", exc)
finally:
if pipeline_was_paused:
if pipeline_was_paused and self._pipeline_resume_allowed(resume_seq):
self._start_run()
self._control_button_busy = False
@@ -333,6 +338,8 @@ class AppWindowPreprocessMixin:
def _start_capture_sequence(self, kind: str) -> None:
"""Start sequential capture session for requested preprocess asset."""
if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False):
return
if self._capture_session is not None:
self._show_error("Another capture sequence is already active", details=self._capture_state_details())
return
@@ -344,9 +351,12 @@ class AppWindowPreprocessMixin:
return
was_running = self._supervisor.is_running()
resume_seq = getattr(self, "_stop_request_seq", 0) + 1
if was_running:
self._log("Pipeline paused for exclusive hardware capture")
self._stop_run()
if not self._pipeline_resume_allowed(resume_seq):
return
self._resume_pipeline_after_capture = was_running
@@ -398,6 +408,8 @@ class AppWindowPreprocessMixin:
def _create_neutral_sets(self) -> None:
"""Save neutral S21 calibration/reference sets for the current radar settings."""
if getattr(self, "_closing", False) or getattr(self, "_stopping_run", False):
return
if self._capture_session is not None:
self._show_error(
"Cannot create neutral sets during active capture sequence",
@@ -412,11 +424,12 @@ class AppWindowPreprocessMixin:
return
pipeline_was_paused = False
resume_seq = getattr(self, "_stop_request_seq", 0) + 1
try:
config = self._build_config()
if not supports_neutral_preprocess_sets(config):
self._show_error(
"Neutral S21 sets are available only for kamil_adc and librevna_multi"
"Neutral S21 sets are not supported for this radar configuration"
)
return
@@ -435,11 +448,14 @@ class AppWindowPreprocessMixin:
self._log("Pipeline paused for neutral-set creation")
self._stop_run()
pipeline_was_paused = True
if not self._pipeline_resume_allowed(resume_seq):
return
calibration, reference = build_neutral_s21_sets(config)
point_count = int(calibration.traces[0].frequency_hz.size)
self._store.save_set("s21_calibration", radar_key, set_name, calibration)
self._store.save_set("s21_reference", radar_key, set_name, reference)
self._store.save_sets(radar_key, set_name, {
"s21_calibration": calibration, "s21_reference": reference,
})
self._selected_preprocess_sets["s21_calibration"] = set_name
self._selected_preprocess_sets["s21_reference"] = set_name
@@ -458,7 +474,7 @@ class AppWindowPreprocessMixin:
except Exception as exc: # noqa: BLE001
self._show_exception("Failed to create neutral S21 sets", exc)
finally:
if pipeline_was_paused:
if pipeline_was_paused and self._pipeline_resume_allowed(resume_seq):
self._start_run()
def _build_single_radar_capture_session(
@@ -556,19 +572,24 @@ class AppWindowPreprocessMixin:
display_name = preprocess_asset_display_name(session.kind)
dialog = self._ensure_preprocess_dialog()
stop_seq = getattr(self, "_stop_request_seq", 0)
dialog.set_status(f"{display_name} batch capture started")
self._log(
f"{display_name} batch capture started: remaining="
f"{session.state().total_count - session.state().captured_count}"
)
try:
while not session.is_complete():
while self._capture_session is session and self._pipeline_resume_allowed(stop_seq) and not session.is_complete():
try:
capture_result = session.capture_current_combo()
except Exception as exc: # noqa: BLE001
self._on_capture_combo_failed(session, exc)
return
self._record_preprocess_capture(session, capture_result)
if not session.is_complete():
self._pump_events_during_drain(0.01)
if self._capture_session is session and not session.is_complete():
dialog.set_status(f"{display_name} batch capture paused; previous captures kept")
finally:
self._end_preprocess_capture()
@@ -578,7 +599,7 @@ class AppWindowPreprocessMixin:
Returns False for a duplicate request (e.g. a click delivered while an
error dialog inside a capture pumps the event loop).
"""
if self._preprocess_capture_busy:
if getattr(self, "_closing", False) or self._preprocess_capture_busy:
self._log("Preprocess combo capture already in progress; ignoring duplicate request.")
return False
self._preprocess_capture_busy = True
python_app/gui/controllers/app_window_recording_mixin.py
+85 -99
View File
@@ -11,11 +11,11 @@ Key properties:
and freed, and at most :data:`_RECORDING_QUEUE_CHUNKS` chunks are ever in flight, so
RAM does not grow with the recording length.
* **Off the GUI thread.** A single background writer thread does all disk I/O; the GUI
poll tick only hands it chunks. The writes never stall acquisition or the web UI. If
poll tick only hands it chunks. If
the disk genuinely cannot keep up, the bounded queue applies brief backpressure
rather than growing memory without limit.
* **Aligned at capture.** Each result is paired with its raw/preprocessed collection by
``collection_id`` as it arrives, so no cross-stage buffering is needed.
* **Complete received stages.** Raw, preprocessed and result collections are saved
independently with their original timestamps, including sweeps without a result.
* **Only new sweeps.** Collections produced before the arm instant are ignored (gated
on ``monotonic_ns``), so a pre-existing ring backlog is not recorded.
* **No double-arm.** Arming is refused while a recording is still in progress.
@@ -29,6 +29,7 @@ from contextlib import suppress
import logging
from pathlib import Path
import queue
import shutil
import threading
import time
@@ -39,9 +40,6 @@ _RECORDING_CHUNK_SIZE = 200
# Max chunks queued for the writer thread before the producer applies backpressure;
# bounds in-flight memory to roughly this many chunks of measurements.
_RECORDING_QUEUE_CHUNKS = 8
# Safety bound on raw/preprocessed collections still waiting for their result (normally
# a handful). If the processor stalls, the oldest are dropped so the maps stay bounded.
_RECORDING_MAX_PENDING = 512
class AppWindowRecordingMixin:
@@ -54,13 +52,12 @@ class AppWindowRecordingMixin:
self._recording_target = 0
self._recording_since_ns = 0
self._recording_enqueued = 0
self._recording_received_count = 0
self._recording_writer = None
self._recording_queue: queue.Queue | None = None
self._recording_thread: threading.Thread | None = None
self._recording_stop = threading.Event()
self._recording_write_error: str | None = None
self._recording_pending_raw: dict[int, object] = {}
self._recording_pending_pre: dict[int, object] = {}
self._recording_chunk_raw: list = []
self._recording_chunk_pre: list = []
self._recording_chunk_results: list = []
@@ -82,6 +79,9 @@ class AppWindowRecordingMixin:
return
target = int(self._record_count.value())
if target <= 0:
self._show_error("Recording count must be greater than zero")
return
destination = self._snapshot_destination_dir()
if destination.exists():
@@ -91,6 +91,7 @@ class AppWindowRecordingMixin:
)
return
writer = None
try:
writer = self._store.create_snapshot_stream(
Path(self._save_path_input.text().strip()).expanduser(),
@@ -102,6 +103,9 @@ class AppWindowRecordingMixin:
self._snapshot_config_profile_path(writer.directory), allow_overwrite=False
)
except Exception as exc: # noqa: BLE001
if writer is not None:
with suppress(OSError):
shutil.rmtree(writer.directory)
self._show_exception("Failed to start disk recording", exc)
return
@@ -109,26 +113,41 @@ class AppWindowRecordingMixin:
self._recording_target = target
self._recording_since_ns = time.monotonic_ns()
self._recording_enqueued = 0
self._recording_received_count = 0
self._recording_write_error = None
self._recording_pending_raw = {}
self._recording_pending_pre = {}
self._recording_chunk_raw = []
self._recording_chunk_pre = []
self._recording_chunk_results = []
self._recording_queue = queue.Queue(maxsize=_RECORDING_QUEUE_CHUNKS)
self._recording_stop = threading.Event()
self._recording_thread = threading.Thread(
target=self._recording_writer_loop,
args=(writer, self._recording_queue, self._recording_stop),
name="disk-recorder",
daemon=True,
)
self._recording_thread.start()
try:
self._recording_thread = threading.Thread(
target=self._recording_writer_loop,
args=(writer, self._recording_queue, self._recording_stop),
name="disk-recorder",
daemon=False,
)
self._recording_thread.start()
except Exception as exc: # noqa: BLE001 - thread resource exhaustion must reach the operator
self._init_recording_state()
with suppress(OSError):
shutil.rmtree(writer.directory)
self._show_exception("Failed to start disk recording", exc)
return
self._recording_active = True
self._recording_collecting = True
if not self._supervisor.is_running():
self._start_run()
if not self._supervisor.is_running():
# Startup can fail after producing data. Flush it, and release the
# destination only when no collection was ever accepted.
self._finalize_recording_on_stop()
if not self._recording_active and not self._recording_received_count:
with suppress(OSError):
shutil.rmtree(writer.directory)
self._log("Disk recording canceled because acquisition did not start")
return
self._log(f"Disk recording armed: streaming the next {target} measurement(s) to {writer.directory}")
@@ -139,7 +158,7 @@ class AppWindowRecordingMixin:
) -> None:
"""Write queued chunks to disk until a sentinel, a stop request, or an error.
Runs on a daemon thread; touches only the writer (file I/O) and logging — never
Runs on a worker thread; touches only the writer (file I/O) and logging — never
Qt. A write failure is recorded for the GUI thread to surface and ends the loop.
"""
while not stop.is_set():
@@ -160,75 +179,62 @@ class AppWindowRecordingMixin:
# -- capture (called from the ring-read hooks on the poll tick) -----------
def _record_collection(self, stage: str, collection) -> None:
"""Stream one arriving collection while recording; hand chunks to the writer thread.
``stage`` is ``"raw"``, ``"preprocessed"`` or ``"results"``. Raw/preprocessed
collections are held by ``collection_id`` until their result arrives; the result
pairs them and appends an aligned measurement to the current chunk.
"""
"""Save each received stage, retaining raw/preprocessed sweeps without a result."""
if not self._recording_collecting:
return
produced_ns = int(getattr(collection, "monotonic_ns", 0) or 0)
if produced_ns and produced_ns < self._recording_since_ns:
return
if stage == "raw":
self._stash_pending(self._recording_pending_raw, collection)
self._recording_chunk_raw.append(collection)
elif stage == "preprocessed":
self._stash_pending(self._recording_pending_pre, collection)
self._recording_chunk_pre.append(collection)
elif stage == "results":
self._record_result(collection)
@staticmethod
def _stash_pending(pending: dict, collection) -> None:
"""Hold a raw/preprocessed collection by id until its result arrives (bounded)."""
pending[int(collection.collection_id)] = collection
while len(pending) > _RECORDING_MAX_PENDING:
pending.pop(next(iter(pending))) # drop oldest; its result never came
def _record_result(self, result) -> None:
"""Pair a result with its raw/preprocessed, buffer it, and flush/finish as needed."""
if self._recording_collected_count() >= self._recording_target:
return # already have the full target accepted
collection_id = int(result.collection_id)
raw = self._recording_pending_raw.pop(collection_id, None)
preprocessed = self._recording_pending_pre.pop(collection_id, None)
if raw is not None:
self._recording_chunk_raw.append(raw)
if preprocessed is not None:
self._recording_chunk_pre.append(preprocessed)
self._recording_chunk_results.append(result)
if len(self._recording_chunk_results) >= _RECORDING_CHUNK_SIZE:
if not self._flush_recording_chunk():
if self._recording_collected_count() >= self._recording_target:
return
self._recording_chunk_results.append(collection)
else:
raise ValueError(f"Unknown recording stage: {stage}")
self._recording_received_count += 1
chunk_size = max(len(self._recording_chunk_raw), len(self._recording_chunk_pre), len(self._recording_chunk_results))
if chunk_size >= _RECORDING_CHUNK_SIZE and not self._flush_recording_chunk():
return
if self._recording_collected_count() >= self._recording_target:
if not self._flush_recording_chunk():
return
self._stop_collecting()
if self._flush_recording_chunk():
self._stop_collecting()
def _enqueue_recording_work(self, work) -> bool:
"""Apply bounded backpressure while checking that the consumer remains healthy."""
while self._recording_thread is not None:
if self._recording_write_error is not None or not self._recording_thread.is_alive():
self._abort_recording_after_write_error()
return False
try:
self._recording_queue.put(work, timeout=0.1)
return True
except queue.Full:
continue
return False
def _flush_recording_chunk(self) -> bool:
"""Hand the buffered chunk to the writer thread. Returns ``False`` if it aborted."""
if not self._recording_chunk_results:
"""Queue all received stages, including a chunk containing no results yet."""
if not (self._recording_chunk_raw or self._recording_chunk_pre or self._recording_chunk_results):
return True
if self._recording_write_error is not None or not self._recording_thread.is_alive():
self._abort_recording_after_write_error() # never block on a dead consumer
return False
chunk = (self._recording_chunk_raw, self._recording_chunk_pre, self._recording_chunk_results)
if not self._enqueue_recording_work(chunk):
return False
self._recording_enqueued += len(self._recording_chunk_results)
self._recording_chunk_raw = []
self._recording_chunk_pre = []
self._recording_chunk_results = []
self._recording_queue.put(chunk) # brief backpressure only if the disk lags
return True
def _stop_collecting(self) -> None:
"""Stop accepting measurements and tell the writer to drain and exit."""
"""Queue an exit sentinel after every accepted chunk, including a full queue."""
self._recording_collecting = False
with suppress(Exception):
self._recording_queue.put_nowait(None) # sentinel after the last chunk
self._enqueue_recording_work(None)
# -- completion / teardown (GUI thread) -----------------------------------
@@ -249,35 +255,19 @@ class AppWindowRecordingMixin:
self._log(f"Disk recording complete: {written} measurement(s) written to {directory}")
def _finalize_recording_on_stop(self) -> None:
"""Flush the partial chunk and finish the recording when the run is stopped.
Pressing Stop mid-recording writes everything collected so far — including a
not-yet-full chunk — to disk and ends the recording, so no buffered measurement
is lost. The dataset then holds exactly what was seen before Stop. Called from
``_stop_run`` after its ring drains, so the last in-flight measurements are
already captured. The writer is joined briefly (bounded, like the stop drains)
so completion is deterministic without waiting on a later poll tick.
"""
if not self._recording_active or not self._recording_collecting:
"""Flush accepted data on Stop and retain the writer until its queue is drained."""
if not self._recording_active:
return
if not self._flush_recording_chunk():
return # writer already failed and was surfaced/reset
self._stop_collecting()
if self._recording_collecting:
if not self._flush_recording_chunk():
return
self._stop_collecting()
thread = self._recording_thread
directory = self._recording_writer.directory if self._recording_writer is not None else "?"
if thread is not None:
thread.join(timeout=3.0)
write_error = self._recording_write_error
written = self._recording_enqueued
self._reset_recording_state()
if write_error is not None:
self._show_error(
"Disk recording stopped with a write error",
details=f"dataset={directory}\nerror={write_error}",
)
else:
self._log(f"Disk recording stopped: {written} measurement(s) written to {directory}")
self._poll_recording_writer()
if self._recording_active:
self._log("Disk recording is finishing queued writes; the dataset is not complete yet")
def _abort_recording_after_write_error(self) -> None:
"""Surface a writer-thread failure on the GUI thread and disarm."""
@@ -302,21 +292,17 @@ class AppWindowRecordingMixin:
self._recording_thread = None
self._recording_write_error = None
self._recording_enqueued = 0
self._recording_pending_raw = {}
self._recording_pending_pre = {}
self._recording_chunk_raw = []
self._recording_chunk_pre = []
self._recording_chunk_results = []
def _shutdown_recording(self) -> None:
"""Stop the writer thread on app teardown, briefly joining so it is not orphaned."""
thread = self._recording_thread
self._recording_stop.set()
if self._recording_queue is not None:
with suppress(Exception):
self._recording_queue.put_nowait(None)
if thread is not None and thread.is_alive():
thread.join(timeout=2.0)
"""Flush on close; a non-daemon writer also drains after the GUI event loop exits."""
self._finalize_recording_on_stop()
if self._recording_active:
# Do not reset a live writer: setting its stop flag would discard its queue.
# Python waits for this non-daemon thread during process exit.
return
self._reset_recording_state()
# -- status ---------------------------------------------------------------
python_app/gui/controllers/app_window_snapshot_mixin.py
+35 -58
View File
@@ -6,7 +6,8 @@ from pathlib import Path
import time
from PyQt6.QtWidgets import QFileDialog
from python_app.gui.runtime.history import record_result_history, remove_last_aligned_histories
from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT
from python_app.gui.runtime.history import remove_last_aligned_histories
from python_app.storage.npz.paths import radar_config_filename_prefix
@@ -39,8 +40,10 @@ class AppWindowSnapshotMixin:
return output_dir / "config_profile.json"
def _save_snapshot(self) -> None:
"""Save the last-N runtime measurements as a numpy snapshot (the "Save Dataset" button)."""
"""Save up to 1000 recent runtime measurements as a numpy snapshot."""
self._drain_runtime_rings_for_snapshot()
if getattr(self, "_closing", False):
return
if not self._raw_history and not self._pre_history and not self._result_history:
self._show_error("No runtime data is available for save", details=self._runtime_history_details())
@@ -50,7 +53,8 @@ class AppWindowSnapshotMixin:
list(self._raw_history),
list(self._pre_history),
list(self._result_history),
int(self._save_count.value()),
GUI_SAVE_HISTORY_LIMIT,
include_unaligned=True,
)
def _snapshot_destination_dir(self) -> Path:
@@ -72,10 +76,12 @@ class AppWindowSnapshotMixin:
preprocessed_history: list,
result_history: list,
last_n: int,
*,
include_unaligned: bool = False,
) -> None:
"""Save the given aligned histories as a numpy snapshot + adjacent config profile.
"""Save the given histories as a numpy snapshot + adjacent config profile.
Shared by the manual "Save Dataset" button (which passes the runtime history)
Shared by the manual "Save Last 1000" button (which passes the runtime history)
and the on-the-fly disk recorder (which passes the freshly recorded buffers),
so both produce byte-identical dataset layouts and identical error reporting.
"""
@@ -90,6 +96,7 @@ class AppWindowSnapshotMixin:
result_history,
last_n,
name_prefix=self._radar_config_name_prefix(),
include_unaligned=include_unaligned,
)
config_profile_path = self._snapshot_config_profile_path(snapshot_dir)
try:
@@ -124,13 +131,15 @@ class AppWindowSnapshotMixin:
def _save_vna_history_json(self) -> None:
"""Save one VNA-history JSON per available combo in runtime history."""
self._drain_runtime_rings_for_snapshot()
if getattr(self, "_closing", False):
return
if not self._raw_history and not self._pre_history and not self._result_history:
self._show_error("No runtime data is available for save", details=self._runtime_history_details())
return
try:
last_n = int(self._save_count.value())
last_n = GUI_SAVE_HISTORY_LIMIT
channel = "s21"
output_root = Path(self._save_path_input.text().strip()).expanduser()
output_name = self._save_name_input.text().strip()
@@ -144,6 +153,7 @@ class AppWindowSnapshotMixin:
channel=channel,
primary_stage="preprocessed",
name_prefix=self._radar_config_name_prefix(),
include_unaligned=True,
)
output_stem = str(summary.get("output_stem", "")).strip()
if not output_stem:
@@ -210,6 +220,7 @@ class AppWindowSnapshotMixin:
return
resume_acquisition = self._supervisor.is_running()
resume_seq = getattr(self, "_stop_request_seq", 0) + 1
dropped_raw = 0
dropped_pre = 0
dropped_results = 0
@@ -220,6 +231,8 @@ class AppWindowSnapshotMixin:
try:
if resume_acquisition:
self._stop_run()
if not self._pipeline_resume_allowed(resume_seq):
return
dropped_raw = self._raw_reader.drop_all() if self._raw_reader is not None else 0
dropped_pre = self._pre_reader.drop_all() if self._pre_reader is not None else 0
@@ -260,7 +273,7 @@ class AppWindowSnapshotMixin:
f"results={dropped_results}"
)
if resume_acquisition:
if resume_acquisition and self._pipeline_resume_allowed(resume_seq):
self._start_run()
except Exception as exc: # noqa: BLE001
self._show_exception(f"Failed to {error_action}", exc)
@@ -280,64 +293,28 @@ class AppWindowSnapshotMixin:
if self._raw_reader is None and self._pre_reader is None and self._result_reader is None:
return
try:
start_raw_count = len(self._raw_history)
start_pre_count = len(self._pre_history)
start_result_count = len(self._result_history)
readers = (self._raw_reader, self._pre_reader, self._result_reader)
observed = [False, False, False]
deadline = time.monotonic() + 1.2
while True:
progress = False
while time.monotonic() < deadline and not getattr(self, "_closing", False):
limit = self._max_pop_per_snapshot_drain
raw = self._read_all_raw(max_collections=limit) if self._raw_reader is not None else None
preprocessed = self._read_all_preprocessed(max_collections=limit)
result = self._read_all_results(max_collections=limit) if self._result_reader is not None else None
latest = (raw, preprocessed, result)
observed = [seen or collection is not None for seen, collection in zip(observed, latest)]
if self._raw_reader is not None:
for _ in range(self._max_pop_per_snapshot_drain):
collection = self._raw_reader.pop_raw_collection()
if collection is None:
break
self._raw_history.append(collection)
progress = True
if self._pre_reader is not None:
for _ in range(self._max_pop_per_snapshot_drain):
collection = self._pre_reader.pop_preprocessed_collection()
if collection is None:
break
self._pre_history.append(collection)
progress = True
if self._result_reader is not None:
for _ in range(self._max_pop_per_snapshot_drain):
collection = self._result_reader.pop_result_collection()
if collection is None:
break
record_result_history(self._result_history, collection)
progress = True
if progress:
if any(collection is not None for collection in latest):
continue
raw_count = len(self._raw_history)
pre_count = len(self._pre_history)
result_count = len(self._result_history)
missing_raw = self._raw_reader is not None and raw_count == start_raw_count
missing_pre = self._pre_reader is not None and pre_count == start_pre_count
missing_results = self._result_reader is not None and result_count == start_result_count
got_results = result_count > start_result_count
got_raw_or_pre = raw_count > start_raw_count or pre_count > start_pre_count
# Event-loop-friendly waits so the keepalive/watchdog timers and
# queued signals keep firing instead of freezing the headless daemon.
if got_results and (missing_raw or missing_pre) and time.monotonic() < deadline:
self._pump_events_during_drain(0.01)
continue
if got_raw_or_pre and missing_results and time.monotonic() < deadline:
self._pump_events_during_drain(0.01)
continue
if (
self._result_reader is not None
and max(raw_count, pre_count) > result_count
and time.monotonic() < deadline
):
missing_stage = any(reader is not None and not seen for reader, seen in zip(readers, observed))
awaiting_results = self._result_reader is not None and (
max(len(self._raw_history), len(self._pre_history)) > len(self._result_history)
)
if (any(observed) and missing_stage) or awaiting_results:
self._pump_events_during_drain(0.01)
continue
break
python_app/gui/controllers/app_window_ui_mixin.py
+12 -2
View File
@@ -10,6 +10,7 @@ Layout is intentionally split into two independent plot surfaces:
from __future__ import annotations
from PyQt6.QtCore import Qt
from PyQt6.QtWidgets import (
QComboBox,
QFrame,
@@ -145,13 +146,22 @@ class AppWindowUiMixin:
def _build_sar3d_plot_page(self) -> None:
"""Create the 3D SAR page: front, side, top and 3D views in a 2x2 grid."""
self._sar3d_plots_container = QWidget()
layout = QGridLayout(self._sar3d_plots_container)
layout = QVBoxLayout(self._sar3d_plots_container)
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(6)
self._sar3d_status_label = QLabel()
self._sar3d_status_label.setTextFormat(Qt.TextFormat.PlainText)
self._sar3d_status_label.setWordWrap(True)
layout.addWidget(self._sar3d_status_label)
self._sar3d_plot_grid = QWidget()
grid = QGridLayout(self._sar3d_plot_grid)
grid.setContentsMargins(0, 0, 0, 0)
grid.setSpacing(6)
layout.addWidget(self._sar3d_plot_grid, stretch=1)
self._sar3d_plots = {}
for index, name in enumerate(SAR3D_VIEWS):
self._sar3d_plots[name] = self._create_plot_widget(background="#0f141c")
layout.addWidget(self._sar3d_plots[name], index // 2, index % 2)
grid.addWidget(self._sar3d_plots[name], index // 2, index % 2)
self._init_sar3d_views()
self._clear_sar3d_plot()
self._plot_stack.addWidget(self._sar3d_plots_container)
python_app/gui/controllers/app_window_web_mixin.py
+116 -25
View File
@@ -44,17 +44,42 @@ class _WebActionCall:
a request/response so failures reach the browser.
"""
__slots__ = ("done", "error")
__slots__ = ("done", "error", "_lock", "_started")
def __init__(self) -> None:
self.done = threading.Event()
self.error: str | None = None
self._lock = threading.Lock()
self._started = False
def begin(self) -> bool:
with self._lock:
if self.done.is_set() or self._started:
return False
self._started = True
return True
def finish(self, error: str | None = None) -> None:
with self._lock:
if not self.done.is_set():
self.error = error
self.done.set()
def cancel(self, error: str, *, queued_only: bool = True) -> bool:
with self._lock:
if self.done.is_set():
return True
if queued_only and self._started:
return False
self.error = error
self.done.set()
return True
# Headless has no shown window, so give the offscreen window a usable size for the
# grabbed plot. In GUI mode the user's real (shown) window size is used as-is.
_HEADLESS_PLOT_SIZE = (1600, 900)
# Grab/encode the plot at most this often (the plot only changes per result/redraw).
_GRAB_INTERVAL_S = 0.2
# Field names a web client may apply: the form's live/display/stable fields plus the
# Field names a web client may apply: the form's live/display fields plus the
# history command. Derived from the single schema so it can never drift from the form.
_LIVE_FIELD_NAMES = web_apply_field_names()
@@ -89,8 +114,7 @@ class AppWindowWebController(QObject):
"""
# Control signals carry a trailing _WebActionCall the GUI slot fills in, so the web
# thread can block on the real outcome. apply_settings stays fire-and-forget: it is
# validated up front and returns the live schema, not a pass/fail.
# thread can block on the real outcome and receive the updated state.
start_requested = pyqtSignal(object)
stop_requested = pyqtSignal(object)
remove_last_requested = pyqtSignal(object)
@@ -99,7 +123,9 @@ class AppWindowWebController(QObject):
start_recording_requested = pyqtSignal(str, str, int, object)
load_config_requested = pyqtSignal(str, object)
save_dataset_requested = pyqtSignal(str, str, object)
apply_settings_requested = pyqtSignal(dict)
select_combo_requested = pyqtSignal(int, int, object)
next_combo_requested = pyqtSignal(object)
apply_settings_requested = pyqtSignal(dict, object)
def __init__(self, run_configs_dir: Path, parent: QObject | None = None) -> None:
super().__init__(parent)
@@ -107,10 +133,13 @@ class AppWindowWebController(QObject):
self._status: dict = {}
self._live_settings: list = []
self._frame: dict | None = None
self._pending_lock = threading.Lock()
self._pending_calls: set[_WebActionCall] = set()
self._closed = False
# -- Snapshot refresh (Qt main thread) -----------------------------------
def update_snapshot(self, *, status: dict, live_settings: dict, frame: dict | None) -> None:
def update_snapshot(self, *, status: dict, live_settings: list, frame: dict | None) -> None:
"""Replace the served snapshots; ``frame`` only when a new one was grabbed."""
self._status = status
self._live_settings = live_settings
@@ -143,11 +172,28 @@ class AppWindowWebController(QObject):
action surfaced an error or did not finish within the timeout.
"""
call = _WebActionCall()
signal.emit(*args, call)
if not call.done.wait(_WEB_ACTION_TIMEOUT_S):
raise WebActionError("The desktop did not complete the action in time")
if call.error is not None:
raise WebActionError(call.error)
with self._pending_lock:
if self._closed:
raise WebActionError("The application is closing")
self._pending_calls.add(call)
try:
signal.emit(*args, call)
if not call.done.wait(_WEB_ACTION_TIMEOUT_S):
if not call.cancel("The desktop did not start the action in time; the request was cancelled"):
raise WebActionError("The desktop action is still running; check its status before retrying")
if call.error is not None:
raise WebActionError(call.error)
finally:
with self._pending_lock:
self._pending_calls.discard(call)
def shutdown(self) -> None:
"""Release waiting HTTP workers and reject queued controls during teardown."""
with self._pending_lock:
self._closed = True
pending = tuple(self._pending_calls)
for call in pending:
call.cancel("The application is closing", queued_only=False)
def start(self) -> None:
self._dispatch(self.start_requested)
@@ -166,13 +212,15 @@ class AppWindowWebController(QObject):
def start_recording(self, path: str, name: str, count: int) -> None:
"""Arm a run + disk recording of the next ``count`` measurements (the desktop button)."""
self._dispatch(self.start_recording_requested, path, name, int(count))
if type(count) is not int or count < 1:
raise ValueError("Recording count must be a positive integer")
self._dispatch(self.start_recording_requested, path, name, count)
def apply_live_settings(self, fields: dict) -> dict:
def apply_live_settings(self, fields: dict) -> list:
unknown = set(fields) - _LIVE_FIELD_NAMES
if unknown:
raise ValueError(f"Unknown live-settings fields: {', '.join(sorted(unknown))}")
self.apply_settings_requested.emit(dict(fields))
self._dispatch(self.apply_settings_requested, dict(fields))
return self.current_live_settings()
def load_config(self, name: str) -> None:
@@ -187,9 +235,19 @@ class AppWindowWebController(QObject):
self._dispatch(self.load_config_requested, name)
def save_dataset(self, path: str, name: str) -> None:
"""Save the runtime dataset to ``path``/``name`` (the desktop "Save Dataset" button)."""
"""Save the last 1000 runtime measurements to ``path``/``name``."""
self._dispatch(self.save_dataset_requested, path, name)
def select_combo(self, input_pos: int, output_pos: int) -> None:
"""Select the matrix B-scan channel through the desktop display controls."""
if any(type(pos) is not int or pos < 0 for pos in (input_pos, output_pos)):
raise ValueError("Channel positions must be nonnegative integers")
self._dispatch(self.select_combo_requested, input_pos, output_pos)
def next_combo(self) -> None:
"""Advance the matrix B-scan display to the next configured channel pair."""
self._dispatch(self.next_combo_requested)
class AppWindowWebMixin:
"""Start/stop the embedded web server and feed it the live plot + settings."""
@@ -245,7 +303,15 @@ class AppWindowWebMixin:
controller.save_dataset_requested.connect(
lambda path, name, call: self._run_web_action(call, self._save_web_dataset, path, name)
)
controller.apply_settings_requested.connect(self._apply_web_live_settings)
controller.select_combo_requested.connect(
lambda inp, out, call: self._run_web_action(call, self._select_display_combo, inp, out)
)
controller.next_combo_requested.connect(
lambda call: self._run_web_action(call, self._select_next_combo)
)
controller.apply_settings_requested.connect(
lambda fields, call: self._run_web_action(call, self._apply_web_live_settings, fields)
)
self._web_controller = controller
self._web_update_snapshot() # seed snapshots before the first request
@@ -268,17 +334,24 @@ class AppWindowWebMixin:
modal error dialog pumping the event loop in GUI mode — saves and restores the
capture slot, so each action only sees its own first error.
"""
if getattr(self, "_closing", False):
call.cancel("The application is closing", queued_only=False)
return
if not call.begin():
return
previous_capture = self._web_action_error_capture
capture: list[str] = []
self._web_action_error_capture = capture
error = None
try:
action(*args)
call.error = capture[0] if capture else None
error = capture[0] if capture else None
self._web_update_snapshot()
except Exception as exc: # noqa: BLE001 - handlers self-report; this is a backstop
call.error = self._exception_summary(exc)
error = self._exception_summary(exc)
finally:
self._web_action_error_capture = previous_capture
call.done.set()
call.finish(error)
def _load_web_config(self, name: str) -> None:
"""Load a run config chosen in the browser through the shared desktop load path.
@@ -289,12 +362,11 @@ class AppWindowWebMixin:
"""
config_path = _safe_run_config_path(self._run_configs_dir, name)
if config_path is None:
self._log_warning(f"Ignored web request to load unknown run config: {name}")
return
raise ValueError(f"Unknown run config: {name}")
self._load_config_from_path(config_path)
def _save_web_dataset(self, path: str, name: str) -> None:
"""Save the dataset from the browser via the same handler as the desktop button.
"""Save the last 1000 measurements via the shared desktop handler.
The web fields are remote editors of the shared save-path/name widgets: the name
is mirrored as-is (blank is valid — it yields a timestamped file), while a blank
@@ -313,11 +385,15 @@ class AppWindowWebMixin:
path keeps the configured destination), the record count is applied to the shared
spinbox, then the unchanged desktop arming action runs.
"""
if type(count) is not int or not self._record_count.minimum() <= count <= self._record_count.maximum():
raise ValueError(
f"Recording count must be an integer from {self._record_count.minimum()} "
f"to {self._record_count.maximum()}"
)
if path.strip():
self._save_path_input.setText(path)
self._save_name_input.setText(name)
if count >= 1:
self._record_count.setValue(min(count, self._record_count.maximum()))
self._record_count.setValue(count)
self._start_run_with_recording()
def _web_update_snapshot(self) -> None:
@@ -349,6 +425,16 @@ class AppWindowWebMixin:
"raw_count": len(self._raw_history),
"preprocessed_count": len(self._pre_history),
"result_count": len(self._result_history),
"sar3d_status": self._sar3d_status_label.text()
if self._processing_mode.currentText() == "sar3d" else "",
"display_combo": {
"enabled": self._is_matrix_radar_model_selected()
and self._processing_mode.currentText() == "bscan",
"input": self._single_combo_input.text(),
"output": self._single_combo_output.text(),
"options": self._available_display_combo_keys()
if self._is_matrix_radar_model_selected() else [],
},
},
live_settings=self._web_settings_schema(),
frame=self._web_grab_frame_if_due(),
@@ -377,8 +463,10 @@ class AppWindowWebMixin:
}
def _grab_plot_png_b64(self) -> str | None:
"""Render the currently-visible plot page to a base64 PNG (exactly as shown)."""
"""Render the active plots; SAR3D status is sent separately as web text."""
widget = self._plot_stack.currentWidget()
if widget is self._sar3d_plots_container:
widget = self._sar3d_plot_grid
if widget is None or widget.width() <= 0 or widget.height() <= 0:
return None
pixmap = widget.grab()
@@ -391,6 +479,9 @@ class AppWindowWebMixin:
def _shutdown_web_ui(self) -> None:
"""Stop the server (and its broadcaster) during teardown."""
controller = getattr(self, "_web_controller", None)
if controller is not None:
controller.shutdown()
server = getattr(self, "_web_server", None)
if server is not None:
with contextlib.suppress(Exception):
python_app/gui/controllers/sections/data_actions_section.py
+2 -12
View File
@@ -22,7 +22,8 @@ def build_data_actions_group(owner) -> QGroupBox:
layout.setSpacing(8)
data_defaults = owner._gui_defaults.data_actions
save_button = QPushButton("Save Dataset")
save_button = QPushButton("Save Last 1000")
save_button.setToolTip("Save up to 1000 most recent measurements with all available data stages.")
save_button.clicked.connect(owner._save_snapshot)
save_button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed)
save_vna_json_button = QPushButton("Save S21 JSON")
@@ -43,10 +44,6 @@ def build_data_actions_group(owner) -> QGroupBox:
)
record_button.clicked.connect(owner._start_run_with_recording)
record_button.setSizePolicy(QSizePolicy.Policy.Expanding, QSizePolicy.Policy.Fixed)
owner._save_count = QSpinBox()
owner._save_count.setMinimum(1)
owner._save_count.setMaximum(10_000)
owner._save_count.setValue(int(data_defaults.save_count))
owner._record_count = QSpinBox()
owner._record_count.setMinimum(1)
owner._record_count.setMaximum(1_000_000)
@@ -64,13 +61,6 @@ def build_data_actions_group(owner) -> QGroupBox:
button_grid.setColumnStretch(1, 1)
layout.addLayout(button_grid)
count_row = QHBoxLayout()
count_row.setSpacing(8)
count_row.addWidget(QLabel("Number of Measurements to Save"))
count_row.addWidget(owner._save_count)
count_row.addStretch(1)
layout.addLayout(count_row)
record_count_row = QHBoxLayout()
record_count_row.setSpacing(8)
record_count_row.addWidget(QLabel("Number of Sweeps to Record"))
python_app/gui/controllers/sections/processing_section.py
+11 -39
View File
@@ -8,9 +8,7 @@ from PyQt6.QtWidgets import (
QDoubleSpinBox,
QFormLayout,
QGroupBox,
QLabel,
QLineEdit,
QPlainTextEdit,
QSizePolicy,
QSpinBox,
QStackedWidget,
@@ -20,6 +18,7 @@ from PyQt6.QtWidgets import (
from python_app.gui.controllers.app_window_config.state_builders import GUI_SAVE_HISTORY_LIMIT
from python_app.gui.controllers.sections.layout_helpers import FormRow, build_two_column_form_widget
from python_app.gui.antenna_geometry_editor import AntennaGeometryEditor
# Same semantics on every GPR-like page; the data_processor applies them.
@@ -30,31 +29,6 @@ _IGNORE_SOCKET_SPEED_TOOLTIP = (
)
def _format_geometry_row(position: int, x_m: float, y_m: float, z_m: float) -> str:
"""Render one geometry row, trimming trailing zero y/z so 1D layouts stay compact."""
if z_m != 0.0:
return f"{position} {x_m:g} {y_m:g} {z_m:g}"
if y_m != 0.0:
return f"{position} {x_m:g} {y_m:g}"
return f"{position} {x_m:g}"
def _format_tx_geometry(owner) -> str:
"""Render Tx geometry defaults into editable line-based text."""
return "\n".join(
_format_geometry_row(int(entry.output_pos), float(entry.x_m), float(entry.y_m), float(entry.z_m))
for entry in owner._defaults_config.gpr.tx_geometry
)
def _format_rx_geometry(owner) -> str:
"""Render Rx geometry defaults into editable line-based text."""
return "\n".join(
_format_geometry_row(int(entry.input_pos), float(entry.x_m), float(entry.y_m), float(entry.z_m))
for entry in owner._defaults_config.gpr.rx_geometry
)
def _build_processing_mode_page(parent: QStackedWidget, rows: list[FormRow], *, split_index: int | None = None) -> QWidget:
"""Create one processing-mode page with a two-column form body."""
page = QWidget(parent)
@@ -199,21 +173,19 @@ def build_processing_group(owner) -> QGroupBox:
# Medium permittivity is a legacy-GPR-only knob (shown on the legacy page below).
# Coherent BP fixes the medium to eps_r = 1 (Horns_motion_3libre.py), so it is not
# offered there. Applies on the next pipeline start (a run_config field).
# offered there.
owner._gpr_relative_permittivity = QDoubleSpinBox()
owner._gpr_relative_permittivity.setDecimals(4)
owner._gpr_relative_permittivity.setRange(0.0001, 1000.0)
owner._gpr_relative_permittivity.setSingleStep(0.05)
owner._gpr_relative_permittivity.setToolTip("Applied on the next pipeline start (Save Config and restart).")
owner._gpr_relative_permittivity.setValue(float(gpr_defaults.relative_permittivity))
owner._gpr_tx_geometry_input = QPlainTextEdit(_format_tx_geometry(owner))
owner._gpr_tx_geometry_input.setPlaceholderText("output_pos x_m [y_m] [z_m]")
owner._gpr_tx_geometry_input.setFixedHeight(78)
owner._gpr_rx_geometry_input = QPlainTextEdit(_format_rx_geometry(owner))
owner._gpr_rx_geometry_input.setPlaceholderText("input_pos x_m [y_m] [z_m]")
owner._gpr_rx_geometry_input.setFixedHeight(78)
owner._gpr_tx_geometry_input = AntennaGeometryEditor(
"output_pos", gpr_defaults.tx_geometry, position_count=owner._defaults_config.output_switch.positions,
)
owner._gpr_rx_geometry_input = AntennaGeometryEditor(
"input_pos", gpr_defaults.rx_geometry, position_count=owner._defaults_config.input_switch.positions,
)
owner._gpr_common_page = _build_processing_mode_page(
group,
@@ -224,9 +196,9 @@ def build_processing_group(owner) -> QGroupBox:
split_index=1,
)
owner._gpr_geometry_hint = QLabel("To apply Tx/Rx geometry changes: Save Config and restart the app")
owner._gpr_geometry_hint.setWordWrap(True)
owner._gpr_common_page.layout().addWidget(owner._gpr_geometry_hint)
owner._gpr_tx_geometry_input.changed.connect(owner._on_gpr_geometry_changed)
owner._gpr_rx_geometry_input.changed.connect(owner._on_gpr_geometry_changed)
owner._gpr_relative_permittivity.valueChanged.connect(owner._on_gpr_geometry_changed)
owner._gpr_input_positions_input = QLineEdit(str(gpr_live_defaults.input_positions))
owner._gpr_input_positions_input.setPlaceholderText("0,1,2")
python_app/gui/controllers/sections/switch_section.py
+4
View File
@@ -45,6 +45,8 @@ def build_switch_group(owner) -> QGroupBox:
owner._single_combo_input.setPlaceholderText("0")
owner._single_combo_select_button = QPushButton("Select")
owner._single_combo_select_button.setCheckable(True)
owner._next_combo_button = QPushButton("Next combination")
owner._next_combo_button.setToolTip("Advance through input/output pairs and redraw the selected B-scan.")
single_row = QHBoxLayout()
single_row.setSpacing(8)
@@ -54,6 +56,7 @@ def build_switch_group(owner) -> QGroupBox:
single_row.addWidget(QLabel("Output"))
single_row.addWidget(owner._single_combo_output)
single_row.addWidget(owner._single_combo_select_button)
single_row.addWidget(owner._next_combo_button)
layout.addLayout(single_row)
owner._combo_select_group = QButtonGroup(group)
@@ -63,6 +66,7 @@ def build_switch_group(owner) -> QGroupBox:
owner._run_combos_select_button.clicked.connect(lambda: owner._set_combo_selection_mode("text"))
owner._single_combo_select_button.clicked.connect(lambda: owner._set_combo_selection_mode("single"))
owner._next_combo_button.clicked.connect(owner._select_next_combo)
# The live processing config has no combo: picking another one only changes which
# B-scan is shown (and, outside matrix mode, the combo of the next run).
owner._single_combo_output.editingFinished.connect(owner._on_bscan_display_combo_changed)
python_app/hardware_full/compact_m_k209_service.py
-274
View File
@@ -1,274 +0,0 @@
"""VISA HiSLIP driver for Compact-M K209 / S2VNA analyzers."""
from __future__ import annotations
from dataclasses import dataclass, field
from typing import Any
import numpy as np
import pyvisa
from python_app.hardware_full.librevna_driver.models import SweepResult
from python_app.models.run_config_model import RadarSweepModel
@dataclass(slots=True)
class CompactMK209InterleavedSweep:
"""Raw corrected K209 traces as interleaved REAL32 arrays."""
frequency_hz: np.ndarray
s11_values: np.ndarray
s21_values: np.ndarray
@dataclass(slots=True)
class CompactMK209Service:
"""Acquire corrected S11/S21 sweeps from a K209 analyzer through VISA HiSLIP."""
resource: str
timeout_ms: int = 20_000
preset_on_open: bool = True
visa_library: str = "@ivi"
_resource_manager: pyvisa.ResourceManager | None = field(init=False, default=None, repr=False)
_instrument: Any | None = field(init=False, default=None, repr=False)
_settings: RadarSweepModel | None = field(init=False, default=None, repr=False)
_frequency_hz: np.ndarray | None = field(init=False, default=None, repr=False)
def __post_init__(self) -> None:
"""Normalize constructor values."""
self.resource = str(self.resource).strip()
if not self.resource:
raise ValueError("K209 VISA resource must not be empty")
self.timeout_ms = int(self.timeout_ms)
if self.timeout_ms <= 0:
raise ValueError("K209 timeout_ms must be > 0")
self.visa_library = str(self.visa_library).strip() or "@ivi"
if self.visa_library == "@py" or self.visa_library.endswith("@py"):
raise ValueError("K209 requires an IVI/Vendor VISA backend, not pyvisa-py")
@property
def is_open(self) -> bool:
"""Return whether the VISA session is open."""
return self._instrument is not None
def open(self) -> None:
"""Open VISA resource and apply stored sweep settings when available."""
if self._instrument is not None:
return
try:
self._resource_manager = pyvisa.ResourceManager(self.visa_library)
self._instrument = self._resource_manager.open_resource(self.resource)
self._instrument.timeout = self.timeout_ms
self._instrument.write_termination = "\n"
self._instrument.read_termination = None
self._instrument.chunk_size = max(int(getattr(self._instrument, "chunk_size", 20_480)), 8 * 1024 * 1024)
self._instrument.write("*CLS")
if self._settings is not None:
self._apply_configuration(self._settings)
except Exception:
self.close()
raise
def close(self) -> None:
"""Close VISA sessions."""
if self._instrument is not None:
self._instrument.close()
self._instrument = None
if self._resource_manager is not None:
self._resource_manager.close()
self._resource_manager = None
def __enter__(self) -> CompactMK209Service:
"""Open and return this service."""
self.open()
return self
def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
"""Close VISA resources."""
self.close()
def query_identity(self) -> str:
"""Read analyzer identity string."""
instrument = self._require_instrument()
return str(instrument.query("*IDN?")).strip()
def query_system_error(self) -> str:
"""Read one analyzer SCPI error queue entry."""
instrument = self._require_instrument()
return str(instrument.query("SYST:ERR?")).strip()
def configure(self, sweep: RadarSweepModel) -> None:
"""Store and apply sweep settings."""
self._validate_sweep(sweep)
self._settings = sweep
self._frequency_hz = None
if self._instrument is None:
return
self._apply_configuration(sweep)
def read_device_limits(self) -> dict[str, float | int]:
"""Read analyzer limits through SCPI capability/service queries."""
opened_here = self._instrument is None
try:
if opened_here:
self.open()
instrument = self._require_instrument()
return {
"min_frequency_hz": float(instrument.query("SERV:SWE:FREQ:MIN?")),
"max_frequency_hz": float(instrument.query("SERV:SWE:FREQ:MAX?")),
"min_ifbw_hz": float(instrument.query("SYST:CAP:IFBW:MIN?")),
"max_ifbw_hz": float(instrument.query("SYST:CAP:IFBW:MAX?")),
"max_points": int(float(instrument.query("SERV:SWE:POIN?"))),
"min_power_dbm": float(instrument.query("SERV:SWE:POW:MIN?")),
"max_power_dbm": float(instrument.query("SERV:SWE:POW:MAX?")),
}
finally:
if opened_here:
self.close()
def frequency_axis(self) -> np.ndarray:
"""Return the configured frequency axis."""
if self._frequency_hz is None:
raise RuntimeError("K209 frequency axis is not configured")
return self._frequency_hz
def acquire_interleaved(self) -> CompactMK209InterleavedSweep:
"""Acquire one corrected sweep without converting interleaved arrays."""
if self._settings is None:
raise RuntimeError("K209 service is not configured")
if self._frequency_hz is None:
raise RuntimeError("K209 frequency axis is not configured")
points = int(self._settings.points)
s11_values, s21_values = self._query_sweep_trace_pair(points)
return CompactMK209InterleavedSweep(
frequency_hz=self._frequency_hz,
s11_values=s11_values,
s21_values=s21_values,
)
def acquire(self) -> SweepResult:
"""Acquire one corrected S11/S21 sweep."""
raw = self.acquire_interleaved()
return SweepResult(
x=raw.frequency_hz.copy(),
traces={
"s11": self._complex_from_interleaved(raw.s11_values),
"s21": self._complex_from_interleaved(raw.s21_values),
},
)
def _apply_configuration(self, sweep: RadarSweepModel) -> None:
instrument = self._require_instrument()
if self.preset_on_open:
instrument.write("SYST:PRES")
self._expect_opc("*OPC?", context="K209 preset")
instrument.write(f"SENS:FREQ:STAR {float(sweep.start_hz):.9f}")
instrument.write(f"SENS:FREQ:STOP {float(sweep.stop_hz):.9f}")
instrument.write(f"SENS:SWE:POIN {int(sweep.points)}")
instrument.write("SENS:SWE:POIN:TIME 0")
instrument.write(f"SENS:BAND {float(sweep.if_bandwidth_hz):.9f}")
instrument.write(f"SOUR:POW {float(sweep.power_dbm):.3f}")
instrument.write("SENS:AVER OFF")
instrument.write("CALC:PAR1:DEF S21")
instrument.write("CALC:PAR1:SEL")
instrument.write("FORM:DATA REAL32")
instrument.write("FORM:BORD SWAP")
instrument.write("INIT:CONT ON")
instrument.write("TRIG:SOUR BUS")
self._expect_opc("*OPC?", context="K209 setup")
self._frequency_hz = self._query_float32_array("SENS:FREQ:DATA?", int(sweep.points))
def _expect_opc(self, command: str, *, context: str) -> None:
instrument = self._require_instrument()
response = str(instrument.query(command)).strip()
if response != "1":
raise RuntimeError(f"{context} returned unexpected *OPC? response: {response!r}")
def _query_float32_array(self, command: str, expected_values: int) -> np.ndarray:
instrument = self._require_instrument()
instrument.write(command)
return self._read_float32_block(command, expected_values)
def _query_sweep_trace_pair(self, points: int) -> tuple[np.ndarray, np.ndarray]:
instrument = self._require_instrument()
instrument.write("TRIG:SING;*OPC?;:SENS:DATA:CORR? S11;:SENS:DATA:CORR? S21")
response = self._read_ascii_token()
if response != "1":
raise RuntimeError(f"K209 sweep returned unexpected *OPC? response: {response!r}")
return (
self._read_float32_block("SENS:DATA:CORR? S11", points * 2),
self._read_float32_block("SENS:DATA:CORR? S21", points * 2),
)
def _read_ascii_token(self) -> str:
token = bytearray()
while True:
byte = self._read_response_bytes(1)
if byte in (b";", b"\n", b"\r"):
if token:
return token.decode("ascii").strip()
continue
token.extend(byte)
if len(token) > 64 * 1024:
raise RuntimeError("K209 ASCII response token is too long")
def _read_float32_block(self, context: str, expected_values: int) -> np.ndarray:
marker = self._read_response_bytes(1)
while marker in (b";", b"\n", b"\r"):
marker = self._read_response_bytes(1)
if marker != b"#":
raise RuntimeError(f"K209 response for {context!r} does not start with IEEE block marker")
width = self._read_response_bytes(1)
if width != b"8":
raise RuntimeError(f"K209 response for {context!r} uses unsupported IEEE block header width")
payload_size = int(self._read_response_bytes(8).decode("ascii"))
expected_size = expected_values * np.dtype(np.float32).itemsize
if payload_size != expected_size:
raise RuntimeError(
f"K209 response for {context!r} returned {payload_size} payload bytes, "
f"expected {expected_size}"
)
payload = self._read_response_bytes(payload_size)
array = np.frombuffer(payload, dtype="<f4")
if array.size != expected_values:
raise RuntimeError(
f"K209 response for {context!r} returned {array.size} float32 values, "
f"expected {expected_values}"
)
return array
def _read_response_bytes(self, count: int) -> bytes:
instrument = self._require_instrument()
data = instrument.read_bytes(count, break_on_termchar=False)
if len(data) != count:
raise RuntimeError(f"K209 response ended after {len(data)} bytes, expected {count}")
return data
def _require_instrument(self) -> Any:
if self._instrument is None:
raise RuntimeError("K209 VISA instrument is not open")
return self._instrument
@staticmethod
def _validate_sweep(sweep: RadarSweepModel) -> None:
if int(sweep.points) < 2:
raise ValueError("K209 sweep points must be >= 2")
if float(sweep.stop_hz) < float(sweep.start_hz):
raise ValueError("K209 sweep stop_hz must be >= start_hz")
if float(sweep.if_bandwidth_hz) <= 0.0:
raise ValueError("K209 IF bandwidth must be > 0")
@staticmethod
def _complex_from_interleaved(values: np.ndarray) -> np.ndarray:
if values.size % 2 != 0:
raise RuntimeError("K209 complex trace payload has odd scalar count")
reshaped = np.asarray(values, dtype=np.float32).reshape((-1, 2))
return (reshaped[:, 0] + 1j * reshaped[:, 1]).astype(np.complex64)
python_app/hardware_full/k209_remote_protocol.py
-90
View File
@@ -1,90 +0,0 @@
"""Binary TCP protocol shared by the K209 remote server and Python client."""
from __future__ import annotations
import socket
import struct
from typing import BinaryIO
import numpy as np
COMMAND_IDENTITY = b"I"
COMMAND_LIMITS = b"L"
COMMAND_CONFIGURE = b"C"
COMMAND_ACQUIRE = b"A"
STATUS_OK = b"O"
STATUS_ERROR = b"E"
DEFAULT_REMOTE_HOST = "127.0.0.1"
DEFAULT_REMOTE_PORT = 50209
CONFIG_STRUCT = struct.Struct("!ddIdd")
LIMITS_STRUCT = struct.Struct("!ddddIdd")
U32_STRUCT = struct.Struct("!I")
def recv_exact(stream: socket.socket | BinaryIO, size: int) -> bytes:
"""Read exactly `size` bytes from a socket-like object."""
chunks: list[bytes] = []
remaining = int(size)
while remaining > 0:
chunk = stream.recv(remaining) if isinstance(stream, socket.socket) else stream.read(remaining)
if not chunk:
raise ConnectionError(f"K209 remote connection closed with {remaining} bytes pending")
chunks.append(chunk)
remaining -= len(chunk)
return b"".join(chunks)
def send_all(stream: socket.socket | BinaryIO, payload: bytes) -> None:
"""Write all payload bytes to a socket-like object."""
if isinstance(stream, socket.socket):
stream.sendall(payload)
return
stream.write(payload)
def send_u32(stream: socket.socket | BinaryIO, value: int) -> None:
"""Send one network-order uint32."""
send_all(stream, U32_STRUCT.pack(int(value)))
def recv_u32(stream: socket.socket | BinaryIO) -> int:
"""Read one network-order uint32."""
return int(U32_STRUCT.unpack(recv_exact(stream, U32_STRUCT.size))[0])
def send_error(stream: socket.socket | BinaryIO, message: str) -> None:
"""Send protocol error response."""
payload = str(message).encode("utf-8", errors="replace")
send_all(stream, STATUS_ERROR)
send_u32(stream, len(payload))
send_all(stream, payload)
def read_status(stream: socket.socket | BinaryIO) -> None:
"""Read response status and raise remote error when needed."""
status = recv_exact(stream, 1)
if status == STATUS_OK:
return
if status == STATUS_ERROR:
message = recv_exact(stream, recv_u32(stream)).decode("utf-8", errors="replace")
raise RuntimeError(f"K209 remote server error: {message}")
raise RuntimeError(f"K209 remote server returned invalid status byte: {status!r}")
def send_float32_array(stream: socket.socket | BinaryIO, values: np.ndarray) -> None:
"""Send a float32 array as a length-prefixed little-endian payload."""
payload = np.asarray(values, dtype="<f4").tobytes(order="C")
send_u32(stream, len(payload))
send_all(stream, payload)
def recv_float32_array(stream: socket.socket | BinaryIO, expected_values: int) -> np.ndarray:
"""Read a length-prefixed little-endian float32 array."""
payload_size = recv_u32(stream)
expected_size = int(expected_values) * np.dtype(np.float32).itemsize
if payload_size != expected_size:
raise RuntimeError(f"K209 remote payload has {payload_size} bytes, expected {expected_size}")
return np.frombuffer(recv_exact(stream, payload_size), dtype="<f4").copy()
python_app/hardware_full/kamil_adc/tty_reader.py
+11 -1
View File
@@ -74,7 +74,13 @@ class KamilAdcTtyReader:
name=f"kamil-adc-tty-reader[{self.tty_path}]",
daemon=True,
)
self._thread.start()
try:
self._thread.start()
except Exception:
self._thread = None
fd, self._fd = self._fd, None
os.close(fd)
raise
logger.info("Kamil ADC TTY reader started on %s", self.tty_path)
def close(self) -> None:
@@ -120,6 +126,8 @@ class KamilAdcTtyReader:
deadline = time.monotonic() + float(timeout_s)
with self._mailbox_cv:
while True:
if self._stop_event.is_set():
raise RuntimeError("Kamil ADC TTY reader is closed")
# Deliver a pending sweep first: if the reader both published a
# sweep and then died, the consumer still sees the good data and
# only meets the error on the next call.
@@ -156,6 +164,8 @@ class KamilAdcTtyReader:
deadline = time.monotonic() + float(timeout_s)
with self._mailbox_cv:
while True:
if self._stop_event.is_set():
raise RuntimeError("Kamil ADC TTY reader is closed")
sweep = self._combo_slots.pop(combo, None)
if sweep is not None:
return sweep
python_app/hardware_full/laser_control/monitoring/session.py
+3 -6
View File
@@ -20,6 +20,8 @@ from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Union
from python_app.file_io import atomic_write_json
_PathLike = Union[str, os.PathLike[str]]
# Default IPC locations. Both are overridable via CLI/API so several runs can use
@@ -46,12 +48,7 @@ class LaserVariationSession:
Writes to a temp file in the same directory then renames, so a concurrent
checker never observes a half-written file.
"""
dest = Path(path)
dest.parent.mkdir(parents=True, exist_ok=True)
tmp = dest.with_name(f"{dest.name}.{os.getpid()}.tmp")
tmp.write_text(json.dumps(asdict(self), indent=2), encoding="utf-8")
os.replace(tmp, dest)
return dest
return atomic_write_json(Path(path), asdict(self))
@classmethod
def load(cls, path: _PathLike = DEFAULT_SESSION_PATH) -> "LaserVariationSession":
python_app/hardware_full/laser_control/protocol.py
+6 -39
View File
@@ -196,6 +196,8 @@ class Protocol:
the bus. Note the lock is advisory (``flock``): every program that opens
this port must also use ``exclusive=True`` for the guarantee to hold.
"""
if self.is_connected:
return
port = self._port_name or self._detect_port()
try:
self._serial = serial.Serial(
@@ -219,9 +221,10 @@ class Protocol:
def disconnect(self) -> None:
"""Close the serial port if open."""
if self._serial and self._serial.is_open:
connection, self._serial = self._serial, None
if connection is not None and connection.is_open:
logger.debug("Closing laser serial port")
self._serial.close()
connection.close()
@property
def is_connected(self) -> bool:
@@ -246,37 +249,6 @@ class Protocol:
raise CommunicationError("Serial port is not connected")
return self._serial.read(length)
# ---- Static encoding helpers (no connection required) ---------------
@staticmethod
def flipfour(value: int) -> int:
"""Byte-swap a 16-bit integer (little-endian word swap)."""
return ((value & 0xFF) << 8) | ((value >> 8) & 0xFF)
@staticmethod
def pack_float(value: float) -> bytes:
return struct.pack('<f', value)
@staticmethod
def pack_uint16(value: int) -> bytes:
return struct.pack('<H', value)
@staticmethod
def calculate_crc(data: bytes) -> int:
"""Return the XOR CRC over all 16-bit words except word 0 and the CRC field.
The command-code word (word 0) is excluded, matching the firmware's CRC
expectation.
"""
hex_str = data.hex()
words = [hex_str[i:i+4] for i in range(0, len(hex_str), 4)]
# Word 0 (command code) is excluded from the CRC.
crc_words = words[1:]
result = int(crc_words[0], 16)
for w in crc_words[1:]:
result ^= int(w, 16)
return result
# ---- Command encoders -----------------------------------------------
@staticmethod
@@ -404,11 +376,6 @@ class Protocol:
"""Build DEFAULT_ENABLE command (0x2222) — reset device."""
return bytearray.fromhex(_flipfour(_int_to_hex4(CMD_DEFAULT_ENABLE)))
@staticmethod
def encode_remove_file() -> bytes:
"""Build REMOVE_FILE command (0x5555) — delete saved data."""
return bytearray.fromhex(_flipfour(_int_to_hex4(CMD_REMOVE_FILE)))
# ---- Response decoders -----------------------------------------------
@staticmethod
@@ -477,4 +444,4 @@ class Protocol:
@staticmethod
def state_to_description(state_hex_str: str) -> str:
"""Return human-readable description for a state hex string."""
return STATE_DESCRIPTIONS.get(state_hex_str, "Unknown or reserved error.")
return STATE_DESCRIPTIONS.get(state_hex_str, "Unknown or reserved error.")
python_app/hardware_full/librevna_backends.py
+13 -88
View File
@@ -3,8 +3,8 @@
from __future__ import annotations
from dataclasses import dataclass, field
import math
from typing import Any, Protocol
from contextlib import suppress
from typing import Any
import numpy as np
@@ -12,29 +12,6 @@ from python_app.hardware_full.librevna_driver.models import SweepResult
from python_app.models.run_config_model import RadarSweepModel
class LibreVnaBackend(Protocol):
"""Minimal backend contract required by `LibreVnaService`."""
def open(self) -> None:
"""Open backend resources."""
def close(self) -> None:
"""Close backend resources."""
@property
def is_open(self) -> bool:
"""Return whether backend currently holds open runtime resources."""
def configure(self, sweep: RadarSweepModel) -> None:
"""Apply sweep settings."""
def read_device_limits(self) -> dict[str, float | int]:
"""Query runtime device limits."""
def acquire(self) -> SweepResult:
"""Acquire one sweep with all available traces."""
@dataclass(slots=True)
class NativeLibreVnaBackend:
"""Native backend using direct USB LibreVNA Python driver."""
@@ -43,15 +20,13 @@ class NativeLibreVnaBackend:
strict_protocol_version: int
_device: Any | None = field(init=False, default=None, repr=False)
_settings: RadarSweepModel | None = field(init=False, default=None, repr=False)
_libre_vna_device: type[Any] | None = field(init=False, default=None, repr=False)
_vna_sweep_settings: type[Any] | None = field(init=False, default=None, repr=False)
_libre_vna_device: type[Any] = field(init=False, repr=False)
_vna_sweep_settings: type[Any] = field(init=False, repr=False)
def __post_init__(self) -> None:
"""Import native driver classes lazily and initialize backend state."""
from python_app.hardware_full.librevna_driver import LibreVNADevice, VNASweepSettings
self._device: LibreVNADevice | None = None
self._settings: RadarSweepModel | None = None
self._libre_vna_device = LibreVNADevice
self._vna_sweep_settings = VNASweepSettings
@@ -60,8 +35,13 @@ class NativeLibreVnaBackend:
if self._device is not None:
return
self._device = self._libre_vna_device()
self._device.connect(serial=self.serial, strict_protocol_version=self.strict_protocol_version, timeout_s=2.0)
device = self._device = self._libre_vna_device()
try:
device.connect(serial=self.serial, strict_protocol_version=self.strict_protocol_version, timeout_s=2.0)
except Exception:
with suppress(Exception):
self.close()
raise
@property
def is_open(self) -> bool:
@@ -72,8 +52,8 @@ class NativeLibreVnaBackend:
"""Close device connection when open."""
if self._device is None:
return
self._device.disconnect()
self._device = None
device, self._device = self._device, None
device.disconnect()
def configure(self, sweep: RadarSweepModel) -> None:
"""Store and apply sweep settings to connected hardware."""
@@ -124,58 +104,3 @@ class NativeLibreVnaBackend:
"s21": np.asarray(result.trace("s21"), dtype=np.complex64),
},
)
@dataclass(slots=True)
class MockLibreVnaBackend:
"""Synthetic backend used for local development and tests."""
_settings: RadarSweepModel | None = field(init=False, default=None, repr=False)
_mock_phase: float = field(init=False, default=0.0, repr=False)
def __post_init__(self) -> None:
"""Initialize mock backend state."""
self._settings = None
self._mock_phase = 0.0
def open(self) -> None:
"""No-op for mock backend."""
def close(self) -> None:
"""No-op for mock backend."""
@property
def is_open(self) -> bool:
"""Mock backend does not hold external resources."""
return False
def configure(self, sweep: RadarSweepModel) -> None:
"""Store sweep settings used by synthetic acquisition."""
self._settings = sweep
def read_device_limits(self) -> dict[str, float | int]:
"""Mock backend does not support native device limits queries."""
raise RuntimeError("LibreVNA Python driver is not available")
def acquire(self) -> SweepResult:
"""Generate synthetic S11 and S21 values using deterministic envelopes."""
if self._settings is None:
raise RuntimeError("Radar service is not configured")
points = self._settings.points
freq = np.linspace(self._settings.start_hz, self._settings.stop_hz, points, dtype=np.float32)
phase = (2.0 * math.pi * np.linspace(0.0, 1.0, points, dtype=np.float32)) + self._mock_phase
s21_envelope = 0.6 + 0.4 * np.sin(phase * 0.5)
s11_envelope = 0.25 + 0.15 * np.cos(phase * 0.75)
s21 = (s21_envelope * np.cos(phase) + 1j * s21_envelope * np.sin(phase)).astype(np.complex64)
reflected_phase = phase * 0.6 + 0.8
s11 = (
s11_envelope * np.cos(reflected_phase) + 1j * s11_envelope * np.sin(reflected_phase)
).astype(np.complex64)
self._mock_phase += 0.05
return SweepResult(
x=freq,
traces={
"s11": s11,
"s21": s21,
},
)
python_app/hardware_full/librevna_driver/models.py
+2
View File
@@ -201,6 +201,8 @@ class SweepResult:
x: np.ndarray
traces: dict[str, np.ndarray] = field(default_factory=dict)
x_label: str = "frequency_hz"
capture_start_ns: int = 0
capture_end_ns: int = 0
def __post_init__(self) -> None:
"""Normalize dtypes and validate shape compatibility."""
python_app/hardware_full/librevna_driver/session.py
+39 -25
View File
@@ -28,6 +28,7 @@ from .protocol import (
from .transport import USBTransport
logger = logging.getLogger(__name__)
_PACKET_QUEUE_MAX = 32768
class _IndexedPayload(Protocol):
@@ -50,7 +51,9 @@ class LibreVNASession:
on_disconnect=self._on_transport_disconnect,
)
self._incoming: dict[PacketType, deque[Packet]] = defaultdict(deque)
# The sweep free-runs between acquisitions, and subscriptions do not
# consume request queues. Retain a bounded tail of stale datapoints.
self._incoming: dict[PacketType, deque[Packet]] = defaultdict(lambda: deque(maxlen=_PACKET_QUEUE_MAX))
self._subscribers: dict[PacketType, set[Callable[[Packet], None]]] = defaultdict(set)
self._incoming_cv = threading.Condition()
@@ -159,8 +162,8 @@ class LibreVNASession:
def disconnect(self) -> None:
"""Disconnect USB transport and clear session state."""
logger.debug("Closing USB transport and clearing session queues")
self._transport.disconnect()
with self._incoming_cv:
self._fatal_error = DeviceDisconnectedError("LibreVNA session is disconnected")
self._incoming.clear()
self._subscribers.clear()
self._incoming_cv.notify_all()
@@ -168,6 +171,7 @@ class LibreVNASession:
self._awaiting_ack = False
self._ack_result = None
self._ack_cv.notify_all()
self._transport.disconnect()
def send(self, packet: Packet, *, require_ack: bool = True, timeout_s: float = 0.5) -> None:
"""Send one protocol packet and optionally wait for ACK/NACK."""
@@ -188,26 +192,26 @@ class LibreVNASession:
self._awaiting_ack = True
self._ack_result = None
self._transport.write(frame, timeout_s=timeout_s)
if require_ack:
deadline = time.monotonic() + timeout_s
with self._ack_cv:
while self._ack_result is None:
self._raise_if_fatal_locked()
remaining = deadline - time.monotonic()
if remaining <= 0:
self._awaiting_ack = False
raise TimeoutError(f"Timeout waiting for ACK for packet {packet.type.name}")
self._ack_cv.wait(timeout=remaining)
result = self._ack_result
self._awaiting_ack = False
self._ack_result = None
if result == PacketType.NACK:
raise NackError(f"Received NACK for packet {packet.type.name}")
logger.debug("ACK received for packet %s", packet.type.name)
try:
self._transport.write(frame, timeout_s=timeout_s)
if require_ack:
deadline = time.monotonic() + timeout_s
with self._ack_cv:
while self._ack_result is None:
self._raise_if_fatal_locked()
remaining = deadline - time.monotonic()
if remaining <= 0:
raise TimeoutError(f"Timeout waiting for ACK for packet {packet.type.name}")
self._ack_cv.wait(timeout=remaining)
result = self._ack_result
if result == PacketType.NACK:
raise NackError(f"Received NACK for packet {packet.type.name}")
logger.debug("ACK received for packet %s", packet.type.name)
finally:
if require_ack:
with self._ack_cv:
self._awaiting_ack = False
self._ack_result = None
def request(
self,
@@ -238,9 +242,10 @@ class LibreVNASession:
payload_type: type[TPacketPayload],
payload_error: str,
) -> list[TPacketPayload]:
"""Collect indexed payloads by `point_number` into ascending order.
"""Collect one sweep from point zero, returning points in ascending order.
Missing points are omitted; callers decide whether this is acceptable.
Ignore the tail before the first point zero. A new point zero discards
an incomplete earlier cycle instead of filling its gaps from a later one.
"""
if expected_points <= 0:
raise ValueError("expected_points must be > 0")
@@ -265,6 +270,11 @@ class LibreVNASession:
f"Received out-of-range point index {point_number}, expected 0..{expected_points - 1}"
)
if point_number == 0:
collected = [None] * expected_points
received = 0
elif collected[0] is None:
continue
if collected[point_number] is None:
received += 1
collected[point_number] = payload
@@ -391,8 +401,12 @@ class LibreVNASession:
callbacks: list[Callable[[Packet], None]] = []
with self._incoming_cv:
self._incoming[packet.type].append(packet)
callbacks = list(self._subscribers.get(packet.type, set()))
queue = self._incoming[packet.type]
if len(queue) == queue.maxlen and not callbacks and packet.type != PacketType.VNA_DATAPOINT:
self._set_fatal_error(ParseError(f"Packet queue overflow for {packet.type.name}"))
return
queue.append(packet)
self._incoming_cv.notify_all()
for callback in callbacks:
python_app/hardware_full/librevna_driver/transport/usb.py
+48 -56
View File
@@ -94,68 +94,60 @@ class USBTransport:
logger.debug("Opening libusb context for connect(serial=%s)", serial)
self._ctx = usb1.USBContext()
selected_handle: usb1.USBDeviceHandle | None = None
selected_serial = ""
try:
selected_handle: usb1.USBDeviceHandle | None = None
selected_serial = ""
for device, _, _ in self._iter_matching_devices(self._ctx):
handle: usb1.USBDeviceHandle | None = None
for device, _, _ in self._iter_matching_devices(self._ctx):
handle: usb1.USBDeviceHandle | None = None
try:
handle = device.open()
found_serial = handle.getSerialNumber() or ""
if serial is not None and found_serial != serial:
continue
selected_handle = handle
selected_serial = found_serial
break
except usb1.USBError as exc:
logger.debug("Skipping USB candidate during connect due to error: %s", exc)
finally:
if handle is not None and handle is not selected_handle:
with suppress(usb1.USBError):
handle.close()
if selected_handle is None:
serial_msg = f" with serial '{serial}'" if serial else ""
raise DeviceDisconnectedError(f"No compatible LibreVNA USB device found{serial_msg}")
self._handle = selected_handle
try:
handle = device.open()
found_serial = handle.getSerialNumber() or ""
if serial is not None and found_serial != serial:
handle.close()
continue
selected_handle = handle
selected_serial = found_serial
break
selected_handle.setAutoDetachKernelDriver(True)
if selected_handle.kernelDriverActive(self.INTERFACE):
selected_handle.detachKernelDriver(self.INTERFACE)
except usb1.USBError as exc:
if handle is not None:
with suppress(usb1.USBError):
handle.close()
logger.debug("Skipping USB candidate during connect due to error: %s", exc)
continue
raise DeviceDisconnectedError(f"Failed to prepare USB kernel driver state: {exc}") from exc
if selected_handle is None:
if self._ctx is not None:
self._ctx.close()
self._ctx = None
serial_msg = f" with serial '{serial}'" if serial else ""
raise DeviceDisconnectedError(f"No compatible LibreVNA USB device found{serial_msg}")
try:
self._claim_interface_with_busy_recovery(selected_handle)
except usb1.USBError as exc:
raise DeviceDisconnectedError(f"Failed to claim USB interface {self.INTERFACE}: {exc}") from exc
try:
selected_handle.setAutoDetachKernelDriver(True)
if selected_handle.kernelDriverActive(self.INTERFACE):
selected_handle.detachKernelDriver(self.INTERFACE)
except usb1.USBError as exc:
selected_handle.close()
if self._ctx is not None:
self._ctx.close()
self._ctx = None
raise DeviceDisconnectedError(f"Failed to prepare USB kernel driver state: {exc}") from exc
self.connected_serial = selected_serial
self._stop_event.clear()
logger.info(
"USB connected (serial=%s, timeout=%.2fs, endpoint_out=0x%02x, endpoint_in=0x%02x)",
self.connected_serial,
timeout_s,
self.DATA_EP_OUT,
self.DATA_EP_IN,
)
try:
self._claim_interface_with_busy_recovery(selected_handle)
except usb1.USBError as exc:
selected_handle.close()
if self._ctx is not None:
self._ctx.close()
self._ctx = None
raise DeviceDisconnectedError(f"Failed to claim USB interface {self.INTERFACE}: {exc}") from exc
self._handle = selected_handle
self.connected_serial = selected_serial
self._stop_event.clear()
logger.info(
"USB connected (serial=%s, timeout=%.2fs, endpoint_out=0x%02x, endpoint_in=0x%02x)",
self.connected_serial,
timeout_s,
self.DATA_EP_OUT,
self.DATA_EP_IN,
)
self._rx_thread = threading.Thread(target=self._rx_loop, name="librevna-usb-rx", daemon=True)
self._rx_thread.start()
self._rx_thread = threading.Thread(target=self._rx_loop, name="librevna-usb-rx", daemon=True)
self._rx_thread.start()
except Exception:
self.disconnect()
raise
def _claim_interface_with_busy_recovery(self, handle: "usb1.USBDeviceHandle") -> None: # type: ignore[name-defined]
"""Claim the data interface, clearing a stale BUSY claim once if needed.
python_app/hardware_full/librevna_service.py
+7 -76
View File
@@ -1,102 +1,38 @@
"""High-level orchestration service for configuring and querying LibreVNA."""
"""Native single-device LibreVNA service used by acquisition and device-limit queries."""
from __future__ import annotations
from dataclasses import dataclass, field
import logging
from python_app.hardware_full.librevna_backends import LibreVnaBackend, MockLibreVnaBackend, NativeLibreVnaBackend
from python_app.hardware_full.librevna_backends import NativeLibreVnaBackend
from python_app.hardware_full.librevna_driver.models import SweepResult
from python_app.models.run_config_model import RadarSweepModel
logger = logging.getLogger(__name__)
@dataclass(slots=True)
class LibreVnaService:
"""Provide stable API for GUI/workflows while hiding backend details."""
"""Provide the native single-device API; factories select mock acquisition."""
serial: str | None = None
strict_protocol_version: int = 14
backend_mode: str = "auto"
_driver_available: bool = field(init=False, default=False, repr=False)
_backend: LibreVnaBackend | None = field(init=False, default=None, repr=False)
_using_mock_backend: bool = field(init=False, default=False, repr=False)
_backend: NativeLibreVnaBackend = field(init=False, repr=False)
def __post_init__(self) -> None:
"""Initialize selected backend and detect driver availability."""
self._driver_available = False
self._backend = None
self._using_mock_backend = False
mode = self.backend_mode.strip().lower()
if mode not in {"auto", "native", "mock"}:
raise ValueError(f"Unsupported LibreVnaService backend mode: {self.backend_mode}")
if mode == "mock":
logger.info("LibreVNA service using mock backend (mode=mock)")
self._backend = MockLibreVnaBackend()
self._using_mock_backend = True
return
try:
self._backend = NativeLibreVnaBackend(
serial=self.serial,
strict_protocol_version=self.strict_protocol_version,
)
self._driver_available = True
logger.info("LibreVNA native backend initialized (serial=%s)", self.serial or "auto")
except Exception as exc:
# 'native' demands real hardware — never substitute synthetic data.
if mode == "native":
raise
# 'auto' falls back to the mock backend ONLY when the native driver
# library itself is unavailable (a dev host without the USB stack) —
# this is not device-absence (that surfaces later from open()). Log it
# loudly so it is never a silent surprise; a deployed appliance should
# set driver_mode='native' to forbid the fallback entirely.
logger.warning("LibreVNA native backend unavailable; falling back to mock (mode=auto): %s", exc)
self._backend = MockLibreVnaBackend()
self._using_mock_backend = True
@property
def driver_available(self) -> bool:
"""Return `True` when native Python LibreVNA driver is available."""
return self._driver_available
if self.backend_mode.strip().lower() not in {"auto", "native"}:
raise ValueError(f"Unsupported native LibreVNA backend mode: {self.backend_mode}")
self._backend = NativeLibreVnaBackend(self.serial, self.strict_protocol_version)
def open(self) -> None:
"""Open backend resources."""
if not self._driver_available and not self._using_mock_backend:
return
if self._backend is None:
return
logger.debug("Opening LibreVNA backend (mock=%s)", self._using_mock_backend)
self._backend.open()
def close(self) -> None:
"""Close backend resources."""
if self._backend is None:
return
logger.debug("Closing LibreVNA backend")
self._backend.close()
def configure(self, sweep: RadarSweepModel) -> None:
"""Apply sweep settings to active backend."""
if self._backend is None:
raise RuntimeError("LibreVNA backend is not initialized")
logger.debug(
"Configuring LibreVNA sweep: %s-%s Hz, %s points",
sweep.start_hz, sweep.stop_hz, sweep.points,
)
self._backend.configure(sweep)
def read_device_limits(self) -> dict[str, float | int]:
"""Read native device limits from connected LibreVNA."""
if not self._driver_available:
raise RuntimeError("LibreVNA Python driver is not available")
if self._backend is None:
raise RuntimeError("LibreVNA backend is not initialized")
opened_here = not self._backend.is_open
try:
if opened_here:
@@ -107,9 +43,4 @@ class LibreVnaService:
self.close()
def acquire(self) -> SweepResult:
"""Acquire one sweep with all available traces from active backend."""
if self._backend is None:
raise RuntimeError("LibreVNA backend is not initialized")
if self._using_mock_backend and not self._driver_available and self.backend_mode != "mock":
raise RuntimeError("Device not found")
return self._backend.acquire()
python_app/hardware_full/matrix_radar_service.py
+37 -53
View File
@@ -1,14 +1,4 @@
"""Factory for matrix-mode (multi-port) radar acquisition services.
Matrix radars acquire the full virtual switch matrix per call via
``acquire_collection`` instead of one combo per ``acquire`` like single-radar
services. The supported models are LibreVNA in synchronized multi-device mode
and the PLANAR SN9000 multi-port analyzer.
Service implementations are imported lazily so a config that targets one
model does not pull in transport dependencies (libusb, pyvisa, etc.) needed
only by the other model.
"""
"""Factory for synchronized LibreVNA matrices and the shared physical mock."""
from __future__ import annotations
@@ -32,57 +22,51 @@ class MatrixRadarService(Protocol):
def acquire_collection(self, collection_id: int = 1) -> SweepCollection:
"""Acquire one complete virtual switch matrix in canonical combo order."""
...
def recover(self) -> None:
"""Reconnect to the radar after a transient acquisition failure."""
def create_matrix_radar_service(config: RunConfigModel) -> MatrixRadarService:
"""Create the matrix radar service implementation for the active config."""
if not config.is_matrix_radar:
def create_acquisition_service(config: RunConfigModel) -> MatrixRadarService:
"""Create a native matrix service or the shared single/multi mock scene."""
if config.radar.driver_mode == "mock" and config.radar.model in {
RunConfigModel.LIBREVNA_MODEL, RunConfigModel.LIBREVNA_MULTI_MODEL,
}:
from python_app.hardware_full.point_target_mock_service import PointTargetMockService
return PointTargetMockService(config)
if config.radar.model != RunConfigModel.LIBREVNA_MULTI_MODEL:
raise RuntimeError("matrix radar service factory requires a matrix-mode radar.model")
model = config.radar.model
if model == RunConfigModel.LIBREVNA_MULTI_MODEL:
from python_app.hardware_full.multi_device_service import MultiDeviceLibreVnaService
from python_app.hardware_full.multi_device_service import MultiDeviceLibreVnaService
inner = MultiDeviceLibreVnaService(
master_serial=config.radar.serial,
slave_serials=list(config.radar.multi_device.slave_serials),
force_external_reference=config.radar.multi_device.force_external_reference,
recovery_attempts=config.radar.multi_device.recovery_attempts,
backend_mode=config.radar.driver_mode,
switch_substage_count=config.matrix_stimulus_switch_positions,
)
out_physical = config.matrix_output_switch_positions
in_physical = config.matrix_input_switch_positions
if out_physical <= 1 and in_physical <= 1:
return inner
inner = MultiDeviceLibreVnaService(
master_serial=config.radar.serial,
slave_serials=list(config.radar.multi_device.slave_serials),
force_external_reference=config.radar.multi_device.force_external_reference,
recovery_attempts=config.radar.multi_device.recovery_attempts,
backend_mode=config.radar.driver_mode,
switch_substage_count=config.matrix_stimulus_switch_positions,
)
out_physical = config.matrix_output_switch_positions
in_physical = config.matrix_input_switch_positions
if out_physical <= 1 and in_physical <= 1:
return inner
from python_app.hardware_full.switched_matrix_radar_service import (
SwitchedMatrixRadarService,
build_physical_switch,
)
from python_app.hardware_full.switched_matrix_radar_service import (
SwitchedMatrixRadarService,
build_physical_switch,
)
return SwitchedMatrixRadarService(
inner=inner,
output_switch=build_physical_switch(config.output_switch, out_physical, config.radar.driver_mode),
input_switch=build_physical_switch(config.input_switch, in_physical, config.radar.driver_mode),
inner_output_positions=config.matrix_virtual_output_positions,
inner_input_positions=RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS,
settling_ms=config.runtime.settling_ms,
)
return SwitchedMatrixRadarService(
inner=inner,
output_switch=build_physical_switch(config.output_switch, out_physical, config.radar.driver_mode),
input_switch=build_physical_switch(config.input_switch, in_physical, config.radar.driver_mode),
inner_output_positions=config.matrix_virtual_output_positions,
inner_input_positions=RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS,
settling_ms=config.runtime.settling_ms,
)
if model == RunConfigModel.SN9000_MODEL:
if config.radar.driver_mode != "native":
raise RuntimeError("SN9000 requires radar.driver_mode='native'")
from python_app.hardware_full.sn9000_service import Sn9000Service
visa_library = config.radar.visa_library or "@ivi"
return Sn9000Service(
host=config.radar.remote_host,
port=config.radar.remote_port,
visa_library=visa_library,
)
raise RuntimeError(f"Unsupported matrix radar model: {model}")
create_matrix_radar_service = create_acquisition_service
python_app/hardware_full/multi_device_service.py
+6 -71
View File
@@ -4,7 +4,6 @@ from __future__ import annotations
from dataclasses import dataclass, field
import logging
import math
import threading
import time
from typing import TYPE_CHECKING
@@ -89,8 +88,6 @@ class MultiDeviceLibreVnaService:
switch_substage_count: int = 1
_controller: "MultiDeviceVnaController | None" = field(init=False, default=None, repr=False)
_sweep_configuration: SweepConfiguration | None = field(init=False, default=None, repr=False)
_using_mock_backend: bool = field(init=False, default=False, repr=False)
_mock_phase: float = field(init=False, default=0.0, repr=False)
def __post_init__(self) -> None:
"""Validate static topology and backend selection."""
@@ -101,10 +98,9 @@ class MultiDeviceLibreVnaService:
self.recovery_attempts = max(0, int(self.recovery_attempts))
mode = self.backend_mode.strip().lower()
if mode not in {"auto", "native", "mock"}:
if mode not in {"auto", "native"}:
raise ValueError(f"Unsupported multi-device backend mode: {self.backend_mode}")
self.backend_mode = mode
self._using_mock_backend = mode == "mock"
self.switch_substage_count = normalize_switch_substage_count(self.switch_substage_count)
@property
@@ -112,14 +108,9 @@ class MultiDeviceLibreVnaService:
"""Return how many stimulus outputs one acquisition produces."""
return RunConfigModel.MULTI_DEVICE_MASTER_PORTS * self.switch_substage_count
@property
def using_mock_backend(self) -> bool:
"""Return whether this service is generating synthetic data."""
return self._using_mock_backend
def open(self) -> None:
"""Open native device transports when not in mock mode."""
if self._using_mock_backend or self._controller is not None:
"""Open native device transports."""
if self._controller is not None:
return
try:
from python_app.hardware_full.librevna_multi_device_driver.controller import MultiDeviceVnaController
@@ -135,11 +126,6 @@ class MultiDeviceLibreVnaService:
self.master_serial, self.slave_serials,
)
except Exception as exc:
# Never silently latch to synthetic data: a deployed appliance must wait
# for the real device, not record fakes. Synthetic data requires an
# explicit backend_mode='mock' (selected in __post_init__); both 'auto'
# and 'native' re-raise so the producer's wait-for-device retry keeps
# trying until the hardware appears.
logger.warning("Multi-device open failed (backend_mode=%s): %s", self.backend_mode, exc)
raise
@@ -176,8 +162,6 @@ class MultiDeviceLibreVnaService:
sleeping, so a shutdown request aborts the loop immediately; an optional
`deadline_monotonic` caps the total wall-time spent here.
"""
if self._using_mock_backend:
return
self.close()
# Try to reopen immediately first, then back off only after a failure: a
@@ -261,12 +245,9 @@ class MultiDeviceLibreVnaService:
if self._sweep_configuration is None:
raise RuntimeError("Multi-device service is not configured")
capture_start_ns = time.monotonic_ns()
if self._using_mock_backend:
collection = self._acquire_mock_collection(collection_id, capture_start_ns)
else:
collection = self._acquire_native_collection_with_recovery(
collection_id, capture_start_ns, stop_event=stop_event
)
collection = self._acquire_native_collection_with_recovery(
collection_id, capture_start_ns, stop_event=stop_event
)
collection.capture_end_ns = time.monotonic_ns()
return collection
@@ -417,52 +398,6 @@ class MultiDeviceLibreVnaService:
capture_start_ns=capture_start_ns,
)
def _acquire_mock_collection(self, collection_id: int, capture_start_ns: int) -> SweepCollection:
assert self._sweep_configuration is not None
mock_sweep_start_ns = time.monotonic_ns()
points = int(self._sweep_configuration.points)
frequencies = np.linspace(
self._sweep_configuration.start_hz,
self._sweep_configuration.stop_hz,
points,
dtype=np.float32,
)
base_phase = 2.0 * math.pi * np.linspace(0.0, 1.0, points, dtype=np.float32) + self._mock_phase
traces: list[TraceData] = []
for output_pos in range(self.output_position_count):
reflected_phase = base_phase * (0.55 + 0.05 * output_pos) + 0.7 * (output_pos + 1)
s11 = (
(0.22 + 0.04 * output_pos) * np.cos(reflected_phase)
+ 1j * (0.22 + 0.04 * output_pos) * np.sin(reflected_phase)
).astype(np.complex64)
for input_pos in range(RunConfigModel.MULTI_DEVICE_INPUT_POSITIONS):
gain = 0.45 + 0.08 * input_pos + 0.03 * output_pos
phase = base_phase * (1.0 + 0.03 * input_pos) + (0.4 * input_pos + 0.9 * output_pos)
s21 = (gain * np.cos(phase) + 1j * gain * np.sin(phase)).astype(np.complex64)
traces.append(
TraceData(
combo=ComboKey(input=input_pos, output=output_pos),
frequency_hz=frequencies,
s11=s11,
s21=s21,
capture_start_ns=mock_sweep_start_ns,
)
)
# One shared window, as on real hardware: every combo of a collection comes
# out of the same synchronized cycle. Stamping each trace as it is built
# would instead spread the ends over the generation loop, which never
# happens natively and would mislead anyone reading mock captures.
mock_sweep_end_ns = time.monotonic_ns()
for trace in traces:
trace.capture_end_ns = mock_sweep_end_ns
self._mock_phase += 0.05
return SweepCollection(
collection_id=int(collection_id),
monotonic_ns=time.monotonic_ns(),
traces=traces,
capture_start_ns=capture_start_ns,
)
@staticmethod
def _required_s_parameter(s_parameters: dict[str, np.ndarray], name: str) -> np.ndarray:
values = s_parameters.get(name)
python_app/hardware_full/point_target_mock_service.py
+121
View File
@@ -0,0 +1,121 @@
"""One moving point reflector shared by single and multi-radar mock acquisition."""
from __future__ import annotations
from dataclasses import dataclass, field
import time
import numpy as np
from python_app.hardware_full.librevna_driver.models import SweepResult
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
from python_app.models.run_config_model import RadarSweepModel, RunConfigModel
_WAVE_SPEED_M_S = 3e8
_FRAME_PERIOD_S = 0.05
_MOTION_PERIOD_S = 8.0
@dataclass(slots=True)
class PointTargetMockService:
"""Simulate bistatic propagation from configured antennas to one moving target.
The target moves back and forth along the array and between depths 3.25–4.25 m.
One monotonic sample time determines every channel of a collection; channel
numbers never contribute an arbitrary gain, range, or phase offset.
"""
config: RunConfigModel
_frequency_hz: np.ndarray | None = field(init=False, default=None, repr=False)
_phase_per_m: np.ndarray | None = field(init=False, default=None, repr=False)
_last_capture_ns: int = field(init=False, default=0, repr=False)
_epoch_ns: int = field(init=False, repr=False)
_combos: tuple[ComboKey, ...] = field(init=False, repr=False)
_active_combo: ComboKey = field(init=False, repr=False)
def __post_init__(self) -> None:
self._epoch_ns = time.monotonic_ns()
self._combos = tuple(ComboKey(c.input, c.output) for c in (
self.config.combos or self.config.build_runtime_combos()
))
self._active_combo = self._combos[0]
def open(self) -> None:
"""Mock acquisition owns no hardware resources."""
def close(self) -> None:
"""Mock acquisition owns no hardware resources."""
def recover(self) -> None:
"""Mock acquisition has no hardware connection to recover."""
def read_device_limits(self) -> dict[str, float | int]:
raise RuntimeError("Mock radar has no hardware device limits")
def configure(self, sweep: RadarSweepModel) -> None:
self._frequency_hz = np.linspace(sweep.start_hz, sweep.stop_hz, sweep.points, dtype=np.float32)
self._phase_per_m = -2j * np.pi * self._frequency_hz.astype(np.float64) / _WAVE_SPEED_M_S
def set_active_combo(self, combo: ComboKey) -> None:
if not (0 <= combo.input < self.config.input_switch.positions
and 0 <= combo.output < self.config.output_switch.positions):
raise ValueError(f"Mock combo is outside the antenna matrix: {combo}")
self._active_combo = ComboKey(combo.input, combo.output)
def acquire(self) -> SweepResult:
collection = self._capture((self._active_combo,), 1)
trace = collection.traces[0]
return SweepResult(
x=trace.frequency_hz, traces={"s11": trace.s11, "s21": trace.s21},
capture_start_ns=trace.capture_start_ns, capture_end_ns=trace.capture_end_ns,
)
def acquire_collection(self, collection_id: int = 1) -> SweepCollection:
return self._capture(self._combos, collection_id)
def acquire_combo_collection(
self, *, input_pos: int, output_pos: int, collection_id: int = 1,
) -> SweepCollection:
self.set_active_combo(ComboKey(input_pos, output_pos))
return self._capture((self._active_combo,), collection_id)
def _target_position(
self, sample_ns: int,
tx: dict[int, tuple[float, float, float]], rx: dict[int, tuple[float, float, float]],
) -> np.ndarray:
coordinates = [*tx.values(), *rx.values()] or [(0.0, 0.0, 0.0)]
xs = [position[0] for position in coordinates]
displacement = np.sin(2 * np.pi * (sample_ns - self._epoch_ns) / 1e9 / _MOTION_PERIOD_S)
return np.array([
(min(xs) + max(xs)) / 2 + max(0.75, (max(xs) - min(xs)) / 2 + 0.25) * displacement,
0.25, 3.75 + 0.5 * displacement,
])
def _capture(self, combos: tuple[ComboKey, ...], collection_id: int) -> SweepCollection:
if self._frequency_hz is None or self._phase_per_m is None:
raise RuntimeError("Mock radar is not configured")
delay_s = _FRAME_PERIOD_S - (time.monotonic_ns() - self._last_capture_ns) / 1e9
if delay_s > 0:
time.sleep(delay_s)
capture_start_ns = time.monotonic_ns()
tx = {p.output_pos: (p.x_m, p.y_m, p.z_m) for p in self.config.gpr.tx_geometry}
rx = {p.input_pos: (p.x_m, p.y_m, p.z_m) for p in self.config.gpr.rx_geometry}
target = self._target_position(capture_start_ns, tx, rx)
traces = []
for combo in combos:
r_tx = float(np.linalg.norm(target - tx.get(combo.output, (0.0, 0.0, 0.0))))
r_rx = float(np.linalg.norm(target - rx.get(combo.input, (0.0, 0.0, 0.0))))
traces.append(TraceData(
combo=combo, frequency_hz=self._frequency_hz,
s11=np.asarray(0.5 / max(1.0, r_tx * r_tx) * np.exp(self._phase_per_m * 2 * r_tx), dtype=np.complex64),
s21=np.asarray(0.5 / max(1.0, r_tx * r_rx) * np.exp(self._phase_per_m * (r_tx + r_rx)), dtype=np.complex64),
capture_start_ns=capture_start_ns,
))
capture_end_ns = time.monotonic_ns()
for trace in traces:
trace.capture_end_ns = capture_end_ns
self._last_capture_ns = capture_start_ns
return SweepCollection(
collection_id=collection_id, monotonic_ns=capture_end_ns, traces=traces,
capture_start_ns=capture_start_ns, capture_end_ns=capture_end_ns,
)
python_app/hardware_full/remote_compact_m_k209_service.py
-172
View File
@@ -1,172 +0,0 @@
"""TCP client for a Compact-M K209 connected to a remote S2VNA host."""
from __future__ import annotations
from dataclasses import dataclass, field
import socket
import numpy as np
from python_app.hardware_full.k209_remote_protocol import (
COMMAND_ACQUIRE,
COMMAND_CONFIGURE,
COMMAND_IDENTITY,
COMMAND_LIMITS,
CONFIG_STRUCT,
DEFAULT_REMOTE_HOST,
DEFAULT_REMOTE_PORT,
LIMITS_STRUCT,
read_status,
recv_exact,
recv_float32_array,
recv_u32,
)
from python_app.hardware_full.librevna_driver.models import SweepResult
from python_app.models.run_config_model import RadarSweepModel
@dataclass(slots=True)
class RemoteCompactMK209Service:
"""Acquire K209 sweeps through a persistent TCP connection."""
host: str = DEFAULT_REMOTE_HOST
port: int = DEFAULT_REMOTE_PORT
timeout_s: float = 20.0
_socket: socket.socket | None = field(init=False, default=None, repr=False)
_settings: RadarSweepModel | None = field(init=False, default=None, repr=False)
_frequency_hz: np.ndarray | None = field(init=False, default=None, repr=False)
def __post_init__(self) -> None:
self.host = str(self.host).strip()
if not self.host:
raise ValueError("K209 remote host must not be empty")
self.port = int(self.port)
if self.port <= 0 or self.port > 65535:
raise ValueError("K209 remote port must be in 1..65535")
self.timeout_s = float(self.timeout_s)
if self.timeout_s <= 0.0:
raise ValueError("K209 remote timeout_s must be > 0")
def open(self) -> None:
"""Open TCP connection and apply stored settings when present."""
if self._socket is not None:
return
sock = socket.create_connection((self.host, self.port), timeout=self.timeout_s)
sock.settimeout(self.timeout_s)
sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self._socket = sock
try:
if self._settings is not None:
self._apply_configuration(self._settings)
except Exception:
self.close()
raise
def close(self) -> None:
"""Close TCP connection."""
if self._socket is None:
return
try:
self._socket.close()
finally:
self._socket = None
def query_identity(self) -> str:
"""Read analyzer identity string through the remote server."""
sock = self._require_socket()
sock.sendall(COMMAND_IDENTITY)
read_status(sock)
return recv_exact(sock, recv_u32(sock)).decode("utf-8", errors="replace").strip()
def read_device_limits(self) -> dict[str, float | int]:
"""Read analyzer limits through the remote server."""
opened_here = self._socket is None
try:
if opened_here:
self.open()
sock = self._require_socket()
sock.sendall(COMMAND_LIMITS)
read_status(sock)
min_freq, max_freq, min_ifbw, max_ifbw, max_points, min_power, max_power = LIMITS_STRUCT.unpack(
recv_exact(sock, LIMITS_STRUCT.size)
)
return {
"min_frequency_hz": float(min_freq),
"max_frequency_hz": float(max_freq),
"min_ifbw_hz": float(min_ifbw),
"max_ifbw_hz": float(max_ifbw),
"max_points": int(max_points),
"min_power_dbm": float(min_power),
"max_power_dbm": float(max_power),
}
finally:
if opened_here:
self.close()
def configure(self, sweep: RadarSweepModel) -> None:
"""Store and apply sweep settings."""
self._validate_sweep(sweep)
self._settings = sweep
self._frequency_hz = None
if self._socket is not None:
self._apply_configuration(sweep)
def acquire(self) -> SweepResult:
"""Acquire one corrected S11/S21 sweep."""
if self._settings is None or self._frequency_hz is None:
raise RuntimeError("K209 remote service is not configured")
sock = self._require_socket()
points = int(self._settings.points)
sock.sendall(COMMAND_ACQUIRE)
read_status(sock)
returned_points = recv_u32(sock)
if returned_points != points:
raise RuntimeError(f"K209 remote sweep returned {returned_points} points, expected {points}")
s11_values = recv_float32_array(sock, points * 2)
s21_values = recv_float32_array(sock, points * 2)
return SweepResult(
x=self._frequency_hz.copy(),
traces={
"s11": self._complex_from_interleaved(s11_values),
"s21": self._complex_from_interleaved(s21_values),
},
)
def _apply_configuration(self, sweep: RadarSweepModel) -> None:
sock = self._require_socket()
sock.sendall(COMMAND_CONFIGURE)
sock.sendall(
CONFIG_STRUCT.pack(
float(sweep.start_hz),
float(sweep.stop_hz),
int(sweep.points),
float(sweep.if_bandwidth_hz),
float(sweep.power_dbm),
)
)
read_status(sock)
returned_points = recv_u32(sock)
if returned_points != int(sweep.points):
raise RuntimeError(f"K209 remote config returned {returned_points} points, expected {sweep.points}")
self._frequency_hz = recv_float32_array(sock, int(sweep.points))
def _require_socket(self) -> socket.socket:
if self._socket is None:
raise RuntimeError("K209 remote socket is not open")
return self._socket
@staticmethod
def _validate_sweep(sweep: RadarSweepModel) -> None:
if int(sweep.points) < 2:
raise ValueError("K209 sweep points must be >= 2")
if float(sweep.stop_hz) < float(sweep.start_hz):
raise ValueError("K209 sweep stop_hz must be >= start_hz")
if float(sweep.if_bandwidth_hz) <= 0.0:
raise ValueError("K209 IF bandwidth must be > 0")
@staticmethod
def _complex_from_interleaved(values: np.ndarray) -> np.ndarray:
if values.size % 2 != 0:
raise RuntimeError("K209 remote complex trace payload has odd scalar count")
reshaped = np.asarray(values, dtype=np.float32).reshape((-1, 2))
return (reshaped[:, 0] + 1j * reshaped[:, 1]).astype(np.complex64)
python_app/hardware_full/single_radar_service.py
+6 -11
View File
@@ -8,7 +8,6 @@ from typing import Protocol
from python_app.hardware_full.kamil_adc import KamilAdcService
from python_app.hardware_full.librevna_driver.models import SweepResult
from python_app.hardware_full.librevna_service import LibreVnaService
from python_app.hardware_full.remote_compact_m_k209_service import RemoteCompactMK209Service
from python_app.models.run_config_model import RadarSweepModel, RunConfigModel
logger = logging.getLogger(__name__)
@@ -28,9 +27,11 @@ class SingleRadarService(Protocol):
def read_device_limits(self) -> dict[str, float | int]:
"""Read device capability limits."""
...
def acquire(self) -> SweepResult:
"""Acquire one sweep."""
...
def create_single_radar_service(config: RunConfigModel) -> SingleRadarService:
@@ -44,17 +45,11 @@ def create_single_radar_service(config: RunConfigModel) -> SingleRadarService:
model = config.radar.model or RunConfigModel.LIBREVNA_MODEL
logger.debug("Creating single-radar service for model=%s (driver_mode=%s)", model, config.radar.driver_mode)
if model == RunConfigModel.LIBREVNA_MODEL:
# Forward driver_mode (mirrors the matrix path): 'native' must require real
# hardware and 'mock' must use the synthetic backend — never silently the wrong one.
return LibreVnaService(serial=config.radar.serial or None, backend_mode=config.radar.driver_mode)
if config.radar.driver_mode == "mock":
from python_app.hardware_full.point_target_mock_service import PointTargetMockService
if model == RunConfigModel.COMPACT_M_K209_MODEL:
if config.radar.driver_mode != "native":
raise RuntimeError("Compact-M K209 requires radar.driver_mode='native'")
return RemoteCompactMK209Service(
host=config.radar.remote_host,
port=config.radar.remote_port,
)
return PointTargetMockService(config)
return LibreVnaService(serial=config.radar.serial or None, backend_mode=config.radar.driver_mode)
if model == RunConfigModel.KAMIL_ADC_MODEL:
if config.radar.driver_mode != "native":
python_app/hardware_full/sn9000_service.py
-395
View File
@@ -1,395 +0,0 @@
"""VISA HiSLIP driver for the PLANAR SN9000 (Иридиум series) multi-port VNA.
The SN9000 hardware connects to a host PC over USB 2.0; the PC runs the SNVNA
application, which exposes the SCPI HiSLIP server. This service connects to
that server, configures a 2x4 virtual switch matrix (ports 1 and 2 stimulate,
ports 3..6 receive), and acquires the full matrix per call in a single
synchronized SCPI round trip.
"""
from __future__ import annotations
from dataclasses import dataclass, field
import logging
import time
from typing import Any
import numpy as np
import pyvisa
logger = logging.getLogger(__name__)
from python_app.models.dataset_model import ComboKey, SweepCollection, TraceData
from python_app.models.run_config_model import RadarSweepModel
_OUTPUT_PORT_BY_INDEX: tuple[int, ...] = (1, 2)
_INPUT_PORT_BY_INDEX: tuple[int, ...] = (3, 4, 5, 6)
_REFLECTION_BY_OUTPUT: tuple[str, ...] = tuple(
f"S{port}{port}" for port in _OUTPUT_PORT_BY_INDEX
)
_S_PARAMETER_QUERY_ORDER: tuple[str, ...] = tuple(
name
for output_port in _OUTPUT_PORT_BY_INDEX
for name in (
f"S{output_port}{output_port}",
*(f"S{receiver_port}{output_port}" for receiver_port in _INPUT_PORT_BY_INDEX),
)
)
@dataclass(slots=True)
class Sn9000Service:
"""Acquire the full 2x4 corrected sweep matrix from an SN9000 through VISA HiSLIP."""
host: str = "127.0.0.1"
port: int = 4880
timeout_ms: int = 20_000
preset_on_open: bool = True
visa_library: str = "@ivi"
_resource_manager: pyvisa.ResourceManager | None = field(init=False, default=None, repr=False)
_instrument: Any | None = field(init=False, default=None, repr=False)
_settings: RadarSweepModel | None = field(init=False, default=None, repr=False)
_frequency_hz: np.ndarray | None = field(init=False, default=None, repr=False)
def __post_init__(self) -> None:
"""Normalize and validate constructor values."""
self.host = str(self.host).strip()
if not self.host:
raise ValueError("SN9000 host must not be empty")
self.port = int(self.port)
if self.port <= 0 or self.port > 65535:
raise ValueError("SN9000 port must be in 1..65535")
self.timeout_ms = int(self.timeout_ms)
if self.timeout_ms <= 0:
raise ValueError("SN9000 timeout_ms must be > 0")
self.visa_library = str(self.visa_library).strip() or "@ivi"
@property
def resource(self) -> str:
"""Return the assembled VISA HiSLIP resource string."""
return f"TCPIP0::{self.host}::hislip0,{self.port}::INSTR"
@property
def is_open(self) -> bool:
"""Return whether the VISA session is open."""
return self._instrument is not None
def open(self) -> None:
"""Open the VISA session and apply stored sweep settings when available."""
if self._instrument is not None:
return
logger.info("Opening SN9000 VISA session: %s", self.resource)
try:
self._resource_manager = pyvisa.ResourceManager(self.visa_library)
self._instrument = self._resource_manager.open_resource(self.resource)
self._instrument.timeout = self.timeout_ms
self._instrument.write_termination = "\n"
self._instrument.read_termination = None
self._instrument.chunk_size = max(int(getattr(self._instrument, "chunk_size", 20_480)), 8 * 1024 * 1024)
self._instrument.write("*CLS")
if self._settings is not None:
self._apply_configuration(self._settings)
except Exception:
logger.exception("Failed to open SN9000 VISA session: %s", self.resource)
self.close()
raise
def close(self) -> None:
"""Close VISA sessions."""
if self._instrument is not None:
logger.debug("Closing SN9000 VISA session")
self._instrument.close()
self._instrument = None
if self._resource_manager is not None:
self._resource_manager.close()
self._resource_manager = None
def __enter__(self) -> Sn9000Service:
"""Open the service and return it."""
self.open()
return self
def __exit__(self, exc_type: object, exc: object, traceback: object) -> None:
"""Close VISA resources on context exit."""
self.close()
def recover(self) -> None:
"""Reopen the VISA session after a transient acquisition failure."""
logger.warning("Recovering SN9000 VISA session (close, wait, reopen)")
self.close()
time.sleep(0.25)
self.open()
def query_identity(self) -> str:
"""Read the analyzer identity string."""
instrument = self._require_instrument()
return str(instrument.query("*IDN?")).strip()
def query_system_error(self) -> str:
"""Read one entry from the analyzer SCPI error queue."""
instrument = self._require_instrument()
return str(instrument.query("SYST:ERR?")).strip()
def configure(self, sweep: RadarSweepModel) -> None:
"""Store and apply sweep settings."""
self._validate_sweep(sweep)
self._settings = sweep
self._frequency_hz = None
logger.debug(
"Configuring SN9000 sweep: %s-%s Hz, %s points, IFBW=%s Hz, %s dBm",
sweep.start_hz, sweep.stop_hz, sweep.points, sweep.if_bandwidth_hz, sweep.power_dbm,
)
if self._instrument is None:
return
self._apply_configuration(sweep)
def read_device_limits(self) -> dict[str, float | int]:
"""Read analyzer limits through SCPI capability/service queries."""
opened_here = self._instrument is None
try:
if opened_here:
self.open()
instrument = self._require_instrument()
return {
"min_frequency_hz": float(instrument.query("SERV:SWE:FREQ:MIN?")),
"max_frequency_hz": float(instrument.query("SERV:SWE:FREQ:MAX?")),
# SN9000 SCPI does not expose IFBW capability queries; use the
# documented hardware sequence (1 Hz .. 300 kHz, manual p. 58, 1261).
"min_ifbw_hz": 1.0,
"max_ifbw_hz": 300_000.0,
"max_points": int(float(instrument.query("SERV:SWE:POIN?"))),
"min_power_dbm": float(instrument.query("SERV:SWE:POW:MIN?")),
"max_power_dbm": float(instrument.query("SERV:SWE:POW:MAX?")),
}
finally:
if opened_here:
self.close()
def frequency_axis(self) -> np.ndarray:
"""Return the cached frequency axis from the most recent configuration."""
if self._frequency_hz is None:
raise RuntimeError("SN9000 frequency axis is not configured")
return self._frequency_hz
def acquire_collection(self, collection_id: int = 1) -> SweepCollection:
"""Acquire one full virtual 2x4 matrix in canonical combo order."""
if self._settings is None:
raise RuntimeError("SN9000 service is not configured")
if self._frequency_hz is None:
raise RuntimeError("SN9000 frequency axis is not configured")
points = int(self._settings.points)
capture_start_ns = time.monotonic_ns()
s_parameters = self._query_sweep_s_parameters(points)
traces = self._assemble_traces(
s_parameters,
sweep_start_ns=capture_start_ns,
sweep_end_ns=time.monotonic_ns(),
)
return SweepCollection(
collection_id=int(collection_id),
monotonic_ns=time.monotonic_ns(),
traces=traces,
capture_start_ns=capture_start_ns,
capture_end_ns=time.monotonic_ns(),
)
def _apply_configuration(self, sweep: RadarSweepModel) -> None:
instrument = self._require_instrument()
if self.preset_on_open:
instrument.write("SYST:PRES")
self._expect_opc("*OPC?", context="SN9000 preset")
instrument.write(f"CALC:PAR:COUN {len(_S_PARAMETER_QUERY_ORDER)}")
for trace_index, parameter_name in enumerate(_S_PARAMETER_QUERY_ORDER, start=1):
instrument.write(f"CALC:PAR{trace_index}:DEF {parameter_name}")
instrument.write("SENS:SWE:TYPE LIN")
instrument.write(f"SENS:FREQ:STAR {float(sweep.start_hz):.9f}")
instrument.write(f"SENS:FREQ:STOP {float(sweep.stop_hz):.9f}")
instrument.write(f"SENS:SWE:POIN {int(sweep.points)}")
instrument.write("SENS:SWE:POIN:TIME 0")
instrument.write(f"SENS:BAND {float(sweep.if_bandwidth_hz):.9f}")
instrument.write("SOUR:POW:PORT:COUP ON")
instrument.write(f"SOUR:POW {float(sweep.power_dbm):.3f}")
instrument.write("SENS:AVER OFF")
instrument.write("FORM:DATA REAL32")
instrument.write("FORM:BORD SWAP")
instrument.write("INIT:CONT ON")
instrument.write("TRIG:SOUR BUS")
self._expect_opc("*OPC?", context="SN9000 setup")
self._frequency_hz = self._query_float32_array("SENS:FREQ:DATA?", int(sweep.points))
logger.info(
"SN9000 configured: %d traces, %d points (%s-%s Hz)",
len(_S_PARAMETER_QUERY_ORDER), int(sweep.points), sweep.start_hz, sweep.stop_hz,
)
def _query_sweep_s_parameters(self, points: int) -> dict[str, np.ndarray]:
instrument = self._require_instrument()
if self._uses_pyvisa_py_backend():
# pyvisa-py HiSLIP loses synchronization when a single packet aggregates
# *OPC? plus multiple binary blocks, so issue trigger and data queries
# one at a time. The corrected-data buffer holds the last completed
# sweep, so reading each S-parameter sequentially is safe.
instrument.write("TRIG:SING")
self._expect_opc("*OPC?", context="SN9000 sweep")
complex_values: dict[str, np.ndarray] = {}
for parameter_name in _S_PARAMETER_QUERY_ORDER:
instrument.write(f"SENS:DATA:CORR? {parameter_name}")
interleaved = self._read_float32_block(
f"SENS:DATA:CORR? {parameter_name}", points * 2
)
complex_values[parameter_name] = self._complex_from_interleaved(interleaved)
return complex_values
data_queries = ";".join(f":SENS:DATA:CORR? {name}" for name in _S_PARAMETER_QUERY_ORDER)
instrument.write(f"TRIG:SING;*OPC?;{data_queries}")
opc_token = self._read_ascii_token()
if opc_token != "1":
raise RuntimeError(f"SN9000 sweep returned unexpected *OPC? response: {opc_token!r}")
complex_values = {}
for parameter_name in _S_PARAMETER_QUERY_ORDER:
interleaved = self._read_float32_block(f"SENS:DATA:CORR? {parameter_name}", points * 2)
complex_values[parameter_name] = self._complex_from_interleaved(interleaved)
return complex_values
def _uses_pyvisa_py_backend(self) -> bool:
return self.visa_library == "@py" or self.visa_library.endswith("@py")
def _assemble_traces(
self,
s_parameters: dict[str, np.ndarray],
*,
sweep_start_ns: int,
sweep_end_ns: int,
) -> list[TraceData]:
frequency_hz = self._require_frequency_axis()
traces: list[TraceData] = []
for output_position, output_port in enumerate(_OUTPUT_PORT_BY_INDEX):
reflection = s_parameters[_REFLECTION_BY_OUTPUT[output_position]]
for input_position, receiver_port in enumerate(_INPUT_PORT_BY_INDEX):
transmission = s_parameters[f"S{receiver_port}{output_port}"]
traces.append(
TraceData(
combo=ComboKey(input=input_position, output=output_position),
frequency_hz=frequency_hz,
s11=reflection,
s21=transmission,
# One triggered sweep produces every port pair at once, so
# all combos share the sweep's window.
capture_start_ns=int(sweep_start_ns),
capture_end_ns=int(sweep_end_ns),
)
)
return traces
def _expect_opc(self, command: str, *, context: str) -> None:
instrument = self._require_instrument()
response = str(instrument.query(command)).strip()
if response != "1":
raise RuntimeError(f"{context} returned unexpected *OPC? response: {response!r}")
def _query_float32_array(self, command: str, expected_values: int) -> np.ndarray:
instrument = self._require_instrument()
instrument.write(command)
return self._read_float32_block(command, expected_values)
def _read_ascii_token(self) -> str:
token = bytearray()
while True:
byte = self._read_response_bytes(1)
if byte in (b";", b"\n", b"\r"):
if token:
return token.decode("ascii").strip()
continue
token.extend(byte)
if len(token) > 64 * 1024:
raise RuntimeError("SN9000 ASCII response token is too long")
def _read_float32_block(self, context: str, expected_values: int) -> np.ndarray:
marker = self._read_response_bytes(1)
while marker in (b";", b"\n", b"\r"):
marker = self._read_response_bytes(1)
if marker != b"#":
raise RuntimeError(f"SN9000 response for {context!r} does not start with IEEE block marker")
width = self._read_response_bytes(1)
if width != b"8":
raise RuntimeError(f"SN9000 response for {context!r} uses unsupported IEEE block header width")
payload_size = int(self._read_response_bytes(8).decode("ascii"))
expected_size = expected_values * np.dtype(np.float32).itemsize
if payload_size != expected_size:
raise RuntimeError(
f"SN9000 response for {context!r} returned {payload_size} payload bytes, "
f"expected {expected_size}"
)
payload = self._read_response_bytes(payload_size)
array = np.frombuffer(payload, dtype="<f4")
if array.size != expected_values:
raise RuntimeError(
f"SN9000 response for {context!r} returned {array.size} float32 values, "
f"expected {expected_values}"
)
self._drain_trailing_terminators()
return array
def _drain_trailing_terminators(self) -> None:
"""Consume the SCPI terminator that follows IEEE binary blocks.
SCPI responses end with `\\n`, which over HiSLIP closes the DataEnd
message group. pyvisa-py's HiSLIP layer needs the terminator drained
before the next request, otherwise it loses message-frame
synchronization on subsequent reads.
"""
instrument = self._require_instrument()
deadline = time.monotonic() + 0.2
while time.monotonic() < deadline:
try:
instrument.read_bytes(1, break_on_termchar=True)
return
except pyvisa.errors.VisaIOError as exc:
# Timeouts on a trailing newline are routine; anything else
# likely means HiSLIP framing is out of sync and the next
# request will hang — surface it in the logs.
if exc.error_code != pyvisa.constants.StatusCode.error_timeout:
logger.warning("SN9000 terminator drain failed: %s", exc)
return
def _read_response_bytes(self, count: int) -> bytes:
instrument = self._require_instrument()
data = instrument.read_bytes(count, break_on_termchar=False)
if len(data) != count:
raise RuntimeError(f"SN9000 response ended after {len(data)} bytes, expected {count}")
return data
def _require_instrument(self) -> Any:
if self._instrument is None:
raise RuntimeError("SN9000 VISA instrument is not open")
return self._instrument
def _require_frequency_axis(self) -> np.ndarray:
if self._frequency_hz is None:
raise RuntimeError("SN9000 frequency axis is not configured")
return self._frequency_hz
@staticmethod
def _validate_sweep(sweep: RadarSweepModel) -> None:
if int(sweep.points) < 2:
raise ValueError("SN9000 sweep points must be >= 2")
if float(sweep.stop_hz) < float(sweep.start_hz):
raise ValueError("SN9000 sweep stop_hz must be >= start_hz")
if float(sweep.if_bandwidth_hz) <= 0.0:
raise ValueError("SN9000 IF bandwidth must be > 0")
@staticmethod
def _complex_from_interleaved(values: np.ndarray) -> np.ndarray:
if values.size % 2 != 0:
raise RuntimeError("SN9000 complex trace payload has odd scalar count")
reshaped = np.asarray(values, dtype=np.float32).reshape((-1, 2))
return (reshaped[:, 0] + 1j * reshaped[:, 1]).astype(np.complex64)
python_app/hardware_full/switch_drivers/h7992_driver.py
+12 -12
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
from dataclasses import dataclass, field
from contextlib import suppress
from python_app.hardware_full.switch_drivers.gpio_uapi import GpioOutputLines
@@ -29,12 +30,6 @@ class H7992Driver:
_is_open: bool = field(init=False, default=False, repr=False)
_current_position: int = field(init=False, default=0, repr=False)
def __post_init__(self) -> None:
"""Initialize runtime state."""
self._lines: GpioOutputLines | None = None
self._is_open = False
self._current_position = 0
def open(self) -> None:
"""Open GPIO lines and switch to default position."""
if self._is_open:
@@ -46,16 +41,21 @@ class H7992Driver:
offsets=(self.pin_a, self.pin_b),
consumer=f"radar_{self.name}",
)
self._lines.open()
self._is_open = True
self.switch_to(self.default_position)
try:
self._lines.open()
self._is_open = True
self.switch_to(self.default_position)
except Exception:
with suppress(Exception):
self.close()
raise
def close(self) -> None:
"""Close GPIO lines."""
if self._lines is not None:
self._lines.close()
self._lines = None
lines, self._lines = self._lines, None
self._is_open = False
if lines is not None:
lines.close()
def position_count(self) -> int:
"""Return number of supported positions."""
python_app/hardware_full/switch_drivers/hmc349a_driver.py
+12 -12
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
from dataclasses import dataclass, field
from contextlib import suppress
from python_app.hardware_full.switch_drivers.gpio_uapi import GpioOutputLines
@@ -21,12 +22,6 @@ class HMC349ADriver:
_is_open: bool = field(init=False, default=False, repr=False)
_current_position: int = field(init=False, default=0, repr=False)
def __post_init__(self) -> None:
"""Initialize runtime state."""
self._lines: GpioOutputLines | None = None
self._is_open = False
self._current_position = 0
def open(self) -> None:
"""Open GPIO lines and switch to default position."""
if self._is_open:
@@ -40,16 +35,21 @@ class HMC349ADriver:
offsets=offsets,
consumer=f"radar_{self.name}",
)
self._lines.open()
self._is_open = True
self.switch_to(self.default_position)
try:
self._lines.open()
self._is_open = True
self.switch_to(self.default_position)
except Exception:
with suppress(Exception):
self.close()
raise
def close(self) -> None:
"""Close GPIO lines."""
if self._lines is not None:
self._lines.close()
self._lines = None
lines, self._lines = self._lines, None
self._is_open = False
if lines is not None:
lines.close()
def position_count(self) -> int:
"""Return number of supported positions."""
python_app/hardware_full/switched_matrix_radar_service.py
+14 -11
View File
@@ -38,21 +38,24 @@ class SwitchedMatrixRadarService:
def open(self) -> None:
"""Open the inner radar and both switches."""
self.inner.open()
if self.output_switch is not None:
self.output_switch.open()
if self.input_switch is not None:
self.input_switch.open()
try:
self.inner.open()
if self.output_switch is not None:
self.output_switch.open()
if self.input_switch is not None:
self.input_switch.open()
except Exception:
self.close()
raise
def close(self) -> None:
"""Close switches first, then the inner radar; never raises."""
for switch in (self.input_switch, self.output_switch):
if switch is not None:
for resource in (self.input_switch, self.output_switch, self.inner):
if resource is not None:
try:
switch.close()
resource.close()
except Exception as exc: # noqa: BLE001 — shutdown path
logger.warning("Switch close ignored error: %s", exc)
self.inner.close()
logger.warning("Switched matrix cleanup failed: %s", exc)
def configure(self, sweep: RadarSweepModel) -> None:
"""Apply sweep settings to the inner radar."""
@@ -211,4 +214,4 @@ def build_physical_switch(
driver_mode = "mock" if radar_driver_mode.strip().lower() == "mock" else "native"
return SwitchService.from_model(
replace(model, positions=physical_positions, driver_mode=driver_mode)
)
)
python_app/logging_setup.py
-6
View File
@@ -51,12 +51,6 @@ def coerce_level(value: object) -> int:
return resolved if isinstance(resolved, int) else logging.INFO
def normalize_level_name(value: object) -> str:
"""Return a canonical UPPERCASE level name from the supported set (default INFO)."""
name = str(value).strip().upper()
return name if name in LOG_LEVELS else DEFAULT_LOG_LEVEL
def package_logger() -> logging.Logger:
"""Return the application's root logger."""
return logging.getLogger(PACKAGE_LOGGER_NAME)
python_app/models/gui_profile_codec.py
+3 -117
View File
@@ -6,6 +6,7 @@ import dataclasses
import logging
from typing import Any
from python_app.models.json_fields import finite_number, json_object as _as_dict
from python_app.models.gui_profile_schema import (
GuiBscanStateModel,
GuiDataActionsStateModel,
@@ -24,15 +25,6 @@ from python_app.models.run_config_model import RunConfigModel
logger = logging.getLogger(__name__)
def _as_dict(value: Any, context: str) -> dict[str, Any]:
"""Validate payload node is object-like, treating missing values as empty object."""
if value is None:
return {}
if not isinstance(value, dict):
raise ValueError(f"{context} must be a JSON object")
return value
def _optional_string(object_payload: dict[str, Any], key: str, fallback: str, context: str) -> str:
"""Return optional string field with type validation."""
raw_value = object_payload.get(key, fallback)
@@ -59,10 +51,7 @@ def _optional_int(object_payload: dict[str, Any], key: str, fallback: int, conte
def _optional_float(object_payload: dict[str, Any], key: str, fallback: float, context: str) -> float:
"""Return optional numeric field with type validation."""
raw_value = object_payload.get(key, fallback)
if isinstance(raw_value, bool) or not isinstance(raw_value, (int, float)):
raise ValueError(f"{context}.{key} must be a JSON number")
return float(raw_value)
return finite_number(object_payload.get(key, fallback), f"{context}.{key}")
def _legacy_gpr_mode_from_algorithm(value: str) -> str | None:
@@ -533,12 +522,6 @@ def gui_profile_from_dict(payload: dict[str, Any]) -> GuiProfileModel:
data_actions_object = _as_dict(gui_object.get("data_actions"), "gui.data_actions")
gui.data_actions = GuiDataActionsStateModel(
save_count=_optional_int(
data_actions_object,
"save_count",
gui.data_actions.save_count,
"gui.data_actions",
),
save_path=_optional_string(
data_actions_object,
"save_path",
@@ -598,102 +581,5 @@ def gui_profile_from_dict(payload: dict[str, Any]) -> GuiProfileModel:
def gui_profile_to_dict(model: GuiProfileModel) -> dict[str, Any]:
"""Encode :class:`GuiProfileModel` into JSON-serializable dictionary."""
payload = model.run_config.to_dict()
gui = model.gui if model.gui is not None else GuiStateModel()
payload["gui"] = {
"version": gui.version,
"switches": {
"combo_mode": gui.switches.combo_mode,
"combos_text": gui.switches.combos_text,
"single_input": gui.switches.single_input,
"single_output": gui.switches.single_output,
},
"processing": {
"selected_mode": gui.processing.selected_mode,
"pass_through": {
"show_magnitude": gui.processing.pass_through.show_magnitude,
"show_phase": gui.processing.pass_through.show_phase,
"unwrap_phase": gui.processing.pass_through.unwrap_phase,
"combo_filter": gui.processing.pass_through.combo_filter,
"fixed_y_enabled": gui.processing.pass_through.fixed_y_enabled,
"y_min_db": gui.processing.pass_through.y_min_db,
"y_max_db": gui.processing.pass_through.y_max_db,
},
"bscan": {
"axis": gui.processing.bscan.axis,
"cut_m": gui.processing.bscan.cut_m,
"max_depth_m": gui.processing.bscan.max_depth_m,
"gain": gui.processing.bscan.gain,
"start_freq_mhz": gui.processing.bscan.start_freq_mhz,
"stop_freq_mhz": gui.processing.bscan.stop_freq_mhz,
"subtract_mean_ascan": gui.processing.bscan.subtract_mean_ascan,
"history_window_scans": gui.processing.bscan.history_window_scans,
},
"gpr": {
"input_positions": gui.processing.gpr.input_positions,
"output_positions": gui.processing.gpr.output_positions,
"min_depth_m": gui.processing.gpr.min_depth_m,
"max_depth_m": gui.processing.gpr.max_depth_m,
"range_comp_power": gui.processing.gpr.range_comp_power,
"object_min_frac": gui.processing.gpr.object_min_frac,
"score_mode": gui.processing.gpr.score_mode,
"motion_mode": gui.processing.gpr.motion_mode,
"look_angle_deg": gui.processing.gpr.look_angle_deg,
"direction_sign": gui.processing.gpr.direction_sign,
"speed_m_s": gui.processing.gpr.speed_m_s,
"ignore_socket_speed_enabled": gui.processing.gpr.ignore_socket_speed_enabled,
"max_detected_objects_to_draw": gui.processing.gpr.max_detected_objects_to_draw,
"draw_top_m_objects": gui.processing.gpr.draw_top_m_objects,
"object_approach_min_frames": gui.processing.gpr.object_approach_min_frames,
"start_freq_mhz": gui.processing.gpr.start_freq_mhz,
"stop_freq_mhz": gui.processing.gpr.stop_freq_mhz,
"background_subtract_enabled": gui.processing.gpr.background_subtract_enabled,
"background_mean_count": gui.processing.gpr.background_mean_count,
"remove_sidelobe_objects_enabled": gui.processing.gpr.remove_sidelobe_objects_enabled,
"imaging_plane_y_m": gui.processing.gpr.imaging_plane_y_m,
"render_mode": gui.processing.gpr.render_mode,
"visible_x_min_m": gui.processing.gpr.visible_x_min_m,
"visible_x_max_m": gui.processing.gpr.visible_x_max_m,
"visible_z_min_m": gui.processing.gpr.visible_z_min_m,
"visible_z_max_m": gui.processing.gpr.visible_z_max_m,
},
"legacy_gpr": {
"mode": gui.processing.legacy_gpr.mode,
"input_positions": gui.processing.legacy_gpr.input_positions,
"output_positions": gui.processing.legacy_gpr.output_positions,
"min_depth_m": gui.processing.legacy_gpr.min_depth_m,
"max_depth_m": gui.processing.legacy_gpr.max_depth_m,
"comp_power": gui.processing.legacy_gpr.comp_power,
"start_freq_mhz": gui.processing.legacy_gpr.start_freq_mhz,
"stop_freq_mhz": gui.processing.legacy_gpr.stop_freq_mhz,
"speed_m_s": gui.processing.legacy_gpr.speed_m_s,
"ignore_socket_speed_enabled": gui.processing.legacy_gpr.ignore_socket_speed_enabled,
"look_angle_deg": gui.processing.legacy_gpr.look_angle_deg,
"apply_freq_phase_correction": gui.processing.legacy_gpr.apply_freq_phase_correction,
"reference_mode": gui.processing.legacy_gpr.reference_mode,
"snr_thresh": gui.processing.legacy_gpr.snr_thresh,
"snr_comp_max": gui.processing.legacy_gpr.snr_comp_max,
"background_subtract_enabled": gui.processing.legacy_gpr.background_subtract_enabled,
"background_mean_count": gui.processing.legacy_gpr.background_mean_count,
"render_mode": gui.processing.legacy_gpr.render_mode,
"min_visible_pair_count": gui.processing.legacy_gpr.min_visible_pair_count,
"visible_x_min_m": gui.processing.legacy_gpr.visible_x_min_m,
"visible_x_max_m": gui.processing.legacy_gpr.visible_x_max_m,
"visible_z_min_m": gui.processing.legacy_gpr.visible_z_min_m,
"visible_z_max_m": gui.processing.legacy_gpr.visible_z_max_m,
},
"sar3d": dataclasses.asdict(gui.processing.sar3d),
},
"data_actions": {
"save_count": gui.data_actions.save_count,
"save_path": gui.data_actions.save_path,
"save_name": gui.data_actions.save_name,
"record_count": gui.data_actions.record_count,
},
"preprocess_dialog": {
"set_name": gui.preprocess_dialog.set_name,
"radar_config_dir": gui.preprocess_dialog.radar_config_dir,
"use_all_radar_configs": gui.preprocess_dialog.use_all_radar_configs,
"median_sweep_count": gui.preprocess_dialog.median_sweep_count,
},
}
payload["gui"] = dataclasses.asdict(model.gui if model.gui is not None else GuiStateModel())
return payload
python_app/models/gui_profile_schema.py
+2 -3
View File
@@ -42,8 +42,8 @@ class GuiBscanStateModel:
"""UI-only defaults for B-scan live settings."""
axis: str = "abs"
cut_m: float = 0.824
max_depth_m: float = 1.0
cut_m: float = 0.0
max_depth_m: float = 8.0
gain: float = 1.0
start_freq_mhz: float = 100.0
stop_freq_mhz: float = 8800.0
@@ -164,7 +164,6 @@ class GuiProcessingStateModel:
class GuiDataActionsStateModel:
"""UI-only defaults for snapshot/export controls."""
save_count: int = 10
save_path: str = ""
save_name: str = "snapshot_manual"
# How many freshly acquired measurements the "Start + Record" action writes to
python_app/models/json_fields.py
+62
View File
@@ -0,0 +1,62 @@
"""Shared strict scalar and object readers for configuration codecs."""
from __future__ import annotations
import math
from typing import Any
def json_object(value: Any, context: str) -> dict[str, Any]:
"""Read an object; a missing or null section keeps its defaults."""
if value is None:
return {}
if not isinstance(value, dict):
raise ValueError(f"{context} must be a JSON object")
return value
def _value(payload: dict[str, Any], key: str, default: Any) -> Any:
value = payload.get(key, default)
return default if value is None else value
def read_int(payload: dict[str, Any], key: str, default: int) -> int:
"""Read a JSON integer without truncation or boolean coercion."""
value = _value(payload, key, default)
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError(f"{key} must be a JSON integer")
return value
def read_str(payload: dict[str, Any], key: str, default: str) -> str:
"""Read a JSON string without converting other scalar types."""
value = _value(payload, key, default)
if not isinstance(value, str):
raise ValueError(f"{key} must be a JSON string")
return value
def read_bool(payload: dict[str, Any], key: str, default: bool) -> bool:
"""Read a JSON boolean."""
value = _value(payload, key, default)
if not isinstance(value, bool):
raise ValueError(f"{key} must be a JSON boolean")
return value
def read_float(payload: dict[str, Any], key: str, default: float) -> float:
"""Read a finite JSON number, including overflow-safe integer conversion."""
return finite_number(_value(payload, key, default), key)
def finite_number(value: Any, context: str) -> float:
"""Validate a number without accepting booleans, strings or nonfinite values."""
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError(f"{context} must be a JSON number")
try:
result = float(value)
except OverflowError as exc:
raise ValueError(f"{context} must be a finite number") from exc
if not math.isfinite(result):
raise ValueError(f"{context} must be a finite number")
return result
python_app/models/run_config_codec.py
+14 -84
View File
@@ -3,10 +3,13 @@
from __future__ import annotations
import logging
import math
from typing import Any
from python_app.logging_setup import LOG_LEVELS
from python_app.models.json_fields import (
finite_number, json_object as _as_dict, read_bool as _read_bool, read_float as _read_float,
read_int as _read_int, read_str as _read_str,
)
from python_app.models.run_config_schema import (
ComboModel,
@@ -27,15 +30,6 @@ from python_app.models.run_config_validation import (
logger = logging.getLogger(__name__)
def _as_dict(value: Any, context: str) -> dict[str, Any]:
"""Validate payload node is object-like, treating missing values as empty object."""
if value is None:
return {}
if not isinstance(value, dict):
raise ValueError(f"{context} must be a JSON object")
return value
def _as_list(value: Any, context: str) -> list[Any]:
"""Validate payload node is array-like, treating missing values as an empty list.
@@ -49,71 +43,6 @@ def _as_list(value: Any, context: str) -> list[Any]:
return value
def _read_str(payload: dict[str, Any], key: str, default: str) -> str:
"""Return a payload string, treating an explicit JSON ``null`` as 'use default'.
Keeping the default on ``null`` avoids coercing it to the literal string
``"None"``. JSON arrays/objects reaching a scalar field are rejected as
ValueError to keep the config-error contract uniform.
"""
value = payload.get(key, default)
if value is None:
return default
if isinstance(value, (dict, list)):
raise ValueError(f"{key} must be a JSON string")
return str(value)
def _read_int(payload: dict[str, Any], key: str, default: int) -> int:
"""Return a payload integer, treating an explicit JSON ``null`` as 'use default'.
Accepts only a genuine JSON integer (not bool, not float, not numeric
string); silently truncating ``5.7`` or parsing ``"5"`` would hide a
malformed config. Mirrors ``gui_profile_codec._optional_int`` so the two
codecs agree.
"""
value = payload.get(key, default)
if value is None:
return default
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError(f"{key} must be a JSON integer")
return value
def _read_float(payload: dict[str, Any], key: str, default: float) -> float:
"""Return a payload float, treating an explicit JSON ``null`` as 'use default'.
Accepts only a genuine JSON number (int/float, not bool, not numeric
string); parsing ``"1e9"`` would hide a malformed config. Non-finite values
(NaN/Infinity) are rejected at decode time so the C++ pipeline never receives
a value it cannot honor. Mirrors ``gui_profile_codec._optional_float``.
"""
value = payload.get(key, default)
if value is None:
return default
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise ValueError(f"{key} must be a JSON number")
result = float(value)
if not math.isfinite(result):
raise ValueError(f"{key} must be a finite number")
return result
def _read_bool(payload: dict[str, Any], key: str, default: bool) -> bool:
"""Return a payload boolean, treating an explicit JSON ``null`` as 'use default'.
Keeping the default on ``null`` avoids the silent flip to ``False`` that a
plain ``bool(...)`` coercion would produce. Non-boolean JSON types are
rejected as ValueError.
"""
value = payload.get(key, default)
if value is None:
return default
if not isinstance(value, bool):
raise ValueError(f"{key} must be a JSON boolean")
return value
def _load_preprocess_asset(payload: dict[str, Any], target: PreprocessAssetModel) -> None:
"""Load preprocess asset fields into target model."""
target.set_name = _read_str(payload, "set_name", target.set_name)
@@ -152,6 +81,8 @@ def _load_string_dict(payload: dict[str, Any], key: str, context: str) -> dict[s
def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel:
"""Decode JSON-like payload into :class:`RunConfigModel`."""
if not isinstance(payload, dict):
raise ValueError("Run config root must be a JSON object")
# Schema carries only minimal-safe fallbacks; operational defaults live in run_config.json.
model = RunConfigModel()
@@ -181,11 +112,7 @@ def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel:
model.radar.model = _read_str(radar_payload, "model", model.radar.model)
model.radar.serial = _read_str(radar_payload, "serial", model.radar.serial)
model.radar.remote_host = _read_str(radar_payload, "remote_host", model.radar.remote_host)
model.radar.remote_port = _read_int(radar_payload, "remote_port", model.radar.remote_port)
model.radar.driver_mode = _read_str(radar_payload, "driver_mode", model.radar.driver_mode)
model.radar.mock_signal_hz = _read_float(radar_payload, "mock_signal_hz", model.radar.mock_signal_hz)
model.radar.visa_library = _read_str(radar_payload, "visa_library", model.radar.visa_library)
model.radar.sweep.start_hz = _read_float(sweep_payload, "start_hz", model.radar.sweep.start_hz)
model.radar.sweep.stop_hz = _read_float(sweep_payload, "stop_hz", model.radar.sweep.stop_hz)
@@ -429,7 +356,14 @@ def run_config_from_dict(payload: dict[str, Any]) -> RunConfigModel:
for band in bands_payload:
if not (isinstance(band, (list, tuple)) and len(band) == 2):
raise ValueError("preprocess.notch.bands_hz entries must be [low_hz, high_hz] pairs")
model.preprocess.notch.bands_hz.append((float(band[0]), float(band[1])))
low_hz, high_hz = (finite_number(value, "preprocess.notch.bands_hz[]") for value in band)
if high_hz < low_hz or max(abs(low_hz), abs(high_hz)) > 3.4e38:
raise ValueError("preprocess.notch.bands_hz[] must be ordered finite float32 frequencies")
model.preprocess.notch.bands_hz.append((low_hz, high_hz))
if model.preprocess.notch.taper_width_hz < 0 or model.preprocess.notch.taper_width_hz > 3.4e38:
raise ValueError("preprocess.notch.taper_width_hz must be a nonnegative float32 number")
if model.preprocess.notch.taper_type not in ("cosine", "hard"):
raise ValueError("preprocess.notch.taper_type must be cosine or hard")
model.gpr.relative_permittivity = _read_float(
gpr_payload, "relative_permittivity", model.gpr.relative_permittivity
@@ -514,11 +448,7 @@ def run_config_to_dict(model: RunConfigModel) -> dict[str, Any]:
"radar": {
"model": model.radar.model,
"serial": model.radar.serial,
"remote_host": model.radar.remote_host,
"remote_port": model.radar.remote_port,
"driver_mode": model.radar.driver_mode,
"mock_signal_hz": model.radar.mock_signal_hz,
"visa_library": model.radar.visa_library,
"multi_device": {
"slave_serials": list(model.radar.multi_device.slave_serials),
"force_external_reference": model.radar.multi_device.force_external_reference,
python_app/models/run_config_schema.py
+7 -25
View File
@@ -10,6 +10,7 @@ from pathlib import Path
from typing import Any
logger = logging.getLogger(__name__)
MAX_COMBOS = 4096
@dataclass(slots=True)
@@ -171,11 +172,7 @@ class RadarModel:
model: str = "librevna"
serial: str = ""
remote_host: str = "127.0.0.1"
remote_port: int = 50209
driver_mode: str = "mock"
mock_signal_hz: float = 1_000_000.0
visa_library: str = ""
sweep: RadarSweepModel = field(default_factory=RadarSweepModel)
multi_device: RadarMultiDeviceModel = field(default_factory=RadarMultiDeviceModel)
replay: RadarReplayModel = field(default_factory=RadarReplayModel)
@@ -377,9 +374,7 @@ class RunConfigModel:
LIBREVNA_MODEL = "librevna"
LIBREVNA_MULTI_MODEL = "librevna_multi"
COMPACT_M_K209_MODEL = "compact_m_k209"
KAMIL_ADC_MODEL = "kamil_adc"
SN9000_MODEL = "sn9000"
MULTI_DEVICE_INPUT_POSITIONS = 4
MULTI_DEVICE_OUTPUT_POSITIONS = 2
# Stimulus ports the master itself owns. The virtual output axis is this times the
@@ -390,39 +385,26 @@ class RunConfigModel:
@staticmethod
def build_full_combos(input_positions: int, output_positions: int) -> list[ComboModel]:
"""Build full Cartesian product of input/output switch positions."""
if any(isinstance(value, bool) or not isinstance(value, int) or value <= 0
for value in (input_positions, output_positions)):
raise ValueError("Switch positions must be positive integers")
if input_positions * output_positions > MAX_COMBOS:
raise ValueError(f"Too many combos: limit is {MAX_COMBOS}")
return [
ComboModel(input=input_pos, output=output_pos)
for output_pos in range(output_positions)
for input_pos in range(input_positions)
]
@classmethod
def build_multi_device_virtual_combos(cls) -> list[ComboModel]:
"""Build fixed virtual combo matrix for one master and two slave devices."""
return cls.build_full_combos(
cls.MULTI_DEVICE_INPUT_POSITIONS,
cls.MULTI_DEVICE_OUTPUT_POSITIONS,
)
@classmethod
def build_matrix_radar_virtual_combos(cls) -> list[ComboModel]:
"""Build the canonical 2x4 virtual combo matrix shared by matrix-mode radars."""
return cls.build_multi_device_virtual_combos()
@property
def is_multi_device(self) -> bool:
"""Return whether this config targets synchronized multi-device acquisition."""
return self.radar.model == self.LIBREVNA_MULTI_MODEL
@property
def is_sn9000(self) -> bool:
"""Return whether this config targets the SN9000 multi-port analyzer."""
return self.radar.model == self.SN9000_MODEL
@property
def is_matrix_radar(self) -> bool:
"""Return whether this config acquires the full virtual switch matrix per sweep."""
return self.is_multi_device or self.is_sn9000
return self.is_multi_device
@property
def matrix_output_switch_positions(self) -> int:
python_app/models/run_config_validation.py
+101 -51
View File
@@ -3,14 +3,20 @@
from __future__ import annotations
import logging
import math
from typing import Any
from python_app.models.json_fields import (
read_bool as _require_bool, read_int as _require_int, read_str as _require_str,
)
from python_app.models.run_config_schema import (
MAX_COMBOS,
ComboModel,
ControlButtonModel,
GprModel,
RadarSweepModel,
RingEndpointModel,
RunConfigModel,
SwitchModel,
)
@@ -21,45 +27,6 @@ logger = logging.getLogger(__name__)
# shared-memory mapping, so reject values the C++ side cannot represent.
_UINT32_MAX = (1 << 32) - 1
_RING_SEGMENT_MAX_BYTES = 1 << 40 # 1 TiB upper bound on a single ring mapping.
# Defensive ceiling so a malformed combos string cannot expand into a list that
# stalls the GUI or the downstream acquisition loop.
_MAX_COMBOS = 4096
def _require_int(payload: dict[str, Any], key: str, default: int) -> int:
"""Read a strict JSON integer, treating an explicit ``null`` as 'use default'.
Accepts only a genuine JSON integer (not bool, not float, not numeric string),
because silently truncating ``5.7`` or parsing ``"5"`` would hide a malformed
config. Mirrors ``run_config_codec._read_int`` so every config integer reads
identically.
"""
value = payload.get(key, default)
if value is None: # explicit JSON null -> use the default, never coerce
return default
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError(f"{key} must be a JSON integer")
return value
def _require_str(payload: dict[str, Any], key: str, default: str) -> str:
"""Read a string field, rejecting JSON arrays/objects with a named ValueError."""
value = payload.get(key, default)
if value is None: # explicit JSON null -> use the default, never coerce to "None"
return default
if isinstance(value, (dict, list)):
raise ValueError(f"{key} must be a JSON string")
return str(value)
def _require_bool(payload: dict[str, Any], key: str, default: bool) -> bool:
"""Read a boolean field, rejecting non-boolean JSON types with a named ValueError."""
value = payload.get(key, default)
if value is None: # explicit JSON null -> use the default
return default
if not isinstance(value, bool):
raise ValueError(f"{key} must be a JSON boolean")
return value
def load_switch_payload(
@@ -117,18 +84,79 @@ def validate_ring_endpoint(ring: RingEndpointModel) -> None:
would exceed the maximum single mapping.
"""
ring_name = ring.name or "ring"
for field, value in (("capacity", ring.capacity), ("slot_size_bytes", ring.slot_size_bytes)):
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError(f"rings.{ring_name}.{field} must be an integer")
if ring.capacity <= 0:
raise ValueError(f"rings.{ring_name}.capacity must be > 0")
if ring.slot_size_bytes <= 0:
raise ValueError(f"rings.{ring_name}.slot_size_bytes must be > 0")
if ring.slot_size_bytes > _UINT32_MAX:
raise ValueError(f"rings.{ring_name}.slot_size_bytes exceeds the uint32 wire limit")
if ring.slot_size_bytes % 8:
raise ValueError(f"rings.{ring_name}.slot_size_bytes must be a multiple of 8")
# Overflow-safe: compare against the ceiling without ever forming the full
# product, so an oversized capacity cannot wrap a fixed-width index.
if ring.capacity > _RING_SEGMENT_MAX_BYTES // ring.slot_size_bytes:
if ring.capacity > (_RING_SEGMENT_MAX_BYTES - 64) // (ring.slot_size_bytes + 16):
raise ValueError(
f"rings.{ring_name} capacity * slot_size_bytes exceeds the maximum ring segment size"
)
if ring.capacity > _UINT32_MAX:
raise ValueError(f"rings.{ring_name}.capacity exceeds the uint32 wire limit")
def validate_acquisition_config(config: RunConfigModel) -> None:
"""Reject unusable acquisition settings before starting processes or hardware.
Replay uses the recorded trace dimensions and needs no physical devices; its
producer checks the actual payload size before publishing a frame.
"""
if config.radar.replay.dataset_dir:
return
mode = config.radar.driver_mode
if mode not in ("mock", "native"):
raise ValueError("radar.driver_mode must be mock or native")
if config.radar.model not in (
config.LIBREVNA_MODEL, config.LIBREVNA_MULTI_MODEL, config.KAMIL_ADC_MODEL,
):
raise ValueError(f"Unsupported radar model: {config.radar.model}")
if mode != "native" and config.is_kamil_adc:
raise ValueError(f"{config.radar.model} requires native driver mode")
for switch, physical_positions in (
(config.input_switch, config.matrix_input_switch_positions),
(config.output_switch, config.matrix_output_switch_positions),
):
context = f"switches.{switch.name or switch.radar_port}"
if switch.driver_mode not in ("mock", "native"):
raise ValueError(f"{context}.driver_mode must be mock or native")
if switch.driver not in ("h7992", "hmc349a"):
raise ValueError(f"{context}.driver must be h7992 or hmc349a")
if switch.radar_port not in (1, 2):
raise ValueError(f"{context}.radar_port must be 1 or 2")
if not 0 <= switch.default_position < switch.positions:
raise ValueError(f"{context}.default_position must be within switch positions")
positions, switch_mode = switch.positions, switch.driver_mode
if config.is_matrix_radar and physical_positions > 1:
positions, switch_mode = physical_positions, mode
if switch_mode == "native":
limit = 4 if switch.driver == "h7992" else 2
if not 1 <= positions <= limit:
raise ValueError(f"{context}: native {switch.driver} supports 1..{limit} positions")
if not switch.gpio_chip or switch.pin_a < 0 or (
switch.driver == "h7992" and (switch.pin_b < 0 or switch.pin_b == switch.pin_a)
):
raise ValueError(f"{context}: GPIO chip and distinct nonnegative control pins are required")
points = config.radar.kamil_adc.band.points if config.is_kamil_adc else config.radar.sweep.points
# RAW/PPR header, each combo's frequency/S11/S21 arrays and capture windows.
minimum = 44 + len(config.combos) * (28 + 20 * points)
for name in ("raw", "raw_tap", "preprocessed", "preprocessed_tap"):
ring = getattr(config.rings, name)
if name.endswith("_tap") and not ring.name:
continue
if ring.slot_size_bytes < minimum:
raise ValueError(f"rings.{name}.slot_size_bytes must be at least {minimum} for this sweep")
def validate_sweep_model(sweep: RadarSweepModel) -> None:
@@ -143,9 +171,17 @@ def validate_sweep_model(sweep: RadarSweepModel) -> None:
points = sweep.points
if isinstance(points, bool) or not isinstance(points, int):
raise ValueError("radar.sweep.points must be an integer")
if points <= 0:
raise ValueError("radar.sweep.points must be > 0")
if float(sweep.stop_hz) <= float(sweep.start_hz):
if not 0 < points <= _UINT32_MAX:
raise ValueError("radar.sweep.points must be in [1, uint32 max]")
for field in ("start_hz", "stop_hz", "if_bandwidth_hz", "power_dbm"):
value = getattr(sweep, field)
if isinstance(value, bool) or not isinstance(value, (int, float)) or abs(value) > 3.4e38 or not math.isfinite(value):
raise ValueError(f"radar.sweep.{field} must be a finite float32 number")
if sweep.start_hz < 0:
raise ValueError("radar.sweep.start_hz must be >= 0")
if sweep.if_bandwidth_hz <= 0:
raise ValueError("radar.sweep.if_bandwidth_hz must be > 0")
if sweep.stop_hz <= sweep.start_hz:
raise ValueError("radar.sweep.stop_hz must be > radar.sweep.start_hz")
@@ -167,12 +203,16 @@ def validate_gpr_model(
if sweep is not None:
validate_sweep_model(sweep)
if float(gpr.relative_permittivity) <= 0.0:
if not math.isfinite(gpr.relative_permittivity) or float(gpr.relative_permittivity) <= 0.0:
raise ValueError("gpr.relative_permittivity must be > 0")
seen_output_positions: set[int] = set()
for entry in gpr.tx_geometry:
output_pos = int(entry.output_pos)
if any(not math.isfinite(value) or abs(value) > 3.4e38 for value in (entry.x_m, entry.y_m, entry.z_m)):
raise ValueError("gpr.tx_geometry coordinates must be finite numbers")
output_pos = entry.output_pos
if isinstance(output_pos, bool) or not isinstance(output_pos, int):
raise ValueError("gpr.tx_geometry output_pos must be an integer")
if output_pos < 0 or output_pos >= int(output_switch_positions):
raise ValueError("gpr.tx_geometry output_pos is out of range")
if output_pos in seen_output_positions:
@@ -181,7 +221,11 @@ def validate_gpr_model(
seen_input_positions: set[int] = set()
for entry in gpr.rx_geometry:
input_pos = int(entry.input_pos)
if any(not math.isfinite(value) or abs(value) > 3.4e38 for value in (entry.x_m, entry.y_m, entry.z_m)):
raise ValueError("gpr.rx_geometry coordinates must be finite numbers")
input_pos = entry.input_pos
if isinstance(input_pos, bool) or not isinstance(input_pos, int):
raise ValueError("gpr.rx_geometry input_pos must be an integer")
if input_pos < 0 or input_pos >= int(input_switch_positions):
raise ValueError("gpr.rx_geometry input_pos is out of range")
if input_pos in seen_input_positions:
@@ -202,12 +246,18 @@ def validate_combos(
config error that would otherwise produce missing or doubled traces downstream.
"""
seen: set[tuple[int, int]] = set()
if len(combos) > MAX_COMBOS:
raise ValueError(f"Too many combos: limit is {MAX_COMBOS}")
for combo in combos:
if not 0 <= int(combo.input) < int(input_positions):
for field in ("input", "output"):
value = getattr(combo, field)
if isinstance(value, bool) or not isinstance(value, int):
raise ValueError(f"run.combos {field} must be an integer")
if not 0 <= combo.input < input_positions:
raise ValueError(
f"run.combos input {combo.input} is out of range [0, {input_positions})"
)
if not 0 <= int(combo.output) < int(output_positions):
if not 0 <= combo.output < output_positions:
raise ValueError(
f"run.combos output {combo.output} is out of range [0, {output_positions})"
)
@@ -222,7 +272,7 @@ def parse_combos_from_text(text: str) -> list[ComboModel]:
Returns an empty list for blank input. Raises ValueError (naming the
offending pair) on malformed syntax, empty sides, non-integer values, more
than ``_MAX_COMBOS`` entries, or a non-blank string that yields no combos.
than ``MAX_COMBOS`` entries, or a non-blank string that yields no combos.
"""
cleaned = text.strip()
if not cleaned:
@@ -238,8 +288,8 @@ def parse_combos_from_text(text: str) -> list[ComboModel]:
# Cap the combo count so a pathological string cannot expand into a list
# large enough to stall the GUI or the acquisition loop.
if len(combos) >= _MAX_COMBOS:
raise ValueError(f"Too many combos: limit is {_MAX_COMBOS}")
if len(combos) >= MAX_COMBOS:
raise ValueError(f"Too many combos: limit is {MAX_COMBOS}")
input_text, output_text = (side.strip() for side in pair.split(":", 1))
# Reject empty sides and re-raise non-integer values naming the pair/side.
python_app/orchestration/config_writer.py
+4 -29
View File
@@ -2,12 +2,10 @@
from __future__ import annotations
from contextlib import suppress
import json
import logging
import os
from pathlib import Path
from python_app.file_io import atomic_write_json
from python_app.models.run_config_model import RunConfigModel, parse_combos_from_text
from python_app.orchestration.preprocess_assets import (
PREPROCESS_ASSET_KEYS,
@@ -47,32 +45,9 @@ class ConfigWriter:
asset.bundle_path = str(bundle_path)
def write(self, config: RunConfigModel, output_path: Path) -> Path:
"""Atomically write the run configuration JSON file.
Dump to a sibling ``.tmp``, flush + ``fsync`` to durably commit the bytes on
the SD-card-backed Pi, then ``os.replace()`` atomically onto the destination.
A C++ consumer can therefore never observe a half-written config (which would
abort it with an opaque JSON parse error), even across a crash or power loss.
"""
output_path.parent.mkdir(parents=True, exist_ok=True)
# allow_nan=False: a stray NaN/Infinity must fail loudly here in Python
# rather than serialize to a non-standard token that aborts every C++
# consumer at startup with an opaque JSON parse error.
serialized = json.dumps(config.to_dict(), indent=2, allow_nan=False)
tmp_path = output_path.with_suffix(output_path.suffix + ".tmp")
try:
with open(tmp_path, "w", encoding="utf-8") as handle:
handle.write(serialized)
handle.flush()
os.fsync(handle.fileno())
os.replace(tmp_path, output_path)
except Exception:
logger.exception("Failed to write run config to %s; removing temp file", output_path)
# Never leave a half-written .tmp behind on a write/fsync failure.
with suppress(OSError):
tmp_path.unlink()
raise
logger.debug("Wrote run config to %s (%d bytes)", output_path, len(serialized))
"""Atomically write and sync the run configuration JSON file."""
atomic_write_json(output_path, config.to_dict(), durable=True)
logger.debug("Wrote run config to %s", output_path)
return output_path

Some files were not shown because too many files have changed in this diff Show More