Update GPR and SAR3D processing

This commit is contained in:
Ayzen
2026-10-08 16:24:01 +03:00
parent 390015ec39
commit 3575ca68f5
67 changed files with 1187 additions and 2581 deletions
@@ -15,7 +15,7 @@ import unittest
from unittest import mock
os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
from PyQt6.QtWidgets import QApplication, QCheckBox, QComboBox, QDoubleSpinBox, QLineEdit, QSpinBox # noqa: E402
from PyQt6.QtWidgets import QApplication, QCheckBox, QComboBox, QDoubleSpinBox, QLineEdit, QSlider, QSpinBox # noqa: E402
from PyQt6.QtCore import QSignalBlocker, QTimer # noqa: E402
from python_app.gui.app_window import AppWindow # noqa: E402
@@ -532,11 +532,11 @@ class GuiProfileWriteTest(unittest.TestCase):
value = not widget.isChecked()
elif isinstance(widget, QComboBox):
value = widget.itemText((widget.currentIndex() + 1) % widget.count())
elif isinstance(widget, (QSpinBox, QDoubleSpinBox)):
elif isinstance(widget, (QSpinBox, QDoubleSpinBox, QSlider)):
value = widget.value() + widget.singleStep()
if value > widget.maximum():
value = widget.value() - widget.singleStep()
value = int(value) if isinstance(widget, QSpinBox) else round(value, widget.decimals())
value = int(value) if isinstance(widget, (QSpinBox, QSlider)) else round(value, widget.decimals())
elif isinstance(widget, QLineEdit):
value = "0:0" if binding.spec.web == "combo_filter" else "0" if "positions" in binding.path[-1] else "1"
else:
+11 -5
View File
@@ -110,14 +110,20 @@ class GuiProfileCodecTest(unittest.TestCase):
with self.assertRaises(ValueError): # combos_text must be a JSON string, not a number
GuiProfileModel.from_dict({"gui": {"switches": {"combos_text": 123}}})
def test_legacy_gpr_payload_migrates_selected_mode(self) -> None:
# An old payload that selects 'gpr' but carries a legacy root gpr.mode is migrated.
def test_obsolete_inactive_settings_are_ignored(self) -> None:
decoded = GuiProfileModel.from_dict({
"gui": {"processing": {"selected_mode": "gpr"}},
"gpr": {"relative_permittivity": 4.0, "mode": "point"},
"gui": {"processing": {"selected_mode": "gpr", "legacy_gpr": "obsolete",
"gpr": {"algorithm": "legacy_point"}}},
"gpr": {"relative_permittivity": "obsolete", "mode": "point"},
})
assert decoded.gui is not None
self.assertEqual(decoded.gui.processing.selected_mode, "legacy_gpr")
self.assertEqual(decoded.gui.processing.selected_mode, "gpr")
self.assertNotIn("legacy_gpr", decoded.to_dict()["gui"]["processing"])
self.assertNotIn("relative_permittivity", decoded.to_dict()["gpr"])
def test_removed_processing_mode_is_rejected(self) -> None:
with self.assertRaisesRegex(ValueError, "selected_mode must be one of"):
GuiProfileModel.from_dict({"gui": {"processing": {"selected_mode": "legacy_gpr"}}})
def test_modern_gpr_selection_is_not_migrated(self) -> None:
decoded = GuiProfileModel.from_dict({"gui": {"processing": {"selected_mode": "gpr"}}})
@@ -25,11 +25,11 @@ class LiveSettingsRegressionTest(unittest.TestCase):
config, gui = RunConfigModel(), GuiStateModel()
config.radar.sweep.start_hz, config.radar.sweep.stop_hz = 1e6, 9.5e9
for settings in (gui.processing.bscan, gui.processing.gpr,
gui.processing.legacy_gpr, gui.processing.sar3d):
gui.processing.sar3d):
settings.start_freq_mhz, settings.stop_freq_mhz = 20, 9200
section = build_processing_group(config, gui)
self.addCleanup(section.group.deleteLater)
for mode in ("bscan", "gpr", "legacy_gpr", "sar3d"):
for mode in ("bscan", "gpr", "sar3d"):
for bound, expected in (("start", 20), ("stop", 9200)):
with self.subTest(mode=mode, bound=bound):
editor = widget(section, f"_{mode}_{bound}_freq_mhz", QDoubleSpinBox)
@@ -11,7 +11,7 @@ class LiveSettingsSchemaCoverageTest(unittest.TestCase):
def test_bindings_cover_live_config_and_select_each_page_once(self) -> None:
config_names = {field.name for field in fields(ProcessingLiveConfig)}
runtime = {"reprocess_current_result", "history_clear_before_ns", "history_removed", "settings_revision"}
geometry = {"relative_permittivity", "tx_geometry", "rx_geometry"}
geometry = {"tx_geometry", "rx_geometry"}
modes = next(binding.spec.choices for binding in GUI_FIELDS if binding.spec.live == "processor_mode")
widget_names = [binding.spec.widget for binding in GUI_FIELDS if binding.spec.widget]
self.assertEqual(len(widget_names), len(set(widget_names)))
+2 -53
View File
@@ -1,4 +1,4 @@
"""GPR object filtering, B-scan transforms and live-setting normalization."""
"""GPR payload lookup, B-scan transforms and live-setting normalization."""
from __future__ import annotations
@@ -26,8 +26,6 @@ from python_app.models.dataset_model import (
)
from python_app.gui.result_payloads import (
collection_payload_by_name,
collection_payloads_by_prefix,
filter_object_rows,
gpr_object_rows,
)
from python_app.orchestration.live_processing_config import ProcessingLiveConfig
@@ -43,32 +41,16 @@ def _collection(payloads: Iterable[ResultPayload]) -> ResultCollection:
class CollectionLookupTest(unittest.TestCase):
def test_payload_by_name_matches_name_and_optional_kind(self) -> None:
col = _collection([_table("gpr_points", [[0, 0, 1]]), _table("gpr_region_centers", [[1, 1, 2, 9]])])
col = _collection([_table("gpr_points", [[0, 0, 1]])])
self.assertIs(collection_payload_by_name(col, "gpr_points"), col.collection_payloads[0])
self.assertIsNone(collection_payload_by_name(col, "missing"))
self.assertIsNone(collection_payload_by_name(col, "gpr_points", kind=3)) # wrong kind
def test_payloads_by_prefix_returns_all_in_order(self) -> None:
col = _collection([
ResultPayload(processing_name="gpr_region_mask_0", kind=3, image=np.zeros((1, 1), dtype=np.float32)),
ResultPayload(processing_name="gpr_region_mask_1", kind=3, image=np.zeros((1, 1), dtype=np.float32)),
_table("gpr_points", [[0, 0, 1]]),
])
masks = collection_payloads_by_prefix(col, "gpr_region_mask_", kind=3)
self.assertEqual([p.processing_name for p in masks], ["gpr_region_mask_0", "gpr_region_mask_1"])
self.assertEqual(collection_payloads_by_prefix(col, "nope_"), [])
class GprObjectRowsTest(unittest.TestCase):
def test_extracts_first_three_columns_of_points(self) -> None:
rows = gpr_object_rows(_collection([_table("gpr_points", [[1, 2, 3], [4, 5, 6]])]))
self.assertTrue(np.array_equal(rows, np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)))
def test_falls_back_to_region_centers_and_drops_extra_columns(self) -> None:
# region_centers is [x, z, score, pixel_count]; only the first 3 cols are used.
rows = gpr_object_rows(_collection([_table("gpr_region_centers", [[1, 2, 3, 99]])]))
self.assertTrue(np.array_equal(rows, np.array([[1, 2, 3]], dtype=np.float32)))
def test_empty_when_no_object_payloads(self) -> None:
self.assertEqual(gpr_object_rows(_collection([_table("gpr_accumulator", [[1, 2, 3]])])).shape, (0, 3))
@@ -76,39 +58,6 @@ class GprObjectRowsTest(unittest.TestCase):
self.assertEqual(gpr_object_rows(_collection([_table("gpr_points", [[1, 2]])])).shape, (0, 3))
class FilterObjectRowsTest(unittest.TestCase):
BOUNDS = {"x_bounds": (-2.0, 2.0), "z_bounds": (0.0, 10.0)}
def _filter(self, rows: Sequence[Sequence[float]], *, min_score: float) -> np.ndarray:
return filter_object_rows(np.asarray(rows, dtype=np.float32), min_score=min_score,
**self.BOUNDS)
def test_keeps_in_window_and_at_or_above_threshold(self) -> None:
out = self._filter([[0.0, 5.0, 0.5], [1.0, 1.0, 0.9]], min_score=0.5) # score==threshold kept (inclusive)
self.assertEqual(out.shape[0], 2)
def test_drops_below_threshold(self) -> None:
out = self._filter([[0.0, 5.0, 0.4]], min_score=0.5)
self.assertEqual(out.shape[0], 0)
def test_window_bounds_are_inclusive(self) -> None:
out = self._filter([[2.0, 10.0, 1.0], [-2.0, 0.0, 1.0]], min_score=0.0) # exactly on each edge
self.assertEqual(out.shape[0], 2)
def test_drops_outside_window(self) -> None:
out = self._filter([[2.001, 5.0, 1.0], [0.0, 10.001, 1.0]], min_score=0.0)
self.assertEqual(out.shape[0], 0)
def test_drops_non_finite_rows(self) -> None:
out = self._filter([[np.nan, 5.0, 1.0], [0.0, np.inf, 1.0], [0.0, 5.0, 1.0]], min_score=0.0)
self.assertEqual(out.shape[0], 1)
def test_legacy_pair_count_threshold_uses_same_compare(self) -> None:
# legacy GPR passes an integer pair-count threshold; the >= compare is identical.
out = self._filter([[0.0, 5.0, 3.0], [0.0, 5.0, 2.0]], min_score=3)
self.assertTrue(np.array_equal(out, np.array([[0.0, 5.0, 3.0]], dtype=np.float32)))
class BscanTransformTest(unittest.TestCase):
def test_mean_ascan_subtraction(self) -> None:
history = deque([np.array([1.0, 1.0], dtype=np.float32), np.array([3.0, 3.0], dtype=np.float32)])
@@ -130,6 +130,27 @@ class _ProcessorTestCase(unittest.TestCase):
class FedBackSweepsTest(_ProcessorTestCase):
def test_mean_history_is_not_limited_by_transport_ring_capacity(self) -> None:
self._live.processor_mode = "gpr"
self._live.gpr_input_positions = self._live.gpr_output_positions = [0]
self._live.gpr_background_subtract_enabled = True
self._live.gpr_background_mean_count = 60
self._live.gpr_min_depth_m, self._live.gpr_max_depth_m = .1, 1.5
self._live.gpr_bp_ground_depth_m = 0.
self._live_writer.write(self._live)
start = time.monotonic_ns()
for cid in range(1, 61):
sweep = _sweep(cid, start + cid*1_000_000)
sweep.traces[0].s21 *= (-1.)**np.arange(len(sweep.traces[0].s21))
self._push([sweep])
self._result_for_revision(0, cid)
current = _sweep(61, start + 61_000_000)
current.traces[0].s21[:] = 30.5 * (-1.)**np.arange(len(current.traces[0].s21))
self._push([current])
result = self._result_for_revision(0, 61)
image = next(payload.image for payload in result.collection_payloads if payload.processing_name == "gpr_accumulator")
self.assertFalse(np.any(image)) # Exact cancellation of the 60 preceding spectra (1…60).
def test_remove_targets_exact_sweep_with_reused_ids_and_preserves_missing_targets(self) -> None:
start_ns = time.monotonic_ns()
sweeps = [_sweep(cid, start_ns + index * 1_000_000)
@@ -150,7 +150,7 @@ class ReviewGuiRegressions(unittest.TestCase):
deadline.assert_called_once()
window = isolated_window(self.addCleanup)
window._pipeline.timer.stop()
for mode in ("gpr", "legacy_gpr", "sar3d"):
for mode in ("gpr", "sar3d"):
field = widget(window._view.processing, f"_{mode}_input_positions_input", QLineEdit)
validator = field.validator()
self.assertIsNotNone(validator)
@@ -161,7 +161,8 @@ class ReviewGuiRegressions(unittest.TestCase):
# A trailing comma during editing is harmless to the shared CSV parser.
for table in (np.empty((0, 0)), np.array([[99.]])):
result = ResultCollection(1, 2, collection_payloads=[ResultPayload(processing_name="sar3d_status", kind=4, table=table)])
self.assertFalse(window._plots.sar3d.draw(result))
self.assertFalse(window._plots.sar3d.draw(result, window._config.processing_settings().sar3d,
window._config.geometry()))
self.assertIn("error", window._plots.sar3d.status_label.text().lower())
def test_stop_or_close_cancels_paused_actions_without_error_dialog(self) -> None:
+1 -5
View File
@@ -363,7 +363,7 @@ class RingValidationTest(unittest.TestCase):
class GprValidationTest(unittest.TestCase):
@staticmethod
def _gpr(**overrides: object) -> GprModel:
model = GprModel(relative_permittivity=4.0)
model = GprModel()
for key, value in overrides.items():
setattr(model, key, value)
return model
@@ -371,10 +371,6 @@ class GprValidationTest(unittest.TestCase):
def _validate(self, gpr: GprModel) -> None:
validate_gpr_model(gpr, input_switch_positions=4, output_switch_positions=2)
def test_permittivity_must_be_positive(self) -> None:
with self.assertRaisesRegex(ValueError, "relative_permittivity"):
self._validate(self._gpr(relative_permittivity=0.0))
def test_tx_output_pos_out_of_range(self) -> None:
with self.assertRaisesRegex(ValueError, "output_pos is out of range"):
self._validate(self._gpr(tx_geometry=[GprTxGeometryModel(output_pos=5, x_m=0.0, y_m=0.0, z_m=0.0)]))
+93 -10
View File
@@ -9,11 +9,13 @@ from unittest import mock
import numpy as np
os.environ.setdefault("QT_QPA_PLATFORM", "offscreen")
from PyQt6.QtCore import Qt, QSignalBlocker # noqa: E402
from PyQt6.QtWidgets import QApplication, QCheckBox, QComboBox, QDoubleSpinBox # noqa: E402
from PyQt6.QtCore import QPointF, QRectF, Qt, QSignalBlocker # noqa: E402
from PyQt6.QtWidgets import QApplication, QCheckBox, QComboBox, QDoubleSpinBox, QSlider # noqa: E402
from python_app.gui.controllers.renderers.sar3d import SAR3D_STATES, SAR3D_VIEWS, Sar3dRenderer # noqa: E402
from python_app.models.dataset_model import ComboKey, ResultCollection, ResultPayload, TraceData # noqa: E402
from python_app.models.gui_profile_schema import GuiSar3dStateModel # noqa: E402
from python_app.models.run_config_schema import GprModel, GprTxGeometryModel, GprRxGeometryModel # noqa: E402
from python_app.tests.gui_test_support import widget
@@ -47,12 +49,13 @@ class Sar3dPlotTest(unittest.TestCase):
def setUp(self) -> None:
self.renderer = Sar3dRenderer()
self.settings, self.geometry = GuiSar3dStateModel(), GprModel()
self.addCleanup(self.renderer.widget.close)
def test_all_four_projections_have_visible_images_and_correct_peak_axes(self) -> None:
self.assertTrue(self.renderer.draw(_projections()))
self.assertTrue(self.renderer.draw(_projections(), self.settings, self.geometry))
self.assertEqual(self.renderer.status_label.text(),
"SAR3D: ready — CUDA | max X +0.50, Y -0.20, Z 2.00 m | level 2")
"SAR3D: ready — CUDA | max X +0.50, Y -0.20, Z 2.00 m | level 2 | zoom 100%")
positions = {"sar3d_front": (0.5, -0.2), "sar3d_side": (2, -0.2),
"sar3d_top": (0.5, 2), "sar3d_oblique": (0.5, -0.2)}
for name, position in positions.items():
@@ -60,27 +63,83 @@ class Sar3dPlotTest(unittest.TestCase):
self.assertTrue(image.isVisible(), name)
self.assertGreater(image.boundingRect().width(), 0)
self.assertGreater(image.boundingRect().height(), 0)
xx, yy = self.renderer.peaks[name].getData()
np.testing.assert_allclose([xx[0], yy[0]], position)
xx, yy = self.renderer.targets[name].getData()
assert xx is not None and yy is not None
np.testing.assert_allclose([.5*(xx.min()+xx.max()), .5*(yy.min()+yy.max())], position)
self.assertAlmostEqual(float(np.ptp(xx)), float(np.ptp(yy)))
self.assertTrue(all(label.isVisible() for label in self.renderer.axis_labels.values()))
self.renderer.clear()
self.assertEqual(self.renderer.status_label.text(), "SAR3D: waiting for measurements")
def test_unavailable_warning_clears_previous_images_and_labels_and_shows_reason(self) -> None:
self.renderer.draw(_projections())
self.renderer.draw(_projections(), self.settings, self.geometry)
message = "CUDA unavailable: драйвер <libcuda>"
self.assertFalse(self.renderer.draw(_collection(_status(3, message))))
self.assertFalse(self.renderer.draw(_collection(_status(3, message)), self.settings, self.geometry))
self.assertEqual(self.renderer.status_label.text(), f"SAR3D: warning — {message}")
self.assertEqual(self.renderer.status_label.textFormat(), Qt.TextFormat.PlainText)
self.assertTrue(all(not image.isVisible() for image in self.renderer.images.values()))
self.assertTrue(all(not label.isVisible() for label in self.renderer.axis_labels.values()))
def test_no_signal_and_missing_projections_explain_empty_panels(self) -> None:
self.assertFalse(self.renderer.draw(_collection(_status(1, "No signal in selected range"))))
self.assertFalse(self.renderer.draw(_collection(_status(1, "No signal in selected range")), self.settings, self.geometry))
self.assertIn("no signal", self.renderer.status_label.text())
self.assertFalse(self.renderer.draw(_collection()))
self.assertFalse(self.renderer.draw(_collection(), self.settings, self.geometry))
self.assertEqual(self.renderer.status_label.text(), "SAR3D: waiting for projection data")
def test_response_size_antenna_positions_and_zoom_use_scene_coordinates(self) -> None:
result = _projections()
result.collection_payloads.append(ResultPayload("sar3d_target_width", 4, table=np.array([[.4, .6, .8]])))
self.geometry = GprModel(tx_geometry=[GprTxGeometryModel(0, .7, .3, .4)],
rx_geometry=[GprRxGeometryModel(0, -.7, -.3, .2)])
self.settings.look_angle_deg = 30
self.settings.bp_ground_depth_m = 1.25
self.renderer.widget.resize(800, 600)
self.renderer.widget.show()
self.renderer.draw(result, self.settings, self.geometry)
self.app.processEvents()
baseline = {}
expected_antenna = [.7, 1.25 + .3*np.cos(np.pi/6) - .4*.5, .3*.5 + .4*np.cos(np.pi/6)]
for name, (_, _, _, axes) in SAR3D_VIEWS.items():
xx, yy = self.renderer.targets[name].getData()
assert xx is not None and yy is not None
side = .6 if name in {"sar3d_front", "sar3d_oblique"} else .8
self.assertAlmostEqual(float(np.ptp(xx)), side)
if axes is not None:
tx, _ = self.renderer.antennas[name]
ax, ay = tx.getData()
np.testing.assert_allclose([ax[0], ay[0]], np.asarray(expected_antenna)[list(axes)])
view = self.renderer.plots[name].getViewBox()
for marker in self.renderer.antennas[name]:
ax, ay = marker.getData()
for x, y in zip(ax, ay):
point = view.mapViewToScene(QPointF(float(x), float(y)))
self.assertTrue(view.sceneBoundingRect().contains(QRectF(point.x()-7, point.y()-7, 14, 14)), name)
rect = self.renderer.plots[name].getViewBox().targetRect()
baseline[name] = np.array([[rect.left(), rect.right()], [rect.top(), rect.bottom()]])
self.assertNotIn("sar3d_oblique", self.renderer.antennas)
self.settings.zoom_percent = 250
self.renderer.draw(result, self.settings, self.geometry)
self.app.processEvents()
for name, ranges in baseline.items():
rect = self.renderer.plots[name].getViewBox().targetRect()
zoomed = np.array([[rect.left(), rect.right()], [rect.top(), rect.bottom()]])
np.testing.assert_allclose(np.ptp(zoomed, axis=1), np.ptp(ranges, axis=1)/2.5)
xx, yy = self.renderer.targets[name].getData()
assert xx is not None and yy is not None
peak = np.array([.5*(xx.min()+xx.max()), .5*(yy.min()+yy.max())])
self.assertLess(np.linalg.norm(zoomed.mean(axis=1)-peak), np.linalg.norm(ranges.mean(axis=1)-peak))
def test_cloud_threshold_masks_weak_response_without_changing_levels(self) -> None:
result = _projections()
for payload in result.collection_payloads:
if payload.kind == 3:
payload.image.flat[:2] = [.2, .8]
for threshold, expected in ((.65, [0, .8, 1]), (1., [0, 0, 1])):
self.settings.cloud_threshold = threshold
self.renderer.draw(result, self.settings, self.geometry)
for image in self.renderer.images.values():
np.testing.assert_allclose(np.asarray(image.image).flat[:3], expected)
np.testing.assert_allclose(np.asarray(image.levels), [0, 1])
class Sar3dIntegrationTest(unittest.TestCase):
@@ -100,6 +159,30 @@ class Sar3dIntegrationTest(unittest.TestCase):
self.view._plots.required_revision = None
self.renderer = self.view._plots.sar3d
def test_display_controls_redraw_without_processing_revision_or_live_fields(self) -> None:
result = _projections()
self.view._history.record_result(result)
revision = self.view._live.revision
widget(self.view._view.processing, "_sar3d_zoom_percent", QSlider).setValue(247)
widget(self.view._view.processing, "_sar3d_cloud_threshold", QDoubleSpinBox).setValue(1.)
self.assertEqual(self.view._live.revision, revision)
self.assertIn("zoom 247%", self.renderer.status_label.text())
live = self.view._live.build().to_dict()
self.assertNotIn("sar3d_zoom_percent", live)
self.assertNotIn("sar3d_cloud_threshold", live)
schema = {entry["name"]: entry for entry in self.view._config.web_schema()}
self.assertEqual(schema["sar3d_zoom_percent"]["value"], 247)
self.assertEqual(schema["sar3d_zoom_percent"]["step"], 1)
self.assertEqual(schema["sar3d_cloud_threshold"]["value"], 1.)
def test_inactive_sar_scene_is_validated_before_publishing_gpr_settings(self) -> None:
with QSignalBlocker(self.mode):
self.mode.setCurrentText("gpr")
with QSignalBlocker(widget(self.view._view.processing, "_sar3d_y_min_m", QDoubleSpinBox)):
widget(self.view._view.processing, "_sar3d_y_min_m", QDoubleSpinBox).setValue(3.)
with self.assertRaisesRegex(ValueError, "SAR_Y_RANGE_M"):
self.view._config.build_live_config()
def test_capture_preview_keeps_sar_page_and_uses_saved_trace_settings(self) -> None:
self.view._plots.set_mode("sar3d")
fields = self.view._view.processing
@@ -2,7 +2,7 @@
The data_processor uses the speed a socket client streams unless the live config
sets `ignore_socket_speed`; then `gpr_speed_m_s` from the GUI field applies. Each
GPR-like page (gpr, legacy_gpr, sar3d) has its own checkbox and field, and the
GPR-like page (gpr, sar3d) has its own checkbox and field, and the
field is editable only while it is the one in force.
"""
@@ -31,7 +31,7 @@ class SocketSpeedControlsTest(unittest.TestCase):
def test_socket_authority_controls_editability_and_published_speed(self) -> None:
assert _window is not None
section = _window._view.processing
for mode, (checkbox, speed) in zip(("gpr", "legacy_gpr", "sar3d"), section.speed_controls, strict=True):
for mode, (checkbox, speed) in zip(("gpr", "sar3d"), section.speed_controls, strict=True):
section.mode.setCurrentText(mode)
speed.setValue(1.25)
for ignore in (True, False):