some changes and log fix

This commit is contained in:
2026-07-30 19:58:53 +03:00
parent 68bec25f17
commit 7c6cab07fc
10 changed files with 228 additions and 61 deletions
+1
View File
@@ -227,3 +227,4 @@ SHARE_INTERNET_TO_PI.md
CLAUDE.md
docs/
test_end_2/
@@ -10,7 +10,10 @@ from python_app.orchestration.preprocess_assets import (
preprocess_asset_channel,
preprocess_asset_display_name,
)
from python_app.workflows.kamil_adc_neutral_preprocess import build_kamil_adc_neutral_s21_sets
from python_app.workflows.kamil_adc_neutral_preprocess import (
build_neutral_s21_sets,
supports_neutral_preprocess_sets,
)
from python_app.workflows.multi_radar_capture_workflow import (
MultiRadarCaptureBatch,
MultiRadarSequentialCaptureSession,
@@ -216,8 +219,8 @@ class AppWindowPreprocessMixin:
dialog.undo_last_requested.connect(self._undo_last_capture)
dialog.finalize_sequence_requested.connect(self._finalize_capture_sequence)
dialog.abort_sequence_requested.connect(self._abort_capture_sequence)
dialog.create_kamil_adc_neutral_sets_requested.connect(self._create_kamil_adc_neutral_sets)
dialog.set_kamil_adc_neutral_sets_visible(self._defaults_config.is_kamil_adc)
dialog.create_neutral_sets_requested.connect(self._create_neutral_sets)
dialog.set_neutral_sets_visible(supports_neutral_preprocess_sets(self._defaults_config))
dialog.set_radar_config_summary(
directory_path=self._preprocess_radar_scan_summary.directory_path,
json_file_count=self._preprocess_radar_scan_summary.json_file_count,
@@ -292,7 +295,7 @@ class AppWindowPreprocessMixin:
f"{preprocess_asset_display_name(key)}={len(names)}"
for key, names in available_sets.items()
)
dialog.set_kamil_adc_neutral_sets_visible(self._defaults_config.is_kamil_adc)
dialog.set_neutral_sets_visible(supports_neutral_preprocess_sets(self._defaults_config))
self._log(f"Preprocess set lists refreshed: radar_key={radar_key}, {available_counts}")
if unavailable_selections:
self._log_warning(
@@ -391,8 +394,8 @@ class AppWindowPreprocessMixin:
self._show_exception(f"Failed to start {kind} sequence", exc)
self._resume_pipeline_if_needed()
def _create_kamil_adc_neutral_sets(self) -> None:
"""Save neutral S21 calibration/reference sets for the current Kamil ADC settings."""
def _create_neutral_sets(self) -> None:
"""Save neutral S21 calibration/reference sets for the current radar settings."""
if self._capture_session is not None:
self._show_error(
"Cannot create neutral sets during active capture sequence",
@@ -409,8 +412,10 @@ class AppWindowPreprocessMixin:
pipeline_was_paused = False
try:
config = self._build_config()
if not config.is_kamil_adc:
self._show_error("Neutral S21 sets are available only for kamil_adc")
if not supports_neutral_preprocess_sets(config):
self._show_error(
"Neutral S21 sets are available only for kamil_adc and librevna_multi"
)
return
radar_key = self._radar_key(config)
@@ -425,12 +430,12 @@ class AppWindowPreprocessMixin:
)
if self._supervisor.is_running():
self._log("Pipeline paused for Kamil ADC neutral-set creation")
self._log("Pipeline paused for neutral-set creation")
self._stop_run()
pipeline_was_paused = True
calibration, reference = build_kamil_adc_neutral_s21_sets(config)
point_count = config.radar.kamil_adc.band.points
calibration, reference = build_neutral_s21_sets(config)
point_count = int(calibration.traces[0].frequency_hz.size)
self._store.save_set("s21_calibration", radar_key, set_name, calibration)
self._store.save_set("s21_reference", radar_key, set_name, reference)
@@ -445,11 +450,11 @@ class AppWindowPreprocessMixin:
f"Neutral S21 sets saved: {set_name} ({len(calibration.traces)} combos, {point_count} points)"
)
self._log(
"Kamil ADC neutral S21 sets saved: "
"Neutral S21 sets saved: "
f"set={set_name}, radar_key={radar_key}, combos={len(calibration.traces)}, points={point_count}"
)
except Exception as exc: # noqa: BLE001
self._show_exception("Failed to create Kamil ADC neutral sets", exc)
self._show_exception("Failed to create neutral S21 sets", exc)
finally:
if pipeline_was_paused:
self._start_run()
+13 -12
View File
@@ -46,7 +46,7 @@ class PreprocessDialog(QDialog):
undo_last_requested = pyqtSignal()
finalize_sequence_requested = pyqtSignal()
abort_sequence_requested = pyqtSignal()
create_kamil_adc_neutral_sets_requested = pyqtSignal()
create_neutral_sets_requested = pyqtSignal()
def __init__(self, parent=None) -> None:
"""Initialize window metadata and compose dialog UI."""
@@ -92,17 +92,18 @@ class PreprocessDialog(QDialog):
self._set_name_input = QLineEdit("set_001", group)
refresh_button = QPushButton("Refresh Sets", group)
refresh_button.clicked.connect(self.refresh_requested.emit)
self._kamil_adc_neutral_sets_button = QPushButton("Create Neutral S21 Sets", group)
self._kamil_adc_neutral_sets_button.setToolTip(
"Save S21 calibration=1 and S21 reference=0 for the current Kamil ADC settings."
self._neutral_sets_button = QPushButton("Create Neutral S21 Sets", group)
self._neutral_sets_button.setToolTip(
"Save S21 calibration=1 and S21 reference=0 for the current radar settings, "
"so the pipeline can run before any real calibration exists."
)
self._kamil_adc_neutral_sets_button.clicked.connect(
self.create_kamil_adc_neutral_sets_requested.emit
self._neutral_sets_button.clicked.connect(
self.create_neutral_sets_requested.emit
)
self._kamil_adc_neutral_sets_button.setVisible(False)
self._neutral_sets_button.setVisible(False)
header_row.addWidget(QLabel("Set name"))
header_row.addWidget(self._set_name_input, stretch=1)
header_row.addWidget(self._kamil_adc_neutral_sets_button)
header_row.addWidget(self._neutral_sets_button)
header_row.addWidget(refresh_button)
layout.addLayout(header_row)
layout.addLayout(self._build_median_sweep_row(group))
@@ -413,10 +414,10 @@ class PreprocessDialog(QDialog):
"""Set short human-readable status line."""
self._status_label.setText(message)
def set_kamil_adc_neutral_sets_visible(self, visible: bool) -> None:
"""Show Kamil ADC neutral-set shortcut only in the matching radar mode."""
self._kamil_adc_neutral_sets_button.setVisible(bool(visible))
self._kamil_adc_neutral_sets_button.setEnabled(bool(visible))
def set_neutral_sets_visible(self, visible: bool) -> None:
"""Show the neutral-set shortcut only for radar models that support it."""
self._neutral_sets_button.setVisible(bool(visible))
self._neutral_sets_button.setEnabled(bool(visible))
def reset_preview(self) -> None:
"""Clear preview surfaces and restore default empty-state text when possible."""
@@ -131,7 +131,14 @@ class MultiDeviceVnaController:
if not self._reference_configuration_applied:
self._configure_reference_clocks()
self._drain_all_received_packets()
drain_started_seconds = time.monotonic()
drained_packet_count = self._drain_all_received_packets()
logger.debug(
"timing: drain discarded %d stale packet(s) in %.2f ms (t=%.1f ms)",
drained_packet_count,
(time.monotonic() - drain_started_seconds) * 1e3,
time.monotonic() * 1e3,
)
if (
self._sweep_is_running
@@ -334,12 +341,16 @@ class MultiDeviceVnaController:
self._sweep_is_running = True
logger.debug("Sweep settings applied to all devices; sweep running")
def _drain_all_received_packets(self) -> None:
def _drain_all_received_packets(self) -> int:
"""Empty every device's received-packet queue, in parallel for 2+ devices.
Concurrent draining keeps cross-device timing skew small so a hardware
cycle wrap cannot slip between per-device drains and desynchronize the
cycle counters.
Returns the total number of discarded packets, which the caller logs: a large
count means the host was far behind the free-running stream, a near-zero count
means the drain landed right after a sweep boundary.
"""
# Drain every device queue in parallel rather than one after another:
# serial drain leaves up to a few hundred microseconds of skew between
@@ -349,22 +360,31 @@ class MultiDeviceVnaController:
# so concurrent get_nowait calls do not contend. A single device case
# just runs inline to avoid the thread-spawn overhead.
if len(self._all_devices) < 2:
for device_connection in self._all_devices:
device_connection.drain_received_packets()
return
return sum(
len(device_connection.drain_received_packets())
for device_connection in self._all_devices
)
drained_counts = [0] * len(self._all_devices)
def drain_one_device(device_index: int, device_connection: LibreVnaUsbBulkConnection) -> None:
"""Drain one device's queue and record how many packets it held."""
drained_counts[device_index] = len(device_connection.drain_received_packets())
drain_threads = [
threading.Thread(
target=device_connection.drain_received_packets,
target=drain_one_device,
args=(device_index, device_connection),
name=f"drain-{device_connection.serial_number}",
daemon=True,
)
for device_connection in self._all_devices
for device_index, device_connection in enumerate(self._all_devices)
]
for drain_thread in drain_threads:
drain_thread.start()
for drain_thread in drain_threads:
drain_thread.join()
return sum(drained_counts)
@staticmethod
def _normalize_master_stimulus_ports(master_stimulus_ports: Sequence[int]) -> tuple[int, ...]:
@@ -441,6 +441,7 @@ def collect_complete_running_sweep_cycles(
def build_cycle_tracking_handler(
cycle_aware_handler: Callable[[ParsedVnaDatapoint, int], None],
device_state: _DeviceCollectionState,
device_label: str = "device",
) -> Callable[[ParsedVnaDatapoint], bool]:
"""Wrap a cycle-aware handler with cross-device cycle tracking.
@@ -462,6 +463,11 @@ def collect_complete_running_sweep_cycles(
cycle_tracking_state = {
"current_cycle_index": 0,
"synchronized": False,
# How many mid-sweep points were thrown away before the anchor was found.
# Near zero means the drain landed on a sweep boundary — the case where a
# stale point 0 could still have been in flight; a large count means the
# remainder of the in-progress sweep was safely skipped.
"pre_anchor_skipped": 0,
}
def handle_datapoint(parsed_datapoint: ParsedVnaDatapoint) -> bool:
@@ -475,6 +481,7 @@ def collect_complete_running_sweep_cycles(
if not cycle_tracking_state["synchronized"]:
if current_point_index != 0:
cycle_tracking_state["pre_anchor_skipped"] += 1
return False
# Candidate cycle 0. Commit it only once every device confirms it
# observed point 0 of the SAME physical sweep; otherwise reject the
@@ -484,6 +491,14 @@ def collect_complete_running_sweep_cycles(
report_cycle_misalignment()
return False
cycle_tracking_state["synchronized"] = True
logger.debug(
"timing: %s anchored cycle 0 after skipping %d mid-sweep point(s) of %d "
"(t=%.1f ms)",
device_label,
cycle_tracking_state["pre_anchor_skipped"],
point_count,
time.monotonic() * 1e3,
)
cycle_aware_handler(parsed_datapoint, 0)
return True
@@ -493,6 +508,12 @@ def collect_complete_running_sweep_cycles(
# spurious wrap and desynchronize the cycle counter.
if current_point_index == 0:
cycle_tracking_state["current_cycle_index"] += 1
logger.debug(
"timing: %s first point of NEXT sweep arrived (cycle -> %d, t=%.1f ms)",
device_label,
cycle_tracking_state["current_cycle_index"],
time.monotonic() * 1e3,
)
current_cycle_index = cycle_tracking_state["current_cycle_index"]
if current_cycle_index >= cycle_count:
# The sweep just wrapped past the final requested cycle, closing its
@@ -505,6 +526,14 @@ def collect_complete_running_sweep_cycles(
return False
cycle_aware_handler(parsed_datapoint, current_cycle_index)
if current_point_index == point_count - 1:
logger.debug(
"timing: %s last point of cycle %d arrived (index=%d, t=%.1f ms)",
device_label,
current_cycle_index,
current_point_index,
time.monotonic() * 1e3,
)
return True
return handle_datapoint
@@ -587,7 +616,9 @@ def collect_complete_running_sweep_cycles(
point_index,
] = port_receiver_value
return build_cycle_tracking_handler(handle_slave_datapoint, device_state)
return build_cycle_tracking_handler(
handle_slave_datapoint, device_state, device_label=f"slave{slave_index}"
)
master_device_state = _DeviceCollectionState()
collection_threads = [
@@ -595,7 +626,9 @@ def collect_complete_running_sweep_cycles(
target=collect_datapoints_from_device,
args=(
master_device_connection,
build_cycle_tracking_handler(handle_master_datapoint, master_device_state),
build_cycle_tracking_handler(
handle_master_datapoint, master_device_state, device_label="master"
),
master_device_state,
),
daemon=True,
@@ -2,7 +2,7 @@
from __future__ import annotations
from dataclasses import dataclass, replace
from dataclasses import dataclass, field, replace
import logging
import time
@@ -31,6 +31,10 @@ class SwitchedMatrixRadarService:
inner_output_positions: int
inner_input_positions: int
settling_ms: int = 0
# Monotonic end of the previous inner collection, so the DEBUG timing trace can
# report how long the gap between "sweep collected" and "switch driven" really is
# — that gap is where a stale in-flight point 0 can still slip past the drain.
_last_inner_end_ns: int = field(init=False, default=0, repr=False)
def open(self) -> None:
"""Open the inner radar and both switches."""
@@ -78,17 +82,37 @@ class SwitchedMatrixRadarService:
slots: list[TraceData | None] = [None] * (total_inputs * total_outputs)
for out_k in range(out_steps):
for in_k in range(in_steps):
step_start_ns = time.monotonic_ns()
if self.output_switch is not None:
self.output_switch.switch_to(out_k)
for in_k in range(in_steps):
if self.input_switch is not None:
self.input_switch.switch_to(in_k)
switched_ns = time.monotonic_ns()
# Settle AFTER the last switch change and BEFORE collecting, so the
# cycle we anchor on starts with the RF path already stable.
if self.settling_ms > 0:
time.sleep(self.settling_ms / 1000.0)
settled_ns = time.monotonic_ns()
sub = self.inner.acquire_collection(collection_id)
inner_end_ns = time.monotonic_ns()
logger.debug(
"timing: collection %d step out=%d in=%d | gap_prev_collect_to_switch=%s ms, "
"switch=%.3f ms, settle=%.2f ms, inner_collect=%.2f ms",
collection_id,
out_k,
in_k,
(
f"{(step_start_ns - self._last_inner_end_ns) / 1e6:.2f}"
if self._last_inner_end_ns
else "n/a"
),
(switched_ns - step_start_ns) / 1e6,
(settled_ns - switched_ns) / 1e6,
(inner_end_ns - settled_ns) / 1e6,
)
self._last_inner_end_ns = inner_end_ns
for trace in sub.traces:
input_pos = in_k * self.inner_input_positions + int(trace.combo.input)
output_pos = out_k * self.inner_output_positions + int(trace.combo.output)
+20 -3
View File
@@ -105,7 +105,10 @@ def _read_log_tail(path: Path, max_bytes: int = 16384) -> str:
data = handle.read()
except OSError:
return ""
return data.decode("utf-8", errors="replace").strip()
# Drop NULs: logs written by an older supervisor can carry a sparse hole from
# the pre-O_APPEND truncate bug, and a tail landing in it would otherwise turn
# an exit report (or a rolled `.prev`) into megabytes of NUL padding.
return data.replace(b"\0", b"").decode("utf-8", errors="replace").strip()
class ProcessSupervisor:
@@ -232,8 +235,17 @@ class ProcessSupervisor:
self._roll_log_to_prev(stdout_path)
self._roll_log_to_prev(stderr_path)
stdout_file = open(stdout_path, "wb")
stderr_file = open(stderr_path, "wb")
# O_APPEND ("ab"), not "wb": the child inherits these fds and keeps its own
# file offset. Without O_APPEND, the in-place truncate in
# `_roll_log_if_oversized` leaves that offset far past the new end of file,
# so the next write lands there and the kernel fills everything before it
# with a hole of NUL bytes — the log becomes unreadable and the size cap
# stops working entirely. O_APPEND makes the kernel seek to EOF atomically
# on every write, so a truncate genuinely restarts the file at offset 0.
# `_roll_log_to_prev` above already renamed any previous log away, so not
# truncating on open costs nothing.
stdout_file = open(stdout_path, "ab")
stderr_file = open(stderr_path, "ab")
try:
handle = subprocess.Popen(
command,
@@ -499,6 +511,11 @@ class ProcessSupervisor:
The child holds an open fd to this inode, so a rename would not redirect
its writes. Instead keep one rolled generation via copy-to-`.prev` and
truncate the live inode in place, freeing the allocated disk blocks.
This relies on the child's fd being opened with O_APPEND (see `_spawn`):
only then does the child resume writing at offset 0 after the truncate.
With a plain write fd it would keep writing at its stale offset, punching
a multi-hundred-megabyte NUL hole and defeating the cap.
"""
try:
if path.stat().st_size <= _LOG_MAX_BYTES:
@@ -11,6 +11,7 @@ import threading
import time
from python_app.hardware_full.matrix_radar_service import MatrixRadarService, create_matrix_radar_service
from python_app.logging_setup import coerce_level
from python_app.models.run_config_model import RunConfigModel
from python_app.orchestration.shm import ShmRingWriter
from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collection
@@ -95,6 +96,10 @@ def main() -> int:
config = RunConfigModel.load_from_path(args.config)
config.apply_device_model_constraints()
# Honor the configured verbosity so the DEBUG switch/sweep timing trace can be
# turned on from the profile instead of requiring a code edit. basicConfig above
# only installed the handler; the package logger owns the level.
logging.getLogger("python_app").setLevel(coerce_level(config.logging.level))
if not config.is_matrix_radar:
raise RuntimeError(
"matrix_raw_producer requires a matrix-mode radar.model "
@@ -1,4 +1,15 @@
"""Neutral preprocessing-set helpers for Kamil ADC acquisition."""
"""Neutral preprocessing-set helpers — the "run without calibration" path.
A neutral pair is a calibration set carrying unit S21 (1+0j) and a reference set
carrying zero S21. The C++ through-calibrator divides measured/calibration and the
reference is subtracted, so applying both leaves the measured S21 untouched. That
lets an operator start the pipeline before any real calibration exists, which the
required-asset check in `_start_run` would otherwise refuse.
Supported models: Kamil ADC (axis from the ADC processing grid) and every
VNA-style model, including synchronized multi-device LibreVNA (axis from the
configured linear sweep grid).
"""
from __future__ import annotations
@@ -17,31 +28,65 @@ from python_app.models.run_config_model import ComboModel, RunConfigModel
logger = logging.getLogger(__name__)
def build_kamil_adc_neutral_s21_sets(
def supports_neutral_preprocess_sets(config: RunConfigModel) -> bool:
"""Return whether neutral S21 sets can be generated for this radar model.
Enabled for the Kamil ADC and for synchronized multi-device LibreVNA, the two
models whose emitted frequency axis is fully derivable from the config alone.
Other models still work through `build_neutral_s21_sets`, but are kept out of the
UI shortcut until their axis has been verified against real hardware.
"""
return bool(config.is_kamil_adc or config.is_multi_device)
def neutral_frequency_grid_hz(config: RunConfigModel) -> np.ndarray:
"""Return the exact per-trace frequency axis the configured radar emits.
Neutral sets must line up sample-for-sample with live sweeps, so the axis comes
from the same source the acquisition path uses: the ADC processing grid for Kamil
ADC, and the configured linear sweep grid for every VNA-style model (LibreVNA
single and multi-device, SN9000, Compact-M). The C++ preprocessor re-checks this
axis against the measured one within a tolerance, so a mismatch fails loudly
instead of silently corrupting the correction.
"""
if config.is_kamil_adc:
# Single source of truth for the axis: the same grid the processor emits.
processor = KamilAdcSweepProcessor(
KamilAdcProcessingParams.from_kamil_model(config.radar.kamil_adc)
)
return processor.grid_hz
points = int(config.radar.sweep.points)
if points < 1:
raise ValueError("Neutral sets require radar.sweep.points >= 1")
if points == 1:
return np.array([float(config.radar.sweep.start_hz)], dtype=np.float32)
# Mirrors both acquisition paths: the native collector seeds this same linspace
# and the mock backend generates it outright.
return np.linspace(
float(config.radar.sweep.start_hz),
float(config.radar.sweep.stop_hz),
points,
dtype=np.float32,
)
def build_neutral_s21_sets(
config: RunConfigModel,
) -> tuple[SweepCollection, SweepCollection]:
"""Build neutral S21 calibration/reference collections for the Kamil ADC radar.
"""Build neutral S21 calibration/reference collections for the active radar.
The calibration uses unit S21 (1+0j) and the reference uses zero S21 across
every configured combo, so applying them in the preprocessing pipeline leaves
the input S21 unchanged. The frequency axis is the exact acquisition grid
(``radar.kamil_adc.band``), so neutral sets line up sample-for-sample with
live sweeps. Returns the ``(calibration, reference)`` collections.
Covers every combo in the effective matrix, so a matrix radar widened by real
switches gets a neutral pair for all of its positions and the preprocessor's
``validate_combos()`` is satisfied. Returns ``(calibration, reference)``.
"""
if not config.is_kamil_adc:
raise ValueError("Neutral Kamil ADC sets require radar.model='kamil_adc'")
combos = list(config.combos)
if not combos:
combos = RunConfigModel.build_full_combos(
config.input_switch.positions, config.output_switch.positions
)
combos = config.build_runtime_combos()
if not combos:
raise ValueError("Kamil ADC neutral sets require at least one switch combo")
raise ValueError("Neutral sets require at least one switch combo")
# Single source of truth for the axis: the same grid the processor emits.
processor = KamilAdcSweepProcessor(KamilAdcProcessingParams.from_kamil_model(config.radar.kamil_adc))
frequency_hz = processor.grid_hz
frequency_hz = neutral_frequency_grid_hz(config)
now_ns = time.monotonic_ns()
calibration = _neutral_collection(
@@ -57,11 +102,27 @@ def build_kamil_adc_neutral_s21_sets(
monotonic_ns=now_ns,
)
logger.info(
"Built neutral Kamil ADC S21 sets: combos=%d points=%d", len(combos), int(frequency_hz.size)
"Built neutral S21 sets: model=%s combos=%d points=%d",
config.radar.model,
len(combos),
int(frequency_hz.size),
)
return calibration, reference
def build_kamil_adc_neutral_s21_sets(
config: RunConfigModel,
) -> tuple[SweepCollection, SweepCollection]:
"""Build neutral S21 sets, rejecting anything but the Kamil ADC radar.
Kept as the model-checked entry point for the ADC path; new callers that must
work for several radar models should use `build_neutral_s21_sets` instead.
"""
if not config.is_kamil_adc:
raise ValueError("Neutral Kamil ADC sets require radar.model='kamil_adc'")
return build_neutral_s21_sets(config)
def _neutral_collection(
*,
combos: list[ComboModel],
+1 -1
View File
@@ -109,7 +109,7 @@
"action": "capture_tmp_reference"
},
"logging": {
"level": "info"
"level": "debug"
},
"run": {
"settling_ms": 0,