This commit is contained in:
awe
2026-02-25 20:20:40 +03:00
parent 267ddedb19
commit f1652d072e
12 changed files with 1823 additions and 404 deletions

View File

@ -0,0 +1,227 @@
"""Загрузка эталонов (калибровка/фон) из .npy или бинарных capture-файлов."""
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass
import os
from typing import Iterable, List, Optional, Tuple
import numpy as np
from rfg_adc_plotter.io.sweep_parser_core import BinaryRecordStreamParser, SweepAssembler
from rfg_adc_plotter.types import SweepPacket
@dataclass(frozen=True)
class CaptureParseSummary:
path: str
format: str # "npy" | "bin_capture"
sweeps_total: int
sweeps_valid: int
channels_seen: Tuple[int, ...]
dominant_width: Optional[int]
dominant_n_valid: Optional[int]
aggregation: str
warnings: Tuple[str, ...]
@dataclass(frozen=True)
class ReferenceLoadResult:
vector: np.ndarray
summary: CaptureParseSummary
kind: str # "calibration_envelope" | "background_raw" | "background_processed"
source_type: str # "npy" | "capture"
def detect_reference_file_format(path: str) -> Optional[str]:
"""Определить формат файла эталона: .npy или бинарный capture."""
p = str(path).strip()
if not p or not os.path.isfile(p):
return None
if p.lower().endswith(".npy"):
return "npy"
try:
size = os.path.getsize(p)
except Exception:
return None
if size <= 0 or (size % 8) != 0:
return None
try:
with open(p, "rb") as f:
sample = f.read(min(size, 8 * 2048))
except Exception:
return None
if len(sample) < 8:
return None
# Быстрый sniff aligned-записей: в валидных записях байт 6 == 0x0A.
recs = len(sample) // 8
if recs <= 0:
return None
marker_hits = 0
start_hits = 0
for i in range(0, recs * 8, 8):
b = sample[i : i + 8]
if b[6] == 0x0A:
marker_hits += 1
if b[:6] == b"\xff\xff\xff\xff\xff\xff":
start_hits += 1
if marker_hits >= max(4, int(recs * 0.8)) and start_hits >= 1:
return "bin_capture"
return None
def load_capture_sweeps(path: str, *, fancy: bool = False, logscale: bool = False) -> List[SweepPacket]:
"""Загрузить свипы из бинарного capture-файла в формате --bin."""
parser = BinaryRecordStreamParser()
assembler = SweepAssembler(fancy=fancy, logscale=logscale, debug=False)
sweeps: List[SweepPacket] = []
with open(path, "rb") as f:
while True:
chunk = f.read(65536)
if not chunk:
break
events = parser.feed(chunk)
for ev in events:
packets = assembler.consume_binary_event(ev)
if packets:
sweeps.extend(packets)
tail = assembler.finalize_current()
if tail is not None:
sweeps.append(tail)
return sweeps
def _mode_int(values: Iterable[int]) -> Optional[int]:
vals = [int(v) for v in values]
if not vals:
return None
ctr = Counter(vals)
return int(max(ctr.items(), key=lambda kv: (kv[1], kv[0]))[0])
def aggregate_capture_reference(
sweeps: List[SweepPacket],
*,
channel: int = 0,
method: str = "median",
path: str = "",
) -> Tuple[np.ndarray, CaptureParseSummary]:
"""Отфильтровать и агрегировать свипы из capture в один эталонный вектор."""
ch_target = int(channel)
meth = str(method).strip().lower() or "median"
warnings: list[str] = []
if meth != "median":
warnings.append(f"aggregation '{meth}' не поддерживается, использую median")
meth = "median"
channels_seen: set[int] = set()
candidate_rows: list[np.ndarray] = []
widths: list[int] = []
n_valids: list[int] = []
for sweep, info in sweeps:
chs = info.get("chs") if isinstance(info, dict) else None
ch_set: set[int] = set()
if isinstance(chs, (list, tuple, set)):
for v in chs:
try:
ch_set.add(int(v))
except Exception:
pass
else:
try:
ch_set.add(int(info.get("ch", 0))) # type: ignore[union-attr]
except Exception:
pass
channels_seen.update(ch_set)
if ch_target not in ch_set:
continue
row = np.asarray(sweep, dtype=np.float32).reshape(-1)
candidate_rows.append(row)
widths.append(int(row.size))
n_valids.append(int(np.count_nonzero(np.isfinite(row))))
sweeps_total = len(sweeps)
if not candidate_rows:
summary = CaptureParseSummary(
path=path,
format="bin_capture",
sweeps_total=sweeps_total,
sweeps_valid=0,
channels_seen=tuple(sorted(channels_seen)),
dominant_width=None,
dominant_n_valid=None,
aggregation=meth,
warnings=tuple(warnings + [f"канал ch{ch_target} не найден"]),
)
raise ValueError(summary.warnings[-1])
dominant_width = _mode_int(widths)
dominant_n_valid = _mode_int(n_valids)
if dominant_width is None or dominant_n_valid is None:
summary = CaptureParseSummary(
path=path,
format="bin_capture",
sweeps_total=sweeps_total,
sweeps_valid=0,
channels_seen=tuple(sorted(channels_seen)),
dominant_width=dominant_width,
dominant_n_valid=dominant_n_valid,
aggregation=meth,
warnings=tuple(warnings + ["не удалось определить доминирующие параметры свипа"]),
)
raise ValueError(summary.warnings[-1])
valid_rows: list[np.ndarray] = []
n_valid_threshold = max(1, int(np.floor(0.95 * dominant_n_valid)))
for row in candidate_rows:
if row.size != dominant_width:
continue
n_valid = int(np.count_nonzero(np.isfinite(row)))
if n_valid < n_valid_threshold:
continue
valid_rows.append(row)
if not valid_rows:
warnings.append("после фильтрации не осталось валидных свипов")
summary = CaptureParseSummary(
path=path,
format="bin_capture",
sweeps_total=sweeps_total,
sweeps_valid=0,
channels_seen=tuple(sorted(channels_seen)),
dominant_width=dominant_width,
dominant_n_valid=dominant_n_valid,
aggregation=meth,
warnings=tuple(warnings),
)
raise ValueError(summary.warnings[-1])
# Детерминированная агрегация: медиана по валидным свипам.
stack = np.stack(valid_rows, axis=0).astype(np.float32, copy=False)
vector = np.nanmedian(stack, axis=0).astype(np.float32, copy=False)
if len(valid_rows) < len(candidate_rows):
warnings.append(f"отфильтровано {len(candidate_rows) - len(valid_rows)} неполных/нестандартных свипов")
summary = CaptureParseSummary(
path=path,
format="bin_capture",
sweeps_total=sweeps_total,
sweeps_valid=len(valid_rows),
channels_seen=tuple(sorted(channels_seen)),
dominant_width=dominant_width,
dominant_n_valid=dominant_n_valid,
aggregation=meth,
warnings=tuple(warnings),
)
return vector, summary

View File

@ -0,0 +1,247 @@
"""Переиспользуемые компоненты парсинга бинарных свипов и сборки SweepPacket."""
from __future__ import annotations
from collections import deque
import time
from typing import Iterable, List, Optional, Sequence, Set, Tuple
import numpy as np
from rfg_adc_plotter.constants import DATA_INVERSION_THRESHOLD, LOG_EXP
from rfg_adc_plotter.types import SweepInfo, SweepPacket
# Binary parser events:
# ("start", ch)
# ("point", ch, x, y)
BinaryEvent = Tuple[str, int] | Tuple[str, int, int, int]
def u32_to_i32(v: int) -> int:
"""Преобразование 32-bit слова в знаковое значение."""
return v - 0x1_0000_0000 if (v & 0x8000_0000) else v
class BinaryRecordStreamParser:
"""Инкрементальный парсер бинарных записей протокола (по 8 байт)."""
def __init__(self):
self._buf = bytearray()
self.bytes_consumed: int = 0
self.start_count: int = 0
self.point_count: int = 0
self.desync_count: int = 0
def feed(self, data: bytes) -> List[BinaryEvent]:
if data:
self._buf += data
events: List[BinaryEvent] = []
buf = self._buf
while len(buf) >= 8:
w0 = int(buf[0]) | (int(buf[1]) << 8)
w1 = int(buf[2]) | (int(buf[3]) << 8)
w2 = int(buf[4]) | (int(buf[5]) << 8)
if w0 == 0xFFFF and w1 == 0xFFFF and w2 == 0xFFFF and buf[6] == 0x0A:
ch = int(buf[7])
events.append(("start", ch))
del buf[:8]
self.bytes_consumed += 8
self.start_count += 1
continue
if buf[6] == 0x0A:
ch = int(buf[7])
value_u32 = (w1 << 16) | w2
events.append(("point", ch, int(w0), u32_to_i32(value_u32)))
del buf[:8]
self.bytes_consumed += 8
self.point_count += 1
continue
del buf[:1]
self.bytes_consumed += 1
self.desync_count += 1
return events
def buffered_size(self) -> int:
return len(self._buf)
def clear_buffer_keep_tail(self, max_tail: int = 262_144):
if len(self._buf) > max_tail:
del self._buf[:-max_tail]
class SweepAssembler:
"""Собирает точки в свип и применяет ту же постобработку, что realtime parser."""
def __init__(self, fancy: bool = False, logscale: bool = False, debug: bool = False):
self._fancy = bool(fancy)
self._logscale = bool(logscale)
self._debug = bool(debug)
self._max_width: int = 0
self._sweep_idx: int = 0
self._last_sweep_ts: Optional[float] = None
self._n_valid_hist = deque()
self._xs: list[int] = []
self._ys: list[int] = []
self._cur_channel: Optional[int] = None
self._cur_channels: set[int] = set()
def reset_current(self):
self._xs.clear()
self._ys.clear()
self._cur_channel = None
self._cur_channels.clear()
def add_point(self, ch: int, x: int, y: int):
if self._cur_channel is None:
self._cur_channel = int(ch)
self._cur_channels.add(int(ch))
self._xs.append(int(x))
self._ys.append(int(y))
def start_new_sweep(self, ch: int, now_ts: Optional[float] = None) -> Optional[SweepPacket]:
packet = self.finalize_current(now_ts=now_ts)
self.reset_current()
self._cur_channel = int(ch)
self._cur_channels.add(int(ch))
return packet
def consume_binary_event(self, event: BinaryEvent, now_ts: Optional[float] = None) -> List[SweepPacket]:
out: List[SweepPacket] = []
tag = event[0]
if tag == "start":
packet = self.start_new_sweep(int(event[1]), now_ts=now_ts)
if packet is not None:
out.append(packet)
return out
# point
_tag, ch, x, y = event # type: ignore[misc]
self.add_point(int(ch), int(x), int(y))
return out
def finalize_arrays(
self,
xs: Sequence[int],
ys: Sequence[int],
channels: Optional[Set[int]],
now_ts: Optional[float] = None,
) -> Optional[SweepPacket]:
if self._debug:
if not xs:
import sys
sys.stderr.write("[debug] _finalize_current: xs пуст — свип пропущен\n")
else:
import sys
sys.stderr.write(
f"[debug] _finalize_current: {len(xs)} точек → свип #{self._sweep_idx + 1}\n"
)
if not xs:
return None
ch_list = sorted(channels) if channels else [0]
ch_primary = ch_list[0] if ch_list else 0
max_x = max(int(v) for v in xs)
width = max_x + 1
self._max_width = max(self._max_width, width)
target_width = self._max_width if self._fancy else width
sweep = np.full((target_width,), np.nan, dtype=np.float32)
try:
idx = np.asarray(xs, dtype=np.int64)
vals = np.asarray(ys, dtype=np.float32)
sweep[idx] = vals
except Exception:
for x, y in zip(xs, ys):
xi = int(x)
if 0 <= xi < target_width:
sweep[xi] = float(y)
n_valid_cur = int(np.count_nonzero(np.isfinite(sweep)))
if self._fancy:
try:
known = ~np.isnan(sweep)
if np.any(known):
known_idx = np.nonzero(known)[0]
for i0, i1 in zip(known_idx[:-1], known_idx[1:]):
if i1 - i0 > 1:
avg = (sweep[i0] + sweep[i1]) * 0.5
sweep[i0 + 1 : i1] = avg
first_idx = int(known_idx[0])
last_idx = int(known_idx[-1])
if first_idx > 0:
sweep[:first_idx] = sweep[first_idx]
if last_idx < sweep.size - 1:
sweep[last_idx + 1 :] = sweep[last_idx]
except Exception:
pass
try:
m = float(np.nanmean(sweep))
if np.isfinite(m) and m < DATA_INVERSION_THRESHOLD:
sweep *= -1.0
except Exception:
pass
pre_exp_sweep = None
if self._logscale:
try:
pre_exp_sweep = sweep.copy()
with np.errstate(over="ignore", invalid="ignore"):
sweep = np.power(LOG_EXP, np.asarray(sweep, dtype=np.float64)).astype(np.float32)
sweep[~np.isfinite(sweep)] = np.nan
except Exception:
pass
self._sweep_idx += 1
if len(ch_list) > 1:
import sys
sys.stderr.write(f"[warn] Sweep {self._sweep_idx}: изменялся номер канала: {ch_list}\n")
now = float(time.time() if now_ts is None else now_ts)
if self._last_sweep_ts is None:
dt_ms = float("nan")
else:
dt_ms = (now - self._last_sweep_ts) * 1000.0
self._last_sweep_ts = now
self._n_valid_hist.append((now, n_valid_cur))
while self._n_valid_hist and (now - self._n_valid_hist[0][0]) > 1.0:
self._n_valid_hist.popleft()
if self._n_valid_hist:
n_valid = float(sum(v for _t, v in self._n_valid_hist) / len(self._n_valid_hist))
else:
n_valid = float(n_valid_cur)
if n_valid_cur > 0:
vmin = float(np.nanmin(sweep))
vmax = float(np.nanmax(sweep))
mean = float(np.nanmean(sweep))
std = float(np.nanstd(sweep))
else:
vmin = vmax = mean = std = float("nan")
info: SweepInfo = {
"sweep": self._sweep_idx,
"ch": ch_primary,
"chs": ch_list,
"n_valid": n_valid,
"min": vmin,
"max": vmax,
"mean": mean,
"std": std,
"dt_ms": dt_ms,
}
if pre_exp_sweep is not None:
info["pre_exp_sweep"] = pre_exp_sweep
return (sweep, info)
def finalize_current(self, now_ts: Optional[float] = None) -> Optional[SweepPacket]:
return self.finalize_arrays(self._xs, self._ys, self._cur_channels, now_ts=now_ts)

View File

@ -3,15 +3,12 @@
import sys
import threading
import time
from collections import deque
from queue import Full, Queue
from typing import Optional
import numpy as np
from rfg_adc_plotter.constants import DATA_INVERSION_THRESHOLD, LOG_EXP
from rfg_adc_plotter.io.sweep_parser_core import BinaryRecordStreamParser, SweepAssembler
from rfg_adc_plotter.io.serial_source import SerialChunkReader, SerialLineSource
from rfg_adc_plotter.types import SweepInfo, SweepPacket
from rfg_adc_plotter.types import SweepPacket
class SweepReader(threading.Thread):
@ -38,119 +35,13 @@ class SweepReader(threading.Thread):
self._bin_mode = bool(bin_mode)
self._logscale = bool(logscale)
self._debug = bool(debug)
self._max_width: int = 0
self._sweep_idx: int = 0
self._last_sweep_ts: Optional[float] = None
self._n_valid_hist = deque()
@staticmethod
def _u32_to_i32(v: int) -> int:
"""Преобразование 32-bit слова в знаковое значение."""
return v - 0x1_0000_0000 if (v & 0x8000_0000) else v
self._assembler = SweepAssembler(fancy=self._fancy, logscale=self._logscale, debug=self._debug)
def _finalize_current(self, xs, ys, channels: Optional[set]):
if self._debug:
if not xs:
sys.stderr.write("[debug] _finalize_current: xs пуст — свип пропущен\n")
else:
sys.stderr.write(f"[debug] _finalize_current: {len(xs)} точек → свип #{self._sweep_idx + 1}\n")
if not xs:
packet = self._assembler.finalize_arrays(xs, ys, channels)
if packet is None:
return
ch_list = sorted(channels) if channels else [0]
ch_primary = ch_list[0] if ch_list else 0
max_x = max(xs)
width = max_x + 1
self._max_width = max(self._max_width, width)
target_width = self._max_width if self._fancy else width
sweep = np.full((target_width,), np.nan, dtype=np.float32)
try:
idx = np.asarray(xs, dtype=np.int64)
vals = np.asarray(ys, dtype=np.float32)
sweep[idx] = vals
except Exception:
for x, y in zip(xs, ys):
if 0 <= x < target_width:
sweep[x] = float(y)
finite_pre = np.isfinite(sweep)
n_valid_cur = int(np.count_nonzero(finite_pre))
if self._fancy:
try:
known = ~np.isnan(sweep)
if np.any(known):
known_idx = np.nonzero(known)[0]
for i0, i1 in zip(known_idx[:-1], known_idx[1:]):
if i1 - i0 > 1:
avg = (sweep[i0] + sweep[i1]) * 0.5
sweep[i0 + 1 : i1] = avg
first_idx = int(known_idx[0])
last_idx = int(known_idx[-1])
if first_idx > 0:
sweep[:first_idx] = sweep[first_idx]
if last_idx < sweep.size - 1:
sweep[last_idx + 1 :] = sweep[last_idx]
except Exception:
pass
try:
m = float(np.nanmean(sweep))
if np.isfinite(m) and m < DATA_INVERSION_THRESHOLD:
sweep *= -1.0
except Exception:
pass
pre_exp_sweep = None
if self._logscale:
try:
pre_exp_sweep = sweep.copy()
with np.errstate(over="ignore", invalid="ignore"):
sweep = np.power(LOG_EXP, np.asarray(sweep, dtype=np.float64)).astype(np.float32)
sweep[~np.isfinite(sweep)] = np.nan
except Exception:
pass
self._sweep_idx += 1
if len(ch_list) > 1:
sys.stderr.write(
f"[warn] Sweep {self._sweep_idx}: изменялся номер канала: {ch_list}\n"
)
now = time.time()
if self._last_sweep_ts is None:
dt_ms = float("nan")
else:
dt_ms = (now - self._last_sweep_ts) * 1000.0
self._last_sweep_ts = now
self._n_valid_hist.append((now, n_valid_cur))
while self._n_valid_hist and (now - self._n_valid_hist[0][0]) > 1.0:
self._n_valid_hist.popleft()
if self._n_valid_hist:
n_valid = float(sum(v for _t, v in self._n_valid_hist) / len(self._n_valid_hist))
else:
n_valid = float(n_valid_cur)
if n_valid_cur > 0:
vmin = float(np.nanmin(sweep))
vmax = float(np.nanmax(sweep))
mean = float(np.nanmean(sweep))
std = float(np.nanstd(sweep))
else:
vmin = vmax = mean = std = float("nan")
info: SweepInfo = {
"sweep": self._sweep_idx,
"ch": ch_primary,
"chs": ch_list,
"n_valid": n_valid,
"min": vmin,
"max": vmax,
"mean": mean,
"std": std,
"dt_ms": dt_ms,
}
if pre_exp_sweep is not None:
info["pre_exp_sweep"] = pre_exp_sweep
sweep, info = packet
try:
self._q.put_nowait((sweep, info))
except Full:
@ -263,6 +154,7 @@ class SweepReader(threading.Thread):
ys: list[int] = []
cur_channel: Optional[int] = None
cur_channels: set[int] = set()
parser = BinaryRecordStreamParser()
# Бинарный протокол (4 слова LE u16 = 8 байт на запись):
# старт свипа: 0xFFFF, 0xFFFF, 0xFFFF, (ch<<8)|0x0A
@ -274,7 +166,6 @@ class SweepReader(threading.Thread):
# Признак записи: байт 6 == 0x0A, байт 7 — номер канала.
# При десинхронизации сдвигаемся на 1 БАЙТ (не слово) для самосинхронизации.
buf = bytearray()
_dbg_byte_count = 0
_dbg_desync_count = 0
_dbg_sweep_count = 0
@ -282,20 +173,15 @@ class SweepReader(threading.Thread):
while not self._stop.is_set():
data = chunk_reader.read_available()
if data:
buf += data
events = parser.feed(data)
else:
time.sleep(0.0005)
continue
while len(buf) >= 8:
# Читаем 4 LE u16 слова прямо из байтового буфера
w0 = int(buf[0]) | (int(buf[1]) << 8)
w1 = int(buf[2]) | (int(buf[3]) << 8)
w2 = int(buf[4]) | (int(buf[5]) << 8)
# Старт свипа: три слова 0xFFFF + маркер 0x0A в байте 6, канал в байте 7
if w0 == 0xFFFF and w1 == 0xFFFF and w2 == 0xFFFF and buf[6] == 0x0A:
ch_new = buf[7]
for ev in events:
tag = ev[0]
if tag == "start":
ch_new = int(ev[1])
if self._debug:
sys.stderr.write(f"[debug] BIN: старт свипа, ch={ch_new}\n")
_dbg_sweep_count += 1
@ -305,41 +191,22 @@ class SweepReader(threading.Thread):
cur_channels.clear()
cur_channel = ch_new
cur_channels.add(cur_channel)
del buf[:8]
_dbg_byte_count += 8
continue
# Точка данных: маркер 0x0A в байте 6, канал в байте 7
if buf[6] == 0x0A:
ch_from_term = buf[7]
if cur_channel is None:
cur_channel = ch_from_term
cur_channels.add(cur_channel)
xs.append(w0)
value_u32 = (w1 << 16) | w2
ys.append(self._u32_to_i32(value_u32))
del buf[:8]
_dbg_byte_count += 8
_dbg_point_count += 1
if self._debug and _dbg_point_count <= 3:
sys.stderr.write(
f"[debug] BIN точка: step={w0} hi={w1:#06x} lo={w2:#06x} "
f"ch={ch_from_term} → value={self._u32_to_i32((w1 << 16) | w2)}\n"
)
continue
# Поток не выровнен; сдвигаемся на 1 байт до ресинхронизации.
_dbg_desync_count += 1
_dbg_byte_count += 1
if self._debug and _dbg_desync_count <= 8:
hex6 = " ".join(f"{buf[k]:02x}" for k in range(min(8, len(buf))))
_tag, ch_from_term, step, value_i32 = ev # type: ignore[misc]
if cur_channel is None:
cur_channel = int(ch_from_term)
cur_channels.add(int(cur_channel))
xs.append(int(step))
ys.append(int(value_i32))
_dbg_point_count += 1
if self._debug and _dbg_point_count <= 3:
sys.stderr.write(
f"[debug] BIN десинхронизация #{_dbg_desync_count}: "
f"байты [{hex6}] не совпадают ни с одним шаблоном\n"
f"[debug] BIN точка: step={int(step)} ch={int(ch_from_term)} → value={int(value_i32)}\n"
)
if self._debug and _dbg_desync_count == 9:
sys.stderr.write("[debug] BIN: дальнейшие десинхронизации не выводятся (слишком много)\n")
del buf[:1]
_dbg_byte_count = parser.bytes_consumed
_dbg_desync_count = parser.desync_count
if self._debug and _dbg_byte_count > 0 and _dbg_byte_count % 4000 < 8:
sys.stderr.write(
@ -347,8 +214,8 @@ class SweepReader(threading.Thread):
f"десинхронизаций={_dbg_desync_count}, точек={_dbg_point_count}, свипов={_dbg_sweep_count}\n"
)
if len(buf) > 1_000_000:
del buf[:-262144]
if parser.buffered_size() > 1_000_000:
parser.clear_buffer_keep_tail(262_144)
self._finalize_current(xs, ys, cur_channels)