init commit

This commit is contained in:
Ayzen
2026-03-05 14:42:33 +03:00
commit fd4618b20d
964 changed files with 325114 additions and 0 deletions
+1
View File
@@ -0,0 +1 @@
"""Persistent storage interfaces and implementations."""
+13
View File
@@ -0,0 +1,13 @@
"""NPZ-based dataset storage implementation and serialization helpers."""
from python_app.storage.npz.paths import radar_key_from_config
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, RESULT_MAGIC
from python_app.storage.npz.store import NpzStore
__all__ = [
"NpzStore",
"PREPROC_MAGIC",
"RAW_MAGIC",
"RESULT_MAGIC",
"radar_key_from_config",
]
+52
View File
@@ -0,0 +1,52 @@
"""Path and naming helpers for NPZ snapshot storage."""
from __future__ import annotations
import re
def radar_key_from_config(
model_name: str,
serial: str,
sweep_start_hz: float,
sweep_stop_hz: float,
sweep_points: int,
ifbw_hz: float,
power_dbm: float,
) -> str:
"""Build deterministic key for calibration/reference set lookup."""
serial_part = serial or "no_serial"
start_token = _format_float_for_key(sweep_start_hz)
stop_token = _format_float_for_key(sweep_stop_hz)
ifbw_token = _format_float_for_key(ifbw_hz)
power_token = _format_float_for_key(power_dbm)
return (
f"{model_name}_{serial_part}"
f"_st{start_token}_sp{stop_token}"
f"_p{sweep_points}_if{ifbw_token}_pw{power_token}"
)
def collection_dir_name(index: int, collection_id: int, monotonic_ns: int) -> str:
"""Build canonical collection directory name for snapshot stages."""
return f"{index:04d}_id{int(collection_id)}_ns{int(monotonic_ns)}"
def sanitize_path_component(name: str) -> str:
"""Convert arbitrary string into filesystem-safe component."""
cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", name.strip())
cleaned = cleaned.strip("._")
return cleaned or "snapshot"
def format_float_for_key(value: float) -> str:
"""Format float compactly for deterministic key serialization."""
integer = int(round(value))
if abs(value - float(integer)) < 1e-6:
return str(integer)
return f"{value:.6f}".rstrip("0").rstrip(".")
def _format_float_for_key(value: float) -> str:
"""Private alias preserved for internal compatibility."""
return format_float_for_key(value)
+77
View File
@@ -0,0 +1,77 @@
"""Binary payload serialization for sweep/result collections."""
from __future__ import annotations
import struct
import numpy as np
from python_app.models.dataset_model import ResultCollection, SweepCollection
RAW_MAGIC = 0x31574152
PREPROC_MAGIC = 0x31525050
RESULT_MAGIC = 0x314C5352
def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes:
"""Serialize one raw/preprocessed trace collection into ring-compatible binary format."""
buffer = bytearray()
buffer.extend(
struct.pack("<IQQI", magic, collection.collection_id, collection.monotonic_ns, len(collection.traces))
)
for trace in collection.traces:
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
s21 = np.asarray(trace.s21, dtype=np.complex64)
if freq.size != s21.size:
raise ValueError("Trace frequency and S21 sizes must match")
buffer.extend(struct.pack("<III", trace.combo.input_pos, trace.combo.output_pos, int(freq.size)))
buffer.extend(freq.astype("<f4", copy=False).tobytes())
interleaved = np.empty(freq.size * 2, dtype="<f4")
interleaved[0::2] = s21.real.astype("<f4", copy=False)
interleaved[1::2] = s21.imag.astype("<f4", copy=False)
buffer.extend(interleaved.tobytes())
return bytes(buffer)
def serialize_result_collection(collection: ResultCollection) -> bytes:
"""Serialize one processed collection with result blocks/payloads."""
buffer = bytearray()
buffer.extend(
struct.pack("<IQQI", RESULT_MAGIC, collection.collection_id, collection.monotonic_ns, len(collection.blocks))
)
for block in collection.blocks:
buffer.extend(struct.pack("<II", block.combo.input_pos, block.combo.output_pos))
buffer.extend(struct.pack("<I", len(block.payloads)))
for payload in block.payloads:
name_bytes = payload.processing_name.encode("utf-8")
if len(name_bytes) > 0xFFFF:
raise ValueError("processing_name is too long")
buffer.extend(struct.pack("<BH", payload.kind, len(name_bytes)))
buffer.extend(name_bytes)
if payload.kind == 1:
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
trace = np.asarray(payload.trace, dtype=np.complex64)
if freq.size != trace.size:
raise ValueError("Result trace frequency and values sizes must match")
buffer.extend(struct.pack("<I", int(freq.size)))
buffer.extend(freq.astype("<f4", copy=False).tobytes())
interleaved = np.empty(freq.size * 2, dtype="<f4")
interleaved[0::2] = trace.real.astype("<f4", copy=False)
interleaved[1::2] = trace.imag.astype("<f4", copy=False)
buffer.extend(interleaved.tobytes())
elif payload.kind == 2:
buffer.extend(struct.pack("<f", float(payload.scalar_value)))
else:
raise ValueError(f"Unsupported payload kind: {payload.kind}")
return bytes(buffer)
+225
View File
@@ -0,0 +1,225 @@
"""Snapshot selection and filesystem writers for runtime collection histories."""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any, TypeVar
import numpy as np
from python_app.models.dataset_model import ResultCollection, SweepCollection
from python_app.storage.npz.paths import collection_dir_name, sanitize_path_component
from python_app.storage.npz.serialize import serialize_result_collection, serialize_trace_collection
TCollection = TypeVar("TCollection")
def select_aligned_histories(
raw_history: list[SweepCollection],
preprocessed_history: list[SweepCollection],
result_history: list[ResultCollection],
last_n: int,
) -> tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection], dict[str, Any]]:
"""Select history tails prioritizing currently displayed processed results."""
raw_index, _ = _index_by_collection_sequence(raw_history)
pre_index, _ = _index_by_collection_sequence(preprocessed_history)
result_index, result_pos = _index_by_collection_sequence(result_history)
if result_index:
ordered_result_keys = sorted(result_index, key=lambda key: result_pos[key])
selected_keys = ordered_result_keys[-last_n:]
return (
[raw_index[key] for key in selected_keys if key in raw_index],
[pre_index[key] for key in selected_keys if key in pre_index],
[result_index[key] for key in selected_keys],
{
"selection_mode": "result_tail_with_optional_alignment",
"selected_collection_ids": [int(key[0]) for key in selected_keys],
},
)
return (
raw_history[-last_n:],
preprocessed_history[-last_n:],
result_history[-last_n:],
{
"selection_mode": "independent_tail",
"selected_collection_ids": [],
},
)
def save_trace_history_binary(stage_dir: Path, history: list[SweepCollection], magic: int) -> None:
"""Write binary trace history with lightweight metadata sidecars."""
stage_dir.mkdir(parents=True, exist_ok=True)
for index, collection in enumerate(history):
binary_path = stage_dir / f"{index:04d}.bin"
metadata_path = stage_dir / f"{index:04d}.json"
binary_path.write_bytes(serialize_trace_collection(collection, magic))
metadata_path.write_text(
json.dumps(
{
"collection_id": collection.collection_id,
"monotonic_ns": collection.monotonic_ns,
"trace_count": len(collection.traces),
},
indent=2,
),
encoding="utf-8",
)
def save_result_history_binary(stage_dir: Path, history: list[ResultCollection]) -> None:
"""Write binary processed-result history with metadata sidecars."""
stage_dir.mkdir(parents=True, exist_ok=True)
for index, collection in enumerate(history):
binary_path = stage_dir / f"{index:04d}.bin"
metadata_path = stage_dir / f"{index:04d}.json"
binary_path.write_bytes(serialize_result_collection(collection))
metadata_path.write_text(
json.dumps(
{
"collection_id": collection.collection_id,
"monotonic_ns": collection.monotonic_ns,
"block_count": len(collection.blocks),
},
indent=2,
),
encoding="utf-8",
)
def save_trace_history_numpy(stage_dir: Path, history: list[SweepCollection]) -> None:
"""Write raw/preprocessed collections as NumPy directory tree."""
stage_dir.mkdir(parents=True, exist_ok=True)
for index, collection in enumerate(history):
collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns)
collection_dir.mkdir(parents=True, exist_ok=False)
traces_meta: list[dict[str, int | str]] = []
for trace in collection.traces:
tag = f"i{trace.combo.input_pos}_o{trace.combo.output_pos}"
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
s21 = np.asarray(trace.s21, dtype=np.complex64)
np.save(collection_dir / f"{tag}_freq.npy", freq)
np.save(collection_dir / f"{tag}_s21.npy", s21)
traces_meta.append(
{
"input": int(trace.combo.input_pos),
"output": int(trace.combo.output_pos),
"points": int(freq.size),
"freq_file": f"{tag}_freq.npy",
"s21_file": f"{tag}_s21.npy",
}
)
(collection_dir / "meta.json").write_text(
json.dumps(
{
"collection_id": int(collection.collection_id),
"monotonic_ns": int(collection.monotonic_ns),
"trace_count": len(collection.traces),
"traces": traces_meta,
},
indent=2,
),
encoding="utf-8",
)
def save_result_history_numpy(stage_dir: Path, history: list[ResultCollection]) -> None:
"""Write processed result collections as NumPy directory tree."""
stage_dir.mkdir(parents=True, exist_ok=True)
for index, collection in enumerate(history):
collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns)
collection_dir.mkdir(parents=True, exist_ok=False)
blocks_meta: list[dict[str, int | str | list[dict[str, int | str | float]]]] = []
for block_index, block in enumerate(collection.blocks):
block_dir = collection_dir / f"block_{block_index:03d}_i{block.combo.input_pos}_o{block.combo.output_pos}"
block_dir.mkdir(parents=True, exist_ok=False)
payload_meta: list[dict[str, int | str | float]] = []
for payload_index, payload in enumerate(block.payloads):
safe_name = sanitize_path_component(payload.processing_name or "processor")
base_name = f"{payload_index:03d}_{safe_name}_kind{payload.kind}"
if payload.kind == 1:
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
trace = np.asarray(payload.trace, dtype=np.complex64)
np.save(block_dir / f"{base_name}_freq.npy", freq)
np.save(block_dir / f"{base_name}_trace.npy", trace)
payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"points": int(freq.size),
"freq_file": f"{base_name}_freq.npy",
"trace_file": f"{base_name}_trace.npy",
}
)
elif payload.kind == 2:
scalar = np.asarray([float(payload.scalar_value)], dtype=np.float32)
np.save(block_dir / f"{base_name}_scalar.npy", scalar)
payload_meta.append(
{
"kind": int(payload.kind),
"name": payload.processing_name,
"scalar_file": f"{base_name}_scalar.npy",
"scalar_value": float(payload.scalar_value),
}
)
blocks_meta.append(
{
"input": int(block.combo.input_pos),
"output": int(block.combo.output_pos),
"payload_count": len(block.payloads),
"dir": block_dir.name,
"payloads": payload_meta,
}
)
(collection_dir / "meta.json").write_text(
json.dumps(
{
"collection_id": int(collection.collection_id),
"monotonic_ns": int(collection.monotonic_ns),
"block_count": len(collection.blocks),
"blocks": blocks_meta,
},
indent=2,
),
encoding="utf-8",
)
def validate_snapshot_name(snapshot_name: str) -> str:
"""Normalize snapshot directory stem."""
return sanitize_path_component(snapshot_name)
def _index_by_collection_sequence(
history: list[TCollection],
) -> tuple[dict[tuple[int, int], TCollection], dict[tuple[int, int], int]]:
"""Index collections by latest-first `(collection_id, occurrence)` preserving history position.
Occurrence is counted from the tail (newest item is occurrence `0` for its
collection id). This avoids cross-run misalignment when collection ids are
reused after restarts and one stage keeps longer history than another.
"""
indexed: dict[tuple[int, int], TCollection] = {}
positions: dict[tuple[int, int], int] = {}
seen_count_from_tail_by_id: dict[int, int] = {}
# Walk from newest to oldest so occurrence=0 always means the most recent
# instance for the given collection id.
for index in range(len(history) - 1, -1, -1):
item = history[index]
collection_id = int(getattr(item, "collection_id"))
occurrence = seen_count_from_tail_by_id.get(collection_id, 0)
key = (collection_id, occurrence)
indexed[key] = item
positions[key] = index
seen_count_from_tail_by_id[collection_id] = occurrence + 1
return indexed, positions
+204
View File
@@ -0,0 +1,204 @@
"""Concrete :class:`StoreApi` implementation backed by NPZ files."""
from __future__ import annotations
from datetime import datetime
import json
from pathlib import Path
from typing import Any
import numpy as np
from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData
from python_app.storage.npz.paths import radar_key_from_config, sanitize_path_component
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection
from python_app.storage.npz.snapshot_numpy import (
save_result_history_binary,
save_result_history_numpy,
save_trace_history_binary,
save_trace_history_numpy,
select_aligned_histories,
)
from python_app.storage.store_api import StoreApi
class NpzStore(StoreApi):
"""Persist calibration/reference sets and runtime snapshots using NumPy files."""
def __init__(self, root_dir: Path) -> None:
"""Create store rooted at `root_dir`."""
self._root_dir = root_dir
self._root_dir.mkdir(parents=True, exist_ok=True)
def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None:
"""Persist named calibration/reference set as NPZ and metadata JSON."""
set_dir = self._set_dir(kind, radar_key)
set_dir.mkdir(parents=True, exist_ok=True)
npz_path = set_dir / f"{set_name}.npz"
meta_path = set_dir / f"{set_name}.json"
payload: dict[str, np.ndarray] = {}
combo_records: list[dict[str, str | int]] = []
for trace in collection.traces:
suffix = f"i{trace.combo.input_pos}_o{trace.combo.output_pos}"
freq_key = f"freq_{suffix}"
s21_key = f"s21_{suffix}"
payload[freq_key] = np.asarray(trace.frequency_hz, dtype=np.float32)
payload[s21_key] = np.asarray(trace.s21, dtype=np.complex64)
combo_records.append(
{
"input": trace.combo.input_pos,
"output": trace.combo.output_pos,
"freq_key": freq_key,
"s21_key": s21_key,
}
)
np.savez(npz_path, **payload)
meta = {
"collection_id": int(collection.collection_id),
"monotonic_ns": int(collection.monotonic_ns),
"combos": combo_records,
}
meta_path.write_text(json.dumps(meta, indent=2), encoding="utf-8")
def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection:
"""Load named calibration/reference set from NPZ representation."""
set_dir = self._set_dir(kind, radar_key)
npz_path = set_dir / f"{set_name}.npz"
meta_path = set_dir / f"{set_name}.json"
if not npz_path.exists() or not meta_path.exists():
raise FileNotFoundError(f"Missing set files for {kind}/{radar_key}/{set_name}")
meta = json.loads(meta_path.read_text(encoding="utf-8"))
arrays = np.load(npz_path)
traces: list[TraceData] = []
for combo in meta["combos"]:
freq = np.asarray(arrays[combo["freq_key"]], dtype=np.float32)
s21 = np.asarray(arrays[combo["s21_key"]], dtype=np.complex64)
traces.append(
TraceData(
combo=ComboKey(input_pos=int(combo["input"]), output_pos=int(combo["output"])),
frequency_hz=freq,
s21=s21,
)
)
return SweepCollection(
collection_id=int(meta["collection_id"]),
monotonic_ns=int(meta["monotonic_ns"]),
traces=traces,
)
def list_sets(self, kind: str, radar_key: str) -> list[str]:
"""List available set names for `(kind, radar_key)`."""
set_dir = self._set_dir(kind, radar_key)
if not set_dir.exists():
return []
return sorted(path.stem for path in set_dir.glob("*.npz"))
def has_combo_coverage(self, kind: str, radar_key: str, set_name: str, combos: list[ComboKey]) -> bool:
"""Validate that named set covers all required switch combinations."""
collection = self.load_set(kind, radar_key, set_name)
existing = {(trace.combo.input_pos, trace.combo.output_pos) for trace in collection.traces}
required = {(combo.input_pos, combo.output_pos) for combo in combos}
return required.issubset(existing)
def export_set_bundle(self, kind: str, radar_key: str, set_name: str, output_path: Path) -> Path:
"""Export named set as binary collection bundle for C++ preprocessing stage."""
collection = self.load_set(kind, radar_key, set_name)
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(serialize_trace_collection(collection, RAW_MAGIC))
return output_path
def save_runtime_snapshot(
self,
output_dir: Path,
raw_history: list[SweepCollection],
preprocessed_history: list[SweepCollection],
result_history: list[ResultCollection],
last_n: int,
) -> Path:
"""Save historical runtime collections using binary on-disk format."""
if last_n <= 0:
raise ValueError("last_n must be > 0")
output_dir.mkdir(parents=True, exist_ok=True)
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
snapshot_dir = output_dir / f"snapshot_{timestamp}"
snapshot_dir.mkdir(parents=True, exist_ok=True)
save_trace_history_binary(snapshot_dir / "raw", raw_history[-last_n:], RAW_MAGIC)
save_trace_history_binary(snapshot_dir / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC)
save_result_history_binary(snapshot_dir / "results", result_history[-last_n:])
return snapshot_dir
def save_runtime_snapshot_numpy(
self,
output_root_dir: Path,
snapshot_name: str,
raw_history: list[SweepCollection],
preprocessed_history: list[SweepCollection],
result_history: list[ResultCollection],
last_n: int,
) -> tuple[Path, dict[str, Any]]:
"""Save historical runtime collections in NumPy tree format."""
if last_n <= 0:
raise ValueError("last_n must be > 0")
snapshot_stem = snapshot_name.strip() or datetime.utcnow().strftime("snapshot_%Y%m%d_%H%M%S")
snapshot_stem = sanitize_path_component(snapshot_stem)
output_root_dir.mkdir(parents=True, exist_ok=True)
snapshot_dir = output_root_dir / snapshot_stem
if snapshot_dir.exists():
raise FileExistsError(f"Snapshot directory already exists: {snapshot_dir}")
snapshot_dir.mkdir(parents=True, exist_ok=False)
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
raw_history,
preprocessed_history,
result_history,
last_n,
)
save_trace_history_numpy(snapshot_dir / "raw", selected_raw)
save_trace_history_numpy(snapshot_dir / "preprocessed", selected_preprocessed)
save_result_history_numpy(snapshot_dir / "results", selected_results)
(snapshot_dir / "manifest.json").write_text(
json.dumps(
{
"format": "numpy-directory-v1",
"selection_mode": selection_summary["selection_mode"],
"selected_collection_ids": selection_summary["selected_collection_ids"],
"raw_collections": len(selected_raw),
"preprocessed_collections": len(selected_preprocessed),
"result_collections": len(selected_results),
"last_n_requested": int(last_n),
"raw_history_size": len(raw_history),
"preprocessed_history_size": len(preprocessed_history),
"result_history_size": len(result_history),
},
indent=2,
),
encoding="utf-8",
)
selection_summary["raw_count"] = len(selected_raw)
selection_summary["preprocessed_count"] = len(selected_preprocessed)
selection_summary["result_count"] = len(selected_results)
selection_summary["snapshot_dir"] = str(snapshot_dir)
return snapshot_dir, selection_summary
def _set_dir(self, kind: str, radar_key: str) -> Path:
"""Return directory for set kind and radar key."""
return self._root_dir / kind / radar_key
__all__ = ["NpzStore", "radar_key_from_config"]
+21
View File
@@ -0,0 +1,21 @@
"""Facade exports for NPZ-based storage implementation."""
from python_app.storage.npz.paths import (
collection_dir_name as _collection_dir_name,
format_float_for_key as _format_float_for_key,
radar_key_from_config,
sanitize_path_component as _sanitize_path_component,
)
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, RESULT_MAGIC
from python_app.storage.npz.store import NpzStore
__all__ = [
"NpzStore",
"PREPROC_MAGIC",
"RAW_MAGIC",
"RESULT_MAGIC",
"_collection_dir_name",
"_format_float_for_key",
"_sanitize_path_component",
"radar_key_from_config",
]
+37
View File
@@ -0,0 +1,37 @@
"""Abstract storage API for calibration/reference sets."""
from __future__ import annotations
from abc import ABC, abstractmethod
from pathlib import Path
from python_app.models.dataset_model import ComboKey, SweepCollection
class StoreApi(ABC):
"""Storage contract used by workflows and GUI code."""
@abstractmethod
def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None:
"""Persist a named set."""
raise NotImplementedError
@abstractmethod
def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection:
"""Load a named set."""
raise NotImplementedError
@abstractmethod
def list_sets(self, kind: str, radar_key: str) -> list[str]:
"""List set names for kind/radar key."""
raise NotImplementedError
@abstractmethod
def has_combo_coverage(self, kind: str, radar_key: str, set_name: str, combos: list[ComboKey]) -> bool:
"""Check whether stored set covers requested switch combinations."""
raise NotImplementedError
@abstractmethod
def export_set_bundle(self, kind: str, radar_key: str, set_name: str, output_path: Path) -> Path:
"""Export set into binary bundle file for C++ preprocessing."""
raise NotImplementedError