init commit
This commit is contained in:
Ayzen
@@ -0,0 +1,13 @@
|
||||
"""NPZ-based dataset storage implementation and serialization helpers."""
|
||||
|
||||
from python_app.storage.npz.paths import radar_key_from_config
|
||||
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, RESULT_MAGIC
|
||||
from python_app.storage.npz.store import NpzStore
|
||||
|
||||
__all__ = [
|
||||
"NpzStore",
|
||||
"PREPROC_MAGIC",
|
||||
"RAW_MAGIC",
|
||||
"RESULT_MAGIC",
|
||||
"radar_key_from_config",
|
||||
]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Path and naming helpers for NPZ snapshot storage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def radar_key_from_config(
|
||||
model_name: str,
|
||||
serial: str,
|
||||
sweep_start_hz: float,
|
||||
sweep_stop_hz: float,
|
||||
sweep_points: int,
|
||||
ifbw_hz: float,
|
||||
power_dbm: float,
|
||||
) -> str:
|
||||
"""Build deterministic key for calibration/reference set lookup."""
|
||||
serial_part = serial or "no_serial"
|
||||
start_token = _format_float_for_key(sweep_start_hz)
|
||||
stop_token = _format_float_for_key(sweep_stop_hz)
|
||||
ifbw_token = _format_float_for_key(ifbw_hz)
|
||||
power_token = _format_float_for_key(power_dbm)
|
||||
return (
|
||||
f"{model_name}_{serial_part}"
|
||||
f"_st{start_token}_sp{stop_token}"
|
||||
f"_p{sweep_points}_if{ifbw_token}_pw{power_token}"
|
||||
)
|
||||
|
||||
|
||||
def collection_dir_name(index: int, collection_id: int, monotonic_ns: int) -> str:
|
||||
"""Build canonical collection directory name for snapshot stages."""
|
||||
return f"{index:04d}_id{int(collection_id)}_ns{int(monotonic_ns)}"
|
||||
|
||||
|
||||
def sanitize_path_component(name: str) -> str:
|
||||
"""Convert arbitrary string into filesystem-safe component."""
|
||||
cleaned = re.sub(r"[^A-Za-z0-9._-]+", "_", name.strip())
|
||||
cleaned = cleaned.strip("._")
|
||||
return cleaned or "snapshot"
|
||||
|
||||
|
||||
def format_float_for_key(value: float) -> str:
|
||||
"""Format float compactly for deterministic key serialization."""
|
||||
integer = int(round(value))
|
||||
if abs(value - float(integer)) < 1e-6:
|
||||
return str(integer)
|
||||
return f"{value:.6f}".rstrip("0").rstrip(".")
|
||||
|
||||
|
||||
def _format_float_for_key(value: float) -> str:
|
||||
"""Private alias preserved for internal compatibility."""
|
||||
return format_float_for_key(value)
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Binary payload serialization for sweep/result collections."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.models.dataset_model import ResultCollection, SweepCollection
|
||||
|
||||
RAW_MAGIC = 0x31574152
|
||||
PREPROC_MAGIC = 0x31525050
|
||||
RESULT_MAGIC = 0x314C5352
|
||||
|
||||
|
||||
def serialize_trace_collection(collection: SweepCollection, magic: int) -> bytes:
|
||||
"""Serialize one raw/preprocessed trace collection into ring-compatible binary format."""
|
||||
buffer = bytearray()
|
||||
buffer.extend(
|
||||
struct.pack("<IQQI", magic, collection.collection_id, collection.monotonic_ns, len(collection.traces))
|
||||
)
|
||||
|
||||
for trace in collection.traces:
|
||||
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
|
||||
s21 = np.asarray(trace.s21, dtype=np.complex64)
|
||||
if freq.size != s21.size:
|
||||
raise ValueError("Trace frequency and S21 sizes must match")
|
||||
|
||||
buffer.extend(struct.pack("<III", trace.combo.input_pos, trace.combo.output_pos, int(freq.size)))
|
||||
buffer.extend(freq.astype("<f4", copy=False).tobytes())
|
||||
|
||||
interleaved = np.empty(freq.size * 2, dtype="<f4")
|
||||
interleaved[0::2] = s21.real.astype("<f4", copy=False)
|
||||
interleaved[1::2] = s21.imag.astype("<f4", copy=False)
|
||||
buffer.extend(interleaved.tobytes())
|
||||
|
||||
return bytes(buffer)
|
||||
|
||||
|
||||
def serialize_result_collection(collection: ResultCollection) -> bytes:
|
||||
"""Serialize one processed collection with result blocks/payloads."""
|
||||
buffer = bytearray()
|
||||
buffer.extend(
|
||||
struct.pack("<IQQI", RESULT_MAGIC, collection.collection_id, collection.monotonic_ns, len(collection.blocks))
|
||||
)
|
||||
|
||||
for block in collection.blocks:
|
||||
buffer.extend(struct.pack("<II", block.combo.input_pos, block.combo.output_pos))
|
||||
buffer.extend(struct.pack("<I", len(block.payloads)))
|
||||
|
||||
for payload in block.payloads:
|
||||
name_bytes = payload.processing_name.encode("utf-8")
|
||||
if len(name_bytes) > 0xFFFF:
|
||||
raise ValueError("processing_name is too long")
|
||||
|
||||
buffer.extend(struct.pack("<BH", payload.kind, len(name_bytes)))
|
||||
buffer.extend(name_bytes)
|
||||
|
||||
if payload.kind == 1:
|
||||
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
|
||||
trace = np.asarray(payload.trace, dtype=np.complex64)
|
||||
if freq.size != trace.size:
|
||||
raise ValueError("Result trace frequency and values sizes must match")
|
||||
|
||||
buffer.extend(struct.pack("<I", int(freq.size)))
|
||||
buffer.extend(freq.astype("<f4", copy=False).tobytes())
|
||||
|
||||
interleaved = np.empty(freq.size * 2, dtype="<f4")
|
||||
interleaved[0::2] = trace.real.astype("<f4", copy=False)
|
||||
interleaved[1::2] = trace.imag.astype("<f4", copy=False)
|
||||
buffer.extend(interleaved.tobytes())
|
||||
elif payload.kind == 2:
|
||||
buffer.extend(struct.pack("<f", float(payload.scalar_value)))
|
||||
else:
|
||||
raise ValueError(f"Unsupported payload kind: {payload.kind}")
|
||||
|
||||
return bytes(buffer)
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Snapshot selection and filesystem writers for runtime collection histories."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any, TypeVar
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.models.dataset_model import ResultCollection, SweepCollection
|
||||
from python_app.storage.npz.paths import collection_dir_name, sanitize_path_component
|
||||
from python_app.storage.npz.serialize import serialize_result_collection, serialize_trace_collection
|
||||
|
||||
TCollection = TypeVar("TCollection")
|
||||
|
||||
|
||||
def select_aligned_histories(
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
) -> tuple[list[SweepCollection], list[SweepCollection], list[ResultCollection], dict[str, Any]]:
|
||||
"""Select history tails prioritizing currently displayed processed results."""
|
||||
raw_index, _ = _index_by_collection_sequence(raw_history)
|
||||
pre_index, _ = _index_by_collection_sequence(preprocessed_history)
|
||||
result_index, result_pos = _index_by_collection_sequence(result_history)
|
||||
|
||||
if result_index:
|
||||
ordered_result_keys = sorted(result_index, key=lambda key: result_pos[key])
|
||||
selected_keys = ordered_result_keys[-last_n:]
|
||||
return (
|
||||
[raw_index[key] for key in selected_keys if key in raw_index],
|
||||
[pre_index[key] for key in selected_keys if key in pre_index],
|
||||
[result_index[key] for key in selected_keys],
|
||||
{
|
||||
"selection_mode": "result_tail_with_optional_alignment",
|
||||
"selected_collection_ids": [int(key[0]) for key in selected_keys],
|
||||
},
|
||||
)
|
||||
|
||||
return (
|
||||
raw_history[-last_n:],
|
||||
preprocessed_history[-last_n:],
|
||||
result_history[-last_n:],
|
||||
{
|
||||
"selection_mode": "independent_tail",
|
||||
"selected_collection_ids": [],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def save_trace_history_binary(stage_dir: Path, history: list[SweepCollection], magic: int) -> None:
|
||||
"""Write binary trace history with lightweight metadata sidecars."""
|
||||
stage_dir.mkdir(parents=True, exist_ok=True)
|
||||
for index, collection in enumerate(history):
|
||||
binary_path = stage_dir / f"{index:04d}.bin"
|
||||
metadata_path = stage_dir / f"{index:04d}.json"
|
||||
binary_path.write_bytes(serialize_trace_collection(collection, magic))
|
||||
metadata_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collection_id": collection.collection_id,
|
||||
"monotonic_ns": collection.monotonic_ns,
|
||||
"trace_count": len(collection.traces),
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def save_result_history_binary(stage_dir: Path, history: list[ResultCollection]) -> None:
|
||||
"""Write binary processed-result history with metadata sidecars."""
|
||||
stage_dir.mkdir(parents=True, exist_ok=True)
|
||||
for index, collection in enumerate(history):
|
||||
binary_path = stage_dir / f"{index:04d}.bin"
|
||||
metadata_path = stage_dir / f"{index:04d}.json"
|
||||
binary_path.write_bytes(serialize_result_collection(collection))
|
||||
metadata_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collection_id": collection.collection_id,
|
||||
"monotonic_ns": collection.monotonic_ns,
|
||||
"block_count": len(collection.blocks),
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def save_trace_history_numpy(stage_dir: Path, history: list[SweepCollection]) -> None:
|
||||
"""Write raw/preprocessed collections as NumPy directory tree."""
|
||||
stage_dir.mkdir(parents=True, exist_ok=True)
|
||||
for index, collection in enumerate(history):
|
||||
collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns)
|
||||
collection_dir.mkdir(parents=True, exist_ok=False)
|
||||
|
||||
traces_meta: list[dict[str, int | str]] = []
|
||||
for trace in collection.traces:
|
||||
tag = f"i{trace.combo.input_pos}_o{trace.combo.output_pos}"
|
||||
freq = np.asarray(trace.frequency_hz, dtype=np.float32)
|
||||
s21 = np.asarray(trace.s21, dtype=np.complex64)
|
||||
np.save(collection_dir / f"{tag}_freq.npy", freq)
|
||||
np.save(collection_dir / f"{tag}_s21.npy", s21)
|
||||
traces_meta.append(
|
||||
{
|
||||
"input": int(trace.combo.input_pos),
|
||||
"output": int(trace.combo.output_pos),
|
||||
"points": int(freq.size),
|
||||
"freq_file": f"{tag}_freq.npy",
|
||||
"s21_file": f"{tag}_s21.npy",
|
||||
}
|
||||
)
|
||||
|
||||
(collection_dir / "meta.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collection_id": int(collection.collection_id),
|
||||
"monotonic_ns": int(collection.monotonic_ns),
|
||||
"trace_count": len(collection.traces),
|
||||
"traces": traces_meta,
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def save_result_history_numpy(stage_dir: Path, history: list[ResultCollection]) -> None:
|
||||
"""Write processed result collections as NumPy directory tree."""
|
||||
stage_dir.mkdir(parents=True, exist_ok=True)
|
||||
for index, collection in enumerate(history):
|
||||
collection_dir = stage_dir / collection_dir_name(index, collection.collection_id, collection.monotonic_ns)
|
||||
collection_dir.mkdir(parents=True, exist_ok=False)
|
||||
|
||||
blocks_meta: list[dict[str, int | str | list[dict[str, int | str | float]]]] = []
|
||||
for block_index, block in enumerate(collection.blocks):
|
||||
block_dir = collection_dir / f"block_{block_index:03d}_i{block.combo.input_pos}_o{block.combo.output_pos}"
|
||||
block_dir.mkdir(parents=True, exist_ok=False)
|
||||
|
||||
payload_meta: list[dict[str, int | str | float]] = []
|
||||
for payload_index, payload in enumerate(block.payloads):
|
||||
safe_name = sanitize_path_component(payload.processing_name or "processor")
|
||||
base_name = f"{payload_index:03d}_{safe_name}_kind{payload.kind}"
|
||||
if payload.kind == 1:
|
||||
freq = np.asarray(payload.frequency_hz, dtype=np.float32)
|
||||
trace = np.asarray(payload.trace, dtype=np.complex64)
|
||||
np.save(block_dir / f"{base_name}_freq.npy", freq)
|
||||
np.save(block_dir / f"{base_name}_trace.npy", trace)
|
||||
payload_meta.append(
|
||||
{
|
||||
"kind": int(payload.kind),
|
||||
"name": payload.processing_name,
|
||||
"points": int(freq.size),
|
||||
"freq_file": f"{base_name}_freq.npy",
|
||||
"trace_file": f"{base_name}_trace.npy",
|
||||
}
|
||||
)
|
||||
elif payload.kind == 2:
|
||||
scalar = np.asarray([float(payload.scalar_value)], dtype=np.float32)
|
||||
np.save(block_dir / f"{base_name}_scalar.npy", scalar)
|
||||
payload_meta.append(
|
||||
{
|
||||
"kind": int(payload.kind),
|
||||
"name": payload.processing_name,
|
||||
"scalar_file": f"{base_name}_scalar.npy",
|
||||
"scalar_value": float(payload.scalar_value),
|
||||
}
|
||||
)
|
||||
|
||||
blocks_meta.append(
|
||||
{
|
||||
"input": int(block.combo.input_pos),
|
||||
"output": int(block.combo.output_pos),
|
||||
"payload_count": len(block.payloads),
|
||||
"dir": block_dir.name,
|
||||
"payloads": payload_meta,
|
||||
}
|
||||
)
|
||||
|
||||
(collection_dir / "meta.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"collection_id": int(collection.collection_id),
|
||||
"monotonic_ns": int(collection.monotonic_ns),
|
||||
"block_count": len(collection.blocks),
|
||||
"blocks": blocks_meta,
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def validate_snapshot_name(snapshot_name: str) -> str:
|
||||
"""Normalize snapshot directory stem."""
|
||||
return sanitize_path_component(snapshot_name)
|
||||
|
||||
|
||||
def _index_by_collection_sequence(
|
||||
history: list[TCollection],
|
||||
) -> tuple[dict[tuple[int, int], TCollection], dict[tuple[int, int], int]]:
|
||||
"""Index collections by latest-first `(collection_id, occurrence)` preserving history position.
|
||||
|
||||
Occurrence is counted from the tail (newest item is occurrence `0` for its
|
||||
collection id). This avoids cross-run misalignment when collection ids are
|
||||
reused after restarts and one stage keeps longer history than another.
|
||||
"""
|
||||
indexed: dict[tuple[int, int], TCollection] = {}
|
||||
positions: dict[tuple[int, int], int] = {}
|
||||
seen_count_from_tail_by_id: dict[int, int] = {}
|
||||
|
||||
# Walk from newest to oldest so occurrence=0 always means the most recent
|
||||
# instance for the given collection id.
|
||||
for index in range(len(history) - 1, -1, -1):
|
||||
item = history[index]
|
||||
collection_id = int(getattr(item, "collection_id"))
|
||||
occurrence = seen_count_from_tail_by_id.get(collection_id, 0)
|
||||
key = (collection_id, occurrence)
|
||||
indexed[key] = item
|
||||
positions[key] = index
|
||||
seen_count_from_tail_by_id[collection_id] = occurrence + 1
|
||||
return indexed, positions
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Concrete :class:`StoreApi` implementation backed by NPZ files."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from python_app.models.dataset_model import ComboKey, ResultCollection, SweepCollection, TraceData
|
||||
from python_app.storage.npz.paths import radar_key_from_config, sanitize_path_component
|
||||
from python_app.storage.npz.serialize import PREPROC_MAGIC, RAW_MAGIC, serialize_trace_collection
|
||||
from python_app.storage.npz.snapshot_numpy import (
|
||||
save_result_history_binary,
|
||||
save_result_history_numpy,
|
||||
save_trace_history_binary,
|
||||
save_trace_history_numpy,
|
||||
select_aligned_histories,
|
||||
)
|
||||
from python_app.storage.store_api import StoreApi
|
||||
|
||||
|
||||
class NpzStore(StoreApi):
|
||||
"""Persist calibration/reference sets and runtime snapshots using NumPy files."""
|
||||
|
||||
def __init__(self, root_dir: Path) -> None:
|
||||
"""Create store rooted at `root_dir`."""
|
||||
self._root_dir = root_dir
|
||||
self._root_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def save_set(self, kind: str, radar_key: str, set_name: str, collection: SweepCollection) -> None:
|
||||
"""Persist named calibration/reference set as NPZ and metadata JSON."""
|
||||
set_dir = self._set_dir(kind, radar_key)
|
||||
set_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
npz_path = set_dir / f"{set_name}.npz"
|
||||
meta_path = set_dir / f"{set_name}.json"
|
||||
|
||||
payload: dict[str, np.ndarray] = {}
|
||||
combo_records: list[dict[str, str | int]] = []
|
||||
|
||||
for trace in collection.traces:
|
||||
suffix = f"i{trace.combo.input_pos}_o{trace.combo.output_pos}"
|
||||
freq_key = f"freq_{suffix}"
|
||||
s21_key = f"s21_{suffix}"
|
||||
|
||||
payload[freq_key] = np.asarray(trace.frequency_hz, dtype=np.float32)
|
||||
payload[s21_key] = np.asarray(trace.s21, dtype=np.complex64)
|
||||
combo_records.append(
|
||||
{
|
||||
"input": trace.combo.input_pos,
|
||||
"output": trace.combo.output_pos,
|
||||
"freq_key": freq_key,
|
||||
"s21_key": s21_key,
|
||||
}
|
||||
)
|
||||
|
||||
np.savez(npz_path, **payload)
|
||||
meta = {
|
||||
"collection_id": int(collection.collection_id),
|
||||
"monotonic_ns": int(collection.monotonic_ns),
|
||||
"combos": combo_records,
|
||||
}
|
||||
meta_path.write_text(json.dumps(meta, indent=2), encoding="utf-8")
|
||||
|
||||
def load_set(self, kind: str, radar_key: str, set_name: str) -> SweepCollection:
|
||||
"""Load named calibration/reference set from NPZ representation."""
|
||||
set_dir = self._set_dir(kind, radar_key)
|
||||
npz_path = set_dir / f"{set_name}.npz"
|
||||
meta_path = set_dir / f"{set_name}.json"
|
||||
|
||||
if not npz_path.exists() or not meta_path.exists():
|
||||
raise FileNotFoundError(f"Missing set files for {kind}/{radar_key}/{set_name}")
|
||||
|
||||
meta = json.loads(meta_path.read_text(encoding="utf-8"))
|
||||
arrays = np.load(npz_path)
|
||||
|
||||
traces: list[TraceData] = []
|
||||
for combo in meta["combos"]:
|
||||
freq = np.asarray(arrays[combo["freq_key"]], dtype=np.float32)
|
||||
s21 = np.asarray(arrays[combo["s21_key"]], dtype=np.complex64)
|
||||
traces.append(
|
||||
TraceData(
|
||||
combo=ComboKey(input_pos=int(combo["input"]), output_pos=int(combo["output"])),
|
||||
frequency_hz=freq,
|
||||
s21=s21,
|
||||
)
|
||||
)
|
||||
|
||||
return SweepCollection(
|
||||
collection_id=int(meta["collection_id"]),
|
||||
monotonic_ns=int(meta["monotonic_ns"]),
|
||||
traces=traces,
|
||||
)
|
||||
|
||||
def list_sets(self, kind: str, radar_key: str) -> list[str]:
|
||||
"""List available set names for `(kind, radar_key)`."""
|
||||
set_dir = self._set_dir(kind, radar_key)
|
||||
if not set_dir.exists():
|
||||
return []
|
||||
return sorted(path.stem for path in set_dir.glob("*.npz"))
|
||||
|
||||
def has_combo_coverage(self, kind: str, radar_key: str, set_name: str, combos: list[ComboKey]) -> bool:
|
||||
"""Validate that named set covers all required switch combinations."""
|
||||
collection = self.load_set(kind, radar_key, set_name)
|
||||
existing = {(trace.combo.input_pos, trace.combo.output_pos) for trace in collection.traces}
|
||||
required = {(combo.input_pos, combo.output_pos) for combo in combos}
|
||||
return required.issubset(existing)
|
||||
|
||||
def export_set_bundle(self, kind: str, radar_key: str, set_name: str, output_path: Path) -> Path:
|
||||
"""Export named set as binary collection bundle for C++ preprocessing stage."""
|
||||
collection = self.load_set(kind, radar_key, set_name)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_bytes(serialize_trace_collection(collection, RAW_MAGIC))
|
||||
return output_path
|
||||
|
||||
def save_runtime_snapshot(
|
||||
self,
|
||||
output_dir: Path,
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
) -> Path:
|
||||
"""Save historical runtime collections using binary on-disk format."""
|
||||
if last_n <= 0:
|
||||
raise ValueError("last_n must be > 0")
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
|
||||
snapshot_dir = output_dir / f"snapshot_{timestamp}"
|
||||
snapshot_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
save_trace_history_binary(snapshot_dir / "raw", raw_history[-last_n:], RAW_MAGIC)
|
||||
save_trace_history_binary(snapshot_dir / "preprocessed", preprocessed_history[-last_n:], PREPROC_MAGIC)
|
||||
save_result_history_binary(snapshot_dir / "results", result_history[-last_n:])
|
||||
return snapshot_dir
|
||||
|
||||
def save_runtime_snapshot_numpy(
|
||||
self,
|
||||
output_root_dir: Path,
|
||||
snapshot_name: str,
|
||||
raw_history: list[SweepCollection],
|
||||
preprocessed_history: list[SweepCollection],
|
||||
result_history: list[ResultCollection],
|
||||
last_n: int,
|
||||
) -> tuple[Path, dict[str, Any]]:
|
||||
"""Save historical runtime collections in NumPy tree format."""
|
||||
if last_n <= 0:
|
||||
raise ValueError("last_n must be > 0")
|
||||
|
||||
snapshot_stem = snapshot_name.strip() or datetime.utcnow().strftime("snapshot_%Y%m%d_%H%M%S")
|
||||
snapshot_stem = sanitize_path_component(snapshot_stem)
|
||||
|
||||
output_root_dir.mkdir(parents=True, exist_ok=True)
|
||||
snapshot_dir = output_root_dir / snapshot_stem
|
||||
if snapshot_dir.exists():
|
||||
raise FileExistsError(f"Snapshot directory already exists: {snapshot_dir}")
|
||||
snapshot_dir.mkdir(parents=True, exist_ok=False)
|
||||
|
||||
selected_raw, selected_preprocessed, selected_results, selection_summary = select_aligned_histories(
|
||||
raw_history,
|
||||
preprocessed_history,
|
||||
result_history,
|
||||
last_n,
|
||||
)
|
||||
|
||||
save_trace_history_numpy(snapshot_dir / "raw", selected_raw)
|
||||
save_trace_history_numpy(snapshot_dir / "preprocessed", selected_preprocessed)
|
||||
save_result_history_numpy(snapshot_dir / "results", selected_results)
|
||||
|
||||
(snapshot_dir / "manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"format": "numpy-directory-v1",
|
||||
"selection_mode": selection_summary["selection_mode"],
|
||||
"selected_collection_ids": selection_summary["selected_collection_ids"],
|
||||
"raw_collections": len(selected_raw),
|
||||
"preprocessed_collections": len(selected_preprocessed),
|
||||
"result_collections": len(selected_results),
|
||||
"last_n_requested": int(last_n),
|
||||
"raw_history_size": len(raw_history),
|
||||
"preprocessed_history_size": len(preprocessed_history),
|
||||
"result_history_size": len(result_history),
|
||||
},
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
selection_summary["raw_count"] = len(selected_raw)
|
||||
selection_summary["preprocessed_count"] = len(selected_preprocessed)
|
||||
selection_summary["result_count"] = len(selected_results)
|
||||
selection_summary["snapshot_dir"] = str(snapshot_dir)
|
||||
return snapshot_dir, selection_summary
|
||||
|
||||
def _set_dir(self, kind: str, radar_key: str) -> Path:
|
||||
"""Return directory for set kind and radar key."""
|
||||
return self._root_dir / kind / radar_key
|
||||
|
||||
|
||||
__all__ = ["NpzStore", "radar_key_from_config"]
|
||||
Reference in New Issue
Block a user