some fixes

This commit is contained in:
Ayzen
2026-06-05 14:40:10 +03:00
parent 22942d9dc9
commit bbea744459
35 changed files with 1797 additions and 297 deletions
@@ -56,6 +56,9 @@ class ShmRing {
void close();
static void validate_name(const std::string& name);
// Validates that mapped_size covers the geometry declared in the header
// (fix #50: reject undersized/corrupt mappings to prevent OOB slot access).
static void validate_geometry(const Header& header, std::size_t mapped_size, const std::string& name);
};
} // namespace radar::ipc
@@ -5,6 +5,8 @@
#include <cerrno>
#include <cstring>
#include <fcntl.h>
#include <iostream>
#include <limits>
#include <new>
#include <stdexcept>
#include <string>
@@ -186,6 +188,14 @@ auto ShmRing::open_or_create(
const bool geometry_ok = header->capacity == capacity && header->slot_size_bytes == slot_size_bytes;
if (magic_ok && version_ok && geometry_ok) {
// Fix #50: the requested mapped_size matched the header geometry, but the
// backing file may have been created undersized by another process. Confirm
// st_size covers the geometry before trusting the mapping.
struct stat info {};
if (fstat(fd, &info) != 0) {
throw errno_message("Failed to stat shared memory ring", name);
}
validate_geometry(*header, static_cast<std::size_t>(info.st_size), name);
break;
}
@@ -247,6 +257,8 @@ auto ShmRing::open_existing(const std::string& name) -> ShmRing {
if (header->version != kRingVersion) {
throw std::runtime_error("Shared memory ring version mismatch for " + name);
}
// Fix #50: ensure the mapping actually spans every slot the header describes.
validate_geometry(*header, mapped_size, name);
ShmRing ring{};
ring.fd_ = scoped_fd.release();
@@ -293,17 +305,28 @@ auto ShmRing::push(std::span<const std::uint8_t> payload) -> bool {
}
const auto write_seq = header_->write_seq.load(std::memory_order_relaxed);
const auto read_seq = header_->read_seq.load(std::memory_order_acquire);
auto read_seq = header_->read_seq.load(std::memory_order_acquire);
if (checked_u64_diff(write_seq, read_seq) >= header_->capacity) {
header_->read_seq.store(read_seq + 1U, std::memory_order_release);
header_->dropped.fetch_add(1U, std::memory_order_relaxed);
// Ring full: overwrite the oldest unread slot. Advance read_seq via CAS so a
// concurrent consumer's advance is never clobbered (read_seq never moves back).
header_->read_seq.compare_exchange_strong(
read_seq, read_seq + 1U, std::memory_order_acq_rel, std::memory_order_acquire);
const auto dropped = header_->dropped.fetch_add(1U, std::memory_order_relaxed) + 1U;
if (dropped == 1U || dropped % 100U == 0U) {
std::cerr << "shm_ring: overflow overwrote an unread slot (dropped total=" << dropped << ")\n";
}
}
auto* slot = slot_header(write_seq);
// Seqlock publish: write the payload + size FIRST, then store the slot sequence
// LAST (after a release fence). A consumer that observes sequence == write_seq+1
// is therefore guaranteed a fully-written payload. (Publishing the sequence before
// the copy, as before, let a consumer copy a half-written slot.)
slot->payload_size = static_cast<std::uint32_t>(payload.size());
slot->sequence = write_seq + 1U;
std::memcpy(slot_payload(slot), payload.data(), payload.size());
std::atomic_thread_fence(std::memory_order_release);
slot->sequence = write_seq + 1U;
std::atomic_thread_fence(std::memory_order_release);
header_->write_seq.store(write_seq + 1U, std::memory_order_release);
@@ -323,21 +346,31 @@ auto ShmRing::pop(std::vector<std::uint8_t>& payload) -> bool {
auto* slot = slot_header(read_seq);
if (slot->sequence != read_seq + 1U) {
// Producer overwrote this slot before consumer read it. Resync to latest.
// Slot not published for this seq, or already overwritten (lapped). Resync.
header_->read_seq.store(write_seq, std::memory_order_release);
return false;
}
const auto payload_size = slot->payload_size;
if (payload_size > header_->slot_size_bytes) {
// Corrupt/torn slot: skip it (resync) rather than throw — a single bad slot
// must never abort the long-running consumer.
header_->read_seq.store(write_seq, std::memory_order_release);
throw std::runtime_error("Invalid payload size in shared memory slot");
return false;
}
std::atomic_thread_fence(std::memory_order_acquire);
payload.resize(payload_size);
std::memcpy(payload.data(), slot_payload(slot), payload_size);
// Seqlock verify: if the slot sequence changed during the copy, the producer
// lapped us mid-read and the payload is torn — discard and resync.
std::atomic_thread_fence(std::memory_order_acquire);
if (slot->sequence != read_seq + 1U) {
header_->read_seq.store(write_seq, std::memory_order_release);
return false;
}
header_->read_seq.store(read_seq + 1U, std::memory_order_release);
return true;
}
@@ -377,4 +410,34 @@ void ShmRing::validate_name(const std::string& name) {
}
}
void ShmRing::validate_geometry(const Header& header, std::size_t mapped_size, const std::string& name) {
// Fix #50: derive the expected size from the header's own geometry fields and
// require the real mapping to cover it. A bogus capacity/slot_size or a truncated
// mapping would otherwise yield out-of-bounds slot offsets and a SIGSEGV.
const std::uint32_t capacity = header.capacity;
const std::uint32_t slot_size_bytes = header.slot_size_bytes;
if (capacity == 0U) {
throw std::runtime_error("Shared memory ring capacity is zero for " + name);
}
if (slot_size_bytes == 0U) {
throw std::runtime_error("Shared memory ring slot size is zero for " + name);
}
constexpr auto kMax = std::numeric_limits<std::size_t>::max();
const auto slot_stride = sizeof(SlotHeader) + static_cast<std::size_t>(slot_size_bytes);
// Guard each multiply/add against std::size_t overflow before computing expected_size.
if (slot_stride > kMax / static_cast<std::size_t>(capacity)) {
throw std::runtime_error("Shared memory ring geometry overflows for " + name);
}
const auto slots_size = slot_stride * static_cast<std::size_t>(capacity);
if (slots_size > kMax - sizeof(Header)) {
throw std::runtime_error("Shared memory ring geometry overflows for " + name);
}
const auto expected_size = sizeof(Header) + slots_size;
if (mapped_size < expected_size) {
throw std::runtime_error("Shared memory ring size is too small for " + name);
}
}
} // namespace radar::ipc
@@ -6,6 +6,7 @@
#include <cstddef>
#include <cstdint>
#include <fstream>
#include <iostream>
#include <iterator>
#include <span>
#include <stdexcept>
@@ -26,6 +27,10 @@ enum class RawTraceChannel {
constexpr float kFrequencyRelativeTolerance = 1e-5F;
constexpr float kFrequencyAbsoluteTolerance = 1e-3F;
constexpr float kComplexMagnitudeSquaredEpsilon = 1e-12F;
// #30: OSL denominator must be well conditioned relative to the magnitudes of the terms forming it.
constexpr float kOslRelativeConditioning = 1e-6F;
// #30: maximum fraction of degenerate (fallback) OSL points tolerated per combo before load fails.
constexpr float kOslMaxDegenerateFraction = 0.05F;
[[nodiscard]] auto combo_to_string(const ipc::ComboKey& combo) -> std::string {
return "input=" + std::to_string(combo.input_pos) + " output=" + std::to_string(combo.output_pos);
@@ -139,7 +144,14 @@ void ensure_frequency_axis_match(
std::unordered_map<ipc::ComboKey, ChannelTrace, ipc::ComboKeyHash> traces_by_combo{};
traces_by_combo.reserve(collection.traces.size());
for (const auto& trace : collection.traces) {
traces_by_combo.insert_or_assign(trace.combo, to_channel_trace(trace, channel, bundle_label));
// #47: reject duplicate combos instead of silently overwriting an earlier trace.
const auto inserted =
traces_by_combo.insert({trace.combo, to_channel_trace(trace, channel, bundle_label)});
if (!inserted.second) {
throw std::runtime_error(
bundle_label + " bundle contains duplicate combo " + combo_to_string(trace.combo) + ": " + path
);
}
}
if (traces_by_combo.empty()) {
@@ -212,17 +224,30 @@ void ensure_combo_coverage(
coefficients.source_match.resize(open_trace.frequency_hz.size());
coefficients.reflection_tracking.resize(open_trace.frequency_hz.size());
// #30: track degenerate points so silently-substituted fallback coefficients cannot pass unnoticed.
std::size_t degenerate_points = 0;
for (std::size_t index = 0; index < open_trace.frequency_hz.size(); ++index) {
const auto load = to_std_complex(load_trace.samples[index]);
const auto open_delta = to_std_complex(open_trace.samples[index]) - load;
const auto short_delta = to_std_complex(short_trace.samples[index]) - load;
const auto denominator = open_delta - short_delta;
// #30: require the denominator to be well conditioned both absolutely and relative to the
// magnitudes of the open/short deltas it is formed from, so near-cancellations are rejected.
const auto denominator_norm = std::norm(denominator);
const auto relative_scale = std::max(std::norm(open_delta), std::norm(short_delta));
const auto relative_threshold = relative_scale * (kOslRelativeConditioning * kOslRelativeConditioning);
std::complex<float> source_match = std::complex<float>(0.0F, 0.0F);
std::complex<float> reflection_tracking = std::complex<float>(1.0F, 0.0F);
if (std::norm(denominator) > kComplexMagnitudeSquaredEpsilon) {
if (denominator_norm > kComplexMagnitudeSquaredEpsilon && denominator_norm > relative_threshold) {
source_match = (open_delta + short_delta) / denominator;
reflection_tracking = open_delta * (std::complex<float>(1.0F, 0.0F) - source_match);
} else {
// #30: degenerate point — log the offending combo/frequency and fall back to a pass-through.
++degenerate_points;
std::cerr << "S11 calibration: degenerate OSL point for combo " << combo_to_string(combo)
<< " at " << open_trace.frequency_hz[index] << " Hz (ill-conditioned denominator)\n";
}
coefficients.directivity[index] = load_trace.samples[index];
@@ -230,6 +255,20 @@ void ensure_combo_coverage(
coefficients.reflection_tracking[index] = from_std_complex(reflection_tracking);
}
// #30: refuse to load a calibration whose per-combo fallback fraction exceeds the tolerated threshold.
if (!open_trace.frequency_hz.empty()) {
const auto degenerate_fraction =
static_cast<float>(degenerate_points) / static_cast<float>(open_trace.frequency_hz.size());
if (degenerate_fraction > kOslMaxDegenerateFraction) {
throw std::runtime_error(
"S11 calibration for combo " + combo_to_string(combo) + " has " +
std::to_string(degenerate_points) + " of " +
std::to_string(open_trace.frequency_hz.size()) +
" degenerate OSL points (fraction exceeds tolerance)"
);
}
}
return coefficients;
}
@@ -301,6 +340,8 @@ void S11CalibrationBundle::load(
coefficients_by_combo_.clear();
if (open_path.empty() && short_path.empty() && load_path.empty()) {
// #38: S11 calibration disabled because no OSL paths were provided; surface it instead of silently skipping.
std::cerr << "S11 calibration: no open/short/load bundle paths provided; reflection correction disabled\n";
return;
}
if (open_path.empty() || short_path.empty() || load_path.empty()) {
@@ -384,6 +425,8 @@ void S11ReferenceBundle::load(const std::string& path) {
traces_by_combo_.clear();
if (path.empty()) {
// #38: S11 reference disabled because no path was provided; surface it instead of silently skipping.
std::cerr << "S11 reference: no bundle path provided; reference normalization disabled\n";
return;
}
@@ -4,6 +4,7 @@
#include <chrono>
#include <cstdint>
#include <iostream>
#include <optional>
#include <span>
#include <stdexcept>
#include <thread>
@@ -63,6 +64,10 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
std::uint64_t last_replayed_revision = live_config_loader_.revision();
std::uint64_t last_applied_history_command_seq = 0;
std::uint64_t error_count = 0;
// Fix #55: track the socket-fed speed used for the last reprocess so a change
// arriving without a live-config revision bump still triggers a reprocess of
// the current result (gated below by reprocess_current_result).
std::optional<double> last_reprocessed_socket_speed = std::nullopt;
while (!stop_requested.load(std::memory_order_relaxed)) {
try {
@@ -71,7 +76,17 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
const auto live_revision = live_config_loader_.revision();
auto& processor = resolve_processor(live_config);
if (live_revision != last_replayed_revision) {
// The effective config folds in latest_socket_speed(); a change there is
// a reprocess trigger on its own, even when the live revision is unchanged.
// Mirror the gating in resolve_effective_live_config(): only consider the
// socket speed when it is actually applied to the effective config.
const std::optional<double> current_socket_speed =
(live_config_raw.ignore_socket_speed || locator_server_ == nullptr)
? std::nullopt
: locator_server_->latest_socket_speed();
const bool socket_speed_dirty = current_socket_speed != last_reprocessed_socket_speed;
if (live_revision != last_replayed_revision || socket_speed_dirty) {
if (live_config.history_command_seq > last_applied_history_command_seq) {
if (live_config.history_command == HistoryCommand::RemoveLast) {
if (!preprocessed_history.empty()) {
@@ -108,6 +123,9 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
publish_locator(replay_result, live_config);
}
last_replayed_revision = live_revision;
// Fix #55: record the speed we just reprocessed with so an
// unchanged socket value does not retrigger every iteration.
last_reprocessed_socket_speed = current_socket_speed;
}
if (preprocessed_ring_.pop(bytes)) {
@@ -64,8 +64,9 @@ class ClientSession {
auto operator=(ClientSession&&) -> ClientSession& = delete;
// Start writer/reader threads. The shared `vlc` slot is notified whenever
// an inbound packet carrying a finite `vlc` field arrives.
void start(std::atomic<double>& shared_vlc_slot);
// an inbound packet carrying a finite `vlc` field arrives; the companion
// timestamp slot records when, so the server can expire stale speeds.
void start(std::atomic<double>& shared_vlc_slot, std::atomic<std::int64_t>& shared_vlc_at_ns);
// Enqueue one outbound packet. Disconnects this session if the queue is
// already full or the writer has stopped.
@@ -85,7 +86,7 @@ class ClientSession {
private:
void writer_loop();
void reader_loop(std::atomic<double>& shared_vlc_slot);
void reader_loop(std::atomic<double>& shared_vlc_slot, std::atomic<std::int64_t>& shared_vlc_at_ns);
int socket_fd_;
std::string peer_name_;
@@ -93,6 +94,9 @@ class ClientSession {
std::uint32_t max_payload_bytes_;
std::atomic<bool> stop_requested_{false};
std::atomic<bool> exited_{false};
// Counts how many of the two loops (reader, writer) have finished. The
// second one to finish flips `exited_`, so a writer-only death is reapable.
std::atomic<unsigned int> loops_finished_{0};
std::thread writer_thread_{};
std::thread reader_thread_{};
};
@@ -144,6 +148,10 @@ class TcpServer {
void enroll_client(std::unique_ptr<ClientSession> session);
void broadcast_packet(const std::vector<std::uint8_t>& packet);
void reap_finished_clients();
// Best-effort reap from the publish path: acquires clients_mutex_ via try-lock
// (skips if contended) and joins exited sessions outside the lock. Keeps
// clients_ bounded even when no new connections arrive to trigger a reap.
void try_reap_finished_clients();
void cache_latest_packet(std::vector<std::uint8_t> packet);
[[nodiscard]] auto latest_packet_copy() const -> std::optional<std::vector<std::uint8_t>>;
@@ -160,6 +168,9 @@ class TcpServer {
// Sentinel of "no value yet" is NaN. Lock-free read from data_processor.
std::atomic<double> latest_socket_speed_{};
// Monotonic (steady_clock) nanoseconds at which latest_socket_speed_ was
// last written; used to expire stale socket-fed speeds. 0 means "never set".
std::atomic<std::int64_t> latest_socket_speed_at_ns_{0};
};
} // namespace radar::locator
@@ -95,6 +95,17 @@ constexpr std::uint32_t kMinTableColumns = 3U; // [x_m, z_m, score, ...]
return stream.str();
}
// Wall-clock epoch milliseconds at packet-build time. Unlike the human-readable
// `tim` field (which has no date and rolls over at midnight), this is a
// monotonically increasing generation stamp consumers can use to compute the
// true age of a snapshot and detect a stalled pipeline.
[[nodiscard]] auto epoch_millis_now() -> std::int64_t {
using Clock = std::chrono::system_clock;
return std::chrono::duration_cast<std::chrono::milliseconds>(
Clock::now().time_since_epoch()
).count();
}
void append_u32_little_endian(std::vector<std::uint8_t>& buffer, std::uint32_t value) {
buffer.push_back(static_cast<std::uint8_t>(value & 0xFFU));
buffer.push_back(static_cast<std::uint8_t>((value >> 8U) & 0xFFU));
@@ -181,9 +192,15 @@ auto build_payload_json(
});
}
// Fix #48: stamp every packet with a numeric build-time generation
// (`gen`, epoch ms). A cached snapshot re-sent verbatim to a new client
// keeps its original `gen`, so a consumer can compute the snapshot's true
// age and flag a stalled pipeline (which `tim` alone cannot express, having
// no date and rolling over at midnight).
const Json root{
{"ver", protocol_version},
{"tim", format_timestamp_now()},
{"gen", epoch_millis_now()},
{"sts", status},
{"obs", std::move(obs_array)},
};
@@ -33,6 +33,14 @@ using Json = nlohmann::json;
constexpr std::size_t kPacketHeaderSize = 8U; // device_id u32 LE + payload_len u32 LE.
// Per-recv()/send() socket timeout. Bounds how long a wedged peer can stall a
// reader/writer loop and lets the reader re-check the stop flag periodically.
constexpr long kSocketTimeoutSeconds = 5;
// A socket-fed `vlc` speed older than this is considered stale and ignored, so
// a client that disconnects (or stops sending) cannot pin a fixed speed forever.
constexpr std::chrono::seconds kSocketSpeedStaleAfter{5};
// Best-effort full-write helper: loops over write() until everything is sent
// or an error occurs. Returns false on socket error or peer disconnect.
[[nodiscard]] auto write_all(int socket_fd, const std::uint8_t* data, std::size_t size) -> bool {
@@ -48,6 +56,9 @@ constexpr std::size_t kPacketHeaderSize = 8U; // device_id u32 LE + payload_len
if (errno == EINTR) {
continue;
}
// EAGAIN/EWOULDBLOCK here is a SO_SNDTIMEO send timeout: treat it as
// fatal so a stalled peer tears the session down rather than wedging
// the writer thread indefinitely.
return false;
}
if (chunk == 0) {
@@ -58,15 +69,25 @@ constexpr std::size_t kPacketHeaderSize = 8U; // device_id u32 LE + payload_len
return true;
}
// Best-effort exact-read helper. Returns false if the peer closed the socket
// or an unrecoverable error occurred before all bytes were read.
[[nodiscard]] auto read_exact(int socket_fd, std::uint8_t* data, std::size_t size) -> bool {
// Best-effort exact-read helper. Returns false if the peer closed the socket,
// an unrecoverable error occurred, or stop was requested before all bytes were
// read. A SO_RCVTIMEO timeout (EAGAIN/EWOULDBLOCK) is not fatal: we re-check the
// stop flag and retry so a wedged peer cannot keep the reader thread alive.
[[nodiscard]] auto read_exact(
int socket_fd,
std::uint8_t* data,
std::size_t size,
const std::atomic<bool>& stop_requested
) -> bool {
std::size_t consumed = 0;
while (consumed < size) {
if (stop_requested.load(std::memory_order_acquire)) {
return false;
}
const auto chunk = ::recv(socket_fd, data + consumed, size - consumed, 0);
if (chunk < 0) {
if (errno == EINTR) {
continue;
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {
continue; // Interrupted or recv timeout: re-check stop and retry.
}
return false;
}
@@ -107,6 +128,14 @@ void apply_socket_keepalive(int socket_fd) {
int yes = 1;
(void)::setsockopt(socket_fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes));
(void)::setsockopt(socket_fd, SOL_SOCKET, SO_KEEPALIVE, &yes, sizeof(yes));
// Bound blocking recv()/send() so a wedged peer cannot stall a reader/writer
// thread forever. recv() timeouts are retried (stop-aware); a send() timeout
// is treated as a fatal write failure that tears the session down.
timeval timeout{};
timeout.tv_sec = kSocketTimeoutSeconds;
timeout.tv_usec = 0;
(void)::setsockopt(socket_fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
(void)::setsockopt(socket_fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout));
}
void shutdown_and_close(int& socket_fd) {
@@ -192,9 +221,14 @@ ClientSession::~ClientSession() {
shutdown_and_close(socket_fd_);
}
void ClientSession::start(std::atomic<double>& shared_vlc_slot) {
void ClientSession::start(
std::atomic<double>& shared_vlc_slot,
std::atomic<std::int64_t>& shared_vlc_at_ns
) {
writer_thread_ = std::thread([this]() { writer_loop(); });
reader_thread_ = std::thread([this, &shared_vlc_slot]() { reader_loop(shared_vlc_slot); });
reader_thread_ = std::thread([this, &shared_vlc_slot, &shared_vlc_at_ns]() {
reader_loop(shared_vlc_slot, shared_vlc_at_ns);
});
}
void ClientSession::enqueue(std::vector<std::uint8_t> packet) {
@@ -245,15 +279,24 @@ void ClientSession::writer_loop() {
}
}
request_stop();
// Exited flag is set once both threads finish; reader_loop sets it.
// Mark exited only once BOTH loops have finished; otherwise a writer-only
// death (e.g. a send timeout while the reader still blocks on recv) would
// never be reaped. request_stop() above shuts the socket so the reader's
// recv() returns promptly.
if (loops_finished_.fetch_add(1, std::memory_order_acq_rel) + 1U == 2U) {
exited_.store(true, std::memory_order_release);
}
}
void ClientSession::reader_loop(std::atomic<double>& shared_vlc_slot) {
void ClientSession::reader_loop(
std::atomic<double>& shared_vlc_slot,
std::atomic<std::int64_t>& shared_vlc_at_ns
) {
std::array<std::uint8_t, kPacketHeaderSize> header_buffer{};
std::vector<std::uint8_t> payload_buffer;
while (!stop_requested_.load(std::memory_order_acquire)) {
if (!read_exact(socket_fd_, header_buffer.data(), header_buffer.size())) {
if (!read_exact(socket_fd_, header_buffer.data(), header_buffer.size(), stop_requested_)) {
break;
}
const auto payload_len = decode_u32_little_endian(header_buffer.data() + 4U);
@@ -266,7 +309,8 @@ void ClientSession::reader_loop(std::atomic<double>& shared_vlc_slot) {
}
payload_buffer.assign(payload_len, std::uint8_t{0});
if (payload_len > 0U && !read_exact(socket_fd_, payload_buffer.data(), payload_len)) {
if (payload_len > 0U
&& !read_exact(socket_fd_, payload_buffer.data(), payload_len, stop_requested_)) {
break;
}
@@ -277,7 +321,13 @@ void ClientSession::reader_loop(std::atomic<double>& shared_vlc_slot) {
if (found != json.end() && found->is_number()) {
const double value = found->get<double>();
if (std::isfinite(value)) {
// Stamp the value first, then the time, so a reader that
// observes a fresh timestamp also observes the matching value.
shared_vlc_slot.store(value, std::memory_order_release);
const auto now_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch()
).count();
shared_vlc_at_ns.store(now_ns, std::memory_order_release);
}
}
}
@@ -288,7 +338,11 @@ void ClientSession::reader_loop(std::atomic<double>& shared_vlc_slot) {
}
}
request_stop();
exited_.store(true, std::memory_order_release);
// Mark exited only once BOTH loops have finished (see writer_loop), so a
// reader-only death still waits for the writer before this session is reaped.
if (loops_finished_.fetch_add(1, std::memory_order_acq_rel) + 1U == 2U) {
exited_.store(true, std::memory_order_release);
}
}
// ----- TcpServer ------------------------------------------------------------
@@ -409,11 +463,28 @@ void TcpServer::publish(const ipc::ResultCollection& collection, const FilterPar
auto packet = encode_packet(payload_json, config_.device_id);
cache_latest_packet(packet);
broadcast_packet(packet);
// Reap exited sessions on the publish path too, so clients_ stays bounded
// even when no new connections arrive to trigger an accept-time reap. Uses a
// try-lock and joins outside the mutex, so publish() never blocks on it.
try_reap_finished_clients();
}
auto TcpServer::latest_socket_speed() const -> std::optional<double> {
// Read the timestamp first, then the value, mirroring the writer's order so a
// value seen here is at least as fresh as its timestamp.
const std::int64_t at_ns = latest_socket_speed_at_ns_.load(std::memory_order_acquire);
const double value = latest_socket_speed_.load(std::memory_order_acquire);
if (std::isnan(value)) {
if (at_ns == 0 || std::isnan(value)) {
return std::nullopt;
}
// Expire stale socket-fed speeds: a client that stopped sending (or
// disconnected) must not pin a fixed speed indefinitely.
const auto now_ns = std::chrono::duration_cast<std::chrono::nanoseconds>(
std::chrono::steady_clock::now().time_since_epoch()
).count();
const auto stale_after_ns =
std::chrono::duration_cast<std::chrono::nanoseconds>(kSocketSpeedStaleAfter).count();
if (now_ns - at_ns > stale_after_ns) {
return std::nullopt;
}
return value;
@@ -457,7 +528,7 @@ void TcpServer::acceptor_loop() {
if (snapshot.has_value()) {
session->enqueue(*snapshot);
}
session->start(latest_socket_speed_);
session->start(latest_socket_speed_, latest_socket_speed_at_ns_);
enroll_client(std::move(session));
}
}
@@ -496,6 +567,35 @@ void TcpServer::reap_finished_clients() {
}
}
void TcpServer::try_reap_finished_clients() {
std::vector<std::unique_ptr<ClientSession>> to_join;
{
// try_to_lock: if the acceptor already holds the mutex (it reaps too),
// skip this round rather than block the publish path.
std::unique_lock<std::mutex> guard(clients_mutex_, std::try_to_lock);
if (!guard.owns_lock()) {
return;
}
auto first_dead = std::partition(
clients_.begin(),
clients_.end(),
[](const std::unique_ptr<ClientSession>& session) {
return !session->has_exited();
}
);
for (auto it = first_dead; it != clients_.end(); ++it) {
to_join.push_back(std::move(*it));
}
clients_.erase(first_dead, clients_.end());
}
// Join outside the lock: a finished session's threads are done, so join()
// returns promptly without holding clients_mutex_.
for (auto& session : to_join) {
session->request_stop();
session->join();
}
}
void TcpServer::cache_latest_packet(std::vector<std::uint8_t> packet) {
std::lock_guard<std::mutex> guard(latest_packet_mutex_);
latest_packet_ = std::move(packet);
@@ -5,6 +5,7 @@
#include <chrono>
#include <cmath>
#include <complex>
#include <cstdint>
#include <exception>
#include <random>
#include <string>
@@ -21,7 +22,26 @@ namespace detail = radar::drivers::librevna::detail;
namespace {
constexpr std::uint32_t kNativeAcquireMaxAttempts = 3U;
constexpr auto kNativeSweepResponseTimeout = std::chrono::milliseconds(1500);
// Derive the budget the whole multi-point sweep is allowed to take before the
// first datapoint must arrive (#7): a fixed setup cost plus the expected dwell
// (points / IFBW) with a generous margin, clamped to the overall hard cap. The
// per-gap stall timeout then governs progress once datapoints start flowing.
[[nodiscard]] auto native_initial_sweep_budget(const config::RadarSweepSettings& sweep)
-> std::chrono::milliseconds {
const float points = std::max(1.0F, static_cast<float>(sweep.points));
const float if_bw = std::max(1.0F, sweep.if_bandwidth_hz);
const double dwell_ms =
(static_cast<double>(points) / static_cast<double>(if_bw)) * 1000.0 *
static_cast<double>(detail::kNativeSweepDwellMargin);
const double total_ms = static_cast<double>(detail::kNativeSweepSetupBudgetMs) + dwell_ms;
const auto budget = std::chrono::milliseconds(static_cast<std::int64_t>(total_ms));
return std::clamp<std::chrono::milliseconds>(
budget,
std::chrono::milliseconds(detail::kNativeSweepPerGapTimeoutMs),
std::chrono::milliseconds(detail::kNativeSweepHardCapMs)
);
}
// One synthetic GPR reflector. `range_m` is its physical depth, `reflection`
// is the dimensionless complex reflection coefficient (|Γ| ≤ 1).
@@ -78,12 +98,24 @@ constexpr auto kMockMinimumSweepDuration = std::chrono::microseconds(50);
}
[[nodiscard]] auto is_retryable_native_acquire_error(std::string_view message) -> bool {
constexpr std::array<std::string_view, 5> kRetryableSubstrings = {
// A stop request must not be retried: drain to the caller immediately (#26).
if (message.find("aborted by stop request") != std::string_view::npos) {
return false;
}
constexpr std::array<std::string_view, 8> kRetryableSubstrings = {
"Timeout waiting for expected LibreVNA packet type",
"Timeout waiting for LibreVNA ACK",
"LibreVNA returned NACK",
"Failed to read LibreVNA USB bulk packet",
"Failed to write LibreVNA USB bulk packet",
// #31: a transient USB error often triggers a brief device re-enumeration,
// so the device may be momentarily absent or its handle invalidated when
// we reconnect. Treat these as retryable; the reconnect path absorbs the
// re-enumeration with a short bounded discovery backoff.
"No compatible LibreVNA USB device found",
"LibreVNA native handle is not open",
"LibreVNA USB handle is not open",
};
return std::any_of(
@@ -153,10 +185,28 @@ auto LibreVnaMinimalDriver::acquire_sweep() -> SweepTrace {
if (!is_retryable_native_acquire_error(last_message) || attempt == kNativeAcquireMaxAttempts) {
break;
}
if (detail::native_stop_requested()) {
break; // #26: do not waste reconnect attempts while shutting down.
}
// Recover from transient USB/protocol stalls by reconnecting the device.
close_native();
open_native();
// #31: recover from transient USB/protocol stalls without
// destroying the libusb_context. Release only the interface
// and handle, then reopen against the surviving context so
// we ride out a brief device re-enumeration instead of
// re-initialising libusb from scratch on every retry.
rx_buffer_.clear();
packet_queue_.clear();
if (usb_handle_ != nullptr && interface_claimed_) {
libusb_release_interface(usb_handle_, detail::kUsbInterface);
interface_claimed_ = false;
}
if (usb_handle_ != nullptr) {
libusb_close(usb_handle_);
usb_handle_ = nullptr;
}
protocol_version_ = 0;
device_num_ports_ = 0;
open_native(); // Reuses the surviving usb_context_ and retries discovery.
}
}
@@ -282,12 +332,32 @@ auto LibreVnaMinimalDriver::acquire_native() -> SweepTrace {
std::vector<std::uint8_t> received(settings_.sweep.points, 0U);
std::uint32_t received_count = 0;
const auto deadline = std::chrono::steady_clock::now() + kNativeSweepResponseTimeout;
// #7: the per-datapoint deadline is rebased every time a fresh point lands
// (or the first point must arrive within the size-derived budget), so a
// large sweep is not killed by one fixed ~1.5s timeout. An independent hard
// cap bounds the whole sweep against a device that streams forever.
const auto now = std::chrono::steady_clock::now();
const auto hard_cap_deadline = now + std::chrono::milliseconds(detail::kNativeSweepHardCapMs);
auto gap_deadline = std::min(now + native_initial_sweep_budget(settings_.sweep), hard_cap_deadline);
while (received_count < settings_.sweep.points) {
// #26: honour a stop request observed during the (chunked) USB wait so
// SIGTERM aborts the sweep promptly instead of after the full budget.
if (detail::native_stop_requested()) {
throw std::runtime_error("Native sweep aborted by stop request");
}
if (std::chrono::steady_clock::now() >= hard_cap_deadline) {
throw std::runtime_error(
"Native sweep exceeded hard cap (" + std::to_string(received_count) + "/" +
std::to_string(settings_.sweep.points) + " points received)"
);
}
const auto wait_deadline = std::min(gap_deadline, hard_cap_deadline);
NativePacket packet{};
try {
packet = wait_for_packet(detail::kPacketVnaDatapoint, deadline);
packet = wait_for_packet(detail::kPacketVnaDatapoint, wait_deadline);
} catch (const std::exception& exception) {
throw std::runtime_error(
"Timeout while collecting VNADatapoints (" + std::to_string(received_count) + "/" +
@@ -311,6 +381,10 @@ auto LibreVnaMinimalDriver::acquire_native() -> SweepTrace {
trace.frequency_hz[datapoint.point_number] = datapoint.frequency_hz;
trace.s11[datapoint.point_number] = datapoint.s11;
trace.s21[datapoint.point_number] = datapoint.s21;
// Extend the per-gap stall timeout now that progress was made.
gap_deadline = std::chrono::steady_clock::now() +
std::chrono::milliseconds(detail::kNativeSweepPerGapTimeoutMs);
}
return trace;
@@ -3,10 +3,12 @@
#include <algorithm>
#include <array>
#include <cerrno>
#include <chrono>
#include <cstring>
#include <fcntl.h>
#include <stdexcept>
#include <string>
#include <thread>
#include <utility>
#include <vector>
@@ -104,18 +106,40 @@ namespace {
} // namespace
void LibreVnaMinimalDriver::open_native() {
if (usb_context_ != nullptr || usb_handle_ != nullptr) {
if (usb_handle_ != nullptr) {
throw std::runtime_error("LibreVNA native state is already initialized");
}
const auto init_status = libusb_init(&usb_context_);
if (init_status != LIBUSB_SUCCESS) {
usb_context_ = nullptr;
throw std::runtime_error(libusb_error_message("Failed to initialize libusb", init_status));
// #26: observe SIGINT/SIGTERM (chaining to main()'s handler) so the blocking
// USB loops can bail out promptly during shutdown. Safe to call repeatedly.
detail::install_native_stop_observer();
// #31: keep the libusb_context alive across reconnect retries. Only create a
// fresh context the first time; reconnect paths reuse the surviving context
// and merely reopen the handle, avoiding a full libusb teardown/re-init.
if (usb_context_ == nullptr) {
const auto init_status = libusb_init(&usb_context_);
if (init_status != LIBUSB_SUCCESS) {
usb_context_ = nullptr;
throw std::runtime_error(libusb_error_message("Failed to initialize libusb", init_status));
}
}
try {
auto* selected_handle = find_matching_device_handle(usb_context_, settings_.serial);
// #31: retry discovery with a short bounded backoff so a brief device
// re-enumeration (common right after a transient USB error) is absorbed
// instead of surfacing as a hard "device not found".
libusb_device_handle* selected_handle = nullptr;
for (int attempt = 0; attempt < detail::kNativeDiscoveryRetryAttempts; ++attempt) {
if (detail::native_stop_requested()) {
break; // #26: abandon discovery promptly during shutdown.
}
selected_handle = find_matching_device_handle(usb_context_, settings_.serial);
if (selected_handle != nullptr) {
break;
}
std::this_thread::sleep_for(std::chrono::milliseconds(detail::kNativeDiscoveryRetryDelayMs));
}
if (selected_handle == nullptr) {
const auto serial_hint = settings_.serial.empty() ? std::string() : " for serial '" + settings_.serial + "'";
@@ -215,6 +239,12 @@ void LibreVnaMinimalDriver::send_packet_no_payload(std::uint8_t packet_type, boo
void LibreVnaMinimalDriver::wait_for_ack(std::chrono::steady_clock::time_point deadline) {
while (std::chrono::steady_clock::now() < deadline) {
// #26: bail out promptly on SIGTERM/SIGINT rather than blocking until the
// ACK deadline expires.
if (detail::native_stop_requested()) {
throw std::runtime_error("LibreVNA ACK wait aborted by stop request");
}
std::optional<NativePacket> ack_or_nack{};
for (auto iter = packet_queue_.begin(); iter != packet_queue_.end(); ++iter) {
if (iter->packet_type != detail::kPacketAck && iter->packet_type != detail::kPacketNack) {
@@ -246,6 +276,11 @@ auto LibreVnaMinimalDriver::wait_for_packet(
if (auto packet = pop_packet(expected_type); packet.has_value()) {
return *packet;
}
// #26: re-check the stop flag between short USB polls so a shutdown is
// honoured within a few hundred ms even on a long sweep deadline.
if (detail::native_stop_requested()) {
throw std::runtime_error("LibreVNA packet wait aborted by stop request");
}
pump_usb(deadline);
}
@@ -1,6 +1,8 @@
#pragma once
#include <array>
#include <csignal>
#include <signal.h> // POSIX sigaction/NSIG for the chaining stop observer (#26).
#include <cstddef>
#include <cstdint>
#include <cstring>
@@ -51,6 +53,21 @@ constexpr std::size_t kUsbReadChunkBytes = 16 * 1024;
constexpr int kUsbReadPollMinTimeoutMs = 1;
constexpr int kUsbReadPollMaxTimeoutMs = 50;
// Native sweep timing budget (#7). The native deadline is no longer a single
// fixed value for the whole sweep: instead it is derived from the sweep size
// (fixed setup cost + per-point dwell ≈ points/IFBW) and is extended every
// time a fresh datapoint arrives via a per-gap stall timeout, while an overall
// hard cap bounds a wedged device.
constexpr int kNativeSweepSetupBudgetMs = 500;
constexpr float kNativeSweepDwellMargin = 3.0F;
constexpr int kNativeSweepPerGapTimeoutMs = 1500;
constexpr int kNativeSweepHardCapMs = 60'000;
// Reconnect backoff (#31). After a transient error the device may re-enumerate;
// retry discovery a few times with a short bounded delay before giving up.
constexpr int kNativeDiscoveryRetryAttempts = 10;
constexpr int kNativeDiscoveryRetryDelayMs = 150;
[[nodiscard]] inline auto read_u16_le(std::span<const std::uint8_t> data, std::size_t offset) -> std::uint16_t {
if ((offset + sizeof(std::uint16_t)) > data.size()) {
throw std::runtime_error("Failed to decode uint16 from payload");
@@ -108,5 +125,73 @@ inline void write_u32_le(std::span<std::uint8_t> data, std::size_t offset, std::
data[offset + 3] = static_cast<std::uint8_t>((value >> 24U) & 0xFFU);
}
// Stop responsiveness (#26). The blocking USB loops keep their per-call
// timeouts short and re-poll this flag so a SIGTERM/SIGINT delivered mid-sweep
// is honoured within a few hundred ms instead of only after the whole sweep
// budget has elapsed.
//
// `main()` installs its own SIGINT/SIGTERM handlers before any driver is
// opened, so we chain to (and preserve) whatever handler is already installed
// rather than clobbering it. All state lives in single inline-static slots so
// every translation unit observes the same flag and chain table.
[[nodiscard]] inline auto native_stop_flag() -> volatile std::sig_atomic_t& {
static volatile std::sig_atomic_t flag = 0;
return flag;
}
[[nodiscard]] inline auto native_chained_handlers() -> std::array<struct sigaction, NSIG>& {
static std::array<struct sigaction, NSIG> handlers{};
return handlers;
}
inline void native_stop_signal_handler(int signal_number) {
native_stop_flag() = 1;
// Chain to the previously installed disposition (e.g. main()'s handler) so
// process-level stop semantics are unchanged.
if (signal_number < 0 || signal_number >= NSIG) {
return;
}
const auto& prior = native_chained_handlers()[static_cast<std::size_t>(signal_number)];
if ((prior.sa_flags & SA_SIGINFO) == 0 && prior.sa_handler != nullptr &&
prior.sa_handler != SIG_DFL && prior.sa_handler != SIG_IGN &&
prior.sa_handler != native_stop_signal_handler) {
prior.sa_handler(signal_number);
}
}
// Lazily install the chaining stop observer for one signal. Must be called
// after main() has installed its handlers (i.e. from open_native()).
inline void install_native_stop_observer_for(int signal_number) {
if (signal_number < 0 || signal_number >= NSIG) {
return;
}
struct sigaction current{};
if (sigaction(signal_number, nullptr, &current) != 0) {
return;
}
if (current.sa_handler == native_stop_signal_handler) {
return; // Already installed; do not chain to ourselves.
}
native_chained_handlers()[static_cast<std::size_t>(signal_number)] = current;
struct sigaction action{};
action.sa_handler = native_stop_signal_handler;
sigemptyset(&action.sa_mask);
action.sa_flags = current.sa_flags & ~SA_SIGINFO; // Keep flags such as SA_RESTART.
sigaction(signal_number, &action, nullptr);
}
inline void install_native_stop_observer() {
install_native_stop_observer_for(SIGINT);
install_native_stop_observer_for(SIGTERM);
}
[[nodiscard]] inline auto native_stop_requested() -> bool {
return native_stop_flag() != 0;
}
} // namespace radar::drivers::librevna::detail
@@ -14,6 +14,10 @@
namespace radar::drivers {
namespace {
// A transient ioctl failure (e.g. EINTR/EAGAIN/EBUSY under load) should not be
// immediately fatal; retry a small number of times before giving up.
constexpr int kSwitchIoctlRetries = 3;
// Position mapping for H7992 control pins:
// position -> (A, B)
constexpr std::array<std::array<std::uint8_t, 2>, 4> kPositionToAB = {
@@ -44,6 +48,15 @@ void validate_open_settings(const H7992MinimalDriverSettings& settings) {
H7992MinimalDriver::H7992MinimalDriver(H7992MinimalDriverSettings settings) : settings_(std::move(settings)) {}
H7992MinimalDriver::~H7992MinimalDriver() {
// Safety net: if the owner never called close() (e.g. on crash/unwind),
// still drive the RF path to a known-safe state and release the GPIO lines.
try {
close();
} catch (...) {
}
}
void H7992MinimalDriver::open() {
if (is_open_) {
return;
@@ -134,6 +147,15 @@ void H7992MinimalDriver::open_native() {
}
void H7992MinimalDriver::close_native() {
// Leave the RF path in its known-safe default position before releasing the
// lines, so shutdown/crash never strands the switch in an arbitrary state.
if (line_fd_ >= 0 && settings_.default_position < settings_.positions) {
const auto [pin_a_state, pin_b_state] = kPositionToAB[settings_.default_position];
auto values = make_line_values(pin_a_state, pin_b_state);
// Best-effort: do not throw out of the teardown path.
::ioctl(line_fd_, GPIO_V2_LINE_SET_VALUES_IOCTL, &values);
}
if (line_fd_ >= 0) {
::close(line_fd_);
line_fd_ = -1;
@@ -152,9 +174,16 @@ void H7992MinimalDriver::switch_native(std::uint32_t position) {
const auto [pin_a_state, pin_b_state] = kPositionToAB[position];
auto values = make_line_values(pin_a_state, pin_b_state);
if (::ioctl(line_fd_, GPIO_V2_LINE_SET_VALUES_IOCTL, &values) != 0) {
throw std::runtime_error("Failed to switch GPIO state for " + settings_.name + ": " + std::strerror(errno));
// Retry transient ioctl failures so a momentary hiccup is recoverable
// instead of aborting the whole sweep; only the final attempt is fatal.
int last_errno = 0;
for (int attempt = 0; attempt < kSwitchIoctlRetries; ++attempt) {
if (::ioctl(line_fd_, GPIO_V2_LINE_SET_VALUES_IOCTL, &values) == 0) {
return;
}
last_errno = errno;
}
throw std::runtime_error("Failed to switch GPIO state for " + settings_.name + ": " + std::strerror(last_errno));
}
} // namespace radar::drivers
@@ -30,6 +30,7 @@ struct H7992MinimalDriverSettings {
class H7992MinimalDriver final : public SwitchDriver {
public:
explicit H7992MinimalDriver(H7992MinimalDriverSettings settings);
~H7992MinimalDriver() override; // Leaves RF path safe even if close() was skipped.
void open() override;
void close() override;
@@ -13,6 +13,23 @@
namespace radar::drivers {
namespace {
// A transient ioctl failure (e.g. EINTR/EAGAIN/EBUSY under load) should not be
// immediately fatal; retry a small number of times before giving up.
constexpr int kSwitchIoctlRetries = 3;
// Compute the GPIO line values for a given logical position, honouring inverted
// control logic. Shared by switch_native() and the teardown default-drive path.
[[nodiscard]] auto make_line_values(const HMC349AMinimalDriverSettings& settings, std::uint32_t position)
-> gpio_v2_line_values {
const std::uint8_t requested_state = static_cast<std::uint8_t>(position & 0x01U);
const std::uint8_t control_state = settings.invert_logic ? (requested_state ^ 0x01U) : requested_state;
gpio_v2_line_values values{};
values.mask = (1ULL << 0U);
values.bits = static_cast<std::uint64_t>(control_state);
return values;
}
void validate_open_settings(const HMC349AMinimalDriverSettings& settings) {
if (settings.positions == 0U || settings.positions > 2U) {
throw std::runtime_error("HMC349A switch positions must be in range [1, 2] for " + settings.name);
@@ -27,6 +44,15 @@ void validate_open_settings(const HMC349AMinimalDriverSettings& settings) {
HMC349AMinimalDriver::HMC349AMinimalDriver(HMC349AMinimalDriverSettings settings)
: settings_(std::move(settings)) {}
HMC349AMinimalDriver::~HMC349AMinimalDriver() {
// Safety net: if the owner never called close() (e.g. on crash/unwind),
// still drive the RF path to a known-safe state and release the GPIO lines.
try {
close();
} catch (...) {
}
}
void HMC349AMinimalDriver::open() {
if (is_open_) {
return;
@@ -117,6 +143,14 @@ void HMC349AMinimalDriver::open_native() {
}
void HMC349AMinimalDriver::close_native() {
// Leave the RF path in its known-safe default position before releasing the
// line, so shutdown/crash never strands the switch in an arbitrary state.
if (line_fd_ >= 0 && settings_.default_position < settings_.positions) {
auto values = make_line_values(settings_, settings_.default_position);
// Best-effort: do not throw out of the teardown path.
::ioctl(line_fd_, GPIO_V2_LINE_SET_VALUES_IOCTL, &values);
}
if (line_fd_ >= 0) {
::close(line_fd_);
line_fd_ = -1;
@@ -132,16 +166,18 @@ void HMC349AMinimalDriver::switch_native(std::uint32_t position) {
throw std::runtime_error("Native GPIO line fd is not open for " + settings_.name);
}
const std::uint8_t requested_state = static_cast<std::uint8_t>(position & 0x01U);
const std::uint8_t control_state = settings_.invert_logic ? (requested_state ^ 0x01U) : requested_state;
auto values = make_line_values(settings_, position);
gpio_v2_line_values values{};
values.mask = (1ULL << 0U);
values.bits = static_cast<std::uint64_t>(control_state);
if (::ioctl(line_fd_, GPIO_V2_LINE_SET_VALUES_IOCTL, &values) != 0) {
throw std::runtime_error("Failed to switch GPIO state for " + settings_.name + ": " + std::strerror(errno));
// Retry transient ioctl failures so a momentary hiccup is recoverable
// instead of aborting the whole sweep; only the final attempt is fatal.
int last_errno = 0;
for (int attempt = 0; attempt < kSwitchIoctlRetries; ++attempt) {
if (::ioctl(line_fd_, GPIO_V2_LINE_SET_VALUES_IOCTL, &values) == 0) {
return;
}
last_errno = errno;
}
throw std::runtime_error("Failed to switch GPIO state for " + settings_.name + ": " + std::strerror(last_errno));
}
} // namespace radar::drivers
@@ -30,6 +30,7 @@ struct HMC349AMinimalDriverSettings {
class HMC349AMinimalDriver final : public SwitchDriver {
public:
explicit HMC349AMinimalDriver(HMC349AMinimalDriverSettings settings);
~HMC349AMinimalDriver() override; // Leaves RF path safe even if close() was skipped.
void open() override;
void close() override;
@@ -1,7 +1,10 @@
#include "sweep_orchestrator.hpp"
#include <algorithm>
#include <chrono>
#include <cstdint>
#include <exception>
#include <iostream>
#include <stdexcept>
#include <thread>
#include <vector>
@@ -9,6 +12,16 @@
namespace radar::acq {
namespace {
// Wait-for-device retry tuning (mirror of python matrix_raw_producer._open_radar_with_retry):
// a device that is absent at boot or disappears mid-run must never kill the orchestrator,
// only make it wait. Backoff is capped so a long absence does not busy-spin, and every wait
// is interruptible by stop_requested for a prompt clean exit.
constexpr std::uint32_t kOpenRetryMinMs = 1'000U;
constexpr std::uint32_t kOpenRetryMaxMs = 10'000U;
// Throttle open-failure logging during a long wait so a permanently absent device does not
// flood the process log: log the first failure, then every Nth attempt.
constexpr std::uint64_t kOpenRetryLogEvery = 30U;
[[nodiscard]] inline auto should_stop(const std::atomic<bool>& stop_requested) -> bool {
return stop_requested.load(std::memory_order_relaxed);
}
@@ -20,6 +33,21 @@ void sleep_if_needed_ms(std::uint32_t delay_ms) {
std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms));
}
// Interruptible sleep: returns true if a stop was requested while waiting.
[[nodiscard]] auto interruptible_sleep_ms(std::uint32_t delay_ms, const std::atomic<bool>& stop_requested) -> bool {
constexpr std::uint32_t kPollMs = 50U;
std::uint32_t waited = 0U;
while (waited < delay_ms) {
if (should_stop(stop_requested)) {
return true;
}
const std::uint32_t chunk = std::min(kPollMs, delay_ms - waited);
std::this_thread::sleep_for(std::chrono::milliseconds(chunk));
waited += chunk;
}
return should_stop(stop_requested);
}
void validate_sweep(const drivers::SweepTrace& sweep) {
if (sweep.frequency_hz.size() != sweep.s11.size() || sweep.frequency_hz.size() != sweep.s21.size()) {
throw std::runtime_error("Radar driver returned inconsistent sweep vectors");
@@ -64,6 +92,41 @@ class DriverLifecycleGuard {
active_ = true;
}
// Open all devices, retrying forever with capped exponential backoff until success or
// stop. Used for both the initial open and every in-loop reopen, so a device that is
// absent at boot or disappears mid-run never kills the orchestrator — it just waits.
// Returns false only if a stop was requested before any device became available.
[[nodiscard]] auto open_all_with_retry(const std::atomic<bool>& stop_requested) -> bool {
// Drop any partial state from a previous open before retrying.
close_all();
std::uint64_t attempt = 0;
std::uint32_t delay_ms = kOpenRetryMinMs;
while (!should_stop(stop_requested)) {
try {
open_all();
if (attempt > 0) {
std::cerr << "sweep_orchestrator: devices opened after " << (attempt + 1) << " attempt(s)\n";
}
return true;
} catch (const std::exception& exc) {
// Best-effort: drop any partial open before the next attempt.
close_all();
++attempt;
if (attempt == 1 || attempt % kOpenRetryLogEvery == 0) {
std::cerr << "sweep_orchestrator: devices not available (attempt " << attempt
<< "); retrying up to every " << (kOpenRetryMaxMs / 1'000U)
<< "s until present: " << exc.what() << '\n';
}
if (interruptible_sleep_ms(delay_ms, stop_requested)) {
return false;
}
delay_ms = std::min(delay_ms * 2U, kOpenRetryMaxMs);
}
}
return false;
}
void close_all() {
if (!active_) {
return;
@@ -81,14 +144,47 @@ class DriverLifecycleGuard {
bool active_ = false;
};
void publish_collection(ipc::ShmRing& raw_ring, ipc::ShmRing* raw_tap_ring, const ipc::RawSweepCollection& collection) {
// Worst-case serialized size of a collection given the configured combo count and sweep
// point count, using the trace wire format (see ipc::write_trace_collection/write_trace_block):
// collection header: magic(4) + collection_id(8) + monotonic_ns(8) + trace_count(4)
// + capture_start_ns(8) + capture_end_ns(8) = 40 bytes
// per trace block: input_pos(4) + output_pos(4) + point_count(4) = 12 bytes
// + per point: frequency(4) + s11(8) + s21(8) = 20 bytes
[[nodiscard]] auto worst_case_serialized_bytes(std::size_t combo_count, std::uint32_t sweep_points) -> std::size_t {
constexpr std::size_t kCollectionHeaderBytes = 40U;
constexpr std::size_t kTraceHeaderBytes = 12U;
constexpr std::size_t kBytesPerPoint = 20U;
const std::size_t per_trace = kTraceHeaderBytes + (static_cast<std::size_t>(sweep_points) * kBytesPerPoint);
return kCollectionHeaderBytes + (combo_count * per_trace);
}
// Publish a collection on the primary raw ring; tap pushes are strictly best-effort.
// Never throws: an oversize payload is logged-and-dropped (the slot-size budget is also
// validated up front at startup, so this guards only against unexpected runtime growth).
// Returns true if the primary push succeeded.
[[nodiscard]] auto publish_collection(
ipc::ShmRing& raw_ring,
ipc::ShmRing* raw_tap_ring,
const ipc::RawSweepCollection& collection,
std::uint64_t& oversize_drop_count
) -> bool {
const auto serialized_collection = ipc::serialize_raw_collection(collection);
if (!raw_ring.push(serialized_collection)) {
throw std::runtime_error("Raw ring slot is too small for serialized collection");
// Log-and-drop an oversize payload rather than aborting the long-running loop.
// Throttle so a persistently oversize payload cannot flood the log.
if (oversize_drop_count % 100 == 0) {
std::cerr << "sweep_orchestrator: dropped oversize raw payload (" << serialized_collection.size()
<< " bytes > slot " << raw_ring.slot_size_bytes() << "; drop count="
<< (oversize_drop_count + 1) << ")\n";
}
++oversize_drop_count;
return false;
}
if (raw_tap_ring != nullptr && !raw_tap_ring->push(serialized_collection)) {
throw std::runtime_error("Raw tap ring slot is too small for serialized collection");
// Tap ring is for GUI/debug observers only: a tap failure must never abort the primary path.
if (raw_tap_ring != nullptr) {
(void)raw_tap_ring->push(serialized_collection);
}
return true;
}
} // namespace
@@ -109,25 +205,59 @@ SweepOrchestrator::SweepOrchestrator(
raw_tap_ring_(raw_tap_ring) {}
void SweepOrchestrator::run(const std::atomic<bool>& stop_requested) {
// Fail fast on a config error: a slot that is too small for the worst-case payload can
// never carry a full collection, so report it clearly at startup instead of dropping
// every collection at runtime (fix #27).
const auto worst_case_bytes = worst_case_serialized_bytes(config_.run_combos.size(), config_.radar.sweep.points);
if (worst_case_bytes > raw_ring_.slot_size_bytes()) {
throw std::runtime_error(
"Raw ring slot_size_bytes (" + std::to_string(raw_ring_.slot_size_bytes())
+ ") is too small for the worst-case serialized collection (" + std::to_string(worst_case_bytes)
+ " bytes for " + std::to_string(config_.run_combos.size()) + " combos x "
+ std::to_string(config_.radar.sweep.points) + " points)"
);
}
DriverLifecycleGuard lifecycle_guard(radar_driver_, input_switch_driver_, output_switch_driver_);
// Open devices once and keep them active for the whole acquisition loop.
lifecycle_guard.open_all();
// Wait for the devices to become available before starting (fix #8/#9): an absent device
// makes the orchestrator wait, not exit.
if (!lifecycle_guard.open_all_with_retry(stop_requested)) {
return; // stop requested before any device became available
}
std::uint64_t collection_id = 0;
std::uint64_t oversize_drop_count = 0;
while (!should_stop(stop_requested)) {
auto raw_collection = acquire_one_collection(++collection_id, stop_requested);
if (raw_collection.traces.empty()) {
try {
auto raw_collection = acquire_one_collection(++collection_id, stop_requested);
if (raw_collection.traces.empty()) {
if (!config_.runtime.continuous) {
break;
}
sleep_if_needed_ms(config_.runtime.idle_sleep_ms);
continue;
}
(void)publish_collection(raw_ring_, raw_tap_ring_, raw_collection, oversize_drop_count);
if (!config_.runtime.continuous) {
break;
}
sleep_if_needed_ms(config_.runtime.idle_sleep_ms);
continue;
}
publish_collection(raw_ring_, raw_tap_ring_, raw_collection);
if (!config_.runtime.continuous) {
break;
} catch (const std::exception& exc) {
// Treat any in-loop device error (timeout/NACK/transient USB or socket glitch/
// device-not-found-after-glitch) as recoverable: close, wait for the device to
// come back, and continue. This mirrors the python producer's reconnect-forever
// policy so a long-running headless appliance survives transient hardware hiccups
// instead of exiting. Genuinely fatal config errors are caught at startup above
// (and in main()), reserving non-zero exit for those.
std::cerr << "sweep_orchestrator: acquisition failed; reopening and waiting for the device: "
<< exc.what() << '\n';
if (!config_.runtime.continuous) {
throw; // single-run mode has no recovery path; surface the failure
}
if (!lifecycle_guard.open_all_with_retry(stop_requested)) {
break; // stop requested while waiting to reopen
}
}
}