some fixes

This commit is contained in:
Ayzen
2026-06-04 18:33:38 +03:00
parent eacea436a4
commit 22942d9dc9
26 changed files with 1352 additions and 153 deletions
@@ -3,6 +3,8 @@
#include <algorithm>
#include <chrono>
#include <cmath>
#include <cstdint>
#include <iostream>
#include <span>
#include <stdexcept>
#include <string_view>
@@ -138,13 +140,24 @@ void DataPreprocessor::run(const std::atomic<bool>& stop_requested) {
std::vector<std::uint8_t> serialized_raw{};
ipc::RawSweepCollection raw_collection{};
std::uint64_t error_count = 0;
while (!stop_requested.load(std::memory_order_relaxed)) {
if (!try_pop_raw_collection(serialized_raw, &raw_collection)) {
continue;
try {
if (!try_pop_raw_collection(serialized_raw, &raw_collection)) {
continue;
}
const auto preprocessed_collection = preprocess_collection(raw_collection);
publish_preprocessed_collection(preprocessed_collection);
} catch (const std::exception& exc) {
// A single malformed or transiently-bad collection must not kill the
// long-running preprocessor: drop it and keep serving the next sweep.
// Logging is throttled so a persistent error cannot flood the log.
if (error_count % 100 == 0) {
std::cerr << "data_preprocessor: dropped collection after error (count="
<< (error_count + 1) << "): " << exc.what() << '\n';
}
++error_count;
}
const auto preprocessed_collection = preprocess_collection(raw_collection);
publish_preprocessed_collection(preprocessed_collection);
}
}