some fixes
This commit is contained in:
@@ -3,6 +3,8 @@
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <span>
|
||||
#include <stdexcept>
|
||||
#include <string_view>
|
||||
@@ -138,13 +140,24 @@ void DataPreprocessor::run(const std::atomic<bool>& stop_requested) {
|
||||
|
||||
std::vector<std::uint8_t> serialized_raw{};
|
||||
ipc::RawSweepCollection raw_collection{};
|
||||
std::uint64_t error_count = 0;
|
||||
while (!stop_requested.load(std::memory_order_relaxed)) {
|
||||
if (!try_pop_raw_collection(serialized_raw, &raw_collection)) {
|
||||
continue;
|
||||
try {
|
||||
if (!try_pop_raw_collection(serialized_raw, &raw_collection)) {
|
||||
continue;
|
||||
}
|
||||
const auto preprocessed_collection = preprocess_collection(raw_collection);
|
||||
publish_preprocessed_collection(preprocessed_collection);
|
||||
} catch (const std::exception& exc) {
|
||||
// A single malformed or transiently-bad collection must not kill the
|
||||
// long-running preprocessor: drop it and keep serving the next sweep.
|
||||
// Logging is throttled so a persistent error cannot flood the log.
|
||||
if (error_count % 100 == 0) {
|
||||
std::cerr << "data_preprocessor: dropped collection after error (count="
|
||||
<< (error_count + 1) << "): " << exc.what() << '\n';
|
||||
}
|
||||
++error_count;
|
||||
}
|
||||
|
||||
const auto preprocessed_collection = preprocess_collection(raw_collection);
|
||||
publish_preprocessed_collection(preprocessed_collection);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user