some fixes
This commit is contained in:
@@ -3,6 +3,7 @@
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <span>
|
||||
#include <stdexcept>
|
||||
#include <thread>
|
||||
@@ -61,71 +62,84 @@ void DataProcessor::run(const std::atomic<bool>& stop_requested) {
|
||||
const std::size_t history_limit = replay_history_limit(config_);
|
||||
std::uint64_t last_replayed_revision = live_config_loader_.revision();
|
||||
std::uint64_t last_applied_history_command_seq = 0;
|
||||
std::uint64_t error_count = 0;
|
||||
|
||||
while (!stop_requested.load(std::memory_order_relaxed)) {
|
||||
const auto live_config_raw = live_config_loader_.refresh_if_needed();
|
||||
const auto live_config = resolve_effective_live_config(live_config_raw);
|
||||
const auto live_revision = live_config_loader_.revision();
|
||||
auto& processor = resolve_processor(live_config);
|
||||
try {
|
||||
const auto live_config_raw = live_config_loader_.refresh_if_needed();
|
||||
const auto live_config = resolve_effective_live_config(live_config_raw);
|
||||
const auto live_revision = live_config_loader_.revision();
|
||||
auto& processor = resolve_processor(live_config);
|
||||
|
||||
if (live_revision != last_replayed_revision) {
|
||||
if (live_config.history_command_seq > last_applied_history_command_seq) {
|
||||
if (live_config.history_command == HistoryCommand::RemoveLast) {
|
||||
if (!preprocessed_history.empty()) {
|
||||
preprocessed_history.pop_back();
|
||||
if (live_revision != last_replayed_revision) {
|
||||
if (live_config.history_command_seq > last_applied_history_command_seq) {
|
||||
if (live_config.history_command == HistoryCommand::RemoveLast) {
|
||||
if (!preprocessed_history.empty()) {
|
||||
preprocessed_history.pop_back();
|
||||
}
|
||||
} else if (live_config.history_command == HistoryCommand::ClearAll) {
|
||||
preprocessed_history.clear();
|
||||
}
|
||||
} else if (live_config.history_command == HistoryCommand::ClearAll) {
|
||||
preprocessed_history.clear();
|
||||
last_applied_history_command_seq = live_config.history_command_seq;
|
||||
}
|
||||
last_applied_history_command_seq = live_config.history_command_seq;
|
||||
}
|
||||
|
||||
if (!live_config.reprocess_current_result) {
|
||||
// Socket-fed speed updates should affect only future preprocessed collections,
|
||||
// not replay the current history entry.
|
||||
} else if (should_replay_entire_history(live_config)) {
|
||||
for (std::size_t index = 0; index < preprocessed_history.size(); ++index) {
|
||||
if (!live_config.reprocess_current_result) {
|
||||
// Socket-fed speed updates should affect only future preprocessed collections,
|
||||
// not replay the current history entry.
|
||||
} else if (should_replay_entire_history(live_config)) {
|
||||
for (std::size_t index = 0; index < preprocessed_history.size(); ++index) {
|
||||
const auto replay_result = process_collection(
|
||||
preprocessed_history[index],
|
||||
std::span<const ipc::PreprocessedCollection>(preprocessed_history.data(), index),
|
||||
processor,
|
||||
live_config
|
||||
);
|
||||
publish_result_collection(replay_result, results_ring_);
|
||||
publish_locator(replay_result, live_config);
|
||||
}
|
||||
} else if (!preprocessed_history.empty()) {
|
||||
const auto replay_result = process_collection(
|
||||
preprocessed_history[index],
|
||||
std::span<const ipc::PreprocessedCollection>(preprocessed_history.data(), index),
|
||||
preprocessed_history.back(),
|
||||
std::span<const ipc::PreprocessedCollection>(preprocessed_history.data(), preprocessed_history.size() - 1U),
|
||||
processor,
|
||||
live_config
|
||||
);
|
||||
publish_result_collection(replay_result, results_ring_);
|
||||
publish_locator(replay_result, live_config);
|
||||
}
|
||||
} else if (!preprocessed_history.empty()) {
|
||||
const auto replay_result = process_collection(
|
||||
last_replayed_revision = live_revision;
|
||||
}
|
||||
|
||||
if (preprocessed_ring_.pop(bytes)) {
|
||||
auto preprocessed = ipc::deserialize_preprocessed_collection(bytes);
|
||||
preprocessed_history.push_back(std::move(preprocessed));
|
||||
while (preprocessed_history.size() > history_limit) {
|
||||
preprocessed_history.erase(preprocessed_history.begin());
|
||||
}
|
||||
|
||||
const auto result_collection = process_collection(
|
||||
preprocessed_history.back(),
|
||||
std::span<const ipc::PreprocessedCollection>(preprocessed_history.data(), preprocessed_history.size() - 1U),
|
||||
processor,
|
||||
live_config
|
||||
);
|
||||
publish_result_collection(replay_result, results_ring_);
|
||||
publish_locator(replay_result, live_config);
|
||||
}
|
||||
last_replayed_revision = live_revision;
|
||||
}
|
||||
|
||||
if (preprocessed_ring_.pop(bytes)) {
|
||||
auto preprocessed = ipc::deserialize_preprocessed_collection(bytes);
|
||||
preprocessed_history.push_back(std::move(preprocessed));
|
||||
while (preprocessed_history.size() > history_limit) {
|
||||
preprocessed_history.erase(preprocessed_history.begin());
|
||||
publish_result_collection(result_collection, results_ring_);
|
||||
publish_locator(result_collection, live_config);
|
||||
continue;
|
||||
}
|
||||
|
||||
const auto result_collection = process_collection(
|
||||
preprocessed_history.back(),
|
||||
std::span<const ipc::PreprocessedCollection>(preprocessed_history.data(), preprocessed_history.size() - 1U),
|
||||
processor,
|
||||
live_config
|
||||
);
|
||||
publish_result_collection(result_collection, results_ring_);
|
||||
publish_locator(result_collection, live_config);
|
||||
continue;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(config_.runtime.idle_sleep_ms));
|
||||
} catch (const std::exception& exc) {
|
||||
// A single bad collection (torn ring slot, decode/processing error) must
|
||||
// not kill the long-running processor: drop it and keep going. Throttle
|
||||
// logging and pause briefly so a persistent error cannot busy-spin/flood.
|
||||
if (error_count % 100 == 0) {
|
||||
std::cerr << "data_processor: dropped collection after error (count="
|
||||
<< (error_count + 1) << "): " << exc.what() << '\n';
|
||||
}
|
||||
++error_count;
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(config_.runtime.idle_sleep_ms));
|
||||
}
|
||||
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(config_.runtime.idle_sleep_ms));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user