some fixes
This commit is contained in:
Ayzen
@@ -17,27 +17,64 @@ from python_app.storage.npz.serialize import RAW_MAGIC, serialize_trace_collecti
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Maximum number of acquisitions allowed to fail in a row before we give up and
|
||||
# let the supervisor restart the whole process. Picked high enough to survive
|
||||
# transient USB stalls (each retry triggers a full reset cycle of ~1-2s) but
|
||||
# bounded so a permanently broken device does not loop forever.
|
||||
_MAX_CONSECUTIVE_ACQUIRE_FAILURES = 20
|
||||
# Cooldown applied between a failed acquire and the next reset attempt. Stops
|
||||
# us from busy-spinning when the device keeps refusing to come back.
|
||||
_ACQUIRE_FAILURE_COOLDOWN_S = 1.0
|
||||
# The producer waits for the matrix radar forever: a device that is absent at boot
|
||||
# or disappears mid-run must never kill the producer, only make it wait. Reconnect
|
||||
# uses a capped exponential backoff so a long absence does not busy-spin, and every
|
||||
# wait is interruptible by SIGINT/SIGTERM (stop_requested) for a prompt clean exit.
|
||||
_OPEN_RETRY_MIN_S = 1.0
|
||||
_OPEN_RETRY_MAX_S = 10.0
|
||||
# Throttle open-failure logging during a long wait so a permanently absent device
|
||||
# does not flood the process log: log the first failure, then every Nth attempt.
|
||||
_OPEN_RETRY_LOG_EVERY = 30
|
||||
|
||||
|
||||
def _reset_radar_service(
|
||||
config: RunConfigModel, previous: MatrixRadarService | None
|
||||
) -> MatrixRadarService:
|
||||
"""Close `previous` (best-effort) and return a freshly opened+configured service."""
|
||||
def _open_radar_with_retry(
|
||||
config: RunConfigModel,
|
||||
previous: MatrixRadarService | None,
|
||||
stop_requested: threading.Event,
|
||||
) -> MatrixRadarService | None:
|
||||
"""Open+configure the matrix radar, retrying forever until success or stop.
|
||||
|
||||
Used for both the initial open and every in-loop reconnect, so a device that is
|
||||
absent at boot or disappears mid-run never kills the producer — it just waits.
|
||||
Returns the opened service, or ``None`` if a stop was requested before any device
|
||||
became available. Backoff is capped and every wait is interruptible by SIGTERM.
|
||||
"""
|
||||
if previous is not None:
|
||||
with suppress(Exception):
|
||||
previous.close()
|
||||
radar = create_matrix_radar_service(config)
|
||||
radar.open()
|
||||
radar.configure(config.radar.sweep)
|
||||
return radar
|
||||
|
||||
attempt = 0
|
||||
delay = _OPEN_RETRY_MIN_S
|
||||
while not stop_requested.is_set():
|
||||
radar = create_matrix_radar_service(config)
|
||||
try:
|
||||
radar.open()
|
||||
radar.configure(config.radar.sweep)
|
||||
except Exception as exc: # noqa: BLE001 — waiting for the device is the point
|
||||
with suppress(Exception):
|
||||
radar.close() # drop any partial open before the next attempt
|
||||
attempt += 1
|
||||
if attempt == 1 or attempt % _OPEN_RETRY_LOG_EVERY == 0:
|
||||
logger.warning(
|
||||
"Matrix radar not available (attempt %d); retrying every up to %.0fs "
|
||||
"until the device is present: %s",
|
||||
attempt,
|
||||
_OPEN_RETRY_MAX_S,
|
||||
exc,
|
||||
)
|
||||
if stop_requested.wait(delay):
|
||||
with suppress(Exception):
|
||||
radar.close()
|
||||
return None
|
||||
delay = min(delay * 2.0, _OPEN_RETRY_MAX_S)
|
||||
continue
|
||||
|
||||
if attempt > 0:
|
||||
logger.info("Matrix radar opened after %d attempt(s).", attempt + 1)
|
||||
return radar
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -75,52 +112,26 @@ def main() -> int:
|
||||
)
|
||||
|
||||
radar: MatrixRadarService | None = None
|
||||
consecutive_failures = 0
|
||||
try:
|
||||
radar = _reset_radar_service(config, previous=None)
|
||||
radar = _open_radar_with_retry(config, previous=None, stop_requested=stop_requested)
|
||||
if radar is None:
|
||||
return 0 # asked to stop before a device became available
|
||||
collection_id = 1
|
||||
while not stop_requested.is_set():
|
||||
collection_start = time.monotonic()
|
||||
try:
|
||||
if radar is None:
|
||||
radar = _reset_radar_service(config, previous=None)
|
||||
collection = radar.acquire_collection(collection_id=collection_id)
|
||||
except Exception as exc: # noqa: BLE001 — top-level recovery is the point
|
||||
consecutive_failures += 1
|
||||
if consecutive_failures > _MAX_CONSECUTIVE_ACQUIRE_FAILURES:
|
||||
logger.error(
|
||||
"Matrix radar acquisition failed %d times in a row; giving up. "
|
||||
"Last error: %s",
|
||||
consecutive_failures - 1,
|
||||
exc,
|
||||
)
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 — reconnect forever, never give up
|
||||
logger.warning(
|
||||
"Matrix radar acquisition failed (%d/%d), resetting service: %s",
|
||||
consecutive_failures,
|
||||
_MAX_CONSECUTIVE_ACQUIRE_FAILURES,
|
||||
"Matrix radar acquisition failed; reconnecting and waiting for the device: %s",
|
||||
exc,
|
||||
exc_info=True,
|
||||
)
|
||||
# Cooldown gives slow USB stacks (and the device firmware) time
|
||||
# to settle before the next open() attempt.
|
||||
if stop_requested.wait(_ACQUIRE_FAILURE_COOLDOWN_S):
|
||||
break
|
||||
try:
|
||||
radar = _reset_radar_service(config, previous=radar)
|
||||
except Exception as reset_exc: # noqa: BLE001
|
||||
logger.warning(
|
||||
"Matrix radar reset (%d/%d) failed, will retry: %s",
|
||||
consecutive_failures,
|
||||
_MAX_CONSECUTIVE_ACQUIRE_FAILURES,
|
||||
reset_exc,
|
||||
exc_info=True,
|
||||
)
|
||||
radar = None
|
||||
radar = _open_radar_with_retry(config, previous=radar, stop_requested=stop_requested)
|
||||
if radar is None:
|
||||
break # stop requested while waiting to reconnect
|
||||
continue
|
||||
|
||||
consecutive_failures = 0
|
||||
|
||||
payload = serialize_trace_collection(collection, RAW_MAGIC)
|
||||
if not raw_writer.push(payload):
|
||||
raise RuntimeError(
|
||||
|
||||
Reference in New Issue
Block a user