some fixes and improvements

python_app/hardware_full/multi_device_service.py

@@ -20,6 +20,12 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+# Delays applied between successive USB reopen attempts inside recover(). Picked
+# to give libusb time to re-enumerate a stuck device while staying short enough
+# that a healthy reconnect feels instant. The total worst-case wait is the sum
+# of all entries (1.75 s today) plus the cost of close()/open() themselves.
+_REOPEN_BACKOFF_SECONDS: tuple[float, ...] = (0.25, 0.5, 1.0)
+
 _INPUT_S_PARAMETERS_BY_OUTPUT: dict[int, tuple[str, ...]] = {
     0: ("s31", "s41", "s51", "s61"),
     1: ("s32", "s42", "s52", "s62"),
@@ -77,18 +83,60 @@ class MultiDeviceLibreVnaService:
             self._controller = None
 
     def close(self) -> None:
-        """Close native device transports."""
-        if self._controller is not None:
-            self._controller.close()
-            self._controller = None
+        """Close native device transports; never raises.
+
+        Recovery loops rely on `close()` being safe to call on a half-open or
+        already-broken controller. We swallow any transport-level exception here
+        and just drop the reference so the next `open()` starts fresh.
+        """
+        controller = self._controller
+        self._controller = None
+        if controller is None:
+            return
+        try:
+            controller.close()
+        except Exception as exc:  # noqa: BLE001 — recovery path, never propagate
+            logger.warning("Multi-device close() ignored transport error: %s", exc)
 
     def recover(self) -> None:
-        """Reopen native device transports after a failed acquisition."""
+        """Reopen native device transports after a failed acquisition.
+
+        Tries several short backoffs so a transient USB stall does not kill the
+        producer on the very first retry. Raises the last error only after
+        every attempt failed — the outer acquisition loop is expected to count
+        these as recovery_attempts.
+        """
         if self._using_mock_backend:
             return
         self.close()
-        time.sleep(0.25)
-        self.open()
+
+        last_error: Exception | None = None
+        for attempt_index, delay_s in enumerate(_REOPEN_BACKOFF_SECONDS, start=1):
+            time.sleep(delay_s)
+            try:
+                self.open()
+                if self._controller is not None:
+                    logger.info(
+                        "Multi-device reopen succeeded on attempt %d/%d (after %.2fs)",
+                        attempt_index,
+                        len(_REOPEN_BACKOFF_SECONDS),
+                        delay_s,
+                    )
+                    return
+            except Exception as exc:  # noqa: BLE001 — propagate only the last failure
+                last_error = exc
+                logger.warning(
+                    "Multi-device reopen attempt %d/%d failed after %.2fs: %s",
+                    attempt_index,
+                    len(_REOPEN_BACKOFF_SECONDS),
+                    delay_s,
+                    exc,
+                )
+                self.close()  # tidy partially-opened state before next try
+
+        if last_error is not None:
+            raise last_error
+        raise RuntimeError("Multi-device recover() exhausted all reopen attempts")
 
     def configure(self, sweep: RadarSweepModel) -> None:
         """Store sweep settings for subsequent full-matrix acquisitions."""
@@ -132,7 +180,21 @@ class MultiDeviceLibreVnaService:
                     exc,
                     exc_info=True,
                 )
-                self.recover()
+                # recover() may itself fail when libusb cannot re-enumerate the
+                # device fast enough; treat that as the same kind of recovery
+                # attempt and try again on the next loop iteration, so a
+                # transient USB hiccup cannot kill the whole producer.
+                try:
+                    self.recover()
+                except Exception as recover_exc:  # noqa: BLE001
+                    last_error = recover_exc
+                    logger.warning(
+                        "multi-device recover() failed (%d/%d): %s",
+                        attempt_index + 1,
+                        self.recovery_attempts,
+                        recover_exc,
+                        exc_info=True,
+                    )
 
         assert last_error is not None
         raise last_error