fix: close Audio Router self-echo PUBACK/subscription-delivery race

publish_router_mode registered its self-echo correlation marker before
publishing but removed it in a finally block the instant the QoS 1 publish()
call observed its PUBACK. A broker may legally deliver a Controller's own
subscription copy of that same publish after the PUBACK, since MQTT does not
order the two; when that happened, the Controller mistook its own late echo
for live operator intent and relinquished automatic restoration ownership.

Keep the marker claimable for a short, bounded mqtt.router_echo_grace_seconds
window (default 2s) after the PUBACK instead of clearing it immediately.
Genuine external/operator commands are unaffected: once the grace window
elapses, a matching payload is still correctly treated as operator intent
and ownership is relinquished as before.

Verified with deterministic fake-clock unit tests plus a real Mosquitto
broker integration test (tests/test_router_echo_mosquitto_integration.py,
skipped automatically when Docker is unavailable).

controller/config.py

@@ -33,6 +33,7 @@ class MqttConfig:
     reconnect_max_seconds: int = 30
     command_queue_size: int = 512
     publish_ack_timeout_seconds: float = 2.0
+    router_echo_grace_seconds: float = 2.0
 
 
 @dataclass(frozen=True)
@@ -219,6 +220,11 @@ def load_config(
             "mqtt.publish_ack_timeout_seconds",
             minimum=0.1,
         ),
+        router_echo_grace_seconds=_number(
+            mqtt_raw.get("router_echo_grace_seconds", 2.0),
+            "mqtt.router_echo_grace_seconds",
+            minimum=0.0,
+        ),
     )
     if mqtt.reconnect_max_seconds < mqtt.reconnect_min_seconds:
         raise ConfigError("mqtt.reconnect_max_seconds must be >= reconnect_min_seconds")

controller/mqtt_adapter.py

@@ -6,6 +6,7 @@ import json
 import logging
 import queue
 import threading
+import time
 import uuid
 from dataclasses import dataclass
 from typing import Any, Mapping
@@ -58,6 +59,7 @@ class MqttAdapter:
         *,
         client: Any | None = None,
         events: "queue.Queue[MqttEvent] | None" = None,
+        monotonic=time.monotonic,
     ):
         self.config = config
         self.events: "queue.Queue[MqttEvent]" = events or queue.Queue(
@@ -73,8 +75,16 @@ class MqttAdapter:
         self._started = False
         self._loop_started = False
         self._bootstrap_thread: threading.Thread | None = None
+        self._monotonic = monotonic
         self._router_echo_lock = threading.Lock()
-        self._pending_router_echoes: dict[str, bytes] = {}
+        # token -> (exact payload, grace deadline). The deadline is ``None``
+        # while the correlated publish() call has not yet observed its
+        # PUBACK; such an entry never expires on its own, bounded instead by
+        # the publish ack timeout. Once PUBACK is observed, the entry keeps
+        # a short grace window so a subscription echo delivered just after
+        # the PUBACK is still recognized as ours rather than mistaken for
+        # operator intent.
+        self._pending_router_echoes: dict[str, tuple[bytes, float | None]] = {}
         self.last_error: str | None = None
         self._configure_client()
 
@@ -466,36 +476,63 @@ class MqttAdapter:
     def publish_router_mode(self, mode: str) -> PublishReceipt:
         # A broker normally sends our publication back through the matching
         # subscription. Register the exact payload before publishing because
-        # Paho's network thread may receive that echo before publish() returns.
-        # The marker is removed as soon as the QoS 1 publish call completes;
-        # this intentionally prefers a false ownership loss over mistaking a
-        # later same-value operator command for a Controller echo.
+        # Paho's network thread may receive that echo before publish()
+        # returns. A PUBACK only proves broker receipt of our publish, not
+        # that the broker has already forwarded (or even scheduled) our own
+        # subscription copy of it; that copy can legally arrive after the
+        # PUBACK. So the marker is not removed the instant publish()
+        # completes: it stays claimable for a short bounded grace window
+        # afterward, and is removed immediately once actually claimed or
+        # once the grace window elapses. This closes the specific race where
+        # the Controller's own late echo would otherwise be mistaken for
+        # live operator intent and cause automatic restoration ownership to
+        # be relinquished.
         echo_token = uuid.uuid4().hex
+        payload = mode.encode("utf-8")
         with self._router_echo_lock:
-            self._pending_router_echoes[echo_token] = mode.encode("utf-8")
-        try:
-            return self._publish(
-                self.config.topics.router_mode_set,
-                mode,
-                qos=ROUTER_COMMAND_QOS,
-                retain=ROUTER_COMMAND_RETAIN,
-                require_connection=True,
-                invalidate_transport_on_uncertain=True,
-            )
-        finally:
-            with self._router_echo_lock:
-                self._pending_router_echoes.pop(echo_token, None)
+            self._prune_router_echoes_locked()
+            self._pending_router_echoes[echo_token] = (payload, None)
+        receipt = self._publish(
+            self.config.topics.router_mode_set,
+            mode,
+            qos=ROUTER_COMMAND_QOS,
+            retain=ROUTER_COMMAND_RETAIN,
+            require_connection=True,
+            invalidate_transport_on_uncertain=True,
+        )
+        grace_deadline = (
+            self._monotonic() + self.config.mqtt.router_echo_grace_seconds
+        )
+        with self._router_echo_lock:
+            if echo_token in self._pending_router_echoes:
+                self._pending_router_echoes[echo_token] = (payload, grace_deadline)
+        return receipt
 
     def _claim_router_echo(self, payload: bytes) -> bool:
-        """Claim one exact mode-set echo from an in-flight local publish."""
+        """Claim one exact mode-set echo from an in-flight or recent local publish."""
 
         with self._router_echo_lock:
-            for token, expected in tuple(self._pending_router_echoes.items()):
+            self._prune_router_echoes_locked()
+            for token, (expected, _deadline) in tuple(
+                self._pending_router_echoes.items()
+            ):
                 if expected == payload:
                     del self._pending_router_echoes[token]
                     return True
         return False
 
+    def _prune_router_echoes_locked(self) -> None:
+        """Drop expired grace-window entries. Caller holds ``_router_echo_lock``."""
+
+        now = self._monotonic()
+        expired = [
+            token
+            for token, (_, deadline) in self._pending_router_echoes.items()
+            if deadline is not None and deadline <= now
+        ]
+        for token in expired:
+            del self._pending_router_echoes[token]
+
     def publish_legacy(self, topic: str, payload: str, *, retain: bool = True) -> PublishReceipt:
         return self._publish(
             topic,

controller/service.py

@@ -81,6 +81,7 @@ class ControllerService:
         self.mqtt = mqtt_adapter or MqttAdapter(
             config,
             events=self.events,
+            monotonic=monotonic,
         )
         self.downloads = download_manager
         self.playback = playback