fix: close Audio Router self-echo PUBACK/subscription-delivery race

publish_router_mode registered its self-echo correlation marker before
publishing but removed it in a finally block the instant the QoS 1 publish()
call observed its PUBACK. A broker may legally deliver a Controller's own
subscription copy of that same publish after the PUBACK, since MQTT does not
order the two; when that happened, the Controller mistook its own late echo
for live operator intent and relinquished automatic restoration ownership.

Keep the marker claimable for a short, bounded mqtt.router_echo_grace_seconds
window (default 2s) after the PUBACK instead of clearing it immediately.
Genuine external/operator commands are unaffected: once the grace window
elapses, a matching payload is still correctly treated as operator intent
and ownership is relinquished as before.

Verified with deterministic fake-clock unit tests plus a real Mosquitto
broker integration test (tests/test_router_echo_mosquitto_integration.py,
skipped automatically when Docker is unavailable).

tests/test_config_contracts.py

@@ -81,6 +81,7 @@ class ConfigurationTests(unittest.TestCase):
         self.assertIsNone(config.router.fallback_restore_mode)
         self.assertEqual(config.storage.uncertain_review_after_seconds, 900.0)
         self.assertEqual(config.mqtt.publish_ack_timeout_seconds, 2.0)
+        self.assertEqual(config.mqtt.router_echo_grace_seconds, 2.0)
 
     def test_documented_environment_overrides_apply(self):
         env = {
@@ -152,6 +153,15 @@ class ConfigurationTests(unittest.TestCase):
                 with self.assertRaisesRegex(ConfigError, "must be finite"):
                     load_config(path, environ={})
 
+    def test_negative_router_echo_grace_is_rejected(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / "config.yaml"
+            path.write_text(
+                "mqtt:\n  router_echo_grace_seconds: -1\n", encoding="utf-8"
+            )
+            with self.assertRaisesRegex(ConfigError, "router_echo_grace_seconds"):
+                load_config(path, environ={})
+
     def test_runtime_paths_must_be_absolute(self):
         cases = (
             ("storage:\n  media_base: relative/media\n", "storage.media_base"),

tests/test_mqtt_adapter.py

@@ -10,6 +10,14 @@ from controller.config import ControllerConfig, MqttConfig
 from controller.mqtt_adapter import MqttAdapter
 
 
+class MutableClock:
+    def __init__(self, value=0.0):
+        self.value = value
+
+    def __call__(self):
+        return self.value
+
+
 class FakeMessageInfo:
     def __init__(self, mid, published):
         self.rc = mqtt.MQTT_ERR_SUCCESS
@@ -245,7 +253,7 @@ class MqttAdapterTests(unittest.TestCase):
             ("audio/player/mode/set", "video", 1, False),
         )
 
-    def test_router_self_echo_is_marked_only_while_publish_is_in_flight(self):
+    def test_router_self_echo_is_marked_while_publish_is_in_flight(self):
         self.adapter.start()
         self.adapter._on_connect(self.client, None, None, 0)
         self.events.get_nowait()
@@ -268,20 +276,73 @@ class MqttAdapterTests(unittest.TestCase):
         self.assertTrue(self.adapter.publish_router_mode("video").accepted)
         self.assertTrue(self.events.get_nowait().controller_echo)
 
-        self.client.publish_hook = None
-        self.adapter._on_message(
-            self.client,
-            None,
-            SimpleNamespace(
-                topic="audio/player/mode/set",
-                payload=b"video",
-                qos=1,
-                retain=False,
-                dup=False,
-                mid=32,
-            ),
-        )
-        self.assertFalse(self.events.get_nowait().controller_echo)
+    def test_router_self_echo_delivered_after_puback_is_still_claimed_within_grace(
+        self,
+    ):
+        # Regression: a broker may legally deliver our own subscription echo
+        # after the PUBACK that already acknowledged the same publish. If the
+        # echo marker were cleared the instant publish() observes that
+        # PUBACK, this late echo would be mistaken for live operator intent
+        # and cause automatic Router restoration ownership to be relinquished.
+        clock = MutableClock(1000.0)
+        client = FakeClient()
+        events = queue.Queue()
+        adapter = MqttAdapter(self.config, client=client, events=events, monotonic=clock)
+        try:
+            adapter.start()
+            adapter._on_connect(client, None, None, 0)
+            events.get_nowait()
+
+            self.assertTrue(adapter.publish_router_mode("video").accepted)
+            clock.value += self.config.mqtt.router_echo_grace_seconds - 0.5
+
+            adapter._on_message(
+                client,
+                None,
+                SimpleNamespace(
+                    topic="audio/player/mode/set",
+                    payload=b"video",
+                    qos=1,
+                    retain=False,
+                    dup=False,
+                    mid=32,
+                ),
+            )
+            self.assertTrue(events.get_nowait().controller_echo)
+        finally:
+            adapter.stop()
+
+    def test_router_self_echo_marker_expires_after_grace_window(self):
+        # A genuine external/operator command that happens to repeat the same
+        # value must still be recognized as operator intent once the bounded
+        # grace window has elapsed; the marker must not linger forever.
+        clock = MutableClock(1000.0)
+        client = FakeClient()
+        events = queue.Queue()
+        adapter = MqttAdapter(self.config, client=client, events=events, monotonic=clock)
+        try:
+            adapter.start()
+            adapter._on_connect(client, None, None, 0)
+            events.get_nowait()
+
+            self.assertTrue(adapter.publish_router_mode("video").accepted)
+            clock.value += self.config.mqtt.router_echo_grace_seconds + 0.5
+
+            adapter._on_message(
+                client,
+                None,
+                SimpleNamespace(
+                    topic="audio/player/mode/set",
+                    payload=b"video",
+                    qos=1,
+                    retain=False,
+                    dup=False,
+                    mid=32,
+                ),
+            )
+            self.assertFalse(events.get_nowait().controller_echo)
+        finally:
+            adapter.stop()
 
     def test_retained_router_set_never_consumes_controller_echo_marker(self):
         self.adapter.start()

tests/test_router_echo_mosquitto_integration.py

@@ -0,0 +1,206 @@
+"""Real-broker smoke test for Audio Router self-echo correlation.
+
+The fake MQTT client used by ``tests/test_mqtt_adapter.py`` delivers a
+publish's own echo synchronously, inside the ``publish()`` call itself, so it
+cannot exercise the actual ordering risk: a real broker may legally deliver a
+Controller-originated Router mode-set message back through the Controller's
+own subscription either before or after the PUBACK that already acknowledged
+the same publish. This test runs the real ``MqttAdapter`` against a real
+Mosquitto broker in Docker to prove that ordering no longer matters.
+
+Skipped automatically when Docker (or a Mosquitto broker container) is not
+available, so it never blocks the primary regression suite.
+"""
+
+from __future__ import annotations
+
+import queue
+import shutil
+import socket
+import subprocess
+import tempfile
+import time
+import unittest
+import uuid
+from pathlib import Path
+
+import paho.mqtt.client as mqtt
+
+from controller.config import ControllerConfig, MqttConfig
+from controller.mqtt_adapter import MqttAdapter
+
+
+def _docker_available() -> bool:
+    return shutil.which("docker") is not None
+
+
+def _free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+@unittest.skipUnless(
+    _docker_available(), "docker is not available for a real Mosquitto broker"
+)
+class RouterSelfEchoMosquittoIntegrationTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        cls.port = _free_port()
+        cls.container_name = f"controller-echo-test-{uuid.uuid4().hex[:10]}"
+        cls.tempdir = tempfile.TemporaryDirectory()
+        conf_path = Path(cls.tempdir.name) / "mosquitto.conf"
+        conf_path.write_text("listener 1883\nallow_anonymous true\n", encoding="utf-8")
+        try:
+            subprocess.run(
+                [
+                    "docker",
+                    "run",
+                    "-d",
+                    "--rm",
+                    "--name",
+                    cls.container_name,
+                    "-p",
+                    f"127.0.0.1:{cls.port}:1883",
+                    "-v",
+                    f"{conf_path}:/mosquitto/config/mosquitto.conf:ro",
+                    "eclipse-mosquitto",
+                ],
+                check=True,
+                capture_output=True,
+                text=True,
+                timeout=30,
+            )
+        except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
+            cls.tempdir.cleanup()
+            raise unittest.SkipTest(
+                f"could not start a real Mosquitto broker container: {exc}"
+            )
+        try:
+            cls._wait_for_broker(cls.port)
+        except AssertionError as exc:
+            subprocess.run(
+                ["docker", "rm", "-f", cls.container_name],
+                capture_output=True,
+                timeout=15,
+            )
+            cls.tempdir.cleanup()
+            raise unittest.SkipTest(str(exc))
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        subprocess.run(
+            ["docker", "rm", "-f", cls.container_name],
+            capture_output=True,
+            timeout=15,
+        )
+        cls.tempdir.cleanup()
+
+    @staticmethod
+    def _wait_for_broker(port: int, timeout: float = 15.0) -> None:
+        deadline = time.monotonic() + timeout
+        last_exc: OSError | None = None
+        while time.monotonic() < deadline:
+            try:
+                with socket.create_connection(("127.0.0.1", port), timeout=0.5):
+                    return
+            except OSError as exc:
+                last_exc = exc
+                time.sleep(0.2)
+        raise AssertionError(f"Mosquitto broker never became reachable: {last_exc}")
+
+    def _config(self) -> ControllerConfig:
+        return ControllerConfig(
+            mqtt=MqttConfig(
+                host="127.0.0.1",
+                port=self.port,
+                client_id=f"controller-echo-{uuid.uuid4().hex[:10]}",
+                reconnect_min_seconds=1,
+                reconnect_max_seconds=1,
+                command_queue_size=32,
+                publish_ack_timeout_seconds=5.0,
+                router_echo_grace_seconds=2.0,
+            )
+        )
+
+    def test_controller_recognizes_its_own_router_echo_over_a_real_broker(self) -> None:
+        config = self._config()
+        adapter = MqttAdapter(config)
+        adapter.start()
+        try:
+            self._wait_until(
+                lambda: adapter.connected,
+                "Controller MQTT adapter never connected to the real broker",
+            )
+            receipt = adapter.publish_router_mode("video")
+            self.assertTrue(receipt.accepted, receipt.reason)
+
+            event = self._next_message_event(adapter, timeout=5.0)
+            self.assertEqual(event.topic, config.topics.router_mode_set)
+            self.assertEqual(event.payload, b"video")
+            self.assertTrue(
+                event.controller_echo,
+                "Controller did not recognize its own Router mode-set echo "
+                "delivered by a real broker; the PUBACK-vs-subscription-echo "
+                "race was not closed",
+            )
+        finally:
+            adapter.stop()
+
+    def test_genuine_external_router_command_is_not_treated_as_self_echo(self) -> None:
+        config = self._config()
+        adapter = MqttAdapter(config)
+        adapter.start()
+        operator = mqtt.Client(
+            mqtt.CallbackAPIVersion.VERSION2,
+            client_id=f"operator-{uuid.uuid4().hex[:10]}",
+        )
+        try:
+            self._wait_until(
+                lambda: adapter.connected,
+                "Controller MQTT adapter never connected to the real broker",
+            )
+            operator.connect(config.mqtt.host, config.mqtt.port, keepalive=30)
+            operator.loop_start()
+            info = operator.publish(
+                config.topics.router_mode_set, "service", qos=1, retain=False
+            )
+            info.wait_for_publish(timeout=5.0)
+
+            event = self._next_message_event(adapter, timeout=5.0)
+            self.assertEqual(event.payload, b"service")
+            self.assertFalse(
+                event.controller_echo,
+                "A genuine external Router command was mistaken for a "
+                "Controller echo; automatic restoration ownership would be "
+                "incorrectly preserved instead of relinquished",
+            )
+        finally:
+            operator.loop_stop()
+            operator.disconnect()
+            adapter.stop()
+
+    @staticmethod
+    def _wait_until(predicate, message: str, timeout: float = 10.0) -> None:
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            if predicate():
+                return
+            time.sleep(0.05)
+        raise AssertionError(message)
+
+    @staticmethod
+    def _next_message_event(adapter: MqttAdapter, *, timeout: float):
+        deadline = time.monotonic() + timeout
+        while time.monotonic() < deadline:
+            try:
+                event = adapter.events.get(timeout=0.2)
+            except queue.Empty:
+                continue
+            if event.kind == "message":
+                return event
+        raise AssertionError("no MQTT message event was observed within the timeout")
+
+
+if __name__ == "__main__":
+    unittest.main()