fix: close Audio Router self-echo PUBACK/subscription-delivery race
publish_router_mode registered its self-echo correlation marker before publishing but removed it in a finally block the instant the QoS 1 publish() call observed its PUBACK. A broker may legally deliver a Controller's own subscription copy of that same publish after the PUBACK, since MQTT does not order the two; when that happened, the Controller mistook its own late echo for live operator intent and relinquished automatic restoration ownership. Keep the marker claimable for a short, bounded mqtt.router_echo_grace_seconds window (default 2s) after the PUBACK instead of clearing it immediately. Genuine external/operator commands are unaffected: once the grace window elapses, a matching payload is still correctly treated as operator intent and ownership is relinquished as before. Verified with deterministic fake-clock unit tests plus a real Mosquitto broker integration test (tests/test_router_echo_mosquitto_integration.py, skipped automatically when Docker is unavailable).
This commit is contained in:
Media
@@ -0,0 +1,206 @@
|
||||
"""Real-broker smoke test for Audio Router self-echo correlation.
|
||||
|
||||
The fake MQTT client used by ``tests/test_mqtt_adapter.py`` delivers a
|
||||
publish's own echo synchronously, inside the ``publish()`` call itself, so it
|
||||
cannot exercise the actual ordering risk: a real broker may legally deliver a
|
||||
Controller-originated Router mode-set message back through the Controller's
|
||||
own subscription either before or after the PUBACK that already acknowledged
|
||||
the same publish. This test runs the real ``MqttAdapter`` against a real
|
||||
Mosquitto broker in Docker to prove that ordering no longer matters.
|
||||
|
||||
Skipped automatically when Docker (or a Mosquitto broker container) is not
|
||||
available, so it never blocks the primary regression suite.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import paho.mqtt.client as mqtt
|
||||
|
||||
from controller.config import ControllerConfig, MqttConfig
|
||||
from controller.mqtt_adapter import MqttAdapter
|
||||
|
||||
|
||||
def _docker_available() -> bool:
|
||||
return shutil.which("docker") is not None
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return sock.getsockname()[1]
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
_docker_available(), "docker is not available for a real Mosquitto broker"
|
||||
)
|
||||
class RouterSelfEchoMosquittoIntegrationTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.port = _free_port()
|
||||
cls.container_name = f"controller-echo-test-{uuid.uuid4().hex[:10]}"
|
||||
cls.tempdir = tempfile.TemporaryDirectory()
|
||||
conf_path = Path(cls.tempdir.name) / "mosquitto.conf"
|
||||
conf_path.write_text("listener 1883\nallow_anonymous true\n", encoding="utf-8")
|
||||
try:
|
||||
subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"run",
|
||||
"-d",
|
||||
"--rm",
|
||||
"--name",
|
||||
cls.container_name,
|
||||
"-p",
|
||||
f"127.0.0.1:{cls.port}:1883",
|
||||
"-v",
|
||||
f"{conf_path}:/mosquitto/config/mosquitto.conf:ro",
|
||||
"eclipse-mosquitto",
|
||||
],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
|
||||
cls.tempdir.cleanup()
|
||||
raise unittest.SkipTest(
|
||||
f"could not start a real Mosquitto broker container: {exc}"
|
||||
)
|
||||
try:
|
||||
cls._wait_for_broker(cls.port)
|
||||
except AssertionError as exc:
|
||||
subprocess.run(
|
||||
["docker", "rm", "-f", cls.container_name],
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
)
|
||||
cls.tempdir.cleanup()
|
||||
raise unittest.SkipTest(str(exc))
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
subprocess.run(
|
||||
["docker", "rm", "-f", cls.container_name],
|
||||
capture_output=True,
|
||||
timeout=15,
|
||||
)
|
||||
cls.tempdir.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def _wait_for_broker(port: int, timeout: float = 15.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
last_exc: OSError | None = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", port), timeout=0.5):
|
||||
return
|
||||
except OSError as exc:
|
||||
last_exc = exc
|
||||
time.sleep(0.2)
|
||||
raise AssertionError(f"Mosquitto broker never became reachable: {last_exc}")
|
||||
|
||||
def _config(self) -> ControllerConfig:
|
||||
return ControllerConfig(
|
||||
mqtt=MqttConfig(
|
||||
host="127.0.0.1",
|
||||
port=self.port,
|
||||
client_id=f"controller-echo-{uuid.uuid4().hex[:10]}",
|
||||
reconnect_min_seconds=1,
|
||||
reconnect_max_seconds=1,
|
||||
command_queue_size=32,
|
||||
publish_ack_timeout_seconds=5.0,
|
||||
router_echo_grace_seconds=2.0,
|
||||
)
|
||||
)
|
||||
|
||||
def test_controller_recognizes_its_own_router_echo_over_a_real_broker(self) -> None:
|
||||
config = self._config()
|
||||
adapter = MqttAdapter(config)
|
||||
adapter.start()
|
||||
try:
|
||||
self._wait_until(
|
||||
lambda: adapter.connected,
|
||||
"Controller MQTT adapter never connected to the real broker",
|
||||
)
|
||||
receipt = adapter.publish_router_mode("video")
|
||||
self.assertTrue(receipt.accepted, receipt.reason)
|
||||
|
||||
event = self._next_message_event(adapter, timeout=5.0)
|
||||
self.assertEqual(event.topic, config.topics.router_mode_set)
|
||||
self.assertEqual(event.payload, b"video")
|
||||
self.assertTrue(
|
||||
event.controller_echo,
|
||||
"Controller did not recognize its own Router mode-set echo "
|
||||
"delivered by a real broker; the PUBACK-vs-subscription-echo "
|
||||
"race was not closed",
|
||||
)
|
||||
finally:
|
||||
adapter.stop()
|
||||
|
||||
def test_genuine_external_router_command_is_not_treated_as_self_echo(self) -> None:
|
||||
config = self._config()
|
||||
adapter = MqttAdapter(config)
|
||||
adapter.start()
|
||||
operator = mqtt.Client(
|
||||
mqtt.CallbackAPIVersion.VERSION2,
|
||||
client_id=f"operator-{uuid.uuid4().hex[:10]}",
|
||||
)
|
||||
try:
|
||||
self._wait_until(
|
||||
lambda: adapter.connected,
|
||||
"Controller MQTT adapter never connected to the real broker",
|
||||
)
|
||||
operator.connect(config.mqtt.host, config.mqtt.port, keepalive=30)
|
||||
operator.loop_start()
|
||||
info = operator.publish(
|
||||
config.topics.router_mode_set, "service", qos=1, retain=False
|
||||
)
|
||||
info.wait_for_publish(timeout=5.0)
|
||||
|
||||
event = self._next_message_event(adapter, timeout=5.0)
|
||||
self.assertEqual(event.payload, b"service")
|
||||
self.assertFalse(
|
||||
event.controller_echo,
|
||||
"A genuine external Router command was mistaken for a "
|
||||
"Controller echo; automatic restoration ownership would be "
|
||||
"incorrectly preserved instead of relinquished",
|
||||
)
|
||||
finally:
|
||||
operator.loop_stop()
|
||||
operator.disconnect()
|
||||
adapter.stop()
|
||||
|
||||
@staticmethod
|
||||
def _wait_until(predicate, message: str, timeout: float = 10.0) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if predicate():
|
||||
return
|
||||
time.sleep(0.05)
|
||||
raise AssertionError(message)
|
||||
|
||||
@staticmethod
|
||||
def _next_message_event(adapter: MqttAdapter, *, timeout: float):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
event = adapter.events.get(timeout=0.2)
|
||||
except queue.Empty:
|
||||
continue
|
||||
if event.kind == "message":
|
||||
return event
|
||||
raise AssertionError("no MQTT message event was observed within the timeout")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user