354 lines
12 KiB
Python
354 lines
12 KiB
Python
"""Resilient MQTT transport with explicit Player command guarantees."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import queue
|
|
import threading
|
|
from dataclasses import dataclass
|
|
from typing import Any, Mapping
|
|
|
|
import paho.mqtt.client as mqtt
|
|
|
|
from .config import ControllerConfig
|
|
from .contracts import (
|
|
PLAYER_COMMAND_QOS,
|
|
PLAYER_COMMAND_RETAIN,
|
|
PLAYER_STATE_QOS,
|
|
PLAYER_STATUS_QOS,
|
|
PLAYER_TELEMETRY_QOS,
|
|
ROUTER_COMMAND_QOS,
|
|
ROUTER_COMMAND_RETAIN,
|
|
)
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MqttEvent:
|
|
kind: str
|
|
topic: str | None = None
|
|
payload: bytes = b""
|
|
qos: int = 0
|
|
retain: bool = False
|
|
message_id: int | None = None
|
|
connection_generation: int | None = None
|
|
reason: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class PublishReceipt:
|
|
accepted: bool
|
|
message_id: int | None = None
|
|
reason: str | None = None
|
|
|
|
|
|
class MqttAdapter:
|
|
"""Own Paho lifecycle while keeping all domain work off its callbacks."""
|
|
|
|
def __init__(
|
|
self,
|
|
config: ControllerConfig,
|
|
*,
|
|
client: Any | None = None,
|
|
events: "queue.Queue[MqttEvent] | None" = None,
|
|
):
|
|
self.config = config
|
|
self.events: "queue.Queue[MqttEvent]" = events or queue.Queue(
|
|
maxsize=config.mqtt.command_queue_size
|
|
)
|
|
self.client = client or mqtt.Client(
|
|
mqtt.CallbackAPIVersion.VERSION2,
|
|
client_id=config.mqtt.client_id,
|
|
clean_session=True,
|
|
)
|
|
self._generation_lock = threading.Lock()
|
|
self._connection_generation = 0
|
|
self._connected = threading.Event()
|
|
self._stop_event = threading.Event()
|
|
self._lifecycle_lock = threading.Lock()
|
|
self._started = False
|
|
self._loop_started = False
|
|
self._bootstrap_thread: threading.Thread | None = None
|
|
self.last_error: str | None = None
|
|
self._configure_client()
|
|
|
|
def _configure_client(self) -> None:
|
|
if self.config.mqtt.username is not None:
|
|
self.client.username_pw_set(
|
|
self.config.mqtt.username,
|
|
self.config.mqtt.password,
|
|
)
|
|
self.client.on_connect = self._on_connect
|
|
self.client.on_connect_fail = self._on_connect_fail
|
|
self.client.on_disconnect = self._on_disconnect
|
|
self.client.on_message = self._on_message
|
|
self.client.suppress_exceptions = True
|
|
self.client.manual_ack_set(True)
|
|
self.client.max_queued_messages_set(self.config.mqtt.command_queue_size)
|
|
self.client.reconnect_delay_set(
|
|
min_delay=self.config.mqtt.reconnect_min_seconds,
|
|
max_delay=self.config.mqtt.reconnect_max_seconds,
|
|
)
|
|
|
|
@property
|
|
def connected(self) -> bool:
|
|
return self._connected.is_set()
|
|
|
|
@property
|
|
def connection_generation(self) -> int:
|
|
with self._generation_lock:
|
|
return self._connection_generation
|
|
|
|
@property
|
|
def started(self) -> bool:
|
|
with self._lifecycle_lock:
|
|
return self._started
|
|
|
|
def start(self) -> None:
|
|
"""Start without requiring the broker or network to be reachable."""
|
|
with self._lifecycle_lock:
|
|
if self._started:
|
|
return
|
|
self._started = True
|
|
self._stop_event.clear()
|
|
try:
|
|
self._begin_async_connection()
|
|
except (OSError, mqtt.WebsocketConnectionError) as exc:
|
|
self.last_error = f"initial MQTT setup failed: {exc}"
|
|
LOG.warning("%s; retrying in background", self.last_error)
|
|
self._start_bootstrap_retry()
|
|
|
|
def _begin_async_connection(self) -> None:
|
|
self.client.connect_async(
|
|
self.config.mqtt.host,
|
|
self.config.mqtt.port,
|
|
keepalive=self.config.mqtt.keepalive_seconds,
|
|
)
|
|
with self._lifecycle_lock:
|
|
if not self._loop_started:
|
|
result = self.client.loop_start()
|
|
if result != mqtt.MQTT_ERR_SUCCESS:
|
|
raise RuntimeError(f"failed to start MQTT network loop: {result}")
|
|
self._loop_started = True
|
|
|
|
def _start_bootstrap_retry(self) -> None:
|
|
with self._lifecycle_lock:
|
|
if self._bootstrap_thread is not None and self._bootstrap_thread.is_alive():
|
|
return
|
|
self._bootstrap_thread = threading.Thread(
|
|
target=self._bootstrap_retry,
|
|
name="mqtt-bootstrap-retry",
|
|
daemon=True,
|
|
)
|
|
self._bootstrap_thread.start()
|
|
|
|
def _bootstrap_retry(self) -> None:
|
|
delay = float(self.config.mqtt.reconnect_min_seconds)
|
|
maximum = float(self.config.mqtt.reconnect_max_seconds)
|
|
while not self._stop_event.wait(delay):
|
|
try:
|
|
self._begin_async_connection()
|
|
self.last_error = None
|
|
return
|
|
except (OSError, mqtt.WebsocketConnectionError) as exc:
|
|
self.last_error = f"MQTT setup retry failed: {exc}"
|
|
LOG.warning("%s", self.last_error)
|
|
delay = min(maximum, max(delay * 2, 1.0))
|
|
except RuntimeError as exc:
|
|
self.last_error = str(exc)
|
|
LOG.error("%s", self.last_error)
|
|
return
|
|
|
|
def stop(self) -> None:
|
|
with self._lifecycle_lock:
|
|
if not self._started:
|
|
return
|
|
self._started = False
|
|
loop_started = self._loop_started
|
|
self._loop_started = False
|
|
self._stop_event.set()
|
|
self._connected.clear()
|
|
if loop_started:
|
|
try:
|
|
self.client.disconnect()
|
|
except Exception:
|
|
LOG.exception("MQTT disconnect failed")
|
|
self.client.loop_stop()
|
|
thread = self._bootstrap_thread
|
|
if thread is not None and thread is not threading.current_thread():
|
|
thread.join(timeout=1.0)
|
|
|
|
@staticmethod
|
|
def _reason_success(reason_code: Any) -> bool:
|
|
try:
|
|
return int(reason_code) == 0
|
|
except (TypeError, ValueError):
|
|
return reason_code == 0
|
|
|
|
def _on_connect(self, client, userdata, flags, reason_code, properties=None) -> None:
|
|
try:
|
|
if not self._reason_success(reason_code):
|
|
self.last_error = f"MQTT connect failed: {reason_code}"
|
|
LOG.warning("%s", self.last_error)
|
|
return
|
|
with self._generation_lock:
|
|
self._connection_generation += 1
|
|
generation = self._connection_generation
|
|
self._connected.set()
|
|
self.last_error = None
|
|
subscriptions = (
|
|
(self.config.topics.player_status, PLAYER_STATUS_QOS),
|
|
(self.config.topics.player_telemetry, PLAYER_TELEMETRY_QOS),
|
|
(self.config.topics.player_state, PLAYER_STATE_QOS),
|
|
(self.config.topics.controller_command, 1),
|
|
(self.config.topics.router_mode_state, 1),
|
|
)
|
|
for topic, qos in subscriptions:
|
|
client.subscribe(topic, qos=qos)
|
|
self._enqueue(
|
|
MqttEvent(
|
|
kind="connected",
|
|
connection_generation=generation,
|
|
)
|
|
)
|
|
except BaseException:
|
|
LOG.exception("unexpected MQTT on_connect failure")
|
|
|
|
def _on_connect_fail(self, client, userdata) -> None:
|
|
self.last_error = (
|
|
f"MQTT connection to {self.config.mqtt.host}:{self.config.mqtt.port} failed"
|
|
)
|
|
LOG.warning("%s; Paho will retry", self.last_error)
|
|
|
|
def _on_disconnect(
|
|
self,
|
|
client,
|
|
userdata,
|
|
disconnect_flags,
|
|
reason_code,
|
|
properties=None,
|
|
) -> None:
|
|
self._connected.clear()
|
|
try:
|
|
self._enqueue(
|
|
MqttEvent(
|
|
kind="disconnected",
|
|
connection_generation=self.connection_generation,
|
|
reason=str(reason_code),
|
|
)
|
|
)
|
|
except BaseException:
|
|
LOG.exception("unexpected MQTT on_disconnect failure")
|
|
|
|
def _on_message(self, client, userdata, message) -> None:
|
|
try:
|
|
self._enqueue(
|
|
MqttEvent(
|
|
kind="message",
|
|
topic=str(message.topic),
|
|
payload=bytes(message.payload),
|
|
qos=int(getattr(message, "qos", 0)),
|
|
retain=bool(getattr(message, "retain", False)),
|
|
message_id=getattr(message, "mid", None),
|
|
connection_generation=self.connection_generation,
|
|
)
|
|
)
|
|
except BaseException:
|
|
LOG.exception("unexpected MQTT on_message failure")
|
|
|
|
def _enqueue(self, event: MqttEvent) -> bool:
|
|
while not self._stop_event.is_set():
|
|
try:
|
|
self.events.put(event, timeout=0.2)
|
|
return True
|
|
except queue.Full:
|
|
continue
|
|
return False
|
|
|
|
def acknowledge(self, event: MqttEvent) -> bool:
|
|
if event.message_id is None or event.qos <= 0:
|
|
return True
|
|
if event.connection_generation != self.connection_generation:
|
|
LOG.warning(
|
|
"skipping stale MQTT ack mid=%s from generation=%s",
|
|
event.message_id,
|
|
event.connection_generation,
|
|
)
|
|
return False
|
|
try:
|
|
result = self.client.ack(event.message_id, event.qos)
|
|
except BaseException:
|
|
LOG.exception("MQTT ack failed mid=%s", event.message_id)
|
|
return False
|
|
return result == mqtt.MQTT_ERR_SUCCESS
|
|
|
|
def _publish(
|
|
self,
|
|
topic: str,
|
|
payload: str,
|
|
*,
|
|
qos: int,
|
|
retain: bool,
|
|
require_connection: bool,
|
|
) -> PublishReceipt:
|
|
if require_connection and not self.connected:
|
|
return PublishReceipt(False, reason="MQTT is disconnected")
|
|
try:
|
|
info = self.client.publish(topic, payload, qos=qos, retain=retain)
|
|
except BaseException as exc:
|
|
LOG.exception("MQTT publish failed topic=%s", topic)
|
|
return PublishReceipt(False, reason=str(exc))
|
|
if info.rc != mqtt.MQTT_ERR_SUCCESS:
|
|
return PublishReceipt(False, message_id=getattr(info, "mid", None), reason=f"MQTT rc={info.rc}")
|
|
return PublishReceipt(True, message_id=getattr(info, "mid", None))
|
|
|
|
@staticmethod
|
|
def _json_payload(payload: Mapping[str, Any]) -> str:
|
|
return json.dumps(payload, allow_nan=False, separators=(",", ":"))
|
|
|
|
def publish_player_command(self, payload: Mapping[str, Any]) -> PublishReceipt:
|
|
return self._publish(
|
|
self.config.topics.player_command,
|
|
self._json_payload(payload),
|
|
qos=PLAYER_COMMAND_QOS,
|
|
retain=PLAYER_COMMAND_RETAIN,
|
|
require_connection=True,
|
|
)
|
|
|
|
def publish_router_mode(self, mode: str) -> PublishReceipt:
|
|
return self._publish(
|
|
self.config.topics.router_mode_set,
|
|
mode,
|
|
qos=ROUTER_COMMAND_QOS,
|
|
retain=ROUTER_COMMAND_RETAIN,
|
|
require_connection=True,
|
|
)
|
|
|
|
def publish_legacy(self, topic: str, payload: str, *, retain: bool = True) -> PublishReceipt:
|
|
return self._publish(
|
|
topic,
|
|
payload,
|
|
qos=0,
|
|
retain=retain,
|
|
require_connection=True,
|
|
)
|
|
|
|
def publish_json(
|
|
self,
|
|
topic: str,
|
|
payload: Mapping[str, Any],
|
|
*,
|
|
qos: int = 0,
|
|
retain: bool = False,
|
|
) -> PublishReceipt:
|
|
return self._publish(
|
|
topic,
|
|
self._json_payload(payload),
|
|
qos=qos,
|
|
retain=retain,
|
|
require_connection=True,
|
|
)
|