feat: reduce authoritative player mqtt state
This commit is contained in:
Media
@@ -0,0 +1,353 @@
|
||||
"""Resilient MQTT transport with explicit Player command guarantees."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import queue
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping
|
||||
|
||||
import paho.mqtt.client as mqtt
|
||||
|
||||
from .config import ControllerConfig
|
||||
from .contracts import (
|
||||
PLAYER_COMMAND_QOS,
|
||||
PLAYER_COMMAND_RETAIN,
|
||||
PLAYER_STATE_QOS,
|
||||
PLAYER_STATUS_QOS,
|
||||
PLAYER_TELEMETRY_QOS,
|
||||
ROUTER_COMMAND_QOS,
|
||||
ROUTER_COMMAND_RETAIN,
|
||||
)
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MqttEvent:
|
||||
kind: str
|
||||
topic: str | None = None
|
||||
payload: bytes = b""
|
||||
qos: int = 0
|
||||
retain: bool = False
|
||||
message_id: int | None = None
|
||||
connection_generation: int | None = None
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PublishReceipt:
|
||||
accepted: bool
|
||||
message_id: int | None = None
|
||||
reason: str | None = None
|
||||
|
||||
|
||||
class MqttAdapter:
|
||||
"""Own Paho lifecycle while keeping all domain work off its callbacks."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: ControllerConfig,
|
||||
*,
|
||||
client: Any | None = None,
|
||||
events: "queue.Queue[MqttEvent] | None" = None,
|
||||
):
|
||||
self.config = config
|
||||
self.events: "queue.Queue[MqttEvent]" = events or queue.Queue(
|
||||
maxsize=config.mqtt.command_queue_size
|
||||
)
|
||||
self.client = client or mqtt.Client(
|
||||
mqtt.CallbackAPIVersion.VERSION2,
|
||||
client_id=config.mqtt.client_id,
|
||||
clean_session=True,
|
||||
)
|
||||
self._generation_lock = threading.Lock()
|
||||
self._connection_generation = 0
|
||||
self._connected = threading.Event()
|
||||
self._stop_event = threading.Event()
|
||||
self._lifecycle_lock = threading.Lock()
|
||||
self._started = False
|
||||
self._loop_started = False
|
||||
self._bootstrap_thread: threading.Thread | None = None
|
||||
self.last_error: str | None = None
|
||||
self._configure_client()
|
||||
|
||||
def _configure_client(self) -> None:
|
||||
if self.config.mqtt.username is not None:
|
||||
self.client.username_pw_set(
|
||||
self.config.mqtt.username,
|
||||
self.config.mqtt.password,
|
||||
)
|
||||
self.client.on_connect = self._on_connect
|
||||
self.client.on_connect_fail = self._on_connect_fail
|
||||
self.client.on_disconnect = self._on_disconnect
|
||||
self.client.on_message = self._on_message
|
||||
self.client.suppress_exceptions = True
|
||||
self.client.manual_ack_set(True)
|
||||
self.client.max_queued_messages_set(self.config.mqtt.command_queue_size)
|
||||
self.client.reconnect_delay_set(
|
||||
min_delay=self.config.mqtt.reconnect_min_seconds,
|
||||
max_delay=self.config.mqtt.reconnect_max_seconds,
|
||||
)
|
||||
|
||||
@property
|
||||
def connected(self) -> bool:
|
||||
return self._connected.is_set()
|
||||
|
||||
@property
|
||||
def connection_generation(self) -> int:
|
||||
with self._generation_lock:
|
||||
return self._connection_generation
|
||||
|
||||
@property
|
||||
def started(self) -> bool:
|
||||
with self._lifecycle_lock:
|
||||
return self._started
|
||||
|
||||
def start(self) -> None:
|
||||
"""Start without requiring the broker or network to be reachable."""
|
||||
with self._lifecycle_lock:
|
||||
if self._started:
|
||||
return
|
||||
self._started = True
|
||||
self._stop_event.clear()
|
||||
try:
|
||||
self._begin_async_connection()
|
||||
except (OSError, mqtt.WebsocketConnectionError) as exc:
|
||||
self.last_error = f"initial MQTT setup failed: {exc}"
|
||||
LOG.warning("%s; retrying in background", self.last_error)
|
||||
self._start_bootstrap_retry()
|
||||
|
||||
def _begin_async_connection(self) -> None:
|
||||
self.client.connect_async(
|
||||
self.config.mqtt.host,
|
||||
self.config.mqtt.port,
|
||||
keepalive=self.config.mqtt.keepalive_seconds,
|
||||
)
|
||||
with self._lifecycle_lock:
|
||||
if not self._loop_started:
|
||||
result = self.client.loop_start()
|
||||
if result != mqtt.MQTT_ERR_SUCCESS:
|
||||
raise RuntimeError(f"failed to start MQTT network loop: {result}")
|
||||
self._loop_started = True
|
||||
|
||||
def _start_bootstrap_retry(self) -> None:
|
||||
with self._lifecycle_lock:
|
||||
if self._bootstrap_thread is not None and self._bootstrap_thread.is_alive():
|
||||
return
|
||||
self._bootstrap_thread = threading.Thread(
|
||||
target=self._bootstrap_retry,
|
||||
name="mqtt-bootstrap-retry",
|
||||
daemon=True,
|
||||
)
|
||||
self._bootstrap_thread.start()
|
||||
|
||||
def _bootstrap_retry(self) -> None:
|
||||
delay = float(self.config.mqtt.reconnect_min_seconds)
|
||||
maximum = float(self.config.mqtt.reconnect_max_seconds)
|
||||
while not self._stop_event.wait(delay):
|
||||
try:
|
||||
self._begin_async_connection()
|
||||
self.last_error = None
|
||||
return
|
||||
except (OSError, mqtt.WebsocketConnectionError) as exc:
|
||||
self.last_error = f"MQTT setup retry failed: {exc}"
|
||||
LOG.warning("%s", self.last_error)
|
||||
delay = min(maximum, max(delay * 2, 1.0))
|
||||
except RuntimeError as exc:
|
||||
self.last_error = str(exc)
|
||||
LOG.error("%s", self.last_error)
|
||||
return
|
||||
|
||||
def stop(self) -> None:
|
||||
with self._lifecycle_lock:
|
||||
if not self._started:
|
||||
return
|
||||
self._started = False
|
||||
loop_started = self._loop_started
|
||||
self._loop_started = False
|
||||
self._stop_event.set()
|
||||
self._connected.clear()
|
||||
if loop_started:
|
||||
try:
|
||||
self.client.disconnect()
|
||||
except Exception:
|
||||
LOG.exception("MQTT disconnect failed")
|
||||
self.client.loop_stop()
|
||||
thread = self._bootstrap_thread
|
||||
if thread is not None and thread is not threading.current_thread():
|
||||
thread.join(timeout=1.0)
|
||||
|
||||
@staticmethod
|
||||
def _reason_success(reason_code: Any) -> bool:
|
||||
try:
|
||||
return int(reason_code) == 0
|
||||
except (TypeError, ValueError):
|
||||
return reason_code == 0
|
||||
|
||||
def _on_connect(self, client, userdata, flags, reason_code, properties=None) -> None:
|
||||
try:
|
||||
if not self._reason_success(reason_code):
|
||||
self.last_error = f"MQTT connect failed: {reason_code}"
|
||||
LOG.warning("%s", self.last_error)
|
||||
return
|
||||
with self._generation_lock:
|
||||
self._connection_generation += 1
|
||||
generation = self._connection_generation
|
||||
self._connected.set()
|
||||
self.last_error = None
|
||||
subscriptions = (
|
||||
(self.config.topics.player_status, PLAYER_STATUS_QOS),
|
||||
(self.config.topics.player_telemetry, PLAYER_TELEMETRY_QOS),
|
||||
(self.config.topics.player_state, PLAYER_STATE_QOS),
|
||||
(self.config.topics.controller_command, 1),
|
||||
(self.config.topics.router_mode_state, 1),
|
||||
)
|
||||
for topic, qos in subscriptions:
|
||||
client.subscribe(topic, qos=qos)
|
||||
self._enqueue(
|
||||
MqttEvent(
|
||||
kind="connected",
|
||||
connection_generation=generation,
|
||||
)
|
||||
)
|
||||
except BaseException:
|
||||
LOG.exception("unexpected MQTT on_connect failure")
|
||||
|
||||
def _on_connect_fail(self, client, userdata) -> None:
|
||||
self.last_error = (
|
||||
f"MQTT connection to {self.config.mqtt.host}:{self.config.mqtt.port} failed"
|
||||
)
|
||||
LOG.warning("%s; Paho will retry", self.last_error)
|
||||
|
||||
def _on_disconnect(
|
||||
self,
|
||||
client,
|
||||
userdata,
|
||||
disconnect_flags,
|
||||
reason_code,
|
||||
properties=None,
|
||||
) -> None:
|
||||
self._connected.clear()
|
||||
try:
|
||||
self._enqueue(
|
||||
MqttEvent(
|
||||
kind="disconnected",
|
||||
connection_generation=self.connection_generation,
|
||||
reason=str(reason_code),
|
||||
)
|
||||
)
|
||||
except BaseException:
|
||||
LOG.exception("unexpected MQTT on_disconnect failure")
|
||||
|
||||
def _on_message(self, client, userdata, message) -> None:
|
||||
try:
|
||||
self._enqueue(
|
||||
MqttEvent(
|
||||
kind="message",
|
||||
topic=str(message.topic),
|
||||
payload=bytes(message.payload),
|
||||
qos=int(getattr(message, "qos", 0)),
|
||||
retain=bool(getattr(message, "retain", False)),
|
||||
message_id=getattr(message, "mid", None),
|
||||
connection_generation=self.connection_generation,
|
||||
)
|
||||
)
|
||||
except BaseException:
|
||||
LOG.exception("unexpected MQTT on_message failure")
|
||||
|
||||
def _enqueue(self, event: MqttEvent) -> bool:
|
||||
while not self._stop_event.is_set():
|
||||
try:
|
||||
self.events.put(event, timeout=0.2)
|
||||
return True
|
||||
except queue.Full:
|
||||
continue
|
||||
return False
|
||||
|
||||
def acknowledge(self, event: MqttEvent) -> bool:
|
||||
if event.message_id is None or event.qos <= 0:
|
||||
return True
|
||||
if event.connection_generation != self.connection_generation:
|
||||
LOG.warning(
|
||||
"skipping stale MQTT ack mid=%s from generation=%s",
|
||||
event.message_id,
|
||||
event.connection_generation,
|
||||
)
|
||||
return False
|
||||
try:
|
||||
result = self.client.ack(event.message_id, event.qos)
|
||||
except BaseException:
|
||||
LOG.exception("MQTT ack failed mid=%s", event.message_id)
|
||||
return False
|
||||
return result == mqtt.MQTT_ERR_SUCCESS
|
||||
|
||||
def _publish(
|
||||
self,
|
||||
topic: str,
|
||||
payload: str,
|
||||
*,
|
||||
qos: int,
|
||||
retain: bool,
|
||||
require_connection: bool,
|
||||
) -> PublishReceipt:
|
||||
if require_connection and not self.connected:
|
||||
return PublishReceipt(False, reason="MQTT is disconnected")
|
||||
try:
|
||||
info = self.client.publish(topic, payload, qos=qos, retain=retain)
|
||||
except BaseException as exc:
|
||||
LOG.exception("MQTT publish failed topic=%s", topic)
|
||||
return PublishReceipt(False, reason=str(exc))
|
||||
if info.rc != mqtt.MQTT_ERR_SUCCESS:
|
||||
return PublishReceipt(False, message_id=getattr(info, "mid", None), reason=f"MQTT rc={info.rc}")
|
||||
return PublishReceipt(True, message_id=getattr(info, "mid", None))
|
||||
|
||||
@staticmethod
|
||||
def _json_payload(payload: Mapping[str, Any]) -> str:
|
||||
return json.dumps(payload, allow_nan=False, separators=(",", ":"))
|
||||
|
||||
def publish_player_command(self, payload: Mapping[str, Any]) -> PublishReceipt:
|
||||
return self._publish(
|
||||
self.config.topics.player_command,
|
||||
self._json_payload(payload),
|
||||
qos=PLAYER_COMMAND_QOS,
|
||||
retain=PLAYER_COMMAND_RETAIN,
|
||||
require_connection=True,
|
||||
)
|
||||
|
||||
def publish_router_mode(self, mode: str) -> PublishReceipt:
|
||||
return self._publish(
|
||||
self.config.topics.router_mode_set,
|
||||
mode,
|
||||
qos=ROUTER_COMMAND_QOS,
|
||||
retain=ROUTER_COMMAND_RETAIN,
|
||||
require_connection=True,
|
||||
)
|
||||
|
||||
def publish_legacy(self, topic: str, payload: str, *, retain: bool = True) -> PublishReceipt:
|
||||
return self._publish(
|
||||
topic,
|
||||
payload,
|
||||
qos=0,
|
||||
retain=retain,
|
||||
require_connection=True,
|
||||
)
|
||||
|
||||
def publish_json(
|
||||
self,
|
||||
topic: str,
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
qos: int = 0,
|
||||
retain: bool = False,
|
||||
) -> PublishReceipt:
|
||||
return self._publish(
|
||||
topic,
|
||||
self._json_payload(payload),
|
||||
qos=qos,
|
||||
retain=retain,
|
||||
require_connection=True,
|
||||
)
|
||||
Reference in New Issue
Block a user