feat: coordinate authoritative playback and audio routing

This commit is contained in:
Media
2026-08-06 18:52:01 -05:00
parent 8b3bdae8a0
commit 3ee7f94d72
17 changed files with 4863 additions and 186 deletions
+5 -2
View File
@@ -8,6 +8,8 @@ mqtt:
reconnect_min_seconds: 1 reconnect_min_seconds: 1
reconnect_max_seconds: 30 reconnect_max_seconds: 30
command_queue_size: 512 command_queue_size: 512
# QoS 1 is accepted only after the broker PUBACKs within this bound.
publish_ack_timeout_seconds: 2
topics: topics:
player_command: youtube/player/cmd player_command: youtube/player/cmd
@@ -19,8 +21,9 @@ topics:
playback_base: youtube/media/playback playback_base: youtube/media/playback
router_mode_set: audio/player/mode/set router_mode_set: audio/player/mode/set
router_mode_state: audio/player/mode/state router_mode_state: audio/player/mode/state
# Optional additive result topic. Existing consumers do not depend on it. # Optional additive result/warning topic. Set null to disable it.
controller_result: null # Existing consumers never depend on this topic.
controller_result: youtube/controller/v2/events
storage: storage:
media_base: /mnt/Media/YTPlayer media_base: /mnt/Media/YTPlayer
+11 -1
View File
@@ -31,6 +31,7 @@ class MqttConfig:
reconnect_min_seconds: int = 1 reconnect_min_seconds: int = 1
reconnect_max_seconds: int = 30 reconnect_max_seconds: int = 30
command_queue_size: int = 512 command_queue_size: int = 512
publish_ack_timeout_seconds: float = 2.0
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -195,6 +196,11 @@ def load_config(
reconnect_min_seconds=_integer(mqtt_raw.get("reconnect_min_seconds", 1), "mqtt.reconnect_min_seconds"), reconnect_min_seconds=_integer(mqtt_raw.get("reconnect_min_seconds", 1), "mqtt.reconnect_min_seconds"),
reconnect_max_seconds=_integer(mqtt_raw.get("reconnect_max_seconds", 30), "mqtt.reconnect_max_seconds"), reconnect_max_seconds=_integer(mqtt_raw.get("reconnect_max_seconds", 30), "mqtt.reconnect_max_seconds"),
command_queue_size=_integer(mqtt_raw.get("command_queue_size", 512), "mqtt.command_queue_size"), command_queue_size=_integer(mqtt_raw.get("command_queue_size", 512), "mqtt.command_queue_size"),
publish_ack_timeout_seconds=_number(
mqtt_raw.get("publish_ack_timeout_seconds", 2.0),
"mqtt.publish_ack_timeout_seconds",
minimum=0.1,
),
) )
if mqtt.reconnect_max_seconds < mqtt.reconnect_min_seconds: if mqtt.reconnect_max_seconds < mqtt.reconnect_min_seconds:
raise ConfigError("mqtt.reconnect_max_seconds must be >= reconnect_min_seconds") raise ConfigError("mqtt.reconnect_max_seconds must be >= reconnect_min_seconds")
@@ -211,7 +217,11 @@ def load_config(
playback_base=_text(topics_raw.get("playback_base", "youtube/media/playback"), "topics.playback_base"), playback_base=_text(topics_raw.get("playback_base", "youtube/media/playback"), "topics.playback_base"),
router_mode_set=_text(topics_raw.get("router_mode_set", "audio/player/mode/set"), "topics.router_mode_set"), router_mode_set=_text(topics_raw.get("router_mode_set", "audio/player/mode/set"), "topics.router_mode_set"),
router_mode_state=_text(topics_raw.get("router_mode_state", "audio/player/mode/state"), "topics.router_mode_state"), router_mode_state=_text(topics_raw.get("router_mode_state", "audio/player/mode/state"), "topics.router_mode_state"),
controller_result=_text(topics_raw.get("controller_result"), "topics.controller_result", allow_none=True), controller_result=_text(
topics_raw.get("controller_result", "youtube/controller/v2/events"),
"topics.controller_result",
allow_none=True,
),
) )
configured_slots = _integer(storage_raw.get("slot_count", 10), "storage.slot_count") configured_slots = _integer(storage_raw.get("slot_count", 10), "storage.slot_count")
+1 -1
View File
@@ -25,7 +25,7 @@ class Topics:
playback_base: str = "youtube/media/playback" playback_base: str = "youtube/media/playback"
router_mode_set: str = "audio/player/mode/set" router_mode_set: str = "audio/player/mode/set"
router_mode_state: str = "audio/player/mode/state" router_mode_state: str = "audio/player/mode/state"
controller_result: str | None = None controller_result: str | None = "youtube/controller/v2/events"
def slot_base(self, slot: int) -> str: def slot_base(self, slot: int) -> str:
if slot not in SLOT_IDS: if slot not in SLOT_IDS:
+109 -3
View File
@@ -4,9 +4,11 @@ from __future__ import annotations
import logging import logging
import os import os
import signal
import shutil import shutil
import subprocess import subprocess
import threading import threading
import time
import uuid import uuid
from concurrent.futures import Future, ThreadPoolExecutor from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass from dataclasses import dataclass
@@ -32,6 +34,10 @@ class DownloadEvent:
message: str | None = None message: str | None = None
job_id: str | None = None job_id: str | None = None
generation: int | None = None generation: int | None = None
url: str | None = None
submitter: str | None = None
note: str | None = None
reused_slot: bool = False
EventCallback = Callable[[DownloadEvent], None] EventCallback = Callable[[DownloadEvent], None]
@@ -66,7 +72,7 @@ class DownloadManager:
store: ControllerStore, store: ControllerStore,
*, *,
on_event: EventCallback | None = None, on_event: EventCallback | None = None,
runner: Runner = subprocess.run, runner: Runner | None = None,
executor: ThreadPoolExecutor | None = None, executor: ThreadPoolExecutor | None = None,
): ):
self.config = config self.config = config
@@ -80,6 +86,7 @@ class DownloadManager:
self._owns_executor = executor is None self._owns_executor = executor is None
self._lock = threading.Lock() self._lock = threading.Lock()
self._futures: set[Future[Any]] = set() self._futures: set[Future[Any]] = set()
self._processes: set[subprocess.Popen[str]] = set()
self._stopping = False self._stopping = False
def enqueue( def enqueue(
@@ -106,6 +113,10 @@ class DownloadManager:
state="queued", state="queued",
job_id=reservation.job_id, job_id=reservation.job_id,
generation=reservation.generation, generation=reservation.generation,
url=url,
submitter=submitter,
note=note,
reused_slot=reservation.reused_slot,
) )
) )
try: try:
@@ -184,7 +195,7 @@ class DownloadManager:
url, url,
] ]
try: try:
result = self.runner( result = self._run_command(
arguments, arguments,
check=True, check=True,
capture_output=True, capture_output=True,
@@ -246,7 +257,7 @@ class DownloadManager:
), ),
url, url,
] ]
self.runner( self._run_command(
arguments, arguments,
check=True, check=True,
text=True, text=True,
@@ -322,6 +333,90 @@ class DownloadManager:
return f"yt-dlp failed with exit code {exc.returncode}" return f"yt-dlp failed with exit code {exc.returncode}"
return f"download failed: {exc}" return f"download failed: {exc}"
def _run_command(
self,
arguments: list[str],
*,
check: bool,
text: bool,
timeout: float,
capture_output: bool = False,
) -> subprocess.CompletedProcess[str]:
if self.runner is not None:
return self.runner(
arguments,
check=check,
text=text,
timeout=timeout,
capture_output=capture_output,
)
with self._lock:
if self._stopping:
raise RuntimeError("Controller is shutting down")
process = subprocess.Popen(
arguments,
stdout=subprocess.PIPE if capture_output else None,
stderr=subprocess.PIPE if capture_output else None,
text=text,
start_new_session=True,
)
self._processes.add(process)
try:
try:
stdout, stderr = process.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
self._terminate_process(process, force=False)
try:
stdout, stderr = process.communicate(timeout=2.0)
except subprocess.TimeoutExpired:
self._terminate_process(process, force=True)
stdout, stderr = process.communicate()
raise subprocess.TimeoutExpired(
arguments,
timeout,
output=stdout,
stderr=stderr,
)
completed = subprocess.CompletedProcess(
arguments,
process.returncode,
stdout=stdout,
stderr=stderr,
)
if check and process.returncode:
raise subprocess.CalledProcessError(
process.returncode,
arguments,
output=stdout,
stderr=stderr,
)
return completed
finally:
with self._lock:
self._processes.discard(process)
@staticmethod
def _terminate_process(
process: subprocess.Popen[str],
*,
force: bool,
) -> None:
if process.poll() is not None:
return
try:
os.killpg(
process.pid,
signal.SIGKILL if force else signal.SIGTERM,
)
except ProcessLookupError:
return
except OSError:
try:
process.kill() if force else process.terminate()
except ProcessLookupError:
pass
@staticmethod @staticmethod
def _fsync_directory(path: Path) -> None: def _fsync_directory(path: Path) -> None:
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
@@ -387,6 +482,17 @@ class DownloadManager:
def shutdown(self, *, wait: bool = True) -> None: def shutdown(self, *, wait: bool = True) -> None:
with self._lock: with self._lock:
self._stopping = True self._stopping = True
processes = tuple(self._processes)
for process in processes:
self._terminate_process(process, force=False)
deadline = time.monotonic() + 2.0
for process in processes:
if process.poll() is not None:
continue
try:
process.wait(timeout=max(0.0, deadline - time.monotonic()))
except subprocess.TimeoutExpired:
self._terminate_process(process, force=True)
if self._owns_executor: if self._owns_executor:
self._executor.shutdown(wait=wait, cancel_futures=True) self._executor.shutdown(wait=wait, cancel_futures=True)
+28
View File
@@ -84,10 +84,38 @@ class PlayerObservation:
generation: int generation: int
reconciliation_state: str reconciliation_state: str
reconciliation_due_at: float | None reconciliation_due_at: float | None
reconciliation_id: str | None
last_status_at: float | None last_status_at: float | None
last_telemetry_at: float | None last_telemetry_at: float | None
@dataclass(frozen=True)
class RouterObservation:
mode: str | None
freshness: str
observed_at: float | None
last_warning: str | None
@dataclass(frozen=True)
class AudioLeaseRecord:
lease_id: str
operation_id: str | None
playback_generation: int | None
state: str
previous_mode: str | None
target_mode: str
restoration_mode: str | None
restoration_source: str
controller_changed: bool
confirmation_deadline: float | None
created_at: float
acquired_at: float | None
released_at: float | None
warning: str | None
error: str | None
@dataclass(frozen=True) @dataclass(frozen=True)
class MigrationReport: class MigrationReport:
migrated: bool migrated: bool
+207 -40
View File
@@ -6,6 +6,7 @@ import json
import logging import logging
import queue import queue
import threading import threading
import uuid
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Mapping from typing import Any, Mapping
@@ -33,9 +34,11 @@ class MqttEvent:
payload: bytes = b"" payload: bytes = b""
qos: int = 0 qos: int = 0
retain: bool = False retain: bool = False
duplicate: bool = False
message_id: int | None = None message_id: int | None = None
connection_generation: int | None = None connection_generation: int | None = None
reason: str | None = None reason: str | None = None
controller_echo: bool = False
@dataclass(frozen=True) @dataclass(frozen=True)
@@ -43,6 +46,7 @@ class PublishReceipt:
accepted: bool accepted: bool
message_id: int | None = None message_id: int | None = None
reason: str | None = None reason: str | None = None
uncertain: bool = False
class MqttAdapter: class MqttAdapter:
@@ -59,11 +63,8 @@ class MqttAdapter:
self.events: "queue.Queue[MqttEvent]" = events or queue.Queue( self.events: "queue.Queue[MqttEvent]" = events or queue.Queue(
maxsize=config.mqtt.command_queue_size maxsize=config.mqtt.command_queue_size
) )
self.client = client or mqtt.Client( self._injected_client = client is not None
mqtt.CallbackAPIVersion.VERSION2, self.client = client or self._new_client()
client_id=config.mqtt.client_id,
clean_session=True,
)
self._generation_lock = threading.Lock() self._generation_lock = threading.Lock()
self._connection_generation = 0 self._connection_generation = 0
self._connected = threading.Event() self._connected = threading.Event()
@@ -72,9 +73,18 @@ class MqttAdapter:
self._started = False self._started = False
self._loop_started = False self._loop_started = False
self._bootstrap_thread: threading.Thread | None = None self._bootstrap_thread: threading.Thread | None = None
self._router_echo_lock = threading.Lock()
self._pending_router_echoes: dict[str, bytes] = {}
self.last_error: str | None = None self.last_error: str | None = None
self._configure_client() self._configure_client()
def _new_client(self):
return mqtt.Client(
mqtt.CallbackAPIVersion.VERSION2,
client_id=self.config.mqtt.client_id,
clean_session=True,
)
def _configure_client(self) -> None: def _configure_client(self) -> None:
if self.config.mqtt.username is not None: if self.config.mqtt.username is not None:
self.client.username_pw_set( self.client.username_pw_set(
@@ -116,20 +126,32 @@ class MqttAdapter:
self._stop_event.clear() self._stop_event.clear()
try: try:
self._begin_async_connection() self._begin_async_connection()
except (OSError, mqtt.WebsocketConnectionError) as exc: except (OSError, mqtt.WebsocketConnectionError, RuntimeError) as exc:
self.last_error = f"initial MQTT setup failed: {exc}" self.last_error = f"initial MQTT setup failed: {exc}"
LOG.warning("%s; retrying in background", self.last_error) LOG.warning("%s; retrying in background", self.last_error)
self._start_bootstrap_retry() self._start_bootstrap_retry()
def _begin_async_connection(self) -> None: def _begin_async_connection(self) -> None:
self.client.connect_async( with self._lifecycle_lock:
if not self._started or self._stop_event.is_set():
return
client = self.client
result = client.connect_async(
self.config.mqtt.host, self.config.mqtt.host,
self.config.mqtt.port, self.config.mqtt.port,
keepalive=self.config.mqtt.keepalive_seconds, keepalive=self.config.mqtt.keepalive_seconds,
) )
if result != mqtt.MQTT_ERR_SUCCESS:
raise RuntimeError(f"failed to configure MQTT connection: {result}")
with self._lifecycle_lock: with self._lifecycle_lock:
if (
not self._started
or self._stop_event.is_set()
or client is not self.client
):
return
if not self._loop_started: if not self._loop_started:
result = self.client.loop_start() result = client.loop_start()
if result != mqtt.MQTT_ERR_SUCCESS: if result != mqtt.MQTT_ERR_SUCCESS:
raise RuntimeError(f"failed to start MQTT network loop: {result}") raise RuntimeError(f"failed to start MQTT network loop: {result}")
self._loop_started = True self._loop_started = True
@@ -159,17 +181,21 @@ class MqttAdapter:
delay = min(maximum, max(delay * 2, 1.0)) delay = min(maximum, max(delay * 2, 1.0))
except RuntimeError as exc: except RuntimeError as exc:
self.last_error = str(exc) self.last_error = str(exc)
LOG.error("%s", self.last_error) LOG.warning("%s", self.last_error)
return delay = min(maximum, max(delay * 2, 1.0))
def stop(self) -> None: def stop(self) -> None:
# Signal callbacks/retry work before waiting for the lifecycle lock.
# A callback already inside subscribe/connect then observes shutdown
# and cannot publish a post-stop connected state.
self._stop_event.set()
with self._lifecycle_lock: with self._lifecycle_lock:
if not self._started: if not self._started:
self._connected.clear()
return return
self._started = False self._started = False
loop_started = self._loop_started loop_started = self._loop_started
self._loop_started = False self._loop_started = False
self._stop_event.set()
self._connected.clear() self._connected.clear()
if loop_started: if loop_started:
try: try:
@@ -190,24 +216,43 @@ class MqttAdapter:
def _on_connect(self, client, userdata, flags, reason_code, properties=None) -> None: def _on_connect(self, client, userdata, flags, reason_code, properties=None) -> None:
try: try:
if not self._reason_success(reason_code): with self._lifecycle_lock:
self.last_error = f"MQTT connect failed: {reason_code}" if (
LOG.warning("%s", self.last_error) client is not self.client
or not self._started
or self._stop_event.is_set()
):
return
if not self._reason_success(reason_code):
self.last_error = f"MQTT connect failed: {reason_code}"
LOG.warning("%s", self.last_error)
return
if self._stop_event.is_set():
return
with self._generation_lock:
self._connection_generation += 1
generation = self._connection_generation
self._connected.set()
self.last_error = None
subscriptions = (
(self.config.topics.player_status, PLAYER_STATUS_QOS),
(self.config.topics.player_telemetry, PLAYER_TELEMETRY_QOS),
(self.config.topics.player_state, PLAYER_STATE_QOS),
(self.config.topics.controller_command, 1),
(self.config.topics.router_mode_set, ROUTER_COMMAND_QOS),
(self.config.topics.router_mode_state, 1),
)
for topic, qos in subscriptions:
if self._stop_event.is_set():
self._connected.clear()
return
client.subscribe(topic, qos=qos)
if self._stop_event.is_set():
self._connected.clear()
return
if self._stop_event.is_set():
self._connected.clear()
return return
with self._generation_lock:
self._connection_generation += 1
generation = self._connection_generation
self._connected.set()
self.last_error = None
subscriptions = (
(self.config.topics.player_status, PLAYER_STATUS_QOS),
(self.config.topics.player_telemetry, PLAYER_TELEMETRY_QOS),
(self.config.topics.player_state, PLAYER_STATE_QOS),
(self.config.topics.controller_command, 1),
(self.config.topics.router_mode_state, 1),
)
for topic, qos in subscriptions:
client.subscribe(topic, qos=qos)
self._enqueue( self._enqueue(
MqttEvent( MqttEvent(
kind="connected", kind="connected",
@@ -231,6 +276,8 @@ class MqttAdapter:
reason_code, reason_code,
properties=None, properties=None,
) -> None: ) -> None:
if client is not self.client:
return
self._connected.clear() self._connected.clear()
try: try:
self._enqueue( self._enqueue(
@@ -245,15 +292,33 @@ class MqttAdapter:
def _on_message(self, client, userdata, message) -> None: def _on_message(self, client, userdata, message) -> None:
try: try:
# Capture client identity and generation atomically with session
# replacement. A callback that was already running when an
# uncertain command discards the transport must be able to leave a
# full queue so loop_stop() cannot deadlock waiting for it.
with self._lifecycle_lock:
if client is not self.client:
return
with self._generation_lock:
generation = self._connection_generation
topic = str(message.topic)
payload = bytes(message.payload)
retained = bool(getattr(message, "retain", False))
self._enqueue( self._enqueue(
MqttEvent( MqttEvent(
kind="message", kind="message",
topic=str(message.topic), topic=topic,
payload=bytes(message.payload), payload=payload,
qos=int(getattr(message, "qos", 0)), qos=int(getattr(message, "qos", 0)),
retain=bool(getattr(message, "retain", False)), retain=retained,
duplicate=bool(getattr(message, "dup", False)),
message_id=getattr(message, "mid", None), message_id=getattr(message, "mid", None),
connection_generation=self.connection_generation, connection_generation=generation,
controller_echo=(
topic == self.config.topics.router_mode_set
and not retained
and self._claim_router_echo(payload)
),
) )
) )
except BaseException: except BaseException:
@@ -261,6 +326,11 @@ class MqttAdapter:
def _enqueue(self, event: MqttEvent) -> bool: def _enqueue(self, event: MqttEvent) -> bool:
while not self._stop_event.is_set(): while not self._stop_event.is_set():
if (
event.connection_generation is not None
and event.connection_generation != self.connection_generation
):
return False
try: try:
self.events.put(event, timeout=0.2) self.events.put(event, timeout=0.2)
return True return True
@@ -293,8 +363,9 @@ class MqttAdapter:
qos: int, qos: int,
retain: bool, retain: bool,
require_connection: bool, require_connection: bool,
invalidate_transport_on_uncertain: bool = False,
) -> PublishReceipt: ) -> PublishReceipt:
if require_connection and not self.connected: if require_connection and (not self.started or not self.connected):
return PublishReceipt(False, reason="MQTT is disconnected") return PublishReceipt(False, reason="MQTT is disconnected")
try: try:
info = self.client.publish(topic, payload, qos=qos, retain=retain) info = self.client.publish(topic, payload, qos=qos, retain=retain)
@@ -303,8 +374,79 @@ class MqttAdapter:
return PublishReceipt(False, reason=str(exc)) return PublishReceipt(False, reason=str(exc))
if info.rc != mqtt.MQTT_ERR_SUCCESS: if info.rc != mqtt.MQTT_ERR_SUCCESS:
return PublishReceipt(False, message_id=getattr(info, "mid", None), reason=f"MQTT rc={info.rc}") return PublishReceipt(False, message_id=getattr(info, "mid", None), reason=f"MQTT rc={info.rc}")
if qos > 0 and hasattr(info, "wait_for_publish"):
try:
info.wait_for_publish(
timeout=self.config.mqtt.publish_ack_timeout_seconds
)
acknowledged = bool(info.is_published())
except BaseException as exc:
acknowledged = False
detail = str(exc) or exc.__class__.__name__
else:
detail = (
f"no broker PUBACK within "
f"{self.config.mqtt.publish_ack_timeout_seconds:g} seconds"
)
if not acknowledged:
reason = f"MQTT QoS 1 publication is uncertain: {detail}"
if invalidate_transport_on_uncertain:
self._discard_unacknowledged_transport(reason)
return PublishReceipt(
False,
message_id=getattr(info, "mid", None),
reason=reason,
uncertain=True,
)
return PublishReceipt(True, message_id=getattr(info, "mid", None)) return PublishReceipt(True, message_id=getattr(info, "mid", None))
def _discard_unacknowledged_transport(self, reason: str) -> None:
"""Drop the Paho session so stale QoS 1 commands cannot replay."""
if self._injected_client:
# Test transports do not own a recreatable Paho session. Production
# always follows the replacement path below.
with self._generation_lock:
self._connection_generation += 1
self._connected.clear()
return
with self._lifecycle_lock:
old_client = self.client
old_loop_started = self._loop_started
self._loop_started = False
should_restart = self._started and not self._stop_event.is_set()
self._connected.clear()
with self._generation_lock:
previous_generation = self._connection_generation
self._connection_generation += 1
self.client = self._new_client()
self._configure_client()
try:
old_client.disconnect()
except BaseException:
LOG.exception("failed to disconnect uncertain MQTT transport")
if old_loop_started:
try:
old_client.loop_stop()
except BaseException:
LOG.exception("failed to stop uncertain MQTT transport")
self.last_error = reason
try:
self.events.put_nowait(
MqttEvent(
kind="disconnected",
connection_generation=previous_generation,
reason=reason,
)
)
except queue.Full:
# This method can run on the sole coordinator thread; blocking on
# its own queue would deadlock. The uncertain PublishReceipt still
# drives the operation-specific conservative transition.
LOG.error("MQTT event queue was full while discarding uncertain transport")
if should_restart:
self._start_bootstrap_retry()
@staticmethod @staticmethod
def _json_payload(payload: Mapping[str, Any]) -> str: def _json_payload(payload: Mapping[str, Any]) -> str:
return json.dumps(payload, allow_nan=False, separators=(",", ":")) return json.dumps(payload, allow_nan=False, separators=(",", ":"))
@@ -316,16 +458,41 @@ class MqttAdapter:
qos=PLAYER_COMMAND_QOS, qos=PLAYER_COMMAND_QOS,
retain=PLAYER_COMMAND_RETAIN, retain=PLAYER_COMMAND_RETAIN,
require_connection=True, require_connection=True,
invalidate_transport_on_uncertain=True,
) )
def publish_router_mode(self, mode: str) -> PublishReceipt: def publish_router_mode(self, mode: str) -> PublishReceipt:
return self._publish( # A broker normally sends our publication back through the matching
self.config.topics.router_mode_set, # subscription. Register the exact payload before publishing because
mode, # Paho's network thread may receive that echo before publish() returns.
qos=ROUTER_COMMAND_QOS, # The marker is removed as soon as the QoS 1 publish call completes;
retain=ROUTER_COMMAND_RETAIN, # this intentionally prefers a false ownership loss over mistaking a
require_connection=True, # later same-value operator command for a Controller echo.
) echo_token = uuid.uuid4().hex
with self._router_echo_lock:
self._pending_router_echoes[echo_token] = mode.encode("utf-8")
try:
return self._publish(
self.config.topics.router_mode_set,
mode,
qos=ROUTER_COMMAND_QOS,
retain=ROUTER_COMMAND_RETAIN,
require_connection=True,
invalidate_transport_on_uncertain=True,
)
finally:
with self._router_echo_lock:
self._pending_router_echoes.pop(echo_token, None)
def _claim_router_echo(self, payload: bytes) -> bool:
"""Claim one exact mode-set echo from an in-flight local publish."""
with self._router_echo_lock:
for token, expected in tuple(self._pending_router_echoes.items()):
if expected == payload:
del self._pending_router_echoes[token]
return True
return False
def publish_legacy(self, topic: str, payload: str, *, retain: bool = True) -> PublishReceipt: def publish_legacy(self, topic: str, payload: str, *, retain: bool = True) -> PublishReceipt:
return self._publish( return self._publish(
+400 -73
View File
@@ -3,6 +3,7 @@
from __future__ import annotations from __future__ import annotations
import math import math
import os
import time import time
import uuid import uuid
from dataclasses import dataclass, replace from dataclasses import dataclass, replace
@@ -28,6 +29,7 @@ class PlayerOperation:
slot: int | None slot: int | None
expected_path: str | None expected_path: str | None
deadline: float deadline: float
reconciliation_id: str | None = None
state: str = "awaiting_player" state: str = "awaiting_player"
message: str | None = None message: str | None = None
@@ -71,6 +73,7 @@ class PlaybackReducer:
*, *,
slot: int | None = None, slot: int | None = None,
expected_path: str | None = None, expected_path: str | None = None,
operation_id: str | None = None,
) -> PlayerOperation: ) -> PlayerOperation:
if action not in CONTROL_ACTIONS: if action not in CONTROL_ACTIONS:
raise ValueError(f"unsupported Player action {action!r}") raise ValueError(f"unsupported Player action {action!r}")
@@ -82,14 +85,13 @@ class PlaybackReducer:
) )
self._operation_generation += 1 self._operation_generation += 1
operation = PlayerOperation( operation = PlayerOperation(
operation_id=uuid.uuid4().hex, operation_id=operation_id or uuid.uuid4().hex,
generation=self._operation_generation, generation=self._operation_generation,
action=action, action=action,
slot=slot, slot=slot,
expected_path=expected_path, expected_path=expected_path,
deadline=self._monotonic() + self.operation_timeout_seconds, deadline=self._monotonic() + self.operation_timeout_seconds,
) )
self.pending = operation
if action == "play" and slot is not None: if action == "play" and slot is not None:
self.store.protect_slot( self.store.protect_slot(
slot, slot,
@@ -97,11 +99,40 @@ class PlaybackReducer:
guard_id=f"operation-{operation.operation_id}", guard_id=f"operation-{operation.operation_id}",
details={"operation_id": operation.operation_id}, details={"operation_id": operation.operation_id},
) )
elif action == "play" and expected_path is not None:
due = None
if self.config.storage.uncertain_review_after_seconds > 0:
due = (
self._wall_clock()
+ self.config.storage.uncertain_review_after_seconds
)
observation = self.store.mark_external_uncertain(
expected_path,
review_after=due,
details={
"operation_id": operation.operation_id,
"reason": "play command pending Player evidence",
},
)
operation = replace(
operation,
reconciliation_id=observation.reconciliation_id,
)
self.pending = operation
return operation return operation
def publish_failed(self, reason: str) -> PlaybackEffect | None: def publish_failed(self, reason: str) -> PlaybackEffect | None:
if self.pending is None: if self.pending is None:
return None return None
if (
self.pending.action == "play"
and self.pending.slot is None
and self.pending.reconciliation_id is not None
):
self.store.cancel_external_intent(
self.pending.reconciliation_id,
reason=reason,
)
return self._finish_operation("failed", reason) return self._finish_operation("failed", reason)
def _finish_operation(self, state: str, message: str | None = None) -> PlaybackEffect: def _finish_operation(self, state: str, message: str | None = None) -> PlaybackEffect:
@@ -142,11 +173,64 @@ class PlaybackReducer:
operation = self.pending operation = self.pending
if operation is None: if operation is None:
return False return False
slot_matches = operation.slot is None or operation.slot == slot positive_match = False
path_matches = operation.expected_path is None or operation.expected_path == path if operation.slot is not None and slot is not None:
if operation.slot is not None and operation.expected_path is not None: if operation.slot != slot:
return slot_matches or path_matches return False
return slot_matches and path_matches positive_match = True
if operation.expected_path is not None and path is not None:
if os.path.abspath(operation.expected_path) != os.path.abspath(path):
return False
positive_match = True
return positive_match
def _canonical_slot_for_path(self, path: str | None) -> int | None:
if path is None:
return None
candidate = os.path.abspath(path)
for slot in SLOT_IDS:
slot_directory = os.path.abspath(
self.config.storage.media_base / f"slot{slot}"
)
try:
if os.path.commonpath((candidate, slot_directory)) == slot_directory:
return slot
except ValueError:
continue
return None
def _normalized_identity(
self,
payload: Mapping[str, Any],
) -> tuple[int | None, str | None, str | None]:
slot = self._slot(payload.get("slot"))
path = self._path(payload.get("path"))
derived = self._canonical_slot_for_path(path)
if slot is not None and derived is not None and slot != derived:
return (
slot,
path,
f"Player identity conflicts: slot {slot} does not contain path {path!r}",
)
return (derived if slot is None else slot), path, None
@staticmethod
def _same_playback_identity(
previous: PlayerObservation,
slot: int | None,
path: str | None,
) -> bool:
if previous.active_slot != slot:
return False
if slot is None:
return (
previous.active_path is not None
and path is not None
and os.path.abspath(previous.active_path) == os.path.abspath(path)
)
if previous.active_path is None or path is None:
return True
return os.path.abspath(previous.active_path) == os.path.abspath(path)
def _active_evidence( def _active_evidence(
self, self,
@@ -158,11 +242,11 @@ class PlaybackReducer:
telemetry: Mapping[str, Any] | None = None, telemetry: Mapping[str, Any] | None = None,
) -> PlayerObservation: ) -> PlayerObservation:
previous = self.store.player_observation() previous = self.store.player_observation()
if slot is None: if slot is None and (path is None or path == previous.active_path):
slot = previous.active_slot slot = previous.active_slot
if path is None: if path is None and (slot is None or slot == previous.active_slot):
path = previous.active_path path = previous.active_path
if paused is None and previous.active_slot == slot: if paused is None and self._same_playback_identity(previous, slot, path):
paused = previous.paused paused = previous.paused
phase = "paused" if paused is True else "playing" phase = "paused" if paused is True else "playing"
if slot is not None: if slot is not None:
@@ -171,6 +255,8 @@ class PlaybackReducer:
kind="player_active", kind="player_active",
details={"path": path}, details={"path": path},
) )
elif path is not None:
self.store.reconcile_external_active()
return self.store.set_player_observation( return self.store.set_player_observation(
availability="online", availability="online",
phase=phase, phase=phase,
@@ -179,6 +265,7 @@ class PlaybackReducer:
paused=paused, paused=paused,
reconciliation_state="confirmed", reconciliation_state="confirmed",
reconciliation_due_at=None, reconciliation_due_at=None,
reconciliation_id=None,
status_payload=status, status_payload=status,
telemetry_payload=telemetry, telemetry_payload=telemetry,
) )
@@ -199,6 +286,7 @@ class PlaybackReducer:
paused=False, paused=False,
reconciliation_state="clear", reconciliation_state="clear",
reconciliation_due_at=None, reconciliation_due_at=None,
reconciliation_id=None,
status_payload=status, status_payload=status,
telemetry_payload=telemetry, telemetry_payload=telemetry,
) )
@@ -212,36 +300,63 @@ class PlaybackReducer:
if not isinstance(state, str) or not state: if not isinstance(state, str) or not state:
return [PlaybackEffect("warning", message="Player status is missing state")] return [PlaybackEffect("warning", message="Player status is missing state")]
state = state.strip() state = state.strip()
slot = self._slot(payload.get("slot")) slot, path, identity_warning = self._normalized_identity(payload)
path = self._path(payload.get("path")) if identity_warning is not None:
return [PlaybackEffect("warning", slot=slot, message=identity_warning)]
effects: list[PlaybackEffect] = [] effects: list[PlaybackEffect] = []
if state == "offline": if state == "offline":
previous = self.store.player_observation() previous = self.store.player_observation()
due = None due = None
if ( active_known = (
previous.active_slot is not None previous.active_slot is not None or previous.active_path is not None
and self.config.storage.uncertain_review_after_seconds > 0 )
): if previous.active_slot is not None:
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds if self.config.storage.uncertain_review_after_seconds > 0:
self.store.reconcile_active_guard( due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
guard = self.store.mark_slot_uncertain(
previous.active_slot, previous.active_slot,
kind="restart_uncertain",
review_after=due, review_after=due,
details={"reason": "Player offline"}, details={"reason": "Player offline"},
) )
self.store.set_player_observation( due = guard.review_after
availability="offline", self.store.set_player_observation(
phase="unknown" if previous.active_slot is not None else previous.phase, availability="offline",
active_slot=previous.active_slot, phase="unknown",
active_path=previous.active_path, active_slot=previous.active_slot,
paused=previous.paused, active_path=previous.active_path,
reconciliation_state=( paused=previous.paused,
"awaiting_evidence" if previous.active_slot is not None else "unknown" reconciliation_state=(
), "operator_review"
reconciliation_due_at=due, if guard.state == "review_required"
status_payload=payload, else "awaiting_evidence"
) ),
reconciliation_due_at=due,
reconciliation_id=guard.guard_id,
status_payload=payload,
)
elif previous.active_path is not None:
if self.config.storage.uncertain_review_after_seconds > 0:
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
self.store.mark_external_uncertain(
previous.active_path,
review_after=due,
availability="offline",
details={"reason": "Player offline"},
status_payload=payload,
)
else:
self.store.set_player_observation(
availability="offline",
phase=previous.phase,
active_slot=None,
active_path=None,
paused=previous.paused,
reconciliation_state="unknown",
reconciliation_due_at=None,
reconciliation_id=None,
status_payload=payload,
)
effects.append( effects.append(
PlaybackEffect( PlaybackEffect(
"player_offline", "player_offline",
@@ -261,15 +376,56 @@ class PlaybackReducer:
message="Reconnect idle announcement carried an active path", message="Reconnect idle announcement carried an active path",
) )
) )
if self.pending is not None and self.pending.action == "play":
if self._matches_pending(slot, path):
effects.append(
self._finish_operation(
"confirmed", "matching reconnect evidence"
)
)
else:
effects.append(
self._finish_operation(
"failed",
"Player reconnected with different active playback",
)
)
else: else:
self._idle_evidence(status=payload) self._idle_evidence(status=payload)
effects.append(PlaybackEffect("player_idle")) effects.append(PlaybackEffect("player_idle"))
if self.pending is not None and self.pending.action == "stop": if self.pending is not None:
effects.append(self._finish_operation("confirmed")) if self.pending.action == "stop":
effects.append(self._finish_operation("confirmed"))
elif self.pending.action == "play":
effects.append(
self._finish_operation(
"failed", "Player reconnected without active playback"
)
)
return effects return effects
if state == "idle": if state == "idle":
reason = payload.get("reason") reason = payload.get("reason")
if reason not in TERMINAL_REASONS:
previous = self.store.player_observation()
self.store.set_player_observation(
availability="online",
phase=previous.phase,
active_slot=previous.active_slot,
active_path=previous.active_path,
paused=previous.paused,
reconciliation_state=previous.reconciliation_state,
reconciliation_due_at=previous.reconciliation_due_at,
reconciliation_id=previous.reconciliation_id,
status_payload=payload,
)
return [
PlaybackEffect(
"warning",
slot=previous.active_slot,
message="ambiguous Player idle status was not treated as terminal",
)
]
self._idle_evidence(status=payload) self._idle_evidence(status=payload)
effects.append(PlaybackEffect("player_idle", slot=slot, message=str(reason or "idle"))) effects.append(PlaybackEffect("player_idle", slot=slot, message=str(reason or "idle")))
if self.pending is not None: if self.pending is not None:
@@ -280,29 +436,33 @@ class PlaybackReducer:
return effects return effects
if state == "playing": if state == "playing":
previous = self.store.player_observation()
actual_paused = previous.paused if previous.active_slot == slot else None
self._active_evidence( self._active_evidence(
slot=slot, slot=slot,
path=path, path=path,
paused=actual_paused, paused=None,
status=payload, status=payload,
) )
effects.append(PlaybackEffect("player_active", slot=slot)) effects.append(PlaybackEffect("player_active", slot=slot))
if ( if (
self.pending is not None self.pending is not None
and self.pending.action == "play" and self.pending.action == "play"
and self._matches_pending(slot, path)
): ):
effects.append(self._finish_operation("confirmed")) if self._matches_pending(slot, path):
effects.append(self._finish_operation("confirmed"))
else:
effects.append(
self._finish_operation(
"failed",
"Player became active with a different slot or path",
)
)
return effects return effects
if state in {"paused", "toggled"}: if state in {"paused", "toggled"}:
previous = self.store.player_observation()
self._active_evidence( self._active_evidence(
slot=slot, slot=slot,
path=path, path=path,
paused=previous.paused if previous.active_slot == slot else None, paused=None,
status=payload, status=payload,
) )
effects.append( effects.append(
@@ -312,15 +472,24 @@ class PlaybackReducer:
message=f"Player issued {state}; awaiting telemetry", message=f"Player issued {state}; awaiting telemetry",
) )
) )
if self.pending is not None and self.pending.action == "play":
if self._matches_pending(slot, path):
effects.append(self._finish_operation("confirmed", state))
else:
effects.append(
self._finish_operation(
"failed",
"Player control status identified different playback",
)
)
return effects return effects
if state == "busy": if state == "busy":
if slot is not None or path is not None: if slot is not None or path is not None:
previous = self.store.player_observation()
self._active_evidence( self._active_evidence(
slot=slot, slot=slot,
path=path, path=path,
paused=previous.paused if previous.active_slot == slot else None, paused=None,
status=payload, status=payload,
) )
effects.append(PlaybackEffect("player_busy", slot=slot, message=payload.get("message"))) effects.append(PlaybackEffect("player_busy", slot=slot, message=payload.get("message")))
@@ -338,14 +507,29 @@ class PlaybackReducer:
kind="player_active", kind="player_active",
details={"error": payload.get("message")}, details={"error": payload.get("message")},
) )
elif active_path is not None:
self.store.reconcile_external_active()
self.store.set_player_observation( self.store.set_player_observation(
availability="online", availability="online",
phase="error", phase="error",
active_slot=active_slot, active_slot=active_slot,
active_path=active_path, active_path=active_path,
paused=previous.paused, paused=(
reconciliation_state=("confirmed" if active_slot is not None else "unknown"), previous.paused
if self._same_playback_identity(
previous,
active_slot,
active_path,
)
else None
),
reconciliation_state=(
"confirmed"
if active_slot is not None or active_path is not None
else "unknown"
),
reconciliation_due_at=None, reconciliation_due_at=None,
reconciliation_id=None,
status_payload=payload, status_payload=payload,
) )
message = str(payload.get("message") or "Player error") message = str(payload.get("message") or "Player error")
@@ -360,8 +544,9 @@ class PlaybackReducer:
def handle_telemetry(self, payload: Any) -> list[PlaybackEffect]: def handle_telemetry(self, payload: Any) -> list[PlaybackEffect]:
if not isinstance(payload, Mapping) or payload.get("state") != "telemetry": if not isinstance(payload, Mapping) or payload.get("state") != "telemetry":
return [PlaybackEffect("warning", message="invalid Player telemetry envelope")] return [PlaybackEffect("warning", message="invalid Player telemetry envelope")]
slot = self._slot(payload.get("slot")) slot, path, identity_warning = self._normalized_identity(payload)
path = self._path(payload.get("path")) if identity_warning is not None:
return [PlaybackEffect("warning", slot=slot, message=identity_warning)]
effects: list[PlaybackEffect] = [] effects: list[PlaybackEffect] = []
if path is None: if path is None:
self._idle_evidence(telemetry=payload) self._idle_evidence(telemetry=payload)
@@ -383,7 +568,15 @@ class PlaybackReducer:
) )
effects.append(PlaybackEffect("player_telemetry", slot=observation.active_slot)) effects.append(PlaybackEffect("player_telemetry", slot=observation.active_slot))
operation = self.pending operation = self.pending
if operation is None or not self._matches_pending(slot, path): if operation is None:
return effects
if not self._matches_pending(slot, path):
effects.append(
self._finish_operation(
"failed",
"Player telemetry identified a different slot or path",
)
)
return effects return effects
if operation.action == "play": if operation.action == "play":
effects.append(self._finish_operation("confirmed", "matching telemetry")) effects.append(self._finish_operation("confirmed", "matching telemetry"))
@@ -399,48 +592,165 @@ class PlaybackReducer:
operation = self.pending operation = self.pending
if operation is None or self._monotonic() < operation.deadline: if operation is None or self._monotonic() < operation.deadline:
return None return None
if operation.action == "play" and operation.slot is not None: if operation.action == "play":
due = None previous = self.store.player_observation()
if self.config.storage.uncertain_review_after_seconds > 0: has_confirmed_active = (
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds previous.reconciliation_state == "confirmed"
self.store.reconcile_active_guard( and (previous.active_slot is not None or previous.active_path is not None)
operation.slot,
kind="restart_uncertain",
review_after=due,
details={"operation_id": operation.operation_id, "reason": "play timeout"},
)
self.store.set_player_observation(
availability="unknown",
phase="unknown",
active_slot=operation.slot,
active_path=operation.expected_path,
paused=None,
reconciliation_state="awaiting_evidence",
reconciliation_due_at=due,
) )
if has_confirmed_active and self._matches_pending(
previous.active_slot,
previous.active_path,
):
return self._finish_operation(
"confirmed", "matching authoritative Player evidence"
)
if not has_confirmed_active:
self._mark_play_uncertain(operation, "play timeout")
elif operation.action == "stop": elif operation.action == "stop":
previous = self.store.player_observation() previous = self.store.player_observation()
if previous.active_slot is not None: if previous.active_slot is not None:
due = None due = None
if self.config.storage.uncertain_review_after_seconds > 0: if self.config.storage.uncertain_review_after_seconds > 0:
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
self.store.reconcile_active_guard( guard = self.store.mark_slot_uncertain(
previous.active_slot, previous.active_slot,
kind="restart_uncertain",
review_after=due, review_after=due,
details={"operation_id": operation.operation_id, "reason": "stop timeout"}, details={"operation_id": operation.operation_id, "reason": "stop timeout"},
) )
due = guard.review_after
self.store.set_player_observation( self.store.set_player_observation(
availability="unknown", availability="unknown",
phase="unknown", phase="unknown",
active_slot=previous.active_slot, active_slot=previous.active_slot,
active_path=previous.active_path, active_path=previous.active_path,
paused=previous.paused, paused=previous.paused,
reconciliation_state="awaiting_evidence", reconciliation_state=(
"operator_review"
if guard.state == "review_required"
else "awaiting_evidence"
),
reconciliation_due_at=due, reconciliation_due_at=due,
reconciliation_id=guard.guard_id,
)
elif previous.active_path is not None:
due = None
if self.config.storage.uncertain_review_after_seconds > 0:
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
self.store.mark_external_uncertain(
previous.active_path,
review_after=due,
details={
"operation_id": operation.operation_id,
"reason": "stop timeout",
},
) )
return self._finish_operation("timed_out", "no authoritative Player confirmation") return self._finish_operation("timed_out", "no authoritative Player confirmation")
def _mark_play_uncertain(self, operation: PlayerOperation, reason: str) -> None:
due = None
if self.config.storage.uncertain_review_after_seconds > 0:
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
slot = operation.slot or self._canonical_slot_for_path(operation.expected_path)
if slot is not None:
guard = self.store.mark_slot_uncertain(
slot,
review_after=due,
details={"operation_id": operation.operation_id, "reason": reason},
)
due = guard.review_after
self.store.set_player_observation(
availability="unknown",
phase="unknown",
active_slot=slot,
active_path=operation.expected_path,
paused=None,
reconciliation_state=(
"operator_review"
if guard.state == "review_required"
else "awaiting_evidence"
),
reconciliation_due_at=due,
reconciliation_id=guard.guard_id,
)
elif operation.expected_path is not None:
self.store.mark_external_uncertain(
operation.expected_path,
review_after=due,
details={"operation_id": operation.operation_id, "reason": reason},
)
def publication_uncertain(self, reason: str) -> PlaybackEffect | None:
if self.pending is None:
return None
operation = self.pending
if operation.action == "play":
self._mark_play_uncertain(operation, reason)
return self._finish_operation("uncertain", reason)
def mark_transport_uncertain(self, reason: str) -> PlaybackEffect:
previous = self.store.player_observation()
pending_play = (
self.pending
if self.pending is not None and self.pending.action == "play"
else None
)
active_slot = previous.active_slot
active_path = previous.active_path
if active_slot is None and active_path is None and pending_play is not None:
active_slot = pending_play.slot or self._canonical_slot_for_path(
pending_play.expected_path
)
active_path = pending_play.expected_path
due = None
if active_slot is not None:
if self.config.storage.uncertain_review_after_seconds > 0:
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
guard = self.store.mark_slot_uncertain(
active_slot,
review_after=due,
details={"reason": reason},
)
due = guard.review_after
self.store.set_player_observation(
availability="unknown",
phase="unknown",
active_slot=active_slot,
active_path=active_path,
paused=previous.paused,
reconciliation_state=(
"operator_review"
if guard.state == "review_required"
else "awaiting_evidence"
),
reconciliation_due_at=due,
reconciliation_id=guard.guard_id,
)
elif active_path is not None:
if self.config.storage.uncertain_review_after_seconds > 0:
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
self.store.mark_external_uncertain(
active_path,
review_after=due,
details={"reason": reason},
)
else:
self.store.set_player_observation(
availability="unknown",
phase=previous.phase,
active_slot=None,
active_path=None,
paused=previous.paused,
reconciliation_state="unknown",
reconciliation_due_at=None,
reconciliation_id=None,
)
return PlaybackEffect(
"transport_uncertain",
slot=active_slot,
message=reason,
)
def projected_slot_state(self, slot: int) -> str: def projected_slot_state(self, slot: int) -> str:
record = self.store.get_slot(slot) record = self.store.get_slot(slot)
if record.media_state != "ready": if record.media_state != "ready":
@@ -466,9 +776,9 @@ class PlaybackReducer:
return "ready" return "ready"
def mark_due_reconciliation(self) -> list[PlaybackEffect]: def mark_due_reconciliation(self) -> list[PlaybackEffect]:
due = self.store.mark_due_guards_for_review(self._wall_clock()) now = self._wall_clock()
if not due: due = self.store.mark_due_guards_for_review(now)
return [] external_due = self.store.mark_external_reconciliation_due(now)
observation = self.store.player_observation() observation = self.store.player_observation()
if observation.active_slot is not None and any( if observation.active_slot is not None and any(
guard.slot == observation.active_slot for guard in due guard.slot == observation.active_slot for guard in due
@@ -481,15 +791,32 @@ class PlaybackReducer:
paused=observation.paused, paused=observation.paused,
reconciliation_state="operator_review", reconciliation_state="operator_review",
reconciliation_due_at=observation.reconciliation_due_at, reconciliation_due_at=observation.reconciliation_due_at,
reconciliation_id=observation.reconciliation_id,
) )
return [ effects = [
PlaybackEffect( PlaybackEffect(
"operator_review", "operator_review",
slot=guard.slot, slot=guard.slot,
operation_id=guard.guard_id,
message=( message=(
"Playback remains unknown and protected; fresh Player evidence " "Playback remains unknown and protected; fresh Player evidence "
"or an administrative clear is required" "or an administrative clear is required "
f"(review_id={guard.guard_id})"
), ),
) )
for guard in due for guard in due
] ]
if external_due is not None:
review_id, path = external_due
effects.append(
PlaybackEffect(
"operator_review",
operation_id=review_id,
message=(
f"Playback at {path!r} remains unknown and protected; "
"fresh Player evidence or an administrative clear is required "
f"(review_id={review_id})"
),
)
)
return effects
+422
View File
@@ -0,0 +1,422 @@
"""Timeout-bounded Audio Router acquisition and restoration leases."""
from __future__ import annotations
import time
import uuid
from dataclasses import dataclass
from typing import Callable
from .config import ControllerConfig
from .models import AudioLeaseRecord
from .mqtt_adapter import PublishReceipt
from .store import ControllerStore
ModePublisher = Callable[[str], PublishReceipt]
@dataclass(frozen=True)
class RouterDecision:
action: str
lease_id: str | None = None
warning: str | None = None
continued: bool = False
class AudioRouterCoordinator:
"""Own one conservative Router lease for one playback session."""
def __init__(
self,
config: ControllerConfig,
store: ControllerStore,
publish_mode: ModePublisher,
*,
clock=time.monotonic,
):
self.config = config
self.store = store
self.publish_mode = publish_mode
self._clock = clock
def observe_mode(self, mode: str) -> RouterDecision:
mode = mode.strip()
if not mode:
warning = "Audio Router published an empty mode"
self._record_warning(warning)
return RouterDecision("warning", warning=warning)
self.store.set_router_observation(mode=mode, freshness="current")
lease = self.store.live_audio_lease()
if lease is None:
return RouterDecision("observed")
if lease.state == "acquiring":
if mode == lease.target_mode:
self.store.update_audio_lease(
lease.lease_id,
state="held",
confirmation_deadline=None,
acquired=True,
)
return RouterDecision("ready", lease_id=lease.lease_id)
# State is evidence of the Router's current output, not evidence
# that an operator issued a command. It may simply be a periodic
# repeat of the pre-acquisition state, so keep waiting for the
# finite confirmation deadline.
return RouterDecision("observed", lease_id=lease.lease_id)
if lease.state == "held":
if mode != lease.target_mode:
warning = (
f"Audio Router changed from Controller-owned {lease.target_mode!r} "
f"to {mode!r}; automatic restoration was relinquished"
)
self.store.update_audio_lease(
lease.lease_id,
state="ownership_lost",
confirmation_deadline=None,
warning=warning,
released=True,
)
self._record_warning(warning)
return RouterDecision("ownership_lost", lease.lease_id, warning)
return RouterDecision("observed", lease_id=lease.lease_id)
if lease.state == "unconfirmed":
# A late state event must never resurrect a lease that already
# timed out or deliberately continued without confirmation.
return RouterDecision("observed", lease_id=lease.lease_id)
if lease.state == "restore_pending":
if mode == lease.restoration_mode:
self.store.update_audio_lease(
lease.lease_id,
state="released",
confirmation_deadline=None,
released=True,
)
return RouterDecision("restored", lease_id=lease.lease_id)
# The Router may repeat its current state before applying the
# restoration command. Wait for the bounded deadline; explicit
# mode-set intent is handled separately below.
return RouterDecision("observed", lease_id=lease.lease_id)
return RouterDecision("observed", lease_id=lease.lease_id)
def observe_mode_command(
self,
mode: str,
*,
retained: bool = False,
controller_echo: bool = False,
) -> RouterDecision:
"""Observe a Router mode-set publication without treating state as intent.
The MQTT adapter marks an exact echo received while a Controller QoS 1
publication is in flight. Every other live mode-set publication is
conservatively treated as operator intent. Retained commands are old
broker state and can never establish current operator intent.
"""
mode = mode.strip()
if not mode:
warning = "Audio Router published an empty mode command"
self._record_warning(warning)
return RouterDecision("warning", warning=warning)
lease = self.store.live_audio_lease()
lease_id = None if lease is None else lease.lease_id
if retained:
warning = (
f"Retained Audio Router mode command {mode!r} was ignored as stale intent"
)
self._record_warning(warning)
return RouterDecision("retained_ignored", lease_id, warning)
if controller_echo:
return RouterDecision("self_echo", lease_id=lease_id)
if lease is None:
return RouterDecision("operator_intent")
if lease.state == "acquiring":
return self._acquisition_failed(
lease.lease_id,
(
f"Audio Router received an operator mode command for {mode!r} "
f"while {lease.target_mode!r} confirmation was pending; "
"automatic restoration ownership was relinquished"
),
)
if lease.state in {"held", "unconfirmed", "restore_pending"}:
warning = (
f"Audio Router received an operator mode command for {mode!r}; "
"automatic restoration was relinquished"
)
self.store.update_audio_lease(
lease.lease_id,
state="ownership_lost",
confirmation_deadline=None,
warning=warning,
released=True,
)
self._record_warning(warning)
return RouterDecision("ownership_lost", lease.lease_id, warning)
return RouterDecision("operator_intent", lease_id=lease.lease_id)
def acquire(self, operation_id: str) -> RouterDecision:
observation = self.store.router_observation()
known = observation.freshness == "current" and observation.mode is not None
target = self.config.router.required_mode
previous = observation.mode if known else None
lease_id = uuid.uuid4().hex
if known and previous == target:
self.store.create_audio_lease(
lease_id=lease_id,
operation_id=operation_id,
state="held",
previous_mode=previous,
target_mode=target,
restoration_mode=None,
restoration_source="none",
controller_changed=False,
confirmation_deadline=None,
)
self.store.update_audio_lease(
lease_id,
state="held",
acquired=True,
)
return RouterDecision("ready", lease_id=lease_id)
if known:
restoration_mode = previous
restoration_source = "observed_previous"
elif self.config.router.fallback_restore_mode is not None:
restoration_mode = self.config.router.fallback_restore_mode
restoration_source = "configured_fallback"
else:
restoration_mode = None
restoration_source = "none"
policy = self.config.router.when_state_unknown
if not known and policy == "reject":
warning = "Audio Router state is unavailable; playback was rejected by policy"
self.store.create_audio_lease(
lease_id=lease_id,
operation_id=operation_id,
state="abandoned",
previous_mode=None,
target_mode=target,
restoration_mode=restoration_mode,
restoration_source=restoration_source,
controller_changed=False,
confirmation_deadline=None,
warning=warning,
)
self._record_warning(warning)
return RouterDecision("rejected", lease_id, warning)
deadline = self._clock() + self.config.router.confirmation_timeout_seconds
initial_state = "unconfirmed" if not known and policy == "continue" else "acquiring"
self.store.create_audio_lease(
lease_id=lease_id,
operation_id=operation_id,
state=initial_state,
previous_mode=previous,
target_mode=target,
restoration_mode=restoration_mode,
restoration_source=restoration_source,
controller_changed=True,
confirmation_deadline=(deadline if initial_state == "acquiring" else None),
)
receipt = self.publish_mode(target)
if not receipt.accepted:
return self._acquisition_failed(
lease_id,
f"could not request Audio Router mode {target!r}: {receipt.reason}",
)
if initial_state == "unconfirmed":
warning = (
f"Audio Router state is unavailable; requested {target!r} and "
"continued immediately without restoration ownership"
)
self.store.update_audio_lease(
lease_id,
state="unconfirmed",
warning=warning,
)
self._record_warning(warning)
return RouterDecision("ready", lease_id, warning, continued=True)
return RouterDecision("waiting", lease_id=lease_id)
def check_deadline(self) -> RouterDecision | None:
lease = self.store.live_audio_lease()
if (
lease is None
or lease.confirmation_deadline is None
or self._clock() < lease.confirmation_deadline
):
return None
if lease.state == "acquiring":
return self._acquisition_failed(
lease.lease_id,
(
f"Audio Router did not confirm {lease.target_mode!r} within "
f"{self.config.router.confirmation_timeout_seconds:g} seconds"
),
)
if lease.state == "restore_pending":
warning = (
f"Audio Router did not confirm restoration to {lease.restoration_mode!r}; "
"no further automatic change will be attempted"
)
self.store.update_audio_lease(
lease.lease_id,
state="abandoned",
confirmation_deadline=None,
warning=warning,
released=True,
)
self._record_warning(warning)
return RouterDecision("restore_failed", lease.lease_id, warning)
return None
def _acquisition_failed(self, lease_id: str, warning: str) -> RouterDecision:
continue_playback = self.config.router.continue_on_confirmation_failure
self.store.update_audio_lease(
lease_id,
state="unconfirmed" if continue_playback else "abandoned",
confirmation_deadline=None,
warning=warning,
released=not continue_playback,
)
self._record_warning(warning)
return RouterDecision(
"ready" if continue_playback else "rejected",
lease_id,
warning,
continued=continue_playback,
)
def release(self, lease_id: str | None) -> RouterDecision:
if lease_id is None:
return RouterDecision("released")
lease = self.store.get_audio_lease(lease_id)
if lease is None:
return RouterDecision("released")
if lease.state == "held" and not lease.controller_changed:
self.store.update_audio_lease(
lease_id,
state="released",
confirmation_deadline=None,
released=True,
)
return RouterDecision("released", lease_id)
if lease.state != "held":
if lease.state in {"unconfirmed", "acquiring"}:
self.store.update_audio_lease(
lease_id,
state="abandoned",
confirmation_deadline=None,
warning="Audio Router was never confirmed; restoration was skipped",
released=True,
)
return RouterDecision("released", lease_id)
observation = self.store.router_observation()
if observation.freshness != "current" or observation.mode != lease.target_mode:
warning = "Audio Router ownership is no longer current; restoration was skipped"
self.store.update_audio_lease(
lease_id,
state="ownership_lost",
confirmation_deadline=None,
warning=warning,
released=True,
)
self._record_warning(warning)
return RouterDecision("ownership_lost", lease_id, warning)
if lease.restoration_mode is None:
self.store.update_audio_lease(
lease_id,
state="released",
confirmation_deadline=None,
released=True,
)
return RouterDecision("released", lease_id)
receipt = self.publish_mode(lease.restoration_mode)
if not receipt.accepted:
warning = f"could not request Audio Router restoration: {receipt.reason}"
self.store.update_audio_lease(
lease_id,
state="abandoned",
confirmation_deadline=None,
warning=warning,
released=True,
)
self._record_warning(warning)
return RouterDecision("restore_failed", lease_id, warning)
self.store.update_audio_lease(
lease_id,
state="restore_pending",
confirmation_deadline=(
self._clock() + self.config.router.confirmation_timeout_seconds
),
)
return RouterDecision("restore_waiting", lease_id)
def connection_lost(self) -> RouterDecision | None:
observation = self.store.router_observation()
self.store.set_router_observation(
mode=observation.mode,
freshness="stale",
warning="MQTT continuity was lost",
)
lease = self.store.live_audio_lease()
if lease is None:
return None
if lease.state == "acquiring":
return self._acquisition_failed(
lease.lease_id,
"MQTT disconnected while waiting for Audio Router confirmation",
)
warning = "MQTT continuity was lost; automatic Audio Router restoration was relinquished"
self.store.update_audio_lease(
lease.lease_id,
state="ownership_lost",
confirmation_deadline=None,
warning=warning,
released=True,
)
self._record_warning(warning)
return RouterDecision("ownership_lost", lease.lease_id, warning)
def abandon(self, lease_id: str | None, reason: str) -> None:
if lease_id is None:
return
lease = self.store.get_audio_lease(lease_id)
if lease is None or lease.state in {"released", "abandoned", "ownership_lost"}:
return
self.store.update_audio_lease(
lease_id,
state="abandoned",
confirmation_deadline=None,
warning=reason,
released=True,
)
self._record_warning(reason)
def _record_warning(self, warning: str) -> None:
observation = self.store.router_observation()
self.store.set_router_observation(
mode=observation.mode,
freshness=observation.freshness,
warning=warning,
)
self.store.record_audit_event(
"audio_router_warning",
actor="controller",
details={"message": warning},
)
File diff suppressed because it is too large Load Diff
+865 -65
View File
File diff suppressed because it is too large Load Diff
+2
View File
@@ -29,6 +29,7 @@ class ContractCharacterizationTests(unittest.TestCase):
self.assertEqual(topics.playback_base, "youtube/media/playback") self.assertEqual(topics.playback_base, "youtube/media/playback")
self.assertEqual(topics.router_mode_set, "audio/player/mode/set") self.assertEqual(topics.router_mode_set, "audio/player/mode/set")
self.assertEqual(topics.router_mode_state, "audio/player/mode/state") self.assertEqual(topics.router_mode_state, "audio/player/mode/state")
self.assertEqual(topics.controller_result, "youtube/controller/v2/events")
self.assertEqual(PLAYER_COMMAND_QOS, 1) self.assertEqual(PLAYER_COMMAND_QOS, 1)
self.assertIs(PLAYER_COMMAND_RETAIN, False) self.assertIs(PLAYER_COMMAND_RETAIN, False)
self.assertEqual(SLOT_IDS, tuple(range(1, 11))) self.assertEqual(SLOT_IDS, tuple(range(1, 11)))
@@ -79,6 +80,7 @@ class ConfigurationTests(unittest.TestCase):
self.assertIs(config.router.continue_on_confirmation_failure, True) self.assertIs(config.router.continue_on_confirmation_failure, True)
self.assertIsNone(config.router.fallback_restore_mode) self.assertIsNone(config.router.fallback_restore_mode)
self.assertEqual(config.storage.uncertain_review_after_seconds, 900.0) self.assertEqual(config.storage.uncertain_review_after_seconds, 900.0)
self.assertEqual(config.mqtt.publish_ack_timeout_seconds, 2.0)
def test_documented_environment_overrides_apply(self): def test_documented_environment_overrides_apply(self):
env = { env = {
+38
View File
@@ -1,6 +1,8 @@
import os import os
import subprocess import subprocess
import tempfile import tempfile
import threading
import time
import unittest import unittest
from concurrent.futures import Future from concurrent.futures import Future
from pathlib import Path from pathlib import Path
@@ -194,6 +196,42 @@ class DownloadManagerTests(unittest.TestCase):
self.assertFalse(slot.exists()) self.assertFalse(slot.exists())
self.assertEqual(self.store.get_slot(1).media_state, "empty") self.assertEqual(self.store.get_slot(1).media_state, "empty")
def test_shutdown_terminates_tracked_process_without_waiting_for_job_timeout(self):
manager = DownloadManager(self.config, self.store)
errors = []
def run_long_process():
try:
manager._run_command(
["/bin/sh", "-c", "sleep 60"],
check=True,
text=True,
timeout=60,
capture_output=True,
)
except BaseException as exc:
errors.append(exc)
worker = threading.Thread(target=run_long_process)
worker.start()
deadline = time.monotonic() + 2
while time.monotonic() < deadline:
with manager._lock:
if manager._processes:
break
time.sleep(0.01)
else:
self.fail("test process did not start")
started = time.monotonic()
manager.shutdown(wait=False)
worker.join(timeout=3)
self.assertFalse(worker.is_alive())
self.assertLess(time.monotonic() - started, 3)
self.assertTrue(errors)
with manager._lock:
self.assertEqual(manager._processes, set())
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+234 -1
View File
@@ -1,5 +1,6 @@
import json import json
import queue import queue
import threading
import unittest import unittest
from types import SimpleNamespace from types import SimpleNamespace
@@ -9,6 +10,20 @@ from controller.config import ControllerConfig, MqttConfig
from controller.mqtt_adapter import MqttAdapter from controller.mqtt_adapter import MqttAdapter
class FakeMessageInfo:
def __init__(self, mid, published):
self.rc = mqtt.MQTT_ERR_SUCCESS
self.mid = mid
self.published = published
self.waited = []
def wait_for_publish(self, timeout=None):
self.waited.append(timeout)
def is_published(self):
return self.published
class FakeClient: class FakeClient:
def __init__(self): def __init__(self):
self.calls = [] self.calls = []
@@ -17,6 +32,9 @@ class FakeClient:
self.acks = [] self.acks = []
self.suppress_exceptions = False self.suppress_exceptions = False
self.connect_error = None self.connect_error = None
self.loop_result = mqtt.MQTT_ERR_SUCCESS
self.publish_acknowledged = None
self.publish_hook = None
def username_pw_set(self, username, password): def username_pw_set(self, username, password):
self.calls.append(("username_pw_set", username, password)) self.calls.append(("username_pw_set", username, password))
@@ -42,7 +60,7 @@ class FakeClient:
def loop_start(self): def loop_start(self):
self.calls.append(("loop_start",)) self.calls.append(("loop_start",))
return mqtt.MQTT_ERR_SUCCESS return self.loop_result
def loop_stop(self): def loop_stop(self):
self.calls.append(("loop_stop",)) self.calls.append(("loop_stop",))
@@ -56,6 +74,12 @@ class FakeClient:
def publish(self, topic, payload, qos=0, retain=False): def publish(self, topic, payload, qos=0, retain=False):
self.published.append((topic, payload, qos, retain)) self.published.append((topic, payload, qos, retain))
if self.publish_hook is not None:
self.publish_hook(topic, payload, qos, retain)
if self.publish_acknowledged is not None:
return FakeMessageInfo(
len(self.published), self.publish_acknowledged
)
return SimpleNamespace(rc=mqtt.MQTT_ERR_SUCCESS, mid=len(self.published)) return SimpleNamespace(rc=mqtt.MQTT_ERR_SUCCESS, mid=len(self.published))
def ack(self, message_id, qos): def ack(self, message_id, qos):
@@ -106,6 +130,66 @@ class MqttAdapterTests(unittest.TestCase):
self.assertIn("network unreachable", self.adapter.last_error) self.assertIn("network unreachable", self.adapter.last_error)
self.assertNotIn("loop_start", [call[0] for call in self.client.calls]) self.assertNotIn("loop_start", [call[0] for call in self.client.calls])
def test_loop_start_failure_does_not_escape_startup(self):
self.client.loop_result = mqtt.MQTT_ERR_UNKNOWN
self.adapter.start()
self.assertTrue(self.adapter.started)
self.assertIn("failed to start MQTT network loop", self.adapter.last_error)
def test_stop_during_blocked_connect_never_starts_network_loop_afterward(self):
entered = threading.Event()
release = threading.Event()
original_connect = self.client.connect_async
def blocked_connect(*args, **kwargs):
entered.set()
release.wait(timeout=2)
return original_connect(*args, **kwargs)
self.client.connect_async = blocked_connect
starter = threading.Thread(target=self.adapter.start)
starter.start()
self.assertTrue(entered.wait(timeout=1))
self.adapter.stop()
release.set()
starter.join(timeout=2)
self.assertFalse(starter.is_alive())
self.assertNotIn("loop_start", [call[0] for call in self.client.calls])
self.assertFalse(self.adapter.started)
def test_stop_wins_against_connect_callback_blocked_during_subscribe(self):
self.adapter.start()
entered = threading.Event()
release = threading.Event()
original_subscribe = self.client.subscribe
def blocked_subscribe(topic, qos=0):
entered.set()
release.wait(timeout=2)
return original_subscribe(topic, qos=qos)
self.client.subscribe = blocked_subscribe
callback = threading.Thread(
target=self.adapter._on_connect,
args=(self.client, None, None, 0),
)
callback.start()
self.assertTrue(entered.wait(timeout=1))
stopper = threading.Thread(target=self.adapter.stop)
stopper.start()
release.set()
callback.join(timeout=2)
stopper.join(timeout=2)
self.assertFalse(callback.is_alive())
self.assertFalse(stopper.is_alive())
self.assertFalse(self.adapter.started)
self.assertFalse(self.adapter.connected)
self.assertLess(len(self.client.subscriptions), 6)
self.assertTrue(self.events.empty())
self.assertFalse(
self.adapter.publish_player_command({"action": "stop"}).accepted
)
def test_connect_subscribes_to_all_authoritative_inputs(self): def test_connect_subscribes_to_all_authoritative_inputs(self):
self.adapter.start() self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0) self.adapter._on_connect(self.client, None, None, 0)
@@ -117,6 +201,7 @@ class MqttAdapterTests(unittest.TestCase):
("youtube/player/telemetry", 0), ("youtube/player/telemetry", 0),
("youtube/player/state", 0), ("youtube/player/state", 0),
("youtube/controller/cmd", 1), ("youtube/controller/cmd", 1),
("audio/player/mode/set", 1),
("audio/player/mode/state", 1), ("audio/player/mode/state", 1),
], ],
) )
@@ -127,6 +212,7 @@ class MqttAdapterTests(unittest.TestCase):
def test_player_commands_are_exact_qos_one_and_nonretained(self): def test_player_commands_are_exact_qos_one_and_nonretained(self):
disconnected = self.adapter.publish_player_command({"action": "stop"}) disconnected = self.adapter.publish_player_command({"action": "stop"})
self.assertFalse(disconnected.accepted) self.assertFalse(disconnected.accepted)
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0) self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait() self.events.get_nowait()
receipt = self.adapter.publish_player_command( receipt = self.adapter.publish_player_command(
@@ -140,6 +226,7 @@ class MqttAdapterTests(unittest.TestCase):
self.assertIs(retain, False) self.assertIs(retain, False)
def test_router_commands_are_qos_one_nonretained_plain_strings(self): def test_router_commands_are_qos_one_nonretained_plain_strings(self):
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0) self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait() self.events.get_nowait()
receipt = self.adapter.publish_router_mode("video") receipt = self.adapter.publish_router_mode("video")
@@ -149,7 +236,75 @@ class MqttAdapterTests(unittest.TestCase):
("audio/player/mode/set", "video", 1, False), ("audio/player/mode/set", "video", 1, False),
) )
def test_router_self_echo_is_marked_only_while_publish_is_in_flight(self):
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait()
def echo(topic, payload, qos, retain):
self.adapter._on_message(
self.client,
None,
SimpleNamespace(
topic=topic,
payload=payload.encode("utf-8"),
qos=qos,
retain=retain,
dup=False,
mid=31,
),
)
self.client.publish_hook = echo
self.assertTrue(self.adapter.publish_router_mode("video").accepted)
self.assertTrue(self.events.get_nowait().controller_echo)
self.client.publish_hook = None
self.adapter._on_message(
self.client,
None,
SimpleNamespace(
topic="audio/player/mode/set",
payload=b"video",
qos=1,
retain=False,
dup=False,
mid=32,
),
)
self.assertFalse(self.events.get_nowait().controller_echo)
def test_retained_router_set_never_consumes_controller_echo_marker(self):
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait()
def retained_then_live_echo(topic, payload, qos, retain):
for message_retain, mid in ((True, 41), (False, 42)):
self.adapter._on_message(
self.client,
None,
SimpleNamespace(
topic=topic,
payload=payload.encode("utf-8"),
qos=qos,
retain=message_retain,
dup=False,
mid=mid,
),
)
self.client.publish_hook = retained_then_live_echo
self.assertTrue(self.adapter.publish_router_mode("video").accepted)
retained_event = self.events.get_nowait()
live_event = self.events.get_nowait()
self.assertTrue(retained_event.retain)
self.assertFalse(retained_event.controller_echo)
self.assertFalse(live_event.retain)
self.assertTrue(live_event.controller_echo)
def test_qos_message_is_acked_only_after_consumer_calls_acknowledge(self): def test_qos_message_is_acked_only_after_consumer_calls_acknowledge(self):
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0) self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait() self.events.get_nowait()
message = SimpleNamespace( message = SimpleNamespace(
@@ -167,6 +322,7 @@ class MqttAdapterTests(unittest.TestCase):
self.assertEqual(self.client.acks, [(42, 1)]) self.assertEqual(self.client.acks, [(42, 1)])
def test_stale_mid_from_previous_connection_is_not_acked(self): def test_stale_mid_from_previous_connection_is_not_acked(self):
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0) self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait() self.events.get_nowait()
message = SimpleNamespace( message = SimpleNamespace(
@@ -184,6 +340,7 @@ class MqttAdapterTests(unittest.TestCase):
self.assertEqual(self.client.acks, []) self.assertEqual(self.client.acks, [])
def test_retained_flag_is_preserved_for_coordinator_rejection(self): def test_retained_flag_is_preserved_for_coordinator_rejection(self):
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0) self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait() self.events.get_nowait()
message = SimpleNamespace( message = SimpleNamespace(
@@ -196,6 +353,82 @@ class MqttAdapterTests(unittest.TestCase):
self.adapter._on_message(self.client, None, message) self.adapter._on_message(self.client, None, message)
self.assertIs(self.events.get_nowait().retain, True) self.assertIs(self.events.get_nowait().retain, True)
def test_duplicate_flag_is_preserved_for_coordinator_deduplication(self):
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait()
message = SimpleNamespace(
topic="youtube/controller/cmd",
payload=b'{"action":"toggle"}',
qos=1,
retain=False,
dup=True,
mid=9,
)
self.adapter._on_message(self.client, None, message)
self.assertIs(self.events.get_nowait().duplicate, True)
def test_missing_puback_is_uncertain_and_invalidates_connection(self):
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait()
self.client.publish_acknowledged = False
receipt = self.adapter.publish_player_command({"action": "stop"})
self.assertFalse(receipt.accepted)
self.assertTrue(receipt.uncertain)
self.assertIn("no broker PUBACK", receipt.reason)
self.assertFalse(self.adapter.connected)
def test_optional_qos_event_puback_failure_does_not_invalidate_transport(self):
self.adapter.start()
self.adapter._on_connect(self.client, None, None, 0)
self.events.get_nowait()
self.client.publish_acknowledged = False
receipt = self.adapter.publish_json(
"youtube/controller/v2/events",
{"version": 2, "type": "warning"},
qos=1,
retain=False,
)
self.assertFalse(receipt.accepted)
self.assertTrue(receipt.uncertain)
self.assertTrue(self.adapter.connected)
def test_discarded_transport_releases_callback_blocked_on_full_queue(self):
events = queue.Queue(maxsize=1)
client = FakeClient()
adapter = MqttAdapter(self.config, client=client, events=events)
try:
adapter.start()
adapter._on_connect(client, None, None, 0)
self.assertTrue(events.full())
callback = threading.Thread(
target=adapter._on_message,
args=(
client,
None,
SimpleNamespace(
topic="youtube/controller/cmd",
payload=b'{"action":"stop"}',
qos=1,
retain=False,
dup=False,
mid=99,
),
),
)
callback.start()
callback.join(timeout=0.05)
self.assertTrue(callback.is_alive())
adapter._discard_unacknowledged_transport("test transport discard")
callback.join(timeout=1)
self.assertFalse(callback.is_alive())
self.assertEqual(events.qsize(), 1)
self.assertEqual(events.get_nowait().kind, "connected")
finally:
adapter.stop()
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+148
View File
@@ -230,6 +230,154 @@ class PlaybackReducerTests(unittest.TestCase):
effects = self.reducer.handle_status(payload) effects = self.reducer.handle_status(payload)
self.assertEqual(effects[0].kind, "warning") self.assertEqual(effects[0].kind, "warning")
def test_contradictory_slot_and_path_cannot_confirm_play(self):
expected = str(self.config.storage.media_base / "slot1" / "playlist.m3u")
self.reducer.begin_operation("play", slot=1, expected_path=expected)
effects = self.reducer.handle_status(
{
"state": "playing",
"slot": 1,
"path": str(
self.config.storage.media_base / "slot2" / "playlist.m3u"
),
}
)
self.assertEqual(effects[0].kind, "warning")
self.assertIsNotNone(self.reducer.pending)
self.assertEqual(self.store.player_observation().phase, "unknown")
def test_mismatched_active_evidence_fails_intent_and_is_not_overwritten(self):
self.reducer.begin_operation(
"play",
slot=1,
expected_path="/mnt/Media/YTPlayer/slot1/playlist.m3u",
)
effects = self.reducer.handle_status(
{
"state": "playing",
"slot": 2,
"path": "/mnt/Media/YTPlayer/slot2/playlist.m3u",
}
)
self.assertIn("operation_failed", [effect.kind for effect in effects])
self.assertIsNone(self.reducer.pending)
self.monotonic.value += 100
self.assertIsNone(self.reducer.check_timeout())
observation = self.store.player_observation()
self.assertEqual(observation.active_slot, 2)
self.assertEqual(observation.phase, "playing")
self.assertEqual(self.store.list_live_guards()[0].slot, 2)
def test_bare_idle_does_not_clear_live_playback(self):
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
self.reducer.handle_status({"state": "playing", "slot": 1, "path": path})
effects = self.reducer.handle_status({"state": "idle", "slot": 1})
self.assertEqual(effects[0].kind, "warning")
self.assertEqual(self.store.player_observation().phase, "playing")
self.assertEqual(self.store.list_live_guards()[0].slot, 1)
def test_path_only_evidence_derives_and_protects_managed_slot(self):
path = str(self.config.storage.media_base / "slot2" / "playlist.m3u")
self.reducer.handle_telemetry(
{"state": "telemetry", "slot": None, "path": path, "paused": False}
)
observation = self.store.player_observation()
self.assertEqual(observation.active_slot, 2)
self.assertEqual(self.store.list_live_guards()[0].slot, 2)
def test_duplicate_offline_evidence_does_not_postpone_review(self):
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
self.reducer.handle_status({"state": "playing", "slot": 1, "path": path})
self.reducer.handle_status({"state": "offline"})
first_deadline = self.store.list_live_guards()[0].review_after
self.wall.value += 20
self.reducer.handle_status({"state": "offline"})
self.assertEqual(
self.store.list_live_guards()[0].review_after,
first_deadline,
)
def test_repeated_active_telemetry_reuses_guard_without_history_churn(self):
path = str(self.config.storage.media_base / "slot1" / "playlist.m3u")
payload = {
"state": "telemetry",
"slot": 1,
"path": path,
"paused": False,
}
self.reducer.handle_telemetry(payload)
first_guard = self.store.list_live_guards()[0]
for _ in range(20):
self.reducer.handle_telemetry(payload)
self.assertEqual(self.store.list_live_guards()[0].guard_id, first_guard.guard_id)
count = self.store.connection.execute(
"SELECT COUNT(*) FROM slot_guards"
).fetchone()[0]
self.assertEqual(count, 1)
def test_external_play_timeout_transitions_to_review_without_assuming_stop(self):
path = "/srv/operator-playlists/special-event.m3u"
self.reducer.begin_operation("play", expected_path=path)
self.monotonic.value += 6
timeout = self.reducer.check_timeout()
self.assertEqual(timeout.kind, "operation_timed_out")
uncertain = self.store.player_observation()
self.assertEqual(uncertain.active_path, path)
self.assertEqual(uncertain.reconciliation_state, "awaiting_evidence")
self.assertIsNotNone(uncertain.reconciliation_id)
self.wall.value += 31
effects = self.reducer.mark_due_reconciliation()
self.assertEqual(len(effects), 1)
self.assertEqual(effects[0].kind, "operator_review")
self.assertEqual(effects[0].operation_id, uncertain.reconciliation_id)
review = self.store.player_observation()
self.assertEqual(review.reconciliation_state, "operator_review")
self.assertEqual(review.active_path, path)
def test_external_pause_does_not_bleed_to_a_different_path(self):
first_path = "/srv/operator-playlists/first.m3u"
second_path = "/srv/operator-playlists/second.m3u"
self.reducer.handle_telemetry(
{
"state": "telemetry",
"path": first_path,
"paused": True,
}
)
self.assertIs(self.store.player_observation().paused, True)
self.reducer.handle_status(
{"state": "playing", "path": second_path}
)
observation = self.store.player_observation()
self.assertEqual(observation.active_path, second_path)
self.assertIsNone(observation.paused)
self.assertEqual(observation.phase, "playing")
def test_stop_timeout_preserves_existing_operator_review_and_token(self):
path = str(self.config.storage.media_base / "slot1" / "playlist.m3u")
self.reducer.handle_status(
{"state": "playing", "slot": 1, "path": path}
)
self.reducer.handle_status({"state": "offline"})
self.wall.value += 31
self.reducer.mark_due_reconciliation()
before = self.store.player_observation()
guard_before = self.store.list_live_guards()[0]
self.assertEqual(before.reconciliation_state, "operator_review")
self.reducer.begin_operation("stop", slot=1, expected_path=path)
self.monotonic.value += 6
effect = self.reducer.check_timeout()
self.assertEqual(effect.kind, "operation_timed_out")
after = self.store.player_observation()
guard_after = self.store.list_live_guards()[0]
self.assertEqual(after.reconciliation_state, "operator_review")
self.assertEqual(after.reconciliation_id, before.reconciliation_id)
self.assertEqual(guard_after.guard_id, guard_before.guard_id)
self.assertEqual(guard_after.state, "review_required")
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+316
View File
@@ -0,0 +1,316 @@
import tempfile
import unittest
from dataclasses import replace
from pathlib import Path
from controller.config import ControllerConfig, RouterConfig, StorageConfig
from controller.mqtt_adapter import PublishReceipt
from controller.router import AudioRouterCoordinator
from controller.store import ControllerStore
class Clock:
def __init__(self, value=1000.0):
self.value = value
def __call__(self):
return self.value
class ModePublisher:
def __init__(self):
self.modes = []
self.accepted = True
self.reason = None
def __call__(self, mode):
self.modes.append(mode)
return PublishReceipt(self.accepted, message_id=len(self.modes), reason=self.reason)
class AudioRouterCoordinatorTests(unittest.TestCase):
def setUp(self):
self.tempdir = tempfile.TemporaryDirectory()
base = Path(self.tempdir.name)
self.clock = Clock()
self.base_config = ControllerConfig(
storage=StorageConfig(
media_base=base / "media",
data_base=base / "data",
),
router=RouterConfig(
required_mode="video",
confirmation_timeout_seconds=3,
when_state_unknown="attempt",
continue_on_confirmation_failure=True,
fallback_restore_mode=None,
),
)
self.store = ControllerStore(self.base_config.storage, now=self.clock).open()
self.publisher = ModePublisher()
def tearDown(self):
self.store.close()
self.tempdir.cleanup()
def coordinator(self, router_config=None):
config = self.base_config
if router_config is not None:
config = replace(config, router=router_config)
return AudioRouterCoordinator(
config,
self.store,
self.publisher,
clock=self.clock,
)
def test_known_mode_is_confirmed_before_play_and_restored_afterward(self):
router = self.coordinator()
router.observe_mode("service")
decision = router.acquire("play-one")
self.assertEqual(decision.action, "waiting")
self.assertEqual(self.publisher.modes, ["video"])
self.assertEqual(router.observe_mode("video").action, "ready")
lease = self.store.get_audio_lease(decision.lease_id)
self.assertEqual(lease.state, "held")
self.assertEqual(lease.previous_mode, "service")
restore = router.release(decision.lease_id)
self.assertEqual(restore.action, "restore_waiting")
self.assertEqual(self.publisher.modes, ["video", "service"])
self.assertEqual(router.observe_mode("service").action, "restored")
self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "released")
def test_default_unknown_state_timeout_continues_with_warning_not_hang(self):
router = self.coordinator()
decision = router.acquire("play-timeout")
self.assertEqual(decision.action, "waiting")
self.assertEqual(self.publisher.modes, ["video"])
self.clock.value += 2.9
self.assertIsNone(router.check_deadline())
self.clock.value += 0.2
decision = router.check_deadline()
self.assertEqual(decision.action, "ready")
self.assertTrue(decision.continued)
self.assertIn("did not confirm", decision.warning)
lease = self.store.get_audio_lease(decision.lease_id)
self.assertEqual(lease.state, "unconfirmed")
self.assertEqual(router.release(lease.lease_id).action, "released")
self.assertEqual(self.publisher.modes, ["video"])
def test_timeout_can_reject_when_continue_policy_is_disabled(self):
router = self.coordinator(
replace(
self.base_config.router,
continue_on_confirmation_failure=False,
)
)
decision = router.acquire("play-reject")
self.clock.value += 4
decision = router.check_deadline()
self.assertEqual(decision.action, "rejected")
self.assertFalse(decision.continued)
self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "abandoned")
def test_unknown_state_continue_policy_does_not_wait_or_gain_late_ownership(self):
router = self.coordinator(
replace(self.base_config.router, when_state_unknown="continue")
)
decision = router.acquire("play-immediate")
self.assertEqual(decision.action, "ready")
self.assertTrue(decision.continued)
self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "unconfirmed")
self.assertEqual(router.observe_mode("video").action, "observed")
self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "unconfirmed")
def test_unknown_state_reject_policy_does_not_publish(self):
router = self.coordinator(
replace(self.base_config.router, when_state_unknown="reject")
)
decision = router.acquire("play-reject")
self.assertEqual(decision.action, "rejected")
self.assertEqual(self.publisher.modes, [])
def test_already_video_requires_no_change_and_no_restore(self):
router = self.coordinator()
router.observe_mode("video")
decision = router.acquire("play-video")
self.assertEqual(decision.action, "ready")
self.assertEqual(self.publisher.modes, [])
self.assertEqual(router.release(decision.lease_id).action, "released")
self.assertEqual(self.publisher.modes, [])
def test_confirmed_unknown_acquisition_can_use_configured_fallback(self):
router = self.coordinator(
replace(self.base_config.router, fallback_restore_mode="service")
)
decision = router.acquire("play-fallback")
self.assertEqual(router.observe_mode("video").action, "ready")
lease = self.store.get_audio_lease(decision.lease_id)
self.assertEqual(lease.restoration_source, "configured_fallback")
self.assertEqual(router.release(decision.lease_id).action, "restore_waiting")
self.assertEqual(self.publisher.modes, ["video", "service"])
def test_operator_change_is_sticky_even_if_mode_later_returns_to_video(self):
router = self.coordinator()
router.observe_mode("service")
decision = router.acquire("play-operator")
router.observe_mode("video")
lost = router.observe_mode("announcements")
self.assertEqual(lost.action, "ownership_lost")
router.observe_mode("video")
self.assertEqual(router.release(decision.lease_id).action, "released")
self.assertEqual(self.publisher.modes, ["video"])
self.assertEqual(
self.store.get_audio_lease(decision.lease_id).state,
"ownership_lost",
)
def test_connection_loss_relinquishes_confirmed_lease(self):
router = self.coordinator()
router.observe_mode("service")
decision = router.acquire("play-disconnect")
router.observe_mode("video")
lost = router.connection_lost()
self.assertEqual(lost.action, "ownership_lost")
self.assertEqual(router.release(decision.lease_id).action, "released")
self.assertEqual(self.publisher.modes, ["video"])
def test_publish_failure_uses_confirmation_failure_policy(self):
self.publisher.accepted = False
self.publisher.reason = "MQTT disconnected"
router = self.coordinator()
decision = router.acquire("play-publish-fail")
self.assertEqual(decision.action, "ready")
self.assertTrue(decision.continued)
self.assertIn("MQTT disconnected", decision.warning)
def test_restoration_timeout_is_bounded_and_not_retried(self):
router = self.coordinator()
router.observe_mode("service")
decision = router.acquire("play-restore-timeout")
router.observe_mode("video")
router.release(decision.lease_id)
self.clock.value += 4
timeout = router.check_deadline()
self.assertEqual(timeout.action, "restore_failed")
self.assertEqual(self.publisher.modes, ["video", "service"])
self.assertIsNone(router.check_deadline())
def test_repeated_previous_state_during_acquisition_waits_for_deadline(self):
router = self.coordinator()
router.observe_mode("service")
waiting = router.acquire("play-repeated-state")
self.assertEqual(router.observe_mode("service").action, "observed")
self.assertEqual(
self.store.get_audio_lease(waiting.lease_id).state,
"acquiring",
)
self.clock.value += 4
decision = router.check_deadline()
self.assertEqual(decision.action, "ready")
self.assertTrue(decision.continued)
self.assertEqual(
self.store.get_audio_lease(waiting.lease_id).state,
"unconfirmed",
)
router.release(waiting.lease_id)
self.assertEqual(self.publisher.modes, ["video"])
def test_repeated_target_state_is_observation_not_operator_takeover(self):
router = self.coordinator()
router.observe_mode("service")
decision = router.acquire("play-repeated-state")
self.assertEqual(router.observe_mode("video").action, "ready")
self.assertEqual(router.observe_mode("video").action, "observed")
self.assertEqual(
self.store.get_audio_lease(decision.lease_id).state,
"held",
)
self.assertEqual(router.release(decision.lease_id).action, "restore_waiting")
self.assertEqual(self.publisher.modes, ["video", "service"])
def test_controller_mode_set_echo_does_not_relinquish_ownership(self):
router = self.coordinator()
router.observe_mode("service")
decision = router.acquire("play-self-echo")
self.assertEqual(
router.observe_mode_command("video", controller_echo=True).action,
"self_echo",
)
self.assertEqual(router.observe_mode("video").action, "ready")
self.assertEqual(router.release(decision.lease_id).action, "restore_waiting")
self.assertEqual(
router.observe_mode_command("service", controller_echo=True).action,
"self_echo",
)
self.assertEqual(router.observe_mode("service").action, "restored")
def test_manual_mode_set_during_acquisition_uses_continue_policy(self):
router = self.coordinator()
router.observe_mode("service")
waiting = router.acquire("play-manual-acquire")
decision = router.observe_mode_command("announcements")
self.assertEqual(decision.action, "ready")
self.assertTrue(decision.continued)
self.assertEqual(
self.store.get_audio_lease(waiting.lease_id).state,
"unconfirmed",
)
self.assertIsNone(self.store.get_audio_lease(waiting.lease_id).confirmation_deadline)
def test_manual_mode_set_during_acquisition_can_reject(self):
router = self.coordinator(
replace(
self.base_config.router,
continue_on_confirmation_failure=False,
)
)
router.observe_mode("service")
waiting = router.acquire("play-manual-reject")
decision = router.observe_mode_command("announcements")
self.assertEqual(decision.action, "rejected")
self.assertFalse(decision.continued)
self.assertEqual(
self.store.get_audio_lease(waiting.lease_id).state,
"abandoned",
)
def test_manual_mode_set_while_held_relinquishes_restoration(self):
router = self.coordinator()
router.observe_mode("service")
decision = router.acquire("play-manual-held")
router.observe_mode("video")
lost = router.observe_mode_command("video")
self.assertEqual(lost.action, "ownership_lost")
self.assertEqual(router.release(decision.lease_id).action, "released")
self.assertEqual(self.publisher.modes, ["video"])
def test_manual_mode_set_while_restoring_relinquishes_restoration(self):
router = self.coordinator()
router.observe_mode("service")
decision = router.acquire("play-manual-restore")
router.observe_mode("video")
self.assertEqual(router.release(decision.lease_id).action, "restore_waiting")
lost = router.observe_mode_command("announcements")
self.assertEqual(lost.action, "ownership_lost")
self.assertEqual(
self.store.get_audio_lease(decision.lease_id).state,
"ownership_lost",
)
self.assertEqual(self.publisher.modes, ["video", "service"])
def test_retained_mode_set_does_not_claim_live_operator_intent(self):
router = self.coordinator()
router.observe_mode("service")
decision = router.acquire("play-retained-set")
router.observe_mode("video")
ignored = router.observe_mode_command("announcements", retained=True)
self.assertEqual(ignored.action, "retained_ignored")
self.assertEqual(
self.store.get_audio_lease(decision.lease_id).state,
"held",
)
if __name__ == "__main__":
unittest.main()
+742
View File
@@ -0,0 +1,742 @@
import json
import queue
import tempfile
import threading
import time
import unittest
from dataclasses import replace
from pathlib import Path
from controller.config import (
ControllerConfig,
HomeAssistantConfig,
RouterConfig,
StorageConfig,
)
from controller.downloads import canonical_playlist_path
from controller.mqtt_adapter import MqttEvent, PublishReceipt
from controller.service import ControllerService
from controller.store import ControllerStore
class MutableClock:
def __init__(self, value):
self.value = value
def __call__(self):
return self.value
class FakeMqttAdapter:
"""In-memory transport boundary; it never starts a network client."""
def __init__(self, *, connected=True):
self.connected = connected
self.started = False
self.player_commands = []
self.router_modes = []
self.json_messages = []
self.legacy_messages = []
self.acknowledged = []
def start(self):
self.started = True
def stop(self):
self.started = False
def _receipt(self):
if self.connected:
return PublishReceipt(True, message_id=1)
return PublishReceipt(False, reason="MQTT is disconnected")
def publish_player_command(self, payload):
receipt = self._receipt()
if receipt.accepted:
self.player_commands.append(dict(payload))
return receipt
def publish_router_mode(self, mode):
receipt = self._receipt()
if receipt.accepted:
self.router_modes.append(mode)
return receipt
def publish_json(self, topic, payload, *, qos=0, retain=False):
receipt = self._receipt()
if receipt.accepted:
self.json_messages.append((topic, dict(payload), qos, retain))
return receipt
def publish_legacy(self, topic, payload, *, retain=True):
receipt = self._receipt()
if receipt.accepted:
self.legacy_messages.append((topic, payload, retain))
return receipt
def acknowledge(self, event):
self.acknowledged.append(event)
return True
class FakeDownloadManager:
def __init__(self):
self.recovered = False
self.stopped = False
def recover_pending_deletions(self):
self.recovered = True
def expire_due_slots(self):
return []
def shutdown(self, *, wait):
self.stopped = wait
class ControllerServiceTests(unittest.TestCase):
def setUp(self):
self.tempdir = tempfile.TemporaryDirectory()
base = Path(self.tempdir.name)
self.wall = MutableClock(1_700_000_000.0)
self.monotonic = MutableClock(100.0)
self.config = ControllerConfig(
storage=StorageConfig(
media_base=base / "media",
data_base=base / "data",
retention_seconds=3600,
expiry_interval_seconds=3600,
uncertain_review_after_seconds=30,
),
router=RouterConfig(
required_mode="video",
confirmation_timeout_seconds=3,
when_state_unknown="continue",
continue_on_confirmation_failure=True,
fallback_restore_mode=None,
),
homeassistant=HomeAssistantConfig(enabled=False),
)
self.events = queue.Queue()
self.store = None
self.mqtt = None
self.downloads = None
self.service = None
def tearDown(self):
if self.service is not None and self.service.started:
self.service.stop()
self.tempdir.cleanup()
def start_service(self, *, config=None, mqtt=None):
config = config or self.config
self.events = queue.Queue()
self.store = ControllerStore(config.storage, now=self.wall)
self.mqtt = mqtt or FakeMqttAdapter()
self.downloads = FakeDownloadManager()
self.service = ControllerService(
config,
store=self.store,
mqtt_adapter=self.mqtt,
event_queue=self.events,
download_manager=self.downloads,
monotonic=self.monotonic,
wall_clock=self.wall,
)
self.service.start()
self.assertTrue(self.downloads.recovered)
return self.service
def ready_slot(self, slot=1):
reservation = self.store.reserve_download(
f"https://example.test/{slot}",
"tester",
"service test",
job_id=f"ready-{slot}",
)
self.assertEqual(reservation.slot, slot)
self.assertTrue(
self.store.mark_download_running(
reservation.job_id,
reservation.slot,
reservation.generation,
)
)
self.assertTrue(
self.store.complete_download(
reservation.job_id,
reservation.slot,
reservation.generation,
title=f"Slot {slot}",
items=(("One", "one.mp4"),),
)
)
playlist = canonical_playlist_path(self.service.config.storage.media_base, slot)
playlist.parent.mkdir(parents=True, exist_ok=True)
playlist.write_text("one.mp4\n", encoding="utf-8")
return playlist
def send_message(
self,
topic,
payload,
*,
qos=0,
retain=False,
message_id=None,
duplicate=False,
controller_echo=False,
):
if not isinstance(payload, bytes):
payload = (
payload.encode("utf-8")
if isinstance(payload, str)
else json.dumps(payload).encode("utf-8")
)
self.events.put(
MqttEvent(
kind="message",
topic=topic,
payload=payload,
qos=qos,
retain=retain,
duplicate=duplicate,
message_id=message_id,
connection_generation=1,
controller_echo=controller_echo,
)
)
def wait_for(self, predicate, message):
deadline = time.monotonic() + 2.0
while time.monotonic() < deadline:
if predicate():
return
time.sleep(0.01)
self.fail(message)
def result_events(self):
return [
payload
for topic, payload, qos, retain in self.mqtt.json_messages
if topic == self.service.config.topics.controller_result
]
def test_play_waits_for_router_confirmation_and_player_evidence(self):
config = replace(
self.config,
router=replace(self.config.router, when_state_unknown="attempt"),
)
self.start_service(config=config)
playlist = self.ready_slot(1)
self.send_message(config.topics.router_mode_state, "service", qos=1)
self.wait_for(
lambda: self.store.router_observation().mode == "service",
"Router state was not consumed",
)
result = self.service.submit_control({"action": "play", "slot": 1})
self.assertEqual(result["status"], "pending")
self.assertEqual(result["router"], "waiting")
self.assertEqual(self.mqtt.router_modes, ["video"])
self.assertEqual(self.mqtt.player_commands, [])
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
self.send_message(
config.topics.router_mode_set,
"video",
qos=1,
controller_echo=True,
)
self.wait_for(
lambda: len(self.mqtt.acknowledged) == 2,
"Controller Router command echo was not consumed",
)
self.assertEqual(self.store.live_audio_lease().state, "acquiring")
self.assertEqual(self.mqtt.player_commands, [])
self.send_message(config.topics.router_mode_state, "video", qos=1)
self.wait_for(
lambda: len(self.mqtt.player_commands) == 1,
"Player command was not released after Router confirmation",
)
self.assertEqual(
self.mqtt.player_commands,
[{"action": "play", "slot": 1, "label": "Slot 1"}],
)
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
self.send_message(
config.topics.player_status,
{"state": "playing", "slot": 1, "path": str(playlist)},
qos=1,
)
self.wait_for(
lambda: self.store.player_observation().phase == "playing",
"authoritative Player status was not reduced",
)
self.assertEqual(self.service.playback.projected_slot_state(1), "playing")
self.assertIn(
"player_operation_confirmed",
{event["type"] for event in self.result_events()},
)
def test_manual_router_set_during_acquisition_resolves_by_failure_policy(self):
config = replace(
self.config,
router=replace(self.config.router, when_state_unknown="attempt"),
)
self.start_service(config=config)
self.ready_slot(1)
self.send_message(config.topics.router_mode_state, "service", qos=1)
self.wait_for(
lambda: self.store.router_observation().mode == "service",
"Router state was not consumed",
)
result = self.service.submit_control({"action": "play", "slot": 1})
self.assertEqual(result["router"], "waiting")
self.send_message(config.topics.router_mode_set, "announcements", qos=1)
self.wait_for(
lambda: len(self.mqtt.player_commands) == 1,
"manual Router intent did not resolve acquisition",
)
lease = self.store.get_audio_lease(self.service._session_lease_id)
self.assertIsNotNone(lease)
self.assertEqual(lease.state, "unconfirmed")
warnings = [
event
for event in self.result_events()
if event["type"] == "audio_router_warning"
]
self.assertTrue(warnings)
self.assertIs(warnings[-1]["continued"], True)
def test_pause_state_remains_telemetry_authoritative(self):
self.start_service()
playlist = self.ready_slot(1)
self.service.submit_control({"action": "play", "slot": 1})
self.wait_for(
lambda: self.mqtt.player_commands == [
{"action": "play", "slot": 1, "label": "Slot 1"}
],
"play command was not published",
)
self.send_message(
self.config.topics.player_status,
{"state": "playing", "slot": 1, "path": str(playlist)},
qos=1,
)
self.wait_for(
lambda: self.service.playback.pending is None,
"play operation was not confirmed",
)
playing_generation = self.store.player_observation().generation
self.service.submit_control({"action": "pause"})
self.send_message(
self.config.topics.player_status,
{"state": "paused", "slot": 1, "path": str(playlist)},
qos=1,
)
self.wait_for(
lambda: self.store.player_observation().generation > playing_generation,
"pause status was not consumed",
)
self.assertIsNotNone(self.service.playback.pending)
self.assertEqual(self.service.playback.projected_slot_state(1), "playing")
self.send_message(
self.config.topics.player_telemetry,
{
"state": "telemetry",
"slot": 1,
"path": str(playlist),
"paused": True,
},
)
self.wait_for(
lambda: self.service.playback.pending is None,
"pause was not confirmed by telemetry",
)
self.assertEqual(self.store.player_observation().paused, True)
self.assertEqual(self.service.playback.projected_slot_state(1), "paused")
self.assertEqual(self.mqtt.player_commands[-1], {"action": "pause"})
def test_retained_controller_command_is_rejected_then_acked_and_reported(self):
self.start_service()
self.send_message(
self.config.topics.controller_command,
{"action": "play", "path": "/must/not/run"},
qos=1,
retain=True,
message_id=42,
)
self.wait_for(
lambda: len(self.mqtt.acknowledged) == 1,
"retained command was not acknowledged after rejection",
)
self.assertEqual(self.mqtt.player_commands, [])
event = self.result_events()[-1]
self.assertEqual(event["version"], 2)
self.assertEqual(event["type"], "retained_command_rejected")
topic, _, qos, retain = self.mqtt.json_messages[-1]
self.assertEqual(topic, "youtube/controller/v2/events")
self.assertEqual((qos, retain), (1, False))
self.assertEqual(self.mqtt.acknowledged[0].message_id, 42)
def test_optional_result_topic_can_be_disabled_without_affecting_ack(self):
config = replace(
self.config,
topics=replace(self.config.topics, controller_result=None),
)
self.start_service(config=config)
self.send_message(
config.topics.controller_command,
{"action": "delete_slot", "slot": 1},
qos=1,
retain=True,
message_id=7,
)
self.wait_for(
lambda: len(self.mqtt.acknowledged) == 1,
"command was not acknowledged with result topic disabled",
)
self.assertEqual(self.mqtt.json_messages, [])
def test_broker_unavailable_does_not_prevent_service_startup(self):
mqtt = FakeMqttAdapter(connected=False)
self.start_service(mqtt=mqtt)
health = self.service.health()
self.assertTrue(self.service.started)
self.assertTrue(mqtt.started)
self.assertTrue(health["coordinator_alive"])
self.assertEqual(health["status"], "degraded")
self.assertIn("mqtt_disconnected", health["degraded_reasons"])
def test_administrator_can_clear_exact_uncertain_playback_review(self):
self.start_service()
playlist = self.ready_slot(1)
self.store.set_player_observation(
availability="unknown",
phase="unknown",
active_slot=1,
active_path=str(playlist),
paused=None,
reconciliation_state="operator_review",
reconciliation_due_at=None,
reconciliation_id="review-token",
)
guard = self.store.protect_slot(
1,
"restart_uncertain",
guard_id="review-token",
)
review = self.service.health()["reconciliation_reviews"]
self.assertEqual(review[0]["review_id"], "review-token")
result = self.service.submit_control(
{
"action": "reconcile_playback",
"decision": "clear",
"confirm": True,
"slot": 1,
"review_id": review[0]["review_id"],
"actor": "operator",
"reason": "Player host was replaced",
}
)
self.assertEqual(
result,
{"status": "cleared", "slot": 1, "review_id": "review-token"},
)
self.assertEqual(self.store.list_live_guards(), [])
observation = self.store.player_observation()
self.assertEqual(observation.reconciliation_state, "admin_cleared")
self.assertIsNone(observation.active_slot)
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
self.assertEqual(self.result_events()[-1]["type"], "playback_reconciled")
def test_busy_player_keeps_video_route_until_actual_playback_finishes(self):
config = replace(
self.config,
router=replace(self.config.router, when_state_unknown="attempt"),
)
self.start_service(config=config)
self.ready_slot(1)
second_playlist = self.ready_slot(2)
self.send_message(config.topics.router_mode_state, "service", qos=1)
self.wait_for(
lambda: self.store.router_observation().mode == "service",
"Router service mode was not observed",
)
self.service.submit_control({"action": "play", "slot": 1})
self.send_message(config.topics.router_mode_state, "video", qos=1)
self.wait_for(
lambda: len(self.mqtt.player_commands) == 1,
"play command was not released",
)
self.send_message(
config.topics.player_status,
{
"state": "busy",
"slot": 2,
"path": str(second_playlist),
"message": "mpv already playing; stop first",
},
qos=1,
)
self.wait_for(
lambda: self.service.playback.pending is None,
"busy response was not reduced",
)
self.assertEqual(self.store.player_observation().active_slot, 2)
self.assertEqual(self.mqtt.router_modes, ["video"])
lease = self.store.live_audio_lease()
self.assertIsNotNone(lease)
self.assertEqual(lease.state, "held")
def test_disconnected_synchronous_play_failure_is_not_reported_pending(self):
self.start_service(mqtt=FakeMqttAdapter(connected=False))
self.ready_slot(1)
result = self.service.submit_control({"action": "play", "slot": 1})
self.assertEqual(result["status"], "unavailable")
self.assertIsNone(self.service.playback.pending)
self.assertEqual(self.store.list_live_guards(), [])
def test_router_timeout_uses_monotonic_clock_and_continues_with_warning(self):
config = replace(
self.config,
router=replace(self.config.router, when_state_unknown="attempt"),
)
self.start_service(config=config)
self.ready_slot(1)
result = self.service.submit_control({"action": "play", "slot": 1})
self.assertEqual(result["router"], "waiting")
self.wall.value -= 10_000
self.monotonic.value += 4
self.wait_for(
lambda: len(self.mqtt.player_commands) == 1,
"Router timeout did not release play within its finite deadline",
)
warnings = [
event
for event in self.result_events()
if event["type"] == "audio_router_warning"
]
self.assertTrue(warnings)
self.assertIs(warnings[-1]["continued"], True)
def test_duplicate_toggle_is_suppressed_and_translated_idempotently(self):
self.start_service()
playlist = self.ready_slot(1)
self.send_message(
self.config.topics.player_telemetry,
{
"state": "telemetry",
"slot": 1,
"path": str(playlist),
"paused": False,
},
)
self.wait_for(
lambda: self.store.player_observation().active_slot == 1,
"active telemetry was not consumed",
)
command = {"action": "toggle"}
self.send_message(
self.config.topics.controller_command,
command,
qos=1,
message_id=71,
)
self.send_message(
self.config.topics.controller_command,
command,
qos=1,
message_id=71,
duplicate=True,
)
self.wait_for(
lambda: sum(
event.message_id == 71 for event in self.mqtt.acknowledged
)
== 2,
"duplicate command packets were not both acknowledged",
)
self.assertEqual(self.mqtt.player_commands, [{"action": "pause"}])
def test_path_only_managed_play_adds_stable_slot_identity(self):
self.start_service()
playlist = self.ready_slot(1)
result = self.service.submit_control(
{"action": "play", "playlist": str(playlist)}
)
self.assertEqual(result["status"], "pending")
self.assertEqual(
self.mqtt.player_commands,
[{"action": "play", "playlist": str(playlist), "slot": 1}],
)
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
def test_explicit_slot_cannot_be_paired_with_another_managed_slot_path(self):
self.start_service()
self.ready_slot(1)
second_playlist = self.ready_slot(2)
with self.assertRaisesRegex(ValueError, "conflicts"):
self.service.submit_control(
{
"action": "play",
"slot": 1,
"playlist": str(second_playlist),
}
)
self.assertEqual(self.mqtt.player_commands, [])
self.assertEqual(self.store.list_live_guards(), [])
def test_administrator_can_clear_external_playback_review_without_slot(self):
config = replace(
self.config,
topics=replace(self.config.topics, controller_result=None),
)
self.start_service(config=config)
observation = self.store.mark_external_uncertain(
"/srv/operator-playlists/special-event.m3u",
review_after=self.wall.value,
details={"reason": "restart"},
)
self.assertIsNotNone(observation.reconciliation_id)
reviews = self.service.health()["reconciliation_reviews"]
self.assertEqual(len(reviews), 1)
review_id = reviews[0]["review_id"]
result = self.service.submit_control(
{
"action": "reconcile_playback",
"decision": "clear",
"confirm": True,
"review_id": review_id,
"actor": "operator",
"reason": "Player appliance was replaced",
}
)
self.assertEqual(
result,
{"status": "cleared", "review_id": review_id},
)
cleared = self.store.player_observation()
self.assertIsNone(cleared.active_path)
self.assertEqual(cleared.reconciliation_state, "admin_cleared")
def test_external_play_intent_is_durable_immediately_after_puback(self):
self.start_service()
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
result = self.service.submit_control(
{"action": "play", "playlist": str(playlist)}
)
self.assertEqual(result["status"], "pending")
self.assertEqual(
self.mqtt.player_commands,
[{"action": "play", "playlist": str(playlist)}],
)
reviews = self.service.health()["reconciliation_reviews"]
self.assertEqual(len(reviews), 1)
review_id = reviews[0]["review_id"]
self.assertIs(reviews[0]["clearable"], False)
self.service.stop()
self.service = None
with ControllerStore(self.config.storage, now=self.wall).open() as reopened:
reopened.mark_restart_uncertain()
observation = reopened.player_observation()
self.assertEqual(observation.active_path, str(playlist))
self.assertEqual(observation.reconciliation_id, review_id)
def test_live_external_play_fence_cannot_be_admin_cleared(self):
self.start_service()
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
self.service.submit_control(
{"action": "play", "playlist": str(playlist)}
)
review = self.service.health()["reconciliation_reviews"][0]
self.assertIs(review["clearable"], False)
with self.assertRaisesRegex(ValueError, "Player operation is pending"):
self.service.submit_control(
{
"action": "reconcile_playback",
"decision": "clear",
"confirm": True,
"review_id": review["review_id"],
"actor": "operator",
"reason": "clear requested too early",
}
)
self.assertIsNotNone(self.service.playback.pending)
observation = self.store.player_observation()
self.assertEqual(observation.reconciliation_id, review["review_id"])
self.assertEqual(observation.active_path, str(playlist))
def test_definite_external_publish_failure_clears_prepublication_guard(self):
self.start_service(mqtt=FakeMqttAdapter(connected=False))
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
result = self.service.submit_control(
{"action": "play", "playlist": str(playlist)}
)
self.assertEqual(result["status"], "unavailable")
self.assertEqual(self.service.health()["reconciliation_reviews"], [])
observation = self.store.player_observation()
self.assertIsNone(observation.active_path)
self.assertIsNone(observation.reconciliation_id)
def test_unexpected_reducer_failure_is_not_acked_or_deduplicated(self):
self.start_service()
original = self.service.playback.handle_status
failure_seen = threading.Event()
attempts = 0
def fail_once(payload):
nonlocal attempts
attempts += 1
if attempts == 1:
failure_seen.set()
raise RuntimeError("transient reducer failure")
return original(payload)
self.service.playback.handle_status = fail_once
payload = {
"state": "playing",
"slot": 1,
"path": str(self.config.storage.media_base / "slot1" / "playlist.m3u"),
}
self.send_message(
self.config.topics.player_status,
payload,
qos=1,
message_id=88,
)
self.assertTrue(failure_seen.wait(timeout=1))
time.sleep(0.05)
self.assertEqual(self.mqtt.acknowledged, [])
self.send_message(
self.config.topics.player_status,
payload,
qos=1,
message_id=88,
duplicate=True,
)
self.wait_for(
lambda: len(self.mqtt.acknowledged) == 1,
"retried status was not processed and acknowledged",
)
self.assertEqual(attempts, 2)
self.assertEqual(self.store.player_observation().phase, "playing")
if __name__ == "__main__":
unittest.main()
+196
View File
@@ -1,6 +1,7 @@
import concurrent.futures import concurrent.futures
import hashlib import hashlib
import os import os
import sqlite3
import tempfile import tempfile
import unittest import unittest
from pathlib import Path from pathlib import Path
@@ -49,6 +50,37 @@ class FreshStoreTests(StoreTestCase):
self.assertTrue(all(slot.media_state == "empty" for slot in slots)) self.assertTrue(all(slot.media_state == "empty" for slot in slots))
self.assertEqual(store.player_observation().phase, "unknown") self.assertEqual(store.player_observation().phase, "unknown")
def test_schema_one_database_upgrades_external_reconciliation_atomically(self):
with self.open_store() as store:
database_path = store.path
connection = sqlite3.connect(database_path)
try:
connection.execute(
"ALTER TABLE player_observation DROP COLUMN reconciliation_id"
)
connection.execute("DELETE FROM schema_migrations WHERE version = 2")
connection.execute("PRAGMA user_version = 1")
connection.commit()
finally:
connection.close()
with self.open_store() as upgraded:
self.assertEqual(
upgraded.connection.execute("PRAGMA user_version").fetchone()[0],
2,
)
columns = {
row["name"]
for row in upgraded.connection.execute(
"PRAGMA table_info(player_observation)"
).fetchall()
}
self.assertIn("reconciliation_id", columns)
migration = upgraded.connection.execute(
"SELECT name FROM schema_migrations WHERE version = 2"
).fetchone()
self.assertEqual(migration["name"], "external_playback_reconciliation")
def test_slot_reservation_is_committed_before_worker_transition(self): def test_slot_reservation_is_committed_before_worker_transition(self):
with self.open_store() as store: with self.open_store() as store:
reservation = store.reserve_download( reservation = store.reserve_download(
@@ -127,6 +159,25 @@ class FreshStoreTests(StoreTestCase):
self.assertEqual(store.get_slot(2).slot, 2) self.assertEqual(store.get_slot(2).slot, 2)
self.assertEqual(store.get_slot(2).url, "https://example.test/two") self.assertEqual(store.get_slot(2).url, "https://example.test/two")
def test_concurrent_first_open_installs_only_one_valid_database(self):
stores = [ControllerStore(self.config, now=self.clock) for _ in range(2)]
def open_store(store):
store.open()
return [slot.slot for slot in store.list_slots()]
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
results = list(pool.map(open_store, stores))
self.assertEqual(results, [list(range(1, 11)), list(range(1, 11))])
markers = stores[0].connection.execute(
"SELECT name FROM data_migrations ORDER BY name"
).fetchall()
self.assertEqual([row["name"] for row in markers], ["fresh_database_v1"])
finally:
for store in stores:
store.close()
class GuardAndExpiryTests(StoreTestCase): class GuardAndExpiryTests(StoreTestCase):
def _ready_slot(self, store, *, slot_expected=1): def _ready_slot(self, store, *, slot_expected=1):
@@ -196,6 +247,7 @@ class GuardAndExpiryTests(StoreTestCase):
paused=False, paused=False,
reconciliation_state="confirmed", reconciliation_state="confirmed",
reconciliation_due_at=None, reconciliation_due_at=None,
reconciliation_id=None,
) )
guard = store.mark_restart_uncertain() guard = store.mark_restart_uncertain()
self.assertEqual(guard.slot, 1) self.assertEqual(guard.slot, 1)
@@ -204,6 +256,150 @@ class GuardAndExpiryTests(StoreTestCase):
self.assertEqual(observation.reconciliation_state, "awaiting_evidence") self.assertEqual(observation.reconciliation_state, "awaiting_evidence")
self.assertEqual(observation.active_slot, 1) self.assertEqual(observation.active_slot, 1)
def test_external_playback_restart_gets_stable_review_token_and_exact_clear(self):
external_path = "/srv/operator-playlists/live.m3u"
with self.open_store() as store:
store.set_player_observation(
availability="online",
phase="playing",
active_slot=None,
active_path=external_path,
paused=False,
reconciliation_state="confirmed",
reconciliation_due_at=None,
reconciliation_id=None,
)
self.assertIsNone(store.mark_restart_uncertain())
first = store.player_observation()
self.assertEqual(first.reconciliation_state, "awaiting_evidence")
self.assertIsNotNone(first.reconciliation_id)
self.assertEqual(first.reconciliation_due_at, self.clock.value + 30)
self.clock.value += 10
self.assertIsNone(store.mark_restart_uncertain())
repeated = store.player_observation()
self.assertEqual(repeated.reconciliation_id, first.reconciliation_id)
self.assertEqual(repeated.reconciliation_due_at, first.reconciliation_due_at)
self.clock.value = first.reconciliation_due_at + 1
due = store.mark_external_reconciliation_due()
self.assertEqual(due, (first.reconciliation_id, external_path))
self.assertIsNone(store.mark_external_reconciliation_due())
review = store.player_observation()
self.assertEqual(review.reconciliation_state, "operator_review")
self.assertEqual(review.active_path, external_path)
self.assertFalse(
store.clear_external_reconciliation(
"stale-review",
actor="operator",
reason="Player was replaced",
)
)
self.assertTrue(
store.clear_external_reconciliation(
first.reconciliation_id,
actor="operator",
reason="Player was replaced",
)
)
cleared = store.player_observation()
self.assertEqual(cleared.reconciliation_state, "admin_cleared")
self.assertIsNone(cleared.active_path)
self.assertIsNone(cleared.reconciliation_id)
def test_restart_converts_existing_active_guard_and_preserves_deadline(self):
with self.open_store() as store:
self._ready_slot(store)
store.protect_slot(1, "player_active", guard_id="active-before-restart")
store.set_player_observation(
availability="online",
phase="playing",
active_slot=1,
active_path="/mnt/Media/YTPlayer/slot1/playlist.m3u",
paused=False,
reconciliation_state="confirmed",
reconciliation_due_at=None,
reconciliation_id=None,
)
guard = store.mark_restart_uncertain()
self.assertEqual(guard.guard_id, "active-before-restart")
self.assertEqual(guard.kind, "restart_uncertain")
first_deadline = guard.review_after
self.clock.value += 10
repeated = store.mark_restart_uncertain()
self.assertEqual(repeated.guard_id, guard.guard_id)
self.assertEqual(repeated.review_after, first_deadline)
def test_restart_converts_pre_evidence_pending_guard_for_admin_review(self):
with self.open_store() as store:
self._ready_slot(store)
store.protect_slot(1, "pending_play", guard_id="published-before-crash")
guard = store.mark_restart_uncertain()
self.assertEqual(guard.guard_id, "published-before-crash")
self.assertEqual(guard.kind, "restart_uncertain")
self.assertEqual(store.player_observation().active_slot, 1)
self.assertTrue(
store.clear_guard(
1,
guard.guard_id,
actor="operator",
reason="Player replaced",
)
)
def test_guard_review_transition_is_reported_only_once(self):
with self.open_store() as store:
self._ready_slot(store)
store.protect_slot(
1,
"restart_uncertain",
guard_id="one-warning",
review_after=self.clock.value,
)
self.assertEqual(len(store.mark_due_guards_for_review()), 1)
self.assertEqual(store.mark_due_guards_for_review(), [])
def test_clearing_one_legacy_guard_does_not_erase_other_observation(self):
with self.open_store() as store:
self._ready_slot(store)
second = store.reserve_download(
"https://example.test/second", None, None, job_id="second"
)
store.mark_download_running("second", 2, second.generation)
store.complete_download(
"second",
2,
second.generation,
title="Second",
items=(("Second", "second.mp4"),),
)
first_guard = store.protect_slot(
1, "legacy_uncertain", guard_id="legacy-one"
)
store.protect_slot(2, "player_active", guard_id="active-two")
store.set_player_observation(
availability="online",
phase="playing",
active_slot=2,
active_path="/mnt/Media/YTPlayer/slot2/playlist.m3u",
paused=False,
reconciliation_state="confirmed",
reconciliation_due_at=None,
reconciliation_id=None,
)
self.assertTrue(
store.clear_guard(
1,
first_guard.guard_id,
actor="operator",
reason="legacy evidence resolved",
)
)
observation = store.player_observation()
self.assertEqual(observation.active_slot, 2)
self.assertEqual(observation.phase, "playing")
class LegacyMigrationTests(StoreTestCase): class LegacyMigrationTests(StoreTestCase):
def write_legacy(self, data): def write_legacy(self, data):