feat: coordinate authoritative playback and audio routing
This commit is contained in:
+5
-2
@@ -8,6 +8,8 @@ mqtt:
|
|||||||
reconnect_min_seconds: 1
|
reconnect_min_seconds: 1
|
||||||
reconnect_max_seconds: 30
|
reconnect_max_seconds: 30
|
||||||
command_queue_size: 512
|
command_queue_size: 512
|
||||||
|
# QoS 1 is accepted only after the broker PUBACKs within this bound.
|
||||||
|
publish_ack_timeout_seconds: 2
|
||||||
|
|
||||||
topics:
|
topics:
|
||||||
player_command: youtube/player/cmd
|
player_command: youtube/player/cmd
|
||||||
@@ -19,8 +21,9 @@ topics:
|
|||||||
playback_base: youtube/media/playback
|
playback_base: youtube/media/playback
|
||||||
router_mode_set: audio/player/mode/set
|
router_mode_set: audio/player/mode/set
|
||||||
router_mode_state: audio/player/mode/state
|
router_mode_state: audio/player/mode/state
|
||||||
# Optional additive result topic. Existing consumers do not depend on it.
|
# Optional additive result/warning topic. Set null to disable it.
|
||||||
controller_result: null
|
# Existing consumers never depend on this topic.
|
||||||
|
controller_result: youtube/controller/v2/events
|
||||||
|
|
||||||
storage:
|
storage:
|
||||||
media_base: /mnt/Media/YTPlayer
|
media_base: /mnt/Media/YTPlayer
|
||||||
|
|||||||
+11
-1
@@ -31,6 +31,7 @@ class MqttConfig:
|
|||||||
reconnect_min_seconds: int = 1
|
reconnect_min_seconds: int = 1
|
||||||
reconnect_max_seconds: int = 30
|
reconnect_max_seconds: int = 30
|
||||||
command_queue_size: int = 512
|
command_queue_size: int = 512
|
||||||
|
publish_ack_timeout_seconds: float = 2.0
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -195,6 +196,11 @@ def load_config(
|
|||||||
reconnect_min_seconds=_integer(mqtt_raw.get("reconnect_min_seconds", 1), "mqtt.reconnect_min_seconds"),
|
reconnect_min_seconds=_integer(mqtt_raw.get("reconnect_min_seconds", 1), "mqtt.reconnect_min_seconds"),
|
||||||
reconnect_max_seconds=_integer(mqtt_raw.get("reconnect_max_seconds", 30), "mqtt.reconnect_max_seconds"),
|
reconnect_max_seconds=_integer(mqtt_raw.get("reconnect_max_seconds", 30), "mqtt.reconnect_max_seconds"),
|
||||||
command_queue_size=_integer(mqtt_raw.get("command_queue_size", 512), "mqtt.command_queue_size"),
|
command_queue_size=_integer(mqtt_raw.get("command_queue_size", 512), "mqtt.command_queue_size"),
|
||||||
|
publish_ack_timeout_seconds=_number(
|
||||||
|
mqtt_raw.get("publish_ack_timeout_seconds", 2.0),
|
||||||
|
"mqtt.publish_ack_timeout_seconds",
|
||||||
|
minimum=0.1,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
if mqtt.reconnect_max_seconds < mqtt.reconnect_min_seconds:
|
if mqtt.reconnect_max_seconds < mqtt.reconnect_min_seconds:
|
||||||
raise ConfigError("mqtt.reconnect_max_seconds must be >= reconnect_min_seconds")
|
raise ConfigError("mqtt.reconnect_max_seconds must be >= reconnect_min_seconds")
|
||||||
@@ -211,7 +217,11 @@ def load_config(
|
|||||||
playback_base=_text(topics_raw.get("playback_base", "youtube/media/playback"), "topics.playback_base"),
|
playback_base=_text(topics_raw.get("playback_base", "youtube/media/playback"), "topics.playback_base"),
|
||||||
router_mode_set=_text(topics_raw.get("router_mode_set", "audio/player/mode/set"), "topics.router_mode_set"),
|
router_mode_set=_text(topics_raw.get("router_mode_set", "audio/player/mode/set"), "topics.router_mode_set"),
|
||||||
router_mode_state=_text(topics_raw.get("router_mode_state", "audio/player/mode/state"), "topics.router_mode_state"),
|
router_mode_state=_text(topics_raw.get("router_mode_state", "audio/player/mode/state"), "topics.router_mode_state"),
|
||||||
controller_result=_text(topics_raw.get("controller_result"), "topics.controller_result", allow_none=True),
|
controller_result=_text(
|
||||||
|
topics_raw.get("controller_result", "youtube/controller/v2/events"),
|
||||||
|
"topics.controller_result",
|
||||||
|
allow_none=True,
|
||||||
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
configured_slots = _integer(storage_raw.get("slot_count", 10), "storage.slot_count")
|
configured_slots = _integer(storage_raw.get("slot_count", 10), "storage.slot_count")
|
||||||
|
|||||||
@@ -25,7 +25,7 @@ class Topics:
|
|||||||
playback_base: str = "youtube/media/playback"
|
playback_base: str = "youtube/media/playback"
|
||||||
router_mode_set: str = "audio/player/mode/set"
|
router_mode_set: str = "audio/player/mode/set"
|
||||||
router_mode_state: str = "audio/player/mode/state"
|
router_mode_state: str = "audio/player/mode/state"
|
||||||
controller_result: str | None = None
|
controller_result: str | None = "youtube/controller/v2/events"
|
||||||
|
|
||||||
def slot_base(self, slot: int) -> str:
|
def slot_base(self, slot: int) -> str:
|
||||||
if slot not in SLOT_IDS:
|
if slot not in SLOT_IDS:
|
||||||
|
|||||||
+109
-3
@@ -4,9 +4,11 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
|
import signal
|
||||||
import shutil
|
import shutil
|
||||||
import subprocess
|
import subprocess
|
||||||
import threading
|
import threading
|
||||||
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from concurrent.futures import Future, ThreadPoolExecutor
|
from concurrent.futures import Future, ThreadPoolExecutor
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
@@ -32,6 +34,10 @@ class DownloadEvent:
|
|||||||
message: str | None = None
|
message: str | None = None
|
||||||
job_id: str | None = None
|
job_id: str | None = None
|
||||||
generation: int | None = None
|
generation: int | None = None
|
||||||
|
url: str | None = None
|
||||||
|
submitter: str | None = None
|
||||||
|
note: str | None = None
|
||||||
|
reused_slot: bool = False
|
||||||
|
|
||||||
|
|
||||||
EventCallback = Callable[[DownloadEvent], None]
|
EventCallback = Callable[[DownloadEvent], None]
|
||||||
@@ -66,7 +72,7 @@ class DownloadManager:
|
|||||||
store: ControllerStore,
|
store: ControllerStore,
|
||||||
*,
|
*,
|
||||||
on_event: EventCallback | None = None,
|
on_event: EventCallback | None = None,
|
||||||
runner: Runner = subprocess.run,
|
runner: Runner | None = None,
|
||||||
executor: ThreadPoolExecutor | None = None,
|
executor: ThreadPoolExecutor | None = None,
|
||||||
):
|
):
|
||||||
self.config = config
|
self.config = config
|
||||||
@@ -80,6 +86,7 @@ class DownloadManager:
|
|||||||
self._owns_executor = executor is None
|
self._owns_executor = executor is None
|
||||||
self._lock = threading.Lock()
|
self._lock = threading.Lock()
|
||||||
self._futures: set[Future[Any]] = set()
|
self._futures: set[Future[Any]] = set()
|
||||||
|
self._processes: set[subprocess.Popen[str]] = set()
|
||||||
self._stopping = False
|
self._stopping = False
|
||||||
|
|
||||||
def enqueue(
|
def enqueue(
|
||||||
@@ -106,6 +113,10 @@ class DownloadManager:
|
|||||||
state="queued",
|
state="queued",
|
||||||
job_id=reservation.job_id,
|
job_id=reservation.job_id,
|
||||||
generation=reservation.generation,
|
generation=reservation.generation,
|
||||||
|
url=url,
|
||||||
|
submitter=submitter,
|
||||||
|
note=note,
|
||||||
|
reused_slot=reservation.reused_slot,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
@@ -184,7 +195,7 @@ class DownloadManager:
|
|||||||
url,
|
url,
|
||||||
]
|
]
|
||||||
try:
|
try:
|
||||||
result = self.runner(
|
result = self._run_command(
|
||||||
arguments,
|
arguments,
|
||||||
check=True,
|
check=True,
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
@@ -246,7 +257,7 @@ class DownloadManager:
|
|||||||
),
|
),
|
||||||
url,
|
url,
|
||||||
]
|
]
|
||||||
self.runner(
|
self._run_command(
|
||||||
arguments,
|
arguments,
|
||||||
check=True,
|
check=True,
|
||||||
text=True,
|
text=True,
|
||||||
@@ -322,6 +333,90 @@ class DownloadManager:
|
|||||||
return f"yt-dlp failed with exit code {exc.returncode}"
|
return f"yt-dlp failed with exit code {exc.returncode}"
|
||||||
return f"download failed: {exc}"
|
return f"download failed: {exc}"
|
||||||
|
|
||||||
|
def _run_command(
|
||||||
|
self,
|
||||||
|
arguments: list[str],
|
||||||
|
*,
|
||||||
|
check: bool,
|
||||||
|
text: bool,
|
||||||
|
timeout: float,
|
||||||
|
capture_output: bool = False,
|
||||||
|
) -> subprocess.CompletedProcess[str]:
|
||||||
|
if self.runner is not None:
|
||||||
|
return self.runner(
|
||||||
|
arguments,
|
||||||
|
check=check,
|
||||||
|
text=text,
|
||||||
|
timeout=timeout,
|
||||||
|
capture_output=capture_output,
|
||||||
|
)
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
if self._stopping:
|
||||||
|
raise RuntimeError("Controller is shutting down")
|
||||||
|
process = subprocess.Popen(
|
||||||
|
arguments,
|
||||||
|
stdout=subprocess.PIPE if capture_output else None,
|
||||||
|
stderr=subprocess.PIPE if capture_output else None,
|
||||||
|
text=text,
|
||||||
|
start_new_session=True,
|
||||||
|
)
|
||||||
|
self._processes.add(process)
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
stdout, stderr = process.communicate(timeout=timeout)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
self._terminate_process(process, force=False)
|
||||||
|
try:
|
||||||
|
stdout, stderr = process.communicate(timeout=2.0)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
self._terminate_process(process, force=True)
|
||||||
|
stdout, stderr = process.communicate()
|
||||||
|
raise subprocess.TimeoutExpired(
|
||||||
|
arguments,
|
||||||
|
timeout,
|
||||||
|
output=stdout,
|
||||||
|
stderr=stderr,
|
||||||
|
)
|
||||||
|
completed = subprocess.CompletedProcess(
|
||||||
|
arguments,
|
||||||
|
process.returncode,
|
||||||
|
stdout=stdout,
|
||||||
|
stderr=stderr,
|
||||||
|
)
|
||||||
|
if check and process.returncode:
|
||||||
|
raise subprocess.CalledProcessError(
|
||||||
|
process.returncode,
|
||||||
|
arguments,
|
||||||
|
output=stdout,
|
||||||
|
stderr=stderr,
|
||||||
|
)
|
||||||
|
return completed
|
||||||
|
finally:
|
||||||
|
with self._lock:
|
||||||
|
self._processes.discard(process)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _terminate_process(
|
||||||
|
process: subprocess.Popen[str],
|
||||||
|
*,
|
||||||
|
force: bool,
|
||||||
|
) -> None:
|
||||||
|
if process.poll() is not None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
os.killpg(
|
||||||
|
process.pid,
|
||||||
|
signal.SIGKILL if force else signal.SIGTERM,
|
||||||
|
)
|
||||||
|
except ProcessLookupError:
|
||||||
|
return
|
||||||
|
except OSError:
|
||||||
|
try:
|
||||||
|
process.kill() if force else process.terminate()
|
||||||
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _fsync_directory(path: Path) -> None:
|
def _fsync_directory(path: Path) -> None:
|
||||||
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
||||||
@@ -387,6 +482,17 @@ class DownloadManager:
|
|||||||
def shutdown(self, *, wait: bool = True) -> None:
|
def shutdown(self, *, wait: bool = True) -> None:
|
||||||
with self._lock:
|
with self._lock:
|
||||||
self._stopping = True
|
self._stopping = True
|
||||||
|
processes = tuple(self._processes)
|
||||||
|
for process in processes:
|
||||||
|
self._terminate_process(process, force=False)
|
||||||
|
deadline = time.monotonic() + 2.0
|
||||||
|
for process in processes:
|
||||||
|
if process.poll() is not None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
process.wait(timeout=max(0.0, deadline - time.monotonic()))
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
self._terminate_process(process, force=True)
|
||||||
if self._owns_executor:
|
if self._owns_executor:
|
||||||
self._executor.shutdown(wait=wait, cancel_futures=True)
|
self._executor.shutdown(wait=wait, cancel_futures=True)
|
||||||
|
|
||||||
|
|||||||
@@ -84,10 +84,38 @@ class PlayerObservation:
|
|||||||
generation: int
|
generation: int
|
||||||
reconciliation_state: str
|
reconciliation_state: str
|
||||||
reconciliation_due_at: float | None
|
reconciliation_due_at: float | None
|
||||||
|
reconciliation_id: str | None
|
||||||
last_status_at: float | None
|
last_status_at: float | None
|
||||||
last_telemetry_at: float | None
|
last_telemetry_at: float | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RouterObservation:
|
||||||
|
mode: str | None
|
||||||
|
freshness: str
|
||||||
|
observed_at: float | None
|
||||||
|
last_warning: str | None
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AudioLeaseRecord:
|
||||||
|
lease_id: str
|
||||||
|
operation_id: str | None
|
||||||
|
playback_generation: int | None
|
||||||
|
state: str
|
||||||
|
previous_mode: str | None
|
||||||
|
target_mode: str
|
||||||
|
restoration_mode: str | None
|
||||||
|
restoration_source: str
|
||||||
|
controller_changed: bool
|
||||||
|
confirmation_deadline: float | None
|
||||||
|
created_at: float
|
||||||
|
acquired_at: float | None
|
||||||
|
released_at: float | None
|
||||||
|
warning: str | None
|
||||||
|
error: str | None
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class MigrationReport:
|
class MigrationReport:
|
||||||
migrated: bool
|
migrated: bool
|
||||||
|
|||||||
+207
-40
@@ -6,6 +6,7 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
import queue
|
import queue
|
||||||
import threading
|
import threading
|
||||||
|
import uuid
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Mapping
|
from typing import Any, Mapping
|
||||||
|
|
||||||
@@ -33,9 +34,11 @@ class MqttEvent:
|
|||||||
payload: bytes = b""
|
payload: bytes = b""
|
||||||
qos: int = 0
|
qos: int = 0
|
||||||
retain: bool = False
|
retain: bool = False
|
||||||
|
duplicate: bool = False
|
||||||
message_id: int | None = None
|
message_id: int | None = None
|
||||||
connection_generation: int | None = None
|
connection_generation: int | None = None
|
||||||
reason: str | None = None
|
reason: str | None = None
|
||||||
|
controller_echo: bool = False
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -43,6 +46,7 @@ class PublishReceipt:
|
|||||||
accepted: bool
|
accepted: bool
|
||||||
message_id: int | None = None
|
message_id: int | None = None
|
||||||
reason: str | None = None
|
reason: str | None = None
|
||||||
|
uncertain: bool = False
|
||||||
|
|
||||||
|
|
||||||
class MqttAdapter:
|
class MqttAdapter:
|
||||||
@@ -59,11 +63,8 @@ class MqttAdapter:
|
|||||||
self.events: "queue.Queue[MqttEvent]" = events or queue.Queue(
|
self.events: "queue.Queue[MqttEvent]" = events or queue.Queue(
|
||||||
maxsize=config.mqtt.command_queue_size
|
maxsize=config.mqtt.command_queue_size
|
||||||
)
|
)
|
||||||
self.client = client or mqtt.Client(
|
self._injected_client = client is not None
|
||||||
mqtt.CallbackAPIVersion.VERSION2,
|
self.client = client or self._new_client()
|
||||||
client_id=config.mqtt.client_id,
|
|
||||||
clean_session=True,
|
|
||||||
)
|
|
||||||
self._generation_lock = threading.Lock()
|
self._generation_lock = threading.Lock()
|
||||||
self._connection_generation = 0
|
self._connection_generation = 0
|
||||||
self._connected = threading.Event()
|
self._connected = threading.Event()
|
||||||
@@ -72,9 +73,18 @@ class MqttAdapter:
|
|||||||
self._started = False
|
self._started = False
|
||||||
self._loop_started = False
|
self._loop_started = False
|
||||||
self._bootstrap_thread: threading.Thread | None = None
|
self._bootstrap_thread: threading.Thread | None = None
|
||||||
|
self._router_echo_lock = threading.Lock()
|
||||||
|
self._pending_router_echoes: dict[str, bytes] = {}
|
||||||
self.last_error: str | None = None
|
self.last_error: str | None = None
|
||||||
self._configure_client()
|
self._configure_client()
|
||||||
|
|
||||||
|
def _new_client(self):
|
||||||
|
return mqtt.Client(
|
||||||
|
mqtt.CallbackAPIVersion.VERSION2,
|
||||||
|
client_id=self.config.mqtt.client_id,
|
||||||
|
clean_session=True,
|
||||||
|
)
|
||||||
|
|
||||||
def _configure_client(self) -> None:
|
def _configure_client(self) -> None:
|
||||||
if self.config.mqtt.username is not None:
|
if self.config.mqtt.username is not None:
|
||||||
self.client.username_pw_set(
|
self.client.username_pw_set(
|
||||||
@@ -116,20 +126,32 @@ class MqttAdapter:
|
|||||||
self._stop_event.clear()
|
self._stop_event.clear()
|
||||||
try:
|
try:
|
||||||
self._begin_async_connection()
|
self._begin_async_connection()
|
||||||
except (OSError, mqtt.WebsocketConnectionError) as exc:
|
except (OSError, mqtt.WebsocketConnectionError, RuntimeError) as exc:
|
||||||
self.last_error = f"initial MQTT setup failed: {exc}"
|
self.last_error = f"initial MQTT setup failed: {exc}"
|
||||||
LOG.warning("%s; retrying in background", self.last_error)
|
LOG.warning("%s; retrying in background", self.last_error)
|
||||||
self._start_bootstrap_retry()
|
self._start_bootstrap_retry()
|
||||||
|
|
||||||
def _begin_async_connection(self) -> None:
|
def _begin_async_connection(self) -> None:
|
||||||
self.client.connect_async(
|
with self._lifecycle_lock:
|
||||||
|
if not self._started or self._stop_event.is_set():
|
||||||
|
return
|
||||||
|
client = self.client
|
||||||
|
result = client.connect_async(
|
||||||
self.config.mqtt.host,
|
self.config.mqtt.host,
|
||||||
self.config.mqtt.port,
|
self.config.mqtt.port,
|
||||||
keepalive=self.config.mqtt.keepalive_seconds,
|
keepalive=self.config.mqtt.keepalive_seconds,
|
||||||
)
|
)
|
||||||
|
if result != mqtt.MQTT_ERR_SUCCESS:
|
||||||
|
raise RuntimeError(f"failed to configure MQTT connection: {result}")
|
||||||
with self._lifecycle_lock:
|
with self._lifecycle_lock:
|
||||||
|
if (
|
||||||
|
not self._started
|
||||||
|
or self._stop_event.is_set()
|
||||||
|
or client is not self.client
|
||||||
|
):
|
||||||
|
return
|
||||||
if not self._loop_started:
|
if not self._loop_started:
|
||||||
result = self.client.loop_start()
|
result = client.loop_start()
|
||||||
if result != mqtt.MQTT_ERR_SUCCESS:
|
if result != mqtt.MQTT_ERR_SUCCESS:
|
||||||
raise RuntimeError(f"failed to start MQTT network loop: {result}")
|
raise RuntimeError(f"failed to start MQTT network loop: {result}")
|
||||||
self._loop_started = True
|
self._loop_started = True
|
||||||
@@ -159,17 +181,21 @@ class MqttAdapter:
|
|||||||
delay = min(maximum, max(delay * 2, 1.0))
|
delay = min(maximum, max(delay * 2, 1.0))
|
||||||
except RuntimeError as exc:
|
except RuntimeError as exc:
|
||||||
self.last_error = str(exc)
|
self.last_error = str(exc)
|
||||||
LOG.error("%s", self.last_error)
|
LOG.warning("%s", self.last_error)
|
||||||
return
|
delay = min(maximum, max(delay * 2, 1.0))
|
||||||
|
|
||||||
def stop(self) -> None:
|
def stop(self) -> None:
|
||||||
|
# Signal callbacks/retry work before waiting for the lifecycle lock.
|
||||||
|
# A callback already inside subscribe/connect then observes shutdown
|
||||||
|
# and cannot publish a post-stop connected state.
|
||||||
|
self._stop_event.set()
|
||||||
with self._lifecycle_lock:
|
with self._lifecycle_lock:
|
||||||
if not self._started:
|
if not self._started:
|
||||||
|
self._connected.clear()
|
||||||
return
|
return
|
||||||
self._started = False
|
self._started = False
|
||||||
loop_started = self._loop_started
|
loop_started = self._loop_started
|
||||||
self._loop_started = False
|
self._loop_started = False
|
||||||
self._stop_event.set()
|
|
||||||
self._connected.clear()
|
self._connected.clear()
|
||||||
if loop_started:
|
if loop_started:
|
||||||
try:
|
try:
|
||||||
@@ -190,24 +216,43 @@ class MqttAdapter:
|
|||||||
|
|
||||||
def _on_connect(self, client, userdata, flags, reason_code, properties=None) -> None:
|
def _on_connect(self, client, userdata, flags, reason_code, properties=None) -> None:
|
||||||
try:
|
try:
|
||||||
if not self._reason_success(reason_code):
|
with self._lifecycle_lock:
|
||||||
self.last_error = f"MQTT connect failed: {reason_code}"
|
if (
|
||||||
LOG.warning("%s", self.last_error)
|
client is not self.client
|
||||||
|
or not self._started
|
||||||
|
or self._stop_event.is_set()
|
||||||
|
):
|
||||||
|
return
|
||||||
|
if not self._reason_success(reason_code):
|
||||||
|
self.last_error = f"MQTT connect failed: {reason_code}"
|
||||||
|
LOG.warning("%s", self.last_error)
|
||||||
|
return
|
||||||
|
if self._stop_event.is_set():
|
||||||
|
return
|
||||||
|
with self._generation_lock:
|
||||||
|
self._connection_generation += 1
|
||||||
|
generation = self._connection_generation
|
||||||
|
self._connected.set()
|
||||||
|
self.last_error = None
|
||||||
|
subscriptions = (
|
||||||
|
(self.config.topics.player_status, PLAYER_STATUS_QOS),
|
||||||
|
(self.config.topics.player_telemetry, PLAYER_TELEMETRY_QOS),
|
||||||
|
(self.config.topics.player_state, PLAYER_STATE_QOS),
|
||||||
|
(self.config.topics.controller_command, 1),
|
||||||
|
(self.config.topics.router_mode_set, ROUTER_COMMAND_QOS),
|
||||||
|
(self.config.topics.router_mode_state, 1),
|
||||||
|
)
|
||||||
|
for topic, qos in subscriptions:
|
||||||
|
if self._stop_event.is_set():
|
||||||
|
self._connected.clear()
|
||||||
|
return
|
||||||
|
client.subscribe(topic, qos=qos)
|
||||||
|
if self._stop_event.is_set():
|
||||||
|
self._connected.clear()
|
||||||
|
return
|
||||||
|
if self._stop_event.is_set():
|
||||||
|
self._connected.clear()
|
||||||
return
|
return
|
||||||
with self._generation_lock:
|
|
||||||
self._connection_generation += 1
|
|
||||||
generation = self._connection_generation
|
|
||||||
self._connected.set()
|
|
||||||
self.last_error = None
|
|
||||||
subscriptions = (
|
|
||||||
(self.config.topics.player_status, PLAYER_STATUS_QOS),
|
|
||||||
(self.config.topics.player_telemetry, PLAYER_TELEMETRY_QOS),
|
|
||||||
(self.config.topics.player_state, PLAYER_STATE_QOS),
|
|
||||||
(self.config.topics.controller_command, 1),
|
|
||||||
(self.config.topics.router_mode_state, 1),
|
|
||||||
)
|
|
||||||
for topic, qos in subscriptions:
|
|
||||||
client.subscribe(topic, qos=qos)
|
|
||||||
self._enqueue(
|
self._enqueue(
|
||||||
MqttEvent(
|
MqttEvent(
|
||||||
kind="connected",
|
kind="connected",
|
||||||
@@ -231,6 +276,8 @@ class MqttAdapter:
|
|||||||
reason_code,
|
reason_code,
|
||||||
properties=None,
|
properties=None,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
if client is not self.client:
|
||||||
|
return
|
||||||
self._connected.clear()
|
self._connected.clear()
|
||||||
try:
|
try:
|
||||||
self._enqueue(
|
self._enqueue(
|
||||||
@@ -245,15 +292,33 @@ class MqttAdapter:
|
|||||||
|
|
||||||
def _on_message(self, client, userdata, message) -> None:
|
def _on_message(self, client, userdata, message) -> None:
|
||||||
try:
|
try:
|
||||||
|
# Capture client identity and generation atomically with session
|
||||||
|
# replacement. A callback that was already running when an
|
||||||
|
# uncertain command discards the transport must be able to leave a
|
||||||
|
# full queue so loop_stop() cannot deadlock waiting for it.
|
||||||
|
with self._lifecycle_lock:
|
||||||
|
if client is not self.client:
|
||||||
|
return
|
||||||
|
with self._generation_lock:
|
||||||
|
generation = self._connection_generation
|
||||||
|
topic = str(message.topic)
|
||||||
|
payload = bytes(message.payload)
|
||||||
|
retained = bool(getattr(message, "retain", False))
|
||||||
self._enqueue(
|
self._enqueue(
|
||||||
MqttEvent(
|
MqttEvent(
|
||||||
kind="message",
|
kind="message",
|
||||||
topic=str(message.topic),
|
topic=topic,
|
||||||
payload=bytes(message.payload),
|
payload=payload,
|
||||||
qos=int(getattr(message, "qos", 0)),
|
qos=int(getattr(message, "qos", 0)),
|
||||||
retain=bool(getattr(message, "retain", False)),
|
retain=retained,
|
||||||
|
duplicate=bool(getattr(message, "dup", False)),
|
||||||
message_id=getattr(message, "mid", None),
|
message_id=getattr(message, "mid", None),
|
||||||
connection_generation=self.connection_generation,
|
connection_generation=generation,
|
||||||
|
controller_echo=(
|
||||||
|
topic == self.config.topics.router_mode_set
|
||||||
|
and not retained
|
||||||
|
and self._claim_router_echo(payload)
|
||||||
|
),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
except BaseException:
|
except BaseException:
|
||||||
@@ -261,6 +326,11 @@ class MqttAdapter:
|
|||||||
|
|
||||||
def _enqueue(self, event: MqttEvent) -> bool:
|
def _enqueue(self, event: MqttEvent) -> bool:
|
||||||
while not self._stop_event.is_set():
|
while not self._stop_event.is_set():
|
||||||
|
if (
|
||||||
|
event.connection_generation is not None
|
||||||
|
and event.connection_generation != self.connection_generation
|
||||||
|
):
|
||||||
|
return False
|
||||||
try:
|
try:
|
||||||
self.events.put(event, timeout=0.2)
|
self.events.put(event, timeout=0.2)
|
||||||
return True
|
return True
|
||||||
@@ -293,8 +363,9 @@ class MqttAdapter:
|
|||||||
qos: int,
|
qos: int,
|
||||||
retain: bool,
|
retain: bool,
|
||||||
require_connection: bool,
|
require_connection: bool,
|
||||||
|
invalidate_transport_on_uncertain: bool = False,
|
||||||
) -> PublishReceipt:
|
) -> PublishReceipt:
|
||||||
if require_connection and not self.connected:
|
if require_connection and (not self.started or not self.connected):
|
||||||
return PublishReceipt(False, reason="MQTT is disconnected")
|
return PublishReceipt(False, reason="MQTT is disconnected")
|
||||||
try:
|
try:
|
||||||
info = self.client.publish(topic, payload, qos=qos, retain=retain)
|
info = self.client.publish(topic, payload, qos=qos, retain=retain)
|
||||||
@@ -303,8 +374,79 @@ class MqttAdapter:
|
|||||||
return PublishReceipt(False, reason=str(exc))
|
return PublishReceipt(False, reason=str(exc))
|
||||||
if info.rc != mqtt.MQTT_ERR_SUCCESS:
|
if info.rc != mqtt.MQTT_ERR_SUCCESS:
|
||||||
return PublishReceipt(False, message_id=getattr(info, "mid", None), reason=f"MQTT rc={info.rc}")
|
return PublishReceipt(False, message_id=getattr(info, "mid", None), reason=f"MQTT rc={info.rc}")
|
||||||
|
if qos > 0 and hasattr(info, "wait_for_publish"):
|
||||||
|
try:
|
||||||
|
info.wait_for_publish(
|
||||||
|
timeout=self.config.mqtt.publish_ack_timeout_seconds
|
||||||
|
)
|
||||||
|
acknowledged = bool(info.is_published())
|
||||||
|
except BaseException as exc:
|
||||||
|
acknowledged = False
|
||||||
|
detail = str(exc) or exc.__class__.__name__
|
||||||
|
else:
|
||||||
|
detail = (
|
||||||
|
f"no broker PUBACK within "
|
||||||
|
f"{self.config.mqtt.publish_ack_timeout_seconds:g} seconds"
|
||||||
|
)
|
||||||
|
if not acknowledged:
|
||||||
|
reason = f"MQTT QoS 1 publication is uncertain: {detail}"
|
||||||
|
if invalidate_transport_on_uncertain:
|
||||||
|
self._discard_unacknowledged_transport(reason)
|
||||||
|
return PublishReceipt(
|
||||||
|
False,
|
||||||
|
message_id=getattr(info, "mid", None),
|
||||||
|
reason=reason,
|
||||||
|
uncertain=True,
|
||||||
|
)
|
||||||
return PublishReceipt(True, message_id=getattr(info, "mid", None))
|
return PublishReceipt(True, message_id=getattr(info, "mid", None))
|
||||||
|
|
||||||
|
def _discard_unacknowledged_transport(self, reason: str) -> None:
|
||||||
|
"""Drop the Paho session so stale QoS 1 commands cannot replay."""
|
||||||
|
|
||||||
|
if self._injected_client:
|
||||||
|
# Test transports do not own a recreatable Paho session. Production
|
||||||
|
# always follows the replacement path below.
|
||||||
|
with self._generation_lock:
|
||||||
|
self._connection_generation += 1
|
||||||
|
self._connected.clear()
|
||||||
|
return
|
||||||
|
with self._lifecycle_lock:
|
||||||
|
old_client = self.client
|
||||||
|
old_loop_started = self._loop_started
|
||||||
|
self._loop_started = False
|
||||||
|
should_restart = self._started and not self._stop_event.is_set()
|
||||||
|
self._connected.clear()
|
||||||
|
with self._generation_lock:
|
||||||
|
previous_generation = self._connection_generation
|
||||||
|
self._connection_generation += 1
|
||||||
|
self.client = self._new_client()
|
||||||
|
self._configure_client()
|
||||||
|
try:
|
||||||
|
old_client.disconnect()
|
||||||
|
except BaseException:
|
||||||
|
LOG.exception("failed to disconnect uncertain MQTT transport")
|
||||||
|
if old_loop_started:
|
||||||
|
try:
|
||||||
|
old_client.loop_stop()
|
||||||
|
except BaseException:
|
||||||
|
LOG.exception("failed to stop uncertain MQTT transport")
|
||||||
|
self.last_error = reason
|
||||||
|
try:
|
||||||
|
self.events.put_nowait(
|
||||||
|
MqttEvent(
|
||||||
|
kind="disconnected",
|
||||||
|
connection_generation=previous_generation,
|
||||||
|
reason=reason,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
except queue.Full:
|
||||||
|
# This method can run on the sole coordinator thread; blocking on
|
||||||
|
# its own queue would deadlock. The uncertain PublishReceipt still
|
||||||
|
# drives the operation-specific conservative transition.
|
||||||
|
LOG.error("MQTT event queue was full while discarding uncertain transport")
|
||||||
|
if should_restart:
|
||||||
|
self._start_bootstrap_retry()
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _json_payload(payload: Mapping[str, Any]) -> str:
|
def _json_payload(payload: Mapping[str, Any]) -> str:
|
||||||
return json.dumps(payload, allow_nan=False, separators=(",", ":"))
|
return json.dumps(payload, allow_nan=False, separators=(",", ":"))
|
||||||
@@ -316,16 +458,41 @@ class MqttAdapter:
|
|||||||
qos=PLAYER_COMMAND_QOS,
|
qos=PLAYER_COMMAND_QOS,
|
||||||
retain=PLAYER_COMMAND_RETAIN,
|
retain=PLAYER_COMMAND_RETAIN,
|
||||||
require_connection=True,
|
require_connection=True,
|
||||||
|
invalidate_transport_on_uncertain=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
def publish_router_mode(self, mode: str) -> PublishReceipt:
|
def publish_router_mode(self, mode: str) -> PublishReceipt:
|
||||||
return self._publish(
|
# A broker normally sends our publication back through the matching
|
||||||
self.config.topics.router_mode_set,
|
# subscription. Register the exact payload before publishing because
|
||||||
mode,
|
# Paho's network thread may receive that echo before publish() returns.
|
||||||
qos=ROUTER_COMMAND_QOS,
|
# The marker is removed as soon as the QoS 1 publish call completes;
|
||||||
retain=ROUTER_COMMAND_RETAIN,
|
# this intentionally prefers a false ownership loss over mistaking a
|
||||||
require_connection=True,
|
# later same-value operator command for a Controller echo.
|
||||||
)
|
echo_token = uuid.uuid4().hex
|
||||||
|
with self._router_echo_lock:
|
||||||
|
self._pending_router_echoes[echo_token] = mode.encode("utf-8")
|
||||||
|
try:
|
||||||
|
return self._publish(
|
||||||
|
self.config.topics.router_mode_set,
|
||||||
|
mode,
|
||||||
|
qos=ROUTER_COMMAND_QOS,
|
||||||
|
retain=ROUTER_COMMAND_RETAIN,
|
||||||
|
require_connection=True,
|
||||||
|
invalidate_transport_on_uncertain=True,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
with self._router_echo_lock:
|
||||||
|
self._pending_router_echoes.pop(echo_token, None)
|
||||||
|
|
||||||
|
def _claim_router_echo(self, payload: bytes) -> bool:
|
||||||
|
"""Claim one exact mode-set echo from an in-flight local publish."""
|
||||||
|
|
||||||
|
with self._router_echo_lock:
|
||||||
|
for token, expected in tuple(self._pending_router_echoes.items()):
|
||||||
|
if expected == payload:
|
||||||
|
del self._pending_router_echoes[token]
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
def publish_legacy(self, topic: str, payload: str, *, retain: bool = True) -> PublishReceipt:
|
def publish_legacy(self, topic: str, payload: str, *, retain: bool = True) -> PublishReceipt:
|
||||||
return self._publish(
|
return self._publish(
|
||||||
|
|||||||
+400
-73
@@ -3,6 +3,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import math
|
import math
|
||||||
|
import os
|
||||||
import time
|
import time
|
||||||
import uuid
|
import uuid
|
||||||
from dataclasses import dataclass, replace
|
from dataclasses import dataclass, replace
|
||||||
@@ -28,6 +29,7 @@ class PlayerOperation:
|
|||||||
slot: int | None
|
slot: int | None
|
||||||
expected_path: str | None
|
expected_path: str | None
|
||||||
deadline: float
|
deadline: float
|
||||||
|
reconciliation_id: str | None = None
|
||||||
state: str = "awaiting_player"
|
state: str = "awaiting_player"
|
||||||
message: str | None = None
|
message: str | None = None
|
||||||
|
|
||||||
@@ -71,6 +73,7 @@ class PlaybackReducer:
|
|||||||
*,
|
*,
|
||||||
slot: int | None = None,
|
slot: int | None = None,
|
||||||
expected_path: str | None = None,
|
expected_path: str | None = None,
|
||||||
|
operation_id: str | None = None,
|
||||||
) -> PlayerOperation:
|
) -> PlayerOperation:
|
||||||
if action not in CONTROL_ACTIONS:
|
if action not in CONTROL_ACTIONS:
|
||||||
raise ValueError(f"unsupported Player action {action!r}")
|
raise ValueError(f"unsupported Player action {action!r}")
|
||||||
@@ -82,14 +85,13 @@ class PlaybackReducer:
|
|||||||
)
|
)
|
||||||
self._operation_generation += 1
|
self._operation_generation += 1
|
||||||
operation = PlayerOperation(
|
operation = PlayerOperation(
|
||||||
operation_id=uuid.uuid4().hex,
|
operation_id=operation_id or uuid.uuid4().hex,
|
||||||
generation=self._operation_generation,
|
generation=self._operation_generation,
|
||||||
action=action,
|
action=action,
|
||||||
slot=slot,
|
slot=slot,
|
||||||
expected_path=expected_path,
|
expected_path=expected_path,
|
||||||
deadline=self._monotonic() + self.operation_timeout_seconds,
|
deadline=self._monotonic() + self.operation_timeout_seconds,
|
||||||
)
|
)
|
||||||
self.pending = operation
|
|
||||||
if action == "play" and slot is not None:
|
if action == "play" and slot is not None:
|
||||||
self.store.protect_slot(
|
self.store.protect_slot(
|
||||||
slot,
|
slot,
|
||||||
@@ -97,11 +99,40 @@ class PlaybackReducer:
|
|||||||
guard_id=f"operation-{operation.operation_id}",
|
guard_id=f"operation-{operation.operation_id}",
|
||||||
details={"operation_id": operation.operation_id},
|
details={"operation_id": operation.operation_id},
|
||||||
)
|
)
|
||||||
|
elif action == "play" and expected_path is not None:
|
||||||
|
due = None
|
||||||
|
if self.config.storage.uncertain_review_after_seconds > 0:
|
||||||
|
due = (
|
||||||
|
self._wall_clock()
|
||||||
|
+ self.config.storage.uncertain_review_after_seconds
|
||||||
|
)
|
||||||
|
observation = self.store.mark_external_uncertain(
|
||||||
|
expected_path,
|
||||||
|
review_after=due,
|
||||||
|
details={
|
||||||
|
"operation_id": operation.operation_id,
|
||||||
|
"reason": "play command pending Player evidence",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
operation = replace(
|
||||||
|
operation,
|
||||||
|
reconciliation_id=observation.reconciliation_id,
|
||||||
|
)
|
||||||
|
self.pending = operation
|
||||||
return operation
|
return operation
|
||||||
|
|
||||||
def publish_failed(self, reason: str) -> PlaybackEffect | None:
|
def publish_failed(self, reason: str) -> PlaybackEffect | None:
|
||||||
if self.pending is None:
|
if self.pending is None:
|
||||||
return None
|
return None
|
||||||
|
if (
|
||||||
|
self.pending.action == "play"
|
||||||
|
and self.pending.slot is None
|
||||||
|
and self.pending.reconciliation_id is not None
|
||||||
|
):
|
||||||
|
self.store.cancel_external_intent(
|
||||||
|
self.pending.reconciliation_id,
|
||||||
|
reason=reason,
|
||||||
|
)
|
||||||
return self._finish_operation("failed", reason)
|
return self._finish_operation("failed", reason)
|
||||||
|
|
||||||
def _finish_operation(self, state: str, message: str | None = None) -> PlaybackEffect:
|
def _finish_operation(self, state: str, message: str | None = None) -> PlaybackEffect:
|
||||||
@@ -142,11 +173,64 @@ class PlaybackReducer:
|
|||||||
operation = self.pending
|
operation = self.pending
|
||||||
if operation is None:
|
if operation is None:
|
||||||
return False
|
return False
|
||||||
slot_matches = operation.slot is None or operation.slot == slot
|
positive_match = False
|
||||||
path_matches = operation.expected_path is None or operation.expected_path == path
|
if operation.slot is not None and slot is not None:
|
||||||
if operation.slot is not None and operation.expected_path is not None:
|
if operation.slot != slot:
|
||||||
return slot_matches or path_matches
|
return False
|
||||||
return slot_matches and path_matches
|
positive_match = True
|
||||||
|
if operation.expected_path is not None and path is not None:
|
||||||
|
if os.path.abspath(operation.expected_path) != os.path.abspath(path):
|
||||||
|
return False
|
||||||
|
positive_match = True
|
||||||
|
return positive_match
|
||||||
|
|
||||||
|
def _canonical_slot_for_path(self, path: str | None) -> int | None:
|
||||||
|
if path is None:
|
||||||
|
return None
|
||||||
|
candidate = os.path.abspath(path)
|
||||||
|
for slot in SLOT_IDS:
|
||||||
|
slot_directory = os.path.abspath(
|
||||||
|
self.config.storage.media_base / f"slot{slot}"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
if os.path.commonpath((candidate, slot_directory)) == slot_directory:
|
||||||
|
return slot
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _normalized_identity(
|
||||||
|
self,
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
) -> tuple[int | None, str | None, str | None]:
|
||||||
|
slot = self._slot(payload.get("slot"))
|
||||||
|
path = self._path(payload.get("path"))
|
||||||
|
derived = self._canonical_slot_for_path(path)
|
||||||
|
if slot is not None and derived is not None and slot != derived:
|
||||||
|
return (
|
||||||
|
slot,
|
||||||
|
path,
|
||||||
|
f"Player identity conflicts: slot {slot} does not contain path {path!r}",
|
||||||
|
)
|
||||||
|
return (derived if slot is None else slot), path, None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _same_playback_identity(
|
||||||
|
previous: PlayerObservation,
|
||||||
|
slot: int | None,
|
||||||
|
path: str | None,
|
||||||
|
) -> bool:
|
||||||
|
if previous.active_slot != slot:
|
||||||
|
return False
|
||||||
|
if slot is None:
|
||||||
|
return (
|
||||||
|
previous.active_path is not None
|
||||||
|
and path is not None
|
||||||
|
and os.path.abspath(previous.active_path) == os.path.abspath(path)
|
||||||
|
)
|
||||||
|
if previous.active_path is None or path is None:
|
||||||
|
return True
|
||||||
|
return os.path.abspath(previous.active_path) == os.path.abspath(path)
|
||||||
|
|
||||||
def _active_evidence(
|
def _active_evidence(
|
||||||
self,
|
self,
|
||||||
@@ -158,11 +242,11 @@ class PlaybackReducer:
|
|||||||
telemetry: Mapping[str, Any] | None = None,
|
telemetry: Mapping[str, Any] | None = None,
|
||||||
) -> PlayerObservation:
|
) -> PlayerObservation:
|
||||||
previous = self.store.player_observation()
|
previous = self.store.player_observation()
|
||||||
if slot is None:
|
if slot is None and (path is None or path == previous.active_path):
|
||||||
slot = previous.active_slot
|
slot = previous.active_slot
|
||||||
if path is None:
|
if path is None and (slot is None or slot == previous.active_slot):
|
||||||
path = previous.active_path
|
path = previous.active_path
|
||||||
if paused is None and previous.active_slot == slot:
|
if paused is None and self._same_playback_identity(previous, slot, path):
|
||||||
paused = previous.paused
|
paused = previous.paused
|
||||||
phase = "paused" if paused is True else "playing"
|
phase = "paused" if paused is True else "playing"
|
||||||
if slot is not None:
|
if slot is not None:
|
||||||
@@ -171,6 +255,8 @@ class PlaybackReducer:
|
|||||||
kind="player_active",
|
kind="player_active",
|
||||||
details={"path": path},
|
details={"path": path},
|
||||||
)
|
)
|
||||||
|
elif path is not None:
|
||||||
|
self.store.reconcile_external_active()
|
||||||
return self.store.set_player_observation(
|
return self.store.set_player_observation(
|
||||||
availability="online",
|
availability="online",
|
||||||
phase=phase,
|
phase=phase,
|
||||||
@@ -179,6 +265,7 @@ class PlaybackReducer:
|
|||||||
paused=paused,
|
paused=paused,
|
||||||
reconciliation_state="confirmed",
|
reconciliation_state="confirmed",
|
||||||
reconciliation_due_at=None,
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id=None,
|
||||||
status_payload=status,
|
status_payload=status,
|
||||||
telemetry_payload=telemetry,
|
telemetry_payload=telemetry,
|
||||||
)
|
)
|
||||||
@@ -199,6 +286,7 @@ class PlaybackReducer:
|
|||||||
paused=False,
|
paused=False,
|
||||||
reconciliation_state="clear",
|
reconciliation_state="clear",
|
||||||
reconciliation_due_at=None,
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id=None,
|
||||||
status_payload=status,
|
status_payload=status,
|
||||||
telemetry_payload=telemetry,
|
telemetry_payload=telemetry,
|
||||||
)
|
)
|
||||||
@@ -212,36 +300,63 @@ class PlaybackReducer:
|
|||||||
if not isinstance(state, str) or not state:
|
if not isinstance(state, str) or not state:
|
||||||
return [PlaybackEffect("warning", message="Player status is missing state")]
|
return [PlaybackEffect("warning", message="Player status is missing state")]
|
||||||
state = state.strip()
|
state = state.strip()
|
||||||
slot = self._slot(payload.get("slot"))
|
slot, path, identity_warning = self._normalized_identity(payload)
|
||||||
path = self._path(payload.get("path"))
|
if identity_warning is not None:
|
||||||
|
return [PlaybackEffect("warning", slot=slot, message=identity_warning)]
|
||||||
effects: list[PlaybackEffect] = []
|
effects: list[PlaybackEffect] = []
|
||||||
|
|
||||||
if state == "offline":
|
if state == "offline":
|
||||||
previous = self.store.player_observation()
|
previous = self.store.player_observation()
|
||||||
due = None
|
due = None
|
||||||
if (
|
active_known = (
|
||||||
previous.active_slot is not None
|
previous.active_slot is not None or previous.active_path is not None
|
||||||
and self.config.storage.uncertain_review_after_seconds > 0
|
)
|
||||||
):
|
if previous.active_slot is not None:
|
||||||
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
if self.config.storage.uncertain_review_after_seconds > 0:
|
||||||
self.store.reconcile_active_guard(
|
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
||||||
|
guard = self.store.mark_slot_uncertain(
|
||||||
previous.active_slot,
|
previous.active_slot,
|
||||||
kind="restart_uncertain",
|
|
||||||
review_after=due,
|
review_after=due,
|
||||||
details={"reason": "Player offline"},
|
details={"reason": "Player offline"},
|
||||||
)
|
)
|
||||||
self.store.set_player_observation(
|
due = guard.review_after
|
||||||
availability="offline",
|
self.store.set_player_observation(
|
||||||
phase="unknown" if previous.active_slot is not None else previous.phase,
|
availability="offline",
|
||||||
active_slot=previous.active_slot,
|
phase="unknown",
|
||||||
active_path=previous.active_path,
|
active_slot=previous.active_slot,
|
||||||
paused=previous.paused,
|
active_path=previous.active_path,
|
||||||
reconciliation_state=(
|
paused=previous.paused,
|
||||||
"awaiting_evidence" if previous.active_slot is not None else "unknown"
|
reconciliation_state=(
|
||||||
),
|
"operator_review"
|
||||||
reconciliation_due_at=due,
|
if guard.state == "review_required"
|
||||||
status_payload=payload,
|
else "awaiting_evidence"
|
||||||
)
|
),
|
||||||
|
reconciliation_due_at=due,
|
||||||
|
reconciliation_id=guard.guard_id,
|
||||||
|
status_payload=payload,
|
||||||
|
)
|
||||||
|
elif previous.active_path is not None:
|
||||||
|
if self.config.storage.uncertain_review_after_seconds > 0:
|
||||||
|
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
||||||
|
self.store.mark_external_uncertain(
|
||||||
|
previous.active_path,
|
||||||
|
review_after=due,
|
||||||
|
availability="offline",
|
||||||
|
details={"reason": "Player offline"},
|
||||||
|
status_payload=payload,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.store.set_player_observation(
|
||||||
|
availability="offline",
|
||||||
|
phase=previous.phase,
|
||||||
|
active_slot=None,
|
||||||
|
active_path=None,
|
||||||
|
paused=previous.paused,
|
||||||
|
reconciliation_state="unknown",
|
||||||
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id=None,
|
||||||
|
status_payload=payload,
|
||||||
|
)
|
||||||
effects.append(
|
effects.append(
|
||||||
PlaybackEffect(
|
PlaybackEffect(
|
||||||
"player_offline",
|
"player_offline",
|
||||||
@@ -261,15 +376,56 @@ class PlaybackReducer:
|
|||||||
message="Reconnect idle announcement carried an active path",
|
message="Reconnect idle announcement carried an active path",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if self.pending is not None and self.pending.action == "play":
|
||||||
|
if self._matches_pending(slot, path):
|
||||||
|
effects.append(
|
||||||
|
self._finish_operation(
|
||||||
|
"confirmed", "matching reconnect evidence"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
effects.append(
|
||||||
|
self._finish_operation(
|
||||||
|
"failed",
|
||||||
|
"Player reconnected with different active playback",
|
||||||
|
)
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
self._idle_evidence(status=payload)
|
self._idle_evidence(status=payload)
|
||||||
effects.append(PlaybackEffect("player_idle"))
|
effects.append(PlaybackEffect("player_idle"))
|
||||||
if self.pending is not None and self.pending.action == "stop":
|
if self.pending is not None:
|
||||||
effects.append(self._finish_operation("confirmed"))
|
if self.pending.action == "stop":
|
||||||
|
effects.append(self._finish_operation("confirmed"))
|
||||||
|
elif self.pending.action == "play":
|
||||||
|
effects.append(
|
||||||
|
self._finish_operation(
|
||||||
|
"failed", "Player reconnected without active playback"
|
||||||
|
)
|
||||||
|
)
|
||||||
return effects
|
return effects
|
||||||
|
|
||||||
if state == "idle":
|
if state == "idle":
|
||||||
reason = payload.get("reason")
|
reason = payload.get("reason")
|
||||||
|
if reason not in TERMINAL_REASONS:
|
||||||
|
previous = self.store.player_observation()
|
||||||
|
self.store.set_player_observation(
|
||||||
|
availability="online",
|
||||||
|
phase=previous.phase,
|
||||||
|
active_slot=previous.active_slot,
|
||||||
|
active_path=previous.active_path,
|
||||||
|
paused=previous.paused,
|
||||||
|
reconciliation_state=previous.reconciliation_state,
|
||||||
|
reconciliation_due_at=previous.reconciliation_due_at,
|
||||||
|
reconciliation_id=previous.reconciliation_id,
|
||||||
|
status_payload=payload,
|
||||||
|
)
|
||||||
|
return [
|
||||||
|
PlaybackEffect(
|
||||||
|
"warning",
|
||||||
|
slot=previous.active_slot,
|
||||||
|
message="ambiguous Player idle status was not treated as terminal",
|
||||||
|
)
|
||||||
|
]
|
||||||
self._idle_evidence(status=payload)
|
self._idle_evidence(status=payload)
|
||||||
effects.append(PlaybackEffect("player_idle", slot=slot, message=str(reason or "idle")))
|
effects.append(PlaybackEffect("player_idle", slot=slot, message=str(reason or "idle")))
|
||||||
if self.pending is not None:
|
if self.pending is not None:
|
||||||
@@ -280,29 +436,33 @@ class PlaybackReducer:
|
|||||||
return effects
|
return effects
|
||||||
|
|
||||||
if state == "playing":
|
if state == "playing":
|
||||||
previous = self.store.player_observation()
|
|
||||||
actual_paused = previous.paused if previous.active_slot == slot else None
|
|
||||||
self._active_evidence(
|
self._active_evidence(
|
||||||
slot=slot,
|
slot=slot,
|
||||||
path=path,
|
path=path,
|
||||||
paused=actual_paused,
|
paused=None,
|
||||||
status=payload,
|
status=payload,
|
||||||
)
|
)
|
||||||
effects.append(PlaybackEffect("player_active", slot=slot))
|
effects.append(PlaybackEffect("player_active", slot=slot))
|
||||||
if (
|
if (
|
||||||
self.pending is not None
|
self.pending is not None
|
||||||
and self.pending.action == "play"
|
and self.pending.action == "play"
|
||||||
and self._matches_pending(slot, path)
|
|
||||||
):
|
):
|
||||||
effects.append(self._finish_operation("confirmed"))
|
if self._matches_pending(slot, path):
|
||||||
|
effects.append(self._finish_operation("confirmed"))
|
||||||
|
else:
|
||||||
|
effects.append(
|
||||||
|
self._finish_operation(
|
||||||
|
"failed",
|
||||||
|
"Player became active with a different slot or path",
|
||||||
|
)
|
||||||
|
)
|
||||||
return effects
|
return effects
|
||||||
|
|
||||||
if state in {"paused", "toggled"}:
|
if state in {"paused", "toggled"}:
|
||||||
previous = self.store.player_observation()
|
|
||||||
self._active_evidence(
|
self._active_evidence(
|
||||||
slot=slot,
|
slot=slot,
|
||||||
path=path,
|
path=path,
|
||||||
paused=previous.paused if previous.active_slot == slot else None,
|
paused=None,
|
||||||
status=payload,
|
status=payload,
|
||||||
)
|
)
|
||||||
effects.append(
|
effects.append(
|
||||||
@@ -312,15 +472,24 @@ class PlaybackReducer:
|
|||||||
message=f"Player issued {state}; awaiting telemetry",
|
message=f"Player issued {state}; awaiting telemetry",
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
if self.pending is not None and self.pending.action == "play":
|
||||||
|
if self._matches_pending(slot, path):
|
||||||
|
effects.append(self._finish_operation("confirmed", state))
|
||||||
|
else:
|
||||||
|
effects.append(
|
||||||
|
self._finish_operation(
|
||||||
|
"failed",
|
||||||
|
"Player control status identified different playback",
|
||||||
|
)
|
||||||
|
)
|
||||||
return effects
|
return effects
|
||||||
|
|
||||||
if state == "busy":
|
if state == "busy":
|
||||||
if slot is not None or path is not None:
|
if slot is not None or path is not None:
|
||||||
previous = self.store.player_observation()
|
|
||||||
self._active_evidence(
|
self._active_evidence(
|
||||||
slot=slot,
|
slot=slot,
|
||||||
path=path,
|
path=path,
|
||||||
paused=previous.paused if previous.active_slot == slot else None,
|
paused=None,
|
||||||
status=payload,
|
status=payload,
|
||||||
)
|
)
|
||||||
effects.append(PlaybackEffect("player_busy", slot=slot, message=payload.get("message")))
|
effects.append(PlaybackEffect("player_busy", slot=slot, message=payload.get("message")))
|
||||||
@@ -338,14 +507,29 @@ class PlaybackReducer:
|
|||||||
kind="player_active",
|
kind="player_active",
|
||||||
details={"error": payload.get("message")},
|
details={"error": payload.get("message")},
|
||||||
)
|
)
|
||||||
|
elif active_path is not None:
|
||||||
|
self.store.reconcile_external_active()
|
||||||
self.store.set_player_observation(
|
self.store.set_player_observation(
|
||||||
availability="online",
|
availability="online",
|
||||||
phase="error",
|
phase="error",
|
||||||
active_slot=active_slot,
|
active_slot=active_slot,
|
||||||
active_path=active_path,
|
active_path=active_path,
|
||||||
paused=previous.paused,
|
paused=(
|
||||||
reconciliation_state=("confirmed" if active_slot is not None else "unknown"),
|
previous.paused
|
||||||
|
if self._same_playback_identity(
|
||||||
|
previous,
|
||||||
|
active_slot,
|
||||||
|
active_path,
|
||||||
|
)
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
reconciliation_state=(
|
||||||
|
"confirmed"
|
||||||
|
if active_slot is not None or active_path is not None
|
||||||
|
else "unknown"
|
||||||
|
),
|
||||||
reconciliation_due_at=None,
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id=None,
|
||||||
status_payload=payload,
|
status_payload=payload,
|
||||||
)
|
)
|
||||||
message = str(payload.get("message") or "Player error")
|
message = str(payload.get("message") or "Player error")
|
||||||
@@ -360,8 +544,9 @@ class PlaybackReducer:
|
|||||||
def handle_telemetry(self, payload: Any) -> list[PlaybackEffect]:
|
def handle_telemetry(self, payload: Any) -> list[PlaybackEffect]:
|
||||||
if not isinstance(payload, Mapping) or payload.get("state") != "telemetry":
|
if not isinstance(payload, Mapping) or payload.get("state") != "telemetry":
|
||||||
return [PlaybackEffect("warning", message="invalid Player telemetry envelope")]
|
return [PlaybackEffect("warning", message="invalid Player telemetry envelope")]
|
||||||
slot = self._slot(payload.get("slot"))
|
slot, path, identity_warning = self._normalized_identity(payload)
|
||||||
path = self._path(payload.get("path"))
|
if identity_warning is not None:
|
||||||
|
return [PlaybackEffect("warning", slot=slot, message=identity_warning)]
|
||||||
effects: list[PlaybackEffect] = []
|
effects: list[PlaybackEffect] = []
|
||||||
if path is None:
|
if path is None:
|
||||||
self._idle_evidence(telemetry=payload)
|
self._idle_evidence(telemetry=payload)
|
||||||
@@ -383,7 +568,15 @@ class PlaybackReducer:
|
|||||||
)
|
)
|
||||||
effects.append(PlaybackEffect("player_telemetry", slot=observation.active_slot))
|
effects.append(PlaybackEffect("player_telemetry", slot=observation.active_slot))
|
||||||
operation = self.pending
|
operation = self.pending
|
||||||
if operation is None or not self._matches_pending(slot, path):
|
if operation is None:
|
||||||
|
return effects
|
||||||
|
if not self._matches_pending(slot, path):
|
||||||
|
effects.append(
|
||||||
|
self._finish_operation(
|
||||||
|
"failed",
|
||||||
|
"Player telemetry identified a different slot or path",
|
||||||
|
)
|
||||||
|
)
|
||||||
return effects
|
return effects
|
||||||
if operation.action == "play":
|
if operation.action == "play":
|
||||||
effects.append(self._finish_operation("confirmed", "matching telemetry"))
|
effects.append(self._finish_operation("confirmed", "matching telemetry"))
|
||||||
@@ -399,48 +592,165 @@ class PlaybackReducer:
|
|||||||
operation = self.pending
|
operation = self.pending
|
||||||
if operation is None or self._monotonic() < operation.deadline:
|
if operation is None or self._monotonic() < operation.deadline:
|
||||||
return None
|
return None
|
||||||
if operation.action == "play" and operation.slot is not None:
|
if operation.action == "play":
|
||||||
due = None
|
previous = self.store.player_observation()
|
||||||
if self.config.storage.uncertain_review_after_seconds > 0:
|
has_confirmed_active = (
|
||||||
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
previous.reconciliation_state == "confirmed"
|
||||||
self.store.reconcile_active_guard(
|
and (previous.active_slot is not None or previous.active_path is not None)
|
||||||
operation.slot,
|
|
||||||
kind="restart_uncertain",
|
|
||||||
review_after=due,
|
|
||||||
details={"operation_id": operation.operation_id, "reason": "play timeout"},
|
|
||||||
)
|
|
||||||
self.store.set_player_observation(
|
|
||||||
availability="unknown",
|
|
||||||
phase="unknown",
|
|
||||||
active_slot=operation.slot,
|
|
||||||
active_path=operation.expected_path,
|
|
||||||
paused=None,
|
|
||||||
reconciliation_state="awaiting_evidence",
|
|
||||||
reconciliation_due_at=due,
|
|
||||||
)
|
)
|
||||||
|
if has_confirmed_active and self._matches_pending(
|
||||||
|
previous.active_slot,
|
||||||
|
previous.active_path,
|
||||||
|
):
|
||||||
|
return self._finish_operation(
|
||||||
|
"confirmed", "matching authoritative Player evidence"
|
||||||
|
)
|
||||||
|
if not has_confirmed_active:
|
||||||
|
self._mark_play_uncertain(operation, "play timeout")
|
||||||
elif operation.action == "stop":
|
elif operation.action == "stop":
|
||||||
previous = self.store.player_observation()
|
previous = self.store.player_observation()
|
||||||
if previous.active_slot is not None:
|
if previous.active_slot is not None:
|
||||||
due = None
|
due = None
|
||||||
if self.config.storage.uncertain_review_after_seconds > 0:
|
if self.config.storage.uncertain_review_after_seconds > 0:
|
||||||
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
||||||
self.store.reconcile_active_guard(
|
guard = self.store.mark_slot_uncertain(
|
||||||
previous.active_slot,
|
previous.active_slot,
|
||||||
kind="restart_uncertain",
|
|
||||||
review_after=due,
|
review_after=due,
|
||||||
details={"operation_id": operation.operation_id, "reason": "stop timeout"},
|
details={"operation_id": operation.operation_id, "reason": "stop timeout"},
|
||||||
)
|
)
|
||||||
|
due = guard.review_after
|
||||||
self.store.set_player_observation(
|
self.store.set_player_observation(
|
||||||
availability="unknown",
|
availability="unknown",
|
||||||
phase="unknown",
|
phase="unknown",
|
||||||
active_slot=previous.active_slot,
|
active_slot=previous.active_slot,
|
||||||
active_path=previous.active_path,
|
active_path=previous.active_path,
|
||||||
paused=previous.paused,
|
paused=previous.paused,
|
||||||
reconciliation_state="awaiting_evidence",
|
reconciliation_state=(
|
||||||
|
"operator_review"
|
||||||
|
if guard.state == "review_required"
|
||||||
|
else "awaiting_evidence"
|
||||||
|
),
|
||||||
reconciliation_due_at=due,
|
reconciliation_due_at=due,
|
||||||
|
reconciliation_id=guard.guard_id,
|
||||||
|
)
|
||||||
|
elif previous.active_path is not None:
|
||||||
|
due = None
|
||||||
|
if self.config.storage.uncertain_review_after_seconds > 0:
|
||||||
|
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
||||||
|
self.store.mark_external_uncertain(
|
||||||
|
previous.active_path,
|
||||||
|
review_after=due,
|
||||||
|
details={
|
||||||
|
"operation_id": operation.operation_id,
|
||||||
|
"reason": "stop timeout",
|
||||||
|
},
|
||||||
)
|
)
|
||||||
return self._finish_operation("timed_out", "no authoritative Player confirmation")
|
return self._finish_operation("timed_out", "no authoritative Player confirmation")
|
||||||
|
|
||||||
|
def _mark_play_uncertain(self, operation: PlayerOperation, reason: str) -> None:
|
||||||
|
due = None
|
||||||
|
if self.config.storage.uncertain_review_after_seconds > 0:
|
||||||
|
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
||||||
|
slot = operation.slot or self._canonical_slot_for_path(operation.expected_path)
|
||||||
|
if slot is not None:
|
||||||
|
guard = self.store.mark_slot_uncertain(
|
||||||
|
slot,
|
||||||
|
review_after=due,
|
||||||
|
details={"operation_id": operation.operation_id, "reason": reason},
|
||||||
|
)
|
||||||
|
due = guard.review_after
|
||||||
|
self.store.set_player_observation(
|
||||||
|
availability="unknown",
|
||||||
|
phase="unknown",
|
||||||
|
active_slot=slot,
|
||||||
|
active_path=operation.expected_path,
|
||||||
|
paused=None,
|
||||||
|
reconciliation_state=(
|
||||||
|
"operator_review"
|
||||||
|
if guard.state == "review_required"
|
||||||
|
else "awaiting_evidence"
|
||||||
|
),
|
||||||
|
reconciliation_due_at=due,
|
||||||
|
reconciliation_id=guard.guard_id,
|
||||||
|
)
|
||||||
|
elif operation.expected_path is not None:
|
||||||
|
self.store.mark_external_uncertain(
|
||||||
|
operation.expected_path,
|
||||||
|
review_after=due,
|
||||||
|
details={"operation_id": operation.operation_id, "reason": reason},
|
||||||
|
)
|
||||||
|
|
||||||
|
def publication_uncertain(self, reason: str) -> PlaybackEffect | None:
|
||||||
|
if self.pending is None:
|
||||||
|
return None
|
||||||
|
operation = self.pending
|
||||||
|
if operation.action == "play":
|
||||||
|
self._mark_play_uncertain(operation, reason)
|
||||||
|
return self._finish_operation("uncertain", reason)
|
||||||
|
|
||||||
|
def mark_transport_uncertain(self, reason: str) -> PlaybackEffect:
|
||||||
|
previous = self.store.player_observation()
|
||||||
|
pending_play = (
|
||||||
|
self.pending
|
||||||
|
if self.pending is not None and self.pending.action == "play"
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
active_slot = previous.active_slot
|
||||||
|
active_path = previous.active_path
|
||||||
|
if active_slot is None and active_path is None and pending_play is not None:
|
||||||
|
active_slot = pending_play.slot or self._canonical_slot_for_path(
|
||||||
|
pending_play.expected_path
|
||||||
|
)
|
||||||
|
active_path = pending_play.expected_path
|
||||||
|
due = None
|
||||||
|
if active_slot is not None:
|
||||||
|
if self.config.storage.uncertain_review_after_seconds > 0:
|
||||||
|
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
||||||
|
guard = self.store.mark_slot_uncertain(
|
||||||
|
active_slot,
|
||||||
|
review_after=due,
|
||||||
|
details={"reason": reason},
|
||||||
|
)
|
||||||
|
due = guard.review_after
|
||||||
|
self.store.set_player_observation(
|
||||||
|
availability="unknown",
|
||||||
|
phase="unknown",
|
||||||
|
active_slot=active_slot,
|
||||||
|
active_path=active_path,
|
||||||
|
paused=previous.paused,
|
||||||
|
reconciliation_state=(
|
||||||
|
"operator_review"
|
||||||
|
if guard.state == "review_required"
|
||||||
|
else "awaiting_evidence"
|
||||||
|
),
|
||||||
|
reconciliation_due_at=due,
|
||||||
|
reconciliation_id=guard.guard_id,
|
||||||
|
)
|
||||||
|
elif active_path is not None:
|
||||||
|
if self.config.storage.uncertain_review_after_seconds > 0:
|
||||||
|
due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds
|
||||||
|
self.store.mark_external_uncertain(
|
||||||
|
active_path,
|
||||||
|
review_after=due,
|
||||||
|
details={"reason": reason},
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
self.store.set_player_observation(
|
||||||
|
availability="unknown",
|
||||||
|
phase=previous.phase,
|
||||||
|
active_slot=None,
|
||||||
|
active_path=None,
|
||||||
|
paused=previous.paused,
|
||||||
|
reconciliation_state="unknown",
|
||||||
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id=None,
|
||||||
|
)
|
||||||
|
return PlaybackEffect(
|
||||||
|
"transport_uncertain",
|
||||||
|
slot=active_slot,
|
||||||
|
message=reason,
|
||||||
|
)
|
||||||
|
|
||||||
def projected_slot_state(self, slot: int) -> str:
|
def projected_slot_state(self, slot: int) -> str:
|
||||||
record = self.store.get_slot(slot)
|
record = self.store.get_slot(slot)
|
||||||
if record.media_state != "ready":
|
if record.media_state != "ready":
|
||||||
@@ -466,9 +776,9 @@ class PlaybackReducer:
|
|||||||
return "ready"
|
return "ready"
|
||||||
|
|
||||||
def mark_due_reconciliation(self) -> list[PlaybackEffect]:
|
def mark_due_reconciliation(self) -> list[PlaybackEffect]:
|
||||||
due = self.store.mark_due_guards_for_review(self._wall_clock())
|
now = self._wall_clock()
|
||||||
if not due:
|
due = self.store.mark_due_guards_for_review(now)
|
||||||
return []
|
external_due = self.store.mark_external_reconciliation_due(now)
|
||||||
observation = self.store.player_observation()
|
observation = self.store.player_observation()
|
||||||
if observation.active_slot is not None and any(
|
if observation.active_slot is not None and any(
|
||||||
guard.slot == observation.active_slot for guard in due
|
guard.slot == observation.active_slot for guard in due
|
||||||
@@ -481,15 +791,32 @@ class PlaybackReducer:
|
|||||||
paused=observation.paused,
|
paused=observation.paused,
|
||||||
reconciliation_state="operator_review",
|
reconciliation_state="operator_review",
|
||||||
reconciliation_due_at=observation.reconciliation_due_at,
|
reconciliation_due_at=observation.reconciliation_due_at,
|
||||||
|
reconciliation_id=observation.reconciliation_id,
|
||||||
)
|
)
|
||||||
return [
|
effects = [
|
||||||
PlaybackEffect(
|
PlaybackEffect(
|
||||||
"operator_review",
|
"operator_review",
|
||||||
slot=guard.slot,
|
slot=guard.slot,
|
||||||
|
operation_id=guard.guard_id,
|
||||||
message=(
|
message=(
|
||||||
"Playback remains unknown and protected; fresh Player evidence "
|
"Playback remains unknown and protected; fresh Player evidence "
|
||||||
"or an administrative clear is required"
|
"or an administrative clear is required "
|
||||||
|
f"(review_id={guard.guard_id})"
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
for guard in due
|
for guard in due
|
||||||
]
|
]
|
||||||
|
if external_due is not None:
|
||||||
|
review_id, path = external_due
|
||||||
|
effects.append(
|
||||||
|
PlaybackEffect(
|
||||||
|
"operator_review",
|
||||||
|
operation_id=review_id,
|
||||||
|
message=(
|
||||||
|
f"Playback at {path!r} remains unknown and protected; "
|
||||||
|
"fresh Player evidence or an administrative clear is required "
|
||||||
|
f"(review_id={review_id})"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return effects
|
||||||
|
|||||||
@@ -0,0 +1,422 @@
|
|||||||
|
"""Timeout-bounded Audio Router acquisition and restoration leases."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Callable
|
||||||
|
|
||||||
|
from .config import ControllerConfig
|
||||||
|
from .models import AudioLeaseRecord
|
||||||
|
from .mqtt_adapter import PublishReceipt
|
||||||
|
from .store import ControllerStore
|
||||||
|
|
||||||
|
|
||||||
|
ModePublisher = Callable[[str], PublishReceipt]
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RouterDecision:
|
||||||
|
action: str
|
||||||
|
lease_id: str | None = None
|
||||||
|
warning: str | None = None
|
||||||
|
continued: bool = False
|
||||||
|
|
||||||
|
|
||||||
|
class AudioRouterCoordinator:
|
||||||
|
"""Own one conservative Router lease for one playback session."""
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
config: ControllerConfig,
|
||||||
|
store: ControllerStore,
|
||||||
|
publish_mode: ModePublisher,
|
||||||
|
*,
|
||||||
|
clock=time.monotonic,
|
||||||
|
):
|
||||||
|
self.config = config
|
||||||
|
self.store = store
|
||||||
|
self.publish_mode = publish_mode
|
||||||
|
self._clock = clock
|
||||||
|
|
||||||
|
def observe_mode(self, mode: str) -> RouterDecision:
|
||||||
|
mode = mode.strip()
|
||||||
|
if not mode:
|
||||||
|
warning = "Audio Router published an empty mode"
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("warning", warning=warning)
|
||||||
|
self.store.set_router_observation(mode=mode, freshness="current")
|
||||||
|
lease = self.store.live_audio_lease()
|
||||||
|
if lease is None:
|
||||||
|
return RouterDecision("observed")
|
||||||
|
|
||||||
|
if lease.state == "acquiring":
|
||||||
|
if mode == lease.target_mode:
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease.lease_id,
|
||||||
|
state="held",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
acquired=True,
|
||||||
|
)
|
||||||
|
return RouterDecision("ready", lease_id=lease.lease_id)
|
||||||
|
# State is evidence of the Router's current output, not evidence
|
||||||
|
# that an operator issued a command. It may simply be a periodic
|
||||||
|
# repeat of the pre-acquisition state, so keep waiting for the
|
||||||
|
# finite confirmation deadline.
|
||||||
|
return RouterDecision("observed", lease_id=lease.lease_id)
|
||||||
|
|
||||||
|
if lease.state == "held":
|
||||||
|
if mode != lease.target_mode:
|
||||||
|
warning = (
|
||||||
|
f"Audio Router changed from Controller-owned {lease.target_mode!r} "
|
||||||
|
f"to {mode!r}; automatic restoration was relinquished"
|
||||||
|
)
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease.lease_id,
|
||||||
|
state="ownership_lost",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning=warning,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("ownership_lost", lease.lease_id, warning)
|
||||||
|
return RouterDecision("observed", lease_id=lease.lease_id)
|
||||||
|
|
||||||
|
if lease.state == "unconfirmed":
|
||||||
|
# A late state event must never resurrect a lease that already
|
||||||
|
# timed out or deliberately continued without confirmation.
|
||||||
|
return RouterDecision("observed", lease_id=lease.lease_id)
|
||||||
|
|
||||||
|
if lease.state == "restore_pending":
|
||||||
|
if mode == lease.restoration_mode:
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease.lease_id,
|
||||||
|
state="released",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
return RouterDecision("restored", lease_id=lease.lease_id)
|
||||||
|
# The Router may repeat its current state before applying the
|
||||||
|
# restoration command. Wait for the bounded deadline; explicit
|
||||||
|
# mode-set intent is handled separately below.
|
||||||
|
return RouterDecision("observed", lease_id=lease.lease_id)
|
||||||
|
return RouterDecision("observed", lease_id=lease.lease_id)
|
||||||
|
|
||||||
|
def observe_mode_command(
|
||||||
|
self,
|
||||||
|
mode: str,
|
||||||
|
*,
|
||||||
|
retained: bool = False,
|
||||||
|
controller_echo: bool = False,
|
||||||
|
) -> RouterDecision:
|
||||||
|
"""Observe a Router mode-set publication without treating state as intent.
|
||||||
|
|
||||||
|
The MQTT adapter marks an exact echo received while a Controller QoS 1
|
||||||
|
publication is in flight. Every other live mode-set publication is
|
||||||
|
conservatively treated as operator intent. Retained commands are old
|
||||||
|
broker state and can never establish current operator intent.
|
||||||
|
"""
|
||||||
|
|
||||||
|
mode = mode.strip()
|
||||||
|
if not mode:
|
||||||
|
warning = "Audio Router published an empty mode command"
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("warning", warning=warning)
|
||||||
|
|
||||||
|
lease = self.store.live_audio_lease()
|
||||||
|
lease_id = None if lease is None else lease.lease_id
|
||||||
|
if retained:
|
||||||
|
warning = (
|
||||||
|
f"Retained Audio Router mode command {mode!r} was ignored as stale intent"
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("retained_ignored", lease_id, warning)
|
||||||
|
if controller_echo:
|
||||||
|
return RouterDecision("self_echo", lease_id=lease_id)
|
||||||
|
if lease is None:
|
||||||
|
return RouterDecision("operator_intent")
|
||||||
|
|
||||||
|
if lease.state == "acquiring":
|
||||||
|
return self._acquisition_failed(
|
||||||
|
lease.lease_id,
|
||||||
|
(
|
||||||
|
f"Audio Router received an operator mode command for {mode!r} "
|
||||||
|
f"while {lease.target_mode!r} confirmation was pending; "
|
||||||
|
"automatic restoration ownership was relinquished"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
if lease.state in {"held", "unconfirmed", "restore_pending"}:
|
||||||
|
warning = (
|
||||||
|
f"Audio Router received an operator mode command for {mode!r}; "
|
||||||
|
"automatic restoration was relinquished"
|
||||||
|
)
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease.lease_id,
|
||||||
|
state="ownership_lost",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning=warning,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("ownership_lost", lease.lease_id, warning)
|
||||||
|
return RouterDecision("operator_intent", lease_id=lease.lease_id)
|
||||||
|
|
||||||
|
def acquire(self, operation_id: str) -> RouterDecision:
|
||||||
|
observation = self.store.router_observation()
|
||||||
|
known = observation.freshness == "current" and observation.mode is not None
|
||||||
|
target = self.config.router.required_mode
|
||||||
|
previous = observation.mode if known else None
|
||||||
|
lease_id = uuid.uuid4().hex
|
||||||
|
|
||||||
|
if known and previous == target:
|
||||||
|
self.store.create_audio_lease(
|
||||||
|
lease_id=lease_id,
|
||||||
|
operation_id=operation_id,
|
||||||
|
state="held",
|
||||||
|
previous_mode=previous,
|
||||||
|
target_mode=target,
|
||||||
|
restoration_mode=None,
|
||||||
|
restoration_source="none",
|
||||||
|
controller_changed=False,
|
||||||
|
confirmation_deadline=None,
|
||||||
|
)
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="held",
|
||||||
|
acquired=True,
|
||||||
|
)
|
||||||
|
return RouterDecision("ready", lease_id=lease_id)
|
||||||
|
|
||||||
|
if known:
|
||||||
|
restoration_mode = previous
|
||||||
|
restoration_source = "observed_previous"
|
||||||
|
elif self.config.router.fallback_restore_mode is not None:
|
||||||
|
restoration_mode = self.config.router.fallback_restore_mode
|
||||||
|
restoration_source = "configured_fallback"
|
||||||
|
else:
|
||||||
|
restoration_mode = None
|
||||||
|
restoration_source = "none"
|
||||||
|
|
||||||
|
policy = self.config.router.when_state_unknown
|
||||||
|
if not known and policy == "reject":
|
||||||
|
warning = "Audio Router state is unavailable; playback was rejected by policy"
|
||||||
|
self.store.create_audio_lease(
|
||||||
|
lease_id=lease_id,
|
||||||
|
operation_id=operation_id,
|
||||||
|
state="abandoned",
|
||||||
|
previous_mode=None,
|
||||||
|
target_mode=target,
|
||||||
|
restoration_mode=restoration_mode,
|
||||||
|
restoration_source=restoration_source,
|
||||||
|
controller_changed=False,
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning=warning,
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("rejected", lease_id, warning)
|
||||||
|
|
||||||
|
deadline = self._clock() + self.config.router.confirmation_timeout_seconds
|
||||||
|
initial_state = "unconfirmed" if not known and policy == "continue" else "acquiring"
|
||||||
|
self.store.create_audio_lease(
|
||||||
|
lease_id=lease_id,
|
||||||
|
operation_id=operation_id,
|
||||||
|
state=initial_state,
|
||||||
|
previous_mode=previous,
|
||||||
|
target_mode=target,
|
||||||
|
restoration_mode=restoration_mode,
|
||||||
|
restoration_source=restoration_source,
|
||||||
|
controller_changed=True,
|
||||||
|
confirmation_deadline=(deadline if initial_state == "acquiring" else None),
|
||||||
|
)
|
||||||
|
receipt = self.publish_mode(target)
|
||||||
|
if not receipt.accepted:
|
||||||
|
return self._acquisition_failed(
|
||||||
|
lease_id,
|
||||||
|
f"could not request Audio Router mode {target!r}: {receipt.reason}",
|
||||||
|
)
|
||||||
|
|
||||||
|
if initial_state == "unconfirmed":
|
||||||
|
warning = (
|
||||||
|
f"Audio Router state is unavailable; requested {target!r} and "
|
||||||
|
"continued immediately without restoration ownership"
|
||||||
|
)
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="unconfirmed",
|
||||||
|
warning=warning,
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("ready", lease_id, warning, continued=True)
|
||||||
|
return RouterDecision("waiting", lease_id=lease_id)
|
||||||
|
|
||||||
|
def check_deadline(self) -> RouterDecision | None:
|
||||||
|
lease = self.store.live_audio_lease()
|
||||||
|
if (
|
||||||
|
lease is None
|
||||||
|
or lease.confirmation_deadline is None
|
||||||
|
or self._clock() < lease.confirmation_deadline
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
if lease.state == "acquiring":
|
||||||
|
return self._acquisition_failed(
|
||||||
|
lease.lease_id,
|
||||||
|
(
|
||||||
|
f"Audio Router did not confirm {lease.target_mode!r} within "
|
||||||
|
f"{self.config.router.confirmation_timeout_seconds:g} seconds"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if lease.state == "restore_pending":
|
||||||
|
warning = (
|
||||||
|
f"Audio Router did not confirm restoration to {lease.restoration_mode!r}; "
|
||||||
|
"no further automatic change will be attempted"
|
||||||
|
)
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease.lease_id,
|
||||||
|
state="abandoned",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning=warning,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("restore_failed", lease.lease_id, warning)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _acquisition_failed(self, lease_id: str, warning: str) -> RouterDecision:
|
||||||
|
continue_playback = self.config.router.continue_on_confirmation_failure
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="unconfirmed" if continue_playback else "abandoned",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning=warning,
|
||||||
|
released=not continue_playback,
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision(
|
||||||
|
"ready" if continue_playback else "rejected",
|
||||||
|
lease_id,
|
||||||
|
warning,
|
||||||
|
continued=continue_playback,
|
||||||
|
)
|
||||||
|
|
||||||
|
def release(self, lease_id: str | None) -> RouterDecision:
|
||||||
|
if lease_id is None:
|
||||||
|
return RouterDecision("released")
|
||||||
|
lease = self.store.get_audio_lease(lease_id)
|
||||||
|
if lease is None:
|
||||||
|
return RouterDecision("released")
|
||||||
|
if lease.state == "held" and not lease.controller_changed:
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="released",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
return RouterDecision("released", lease_id)
|
||||||
|
if lease.state != "held":
|
||||||
|
if lease.state in {"unconfirmed", "acquiring"}:
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="abandoned",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning="Audio Router was never confirmed; restoration was skipped",
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
return RouterDecision("released", lease_id)
|
||||||
|
|
||||||
|
observation = self.store.router_observation()
|
||||||
|
if observation.freshness != "current" or observation.mode != lease.target_mode:
|
||||||
|
warning = "Audio Router ownership is no longer current; restoration was skipped"
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="ownership_lost",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning=warning,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("ownership_lost", lease_id, warning)
|
||||||
|
if lease.restoration_mode is None:
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="released",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
return RouterDecision("released", lease_id)
|
||||||
|
|
||||||
|
receipt = self.publish_mode(lease.restoration_mode)
|
||||||
|
if not receipt.accepted:
|
||||||
|
warning = f"could not request Audio Router restoration: {receipt.reason}"
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="abandoned",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning=warning,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("restore_failed", lease_id, warning)
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="restore_pending",
|
||||||
|
confirmation_deadline=(
|
||||||
|
self._clock() + self.config.router.confirmation_timeout_seconds
|
||||||
|
),
|
||||||
|
)
|
||||||
|
return RouterDecision("restore_waiting", lease_id)
|
||||||
|
|
||||||
|
def connection_lost(self) -> RouterDecision | None:
|
||||||
|
observation = self.store.router_observation()
|
||||||
|
self.store.set_router_observation(
|
||||||
|
mode=observation.mode,
|
||||||
|
freshness="stale",
|
||||||
|
warning="MQTT continuity was lost",
|
||||||
|
)
|
||||||
|
lease = self.store.live_audio_lease()
|
||||||
|
if lease is None:
|
||||||
|
return None
|
||||||
|
if lease.state == "acquiring":
|
||||||
|
return self._acquisition_failed(
|
||||||
|
lease.lease_id,
|
||||||
|
"MQTT disconnected while waiting for Audio Router confirmation",
|
||||||
|
)
|
||||||
|
warning = "MQTT continuity was lost; automatic Audio Router restoration was relinquished"
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease.lease_id,
|
||||||
|
state="ownership_lost",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning=warning,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
self._record_warning(warning)
|
||||||
|
return RouterDecision("ownership_lost", lease.lease_id, warning)
|
||||||
|
|
||||||
|
def abandon(self, lease_id: str | None, reason: str) -> None:
|
||||||
|
if lease_id is None:
|
||||||
|
return
|
||||||
|
lease = self.store.get_audio_lease(lease_id)
|
||||||
|
if lease is None or lease.state in {"released", "abandoned", "ownership_lost"}:
|
||||||
|
return
|
||||||
|
self.store.update_audio_lease(
|
||||||
|
lease_id,
|
||||||
|
state="abandoned",
|
||||||
|
confirmation_deadline=None,
|
||||||
|
warning=reason,
|
||||||
|
released=True,
|
||||||
|
)
|
||||||
|
self._record_warning(reason)
|
||||||
|
|
||||||
|
def _record_warning(self, warning: str) -> None:
|
||||||
|
observation = self.store.router_observation()
|
||||||
|
self.store.set_router_observation(
|
||||||
|
mode=observation.mode,
|
||||||
|
freshness=observation.freshness,
|
||||||
|
warning=warning,
|
||||||
|
)
|
||||||
|
self.store.record_audit_event(
|
||||||
|
"audio_router_warning",
|
||||||
|
actor="controller",
|
||||||
|
details={"message": warning},
|
||||||
|
)
|
||||||
File diff suppressed because it is too large
Load Diff
+865
-65
File diff suppressed because it is too large
Load Diff
@@ -29,6 +29,7 @@ class ContractCharacterizationTests(unittest.TestCase):
|
|||||||
self.assertEqual(topics.playback_base, "youtube/media/playback")
|
self.assertEqual(topics.playback_base, "youtube/media/playback")
|
||||||
self.assertEqual(topics.router_mode_set, "audio/player/mode/set")
|
self.assertEqual(topics.router_mode_set, "audio/player/mode/set")
|
||||||
self.assertEqual(topics.router_mode_state, "audio/player/mode/state")
|
self.assertEqual(topics.router_mode_state, "audio/player/mode/state")
|
||||||
|
self.assertEqual(topics.controller_result, "youtube/controller/v2/events")
|
||||||
self.assertEqual(PLAYER_COMMAND_QOS, 1)
|
self.assertEqual(PLAYER_COMMAND_QOS, 1)
|
||||||
self.assertIs(PLAYER_COMMAND_RETAIN, False)
|
self.assertIs(PLAYER_COMMAND_RETAIN, False)
|
||||||
self.assertEqual(SLOT_IDS, tuple(range(1, 11)))
|
self.assertEqual(SLOT_IDS, tuple(range(1, 11)))
|
||||||
@@ -79,6 +80,7 @@ class ConfigurationTests(unittest.TestCase):
|
|||||||
self.assertIs(config.router.continue_on_confirmation_failure, True)
|
self.assertIs(config.router.continue_on_confirmation_failure, True)
|
||||||
self.assertIsNone(config.router.fallback_restore_mode)
|
self.assertIsNone(config.router.fallback_restore_mode)
|
||||||
self.assertEqual(config.storage.uncertain_review_after_seconds, 900.0)
|
self.assertEqual(config.storage.uncertain_review_after_seconds, 900.0)
|
||||||
|
self.assertEqual(config.mqtt.publish_ack_timeout_seconds, 2.0)
|
||||||
|
|
||||||
def test_documented_environment_overrides_apply(self):
|
def test_documented_environment_overrides_apply(self):
|
||||||
env = {
|
env = {
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
import unittest
|
import unittest
|
||||||
from concurrent.futures import Future
|
from concurrent.futures import Future
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -194,6 +196,42 @@ class DownloadManagerTests(unittest.TestCase):
|
|||||||
self.assertFalse(slot.exists())
|
self.assertFalse(slot.exists())
|
||||||
self.assertEqual(self.store.get_slot(1).media_state, "empty")
|
self.assertEqual(self.store.get_slot(1).media_state, "empty")
|
||||||
|
|
||||||
|
def test_shutdown_terminates_tracked_process_without_waiting_for_job_timeout(self):
|
||||||
|
manager = DownloadManager(self.config, self.store)
|
||||||
|
errors = []
|
||||||
|
|
||||||
|
def run_long_process():
|
||||||
|
try:
|
||||||
|
manager._run_command(
|
||||||
|
["/bin/sh", "-c", "sleep 60"],
|
||||||
|
check=True,
|
||||||
|
text=True,
|
||||||
|
timeout=60,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
except BaseException as exc:
|
||||||
|
errors.append(exc)
|
||||||
|
|
||||||
|
worker = threading.Thread(target=run_long_process)
|
||||||
|
worker.start()
|
||||||
|
deadline = time.monotonic() + 2
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
with manager._lock:
|
||||||
|
if manager._processes:
|
||||||
|
break
|
||||||
|
time.sleep(0.01)
|
||||||
|
else:
|
||||||
|
self.fail("test process did not start")
|
||||||
|
|
||||||
|
started = time.monotonic()
|
||||||
|
manager.shutdown(wait=False)
|
||||||
|
worker.join(timeout=3)
|
||||||
|
self.assertFalse(worker.is_alive())
|
||||||
|
self.assertLess(time.monotonic() - started, 3)
|
||||||
|
self.assertTrue(errors)
|
||||||
|
with manager._lock:
|
||||||
|
self.assertEqual(manager._processes, set())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+234
-1
@@ -1,5 +1,6 @@
|
|||||||
import json
|
import json
|
||||||
import queue
|
import queue
|
||||||
|
import threading
|
||||||
import unittest
|
import unittest
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
@@ -9,6 +10,20 @@ from controller.config import ControllerConfig, MqttConfig
|
|||||||
from controller.mqtt_adapter import MqttAdapter
|
from controller.mqtt_adapter import MqttAdapter
|
||||||
|
|
||||||
|
|
||||||
|
class FakeMessageInfo:
|
||||||
|
def __init__(self, mid, published):
|
||||||
|
self.rc = mqtt.MQTT_ERR_SUCCESS
|
||||||
|
self.mid = mid
|
||||||
|
self.published = published
|
||||||
|
self.waited = []
|
||||||
|
|
||||||
|
def wait_for_publish(self, timeout=None):
|
||||||
|
self.waited.append(timeout)
|
||||||
|
|
||||||
|
def is_published(self):
|
||||||
|
return self.published
|
||||||
|
|
||||||
|
|
||||||
class FakeClient:
|
class FakeClient:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.calls = []
|
self.calls = []
|
||||||
@@ -17,6 +32,9 @@ class FakeClient:
|
|||||||
self.acks = []
|
self.acks = []
|
||||||
self.suppress_exceptions = False
|
self.suppress_exceptions = False
|
||||||
self.connect_error = None
|
self.connect_error = None
|
||||||
|
self.loop_result = mqtt.MQTT_ERR_SUCCESS
|
||||||
|
self.publish_acknowledged = None
|
||||||
|
self.publish_hook = None
|
||||||
|
|
||||||
def username_pw_set(self, username, password):
|
def username_pw_set(self, username, password):
|
||||||
self.calls.append(("username_pw_set", username, password))
|
self.calls.append(("username_pw_set", username, password))
|
||||||
@@ -42,7 +60,7 @@ class FakeClient:
|
|||||||
|
|
||||||
def loop_start(self):
|
def loop_start(self):
|
||||||
self.calls.append(("loop_start",))
|
self.calls.append(("loop_start",))
|
||||||
return mqtt.MQTT_ERR_SUCCESS
|
return self.loop_result
|
||||||
|
|
||||||
def loop_stop(self):
|
def loop_stop(self):
|
||||||
self.calls.append(("loop_stop",))
|
self.calls.append(("loop_stop",))
|
||||||
@@ -56,6 +74,12 @@ class FakeClient:
|
|||||||
|
|
||||||
def publish(self, topic, payload, qos=0, retain=False):
|
def publish(self, topic, payload, qos=0, retain=False):
|
||||||
self.published.append((topic, payload, qos, retain))
|
self.published.append((topic, payload, qos, retain))
|
||||||
|
if self.publish_hook is not None:
|
||||||
|
self.publish_hook(topic, payload, qos, retain)
|
||||||
|
if self.publish_acknowledged is not None:
|
||||||
|
return FakeMessageInfo(
|
||||||
|
len(self.published), self.publish_acknowledged
|
||||||
|
)
|
||||||
return SimpleNamespace(rc=mqtt.MQTT_ERR_SUCCESS, mid=len(self.published))
|
return SimpleNamespace(rc=mqtt.MQTT_ERR_SUCCESS, mid=len(self.published))
|
||||||
|
|
||||||
def ack(self, message_id, qos):
|
def ack(self, message_id, qos):
|
||||||
@@ -106,6 +130,66 @@ class MqttAdapterTests(unittest.TestCase):
|
|||||||
self.assertIn("network unreachable", self.adapter.last_error)
|
self.assertIn("network unreachable", self.adapter.last_error)
|
||||||
self.assertNotIn("loop_start", [call[0] for call in self.client.calls])
|
self.assertNotIn("loop_start", [call[0] for call in self.client.calls])
|
||||||
|
|
||||||
|
def test_loop_start_failure_does_not_escape_startup(self):
|
||||||
|
self.client.loop_result = mqtt.MQTT_ERR_UNKNOWN
|
||||||
|
self.adapter.start()
|
||||||
|
self.assertTrue(self.adapter.started)
|
||||||
|
self.assertIn("failed to start MQTT network loop", self.adapter.last_error)
|
||||||
|
|
||||||
|
def test_stop_during_blocked_connect_never_starts_network_loop_afterward(self):
|
||||||
|
entered = threading.Event()
|
||||||
|
release = threading.Event()
|
||||||
|
original_connect = self.client.connect_async
|
||||||
|
|
||||||
|
def blocked_connect(*args, **kwargs):
|
||||||
|
entered.set()
|
||||||
|
release.wait(timeout=2)
|
||||||
|
return original_connect(*args, **kwargs)
|
||||||
|
|
||||||
|
self.client.connect_async = blocked_connect
|
||||||
|
starter = threading.Thread(target=self.adapter.start)
|
||||||
|
starter.start()
|
||||||
|
self.assertTrue(entered.wait(timeout=1))
|
||||||
|
self.adapter.stop()
|
||||||
|
release.set()
|
||||||
|
starter.join(timeout=2)
|
||||||
|
self.assertFalse(starter.is_alive())
|
||||||
|
self.assertNotIn("loop_start", [call[0] for call in self.client.calls])
|
||||||
|
self.assertFalse(self.adapter.started)
|
||||||
|
|
||||||
|
def test_stop_wins_against_connect_callback_blocked_during_subscribe(self):
|
||||||
|
self.adapter.start()
|
||||||
|
entered = threading.Event()
|
||||||
|
release = threading.Event()
|
||||||
|
original_subscribe = self.client.subscribe
|
||||||
|
|
||||||
|
def blocked_subscribe(topic, qos=0):
|
||||||
|
entered.set()
|
||||||
|
release.wait(timeout=2)
|
||||||
|
return original_subscribe(topic, qos=qos)
|
||||||
|
|
||||||
|
self.client.subscribe = blocked_subscribe
|
||||||
|
callback = threading.Thread(
|
||||||
|
target=self.adapter._on_connect,
|
||||||
|
args=(self.client, None, None, 0),
|
||||||
|
)
|
||||||
|
callback.start()
|
||||||
|
self.assertTrue(entered.wait(timeout=1))
|
||||||
|
stopper = threading.Thread(target=self.adapter.stop)
|
||||||
|
stopper.start()
|
||||||
|
release.set()
|
||||||
|
callback.join(timeout=2)
|
||||||
|
stopper.join(timeout=2)
|
||||||
|
self.assertFalse(callback.is_alive())
|
||||||
|
self.assertFalse(stopper.is_alive())
|
||||||
|
self.assertFalse(self.adapter.started)
|
||||||
|
self.assertFalse(self.adapter.connected)
|
||||||
|
self.assertLess(len(self.client.subscriptions), 6)
|
||||||
|
self.assertTrue(self.events.empty())
|
||||||
|
self.assertFalse(
|
||||||
|
self.adapter.publish_player_command({"action": "stop"}).accepted
|
||||||
|
)
|
||||||
|
|
||||||
def test_connect_subscribes_to_all_authoritative_inputs(self):
|
def test_connect_subscribes_to_all_authoritative_inputs(self):
|
||||||
self.adapter.start()
|
self.adapter.start()
|
||||||
self.adapter._on_connect(self.client, None, None, 0)
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
@@ -117,6 +201,7 @@ class MqttAdapterTests(unittest.TestCase):
|
|||||||
("youtube/player/telemetry", 0),
|
("youtube/player/telemetry", 0),
|
||||||
("youtube/player/state", 0),
|
("youtube/player/state", 0),
|
||||||
("youtube/controller/cmd", 1),
|
("youtube/controller/cmd", 1),
|
||||||
|
("audio/player/mode/set", 1),
|
||||||
("audio/player/mode/state", 1),
|
("audio/player/mode/state", 1),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
@@ -127,6 +212,7 @@ class MqttAdapterTests(unittest.TestCase):
|
|||||||
def test_player_commands_are_exact_qos_one_and_nonretained(self):
|
def test_player_commands_are_exact_qos_one_and_nonretained(self):
|
||||||
disconnected = self.adapter.publish_player_command({"action": "stop"})
|
disconnected = self.adapter.publish_player_command({"action": "stop"})
|
||||||
self.assertFalse(disconnected.accepted)
|
self.assertFalse(disconnected.accepted)
|
||||||
|
self.adapter.start()
|
||||||
self.adapter._on_connect(self.client, None, None, 0)
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
self.events.get_nowait()
|
self.events.get_nowait()
|
||||||
receipt = self.adapter.publish_player_command(
|
receipt = self.adapter.publish_player_command(
|
||||||
@@ -140,6 +226,7 @@ class MqttAdapterTests(unittest.TestCase):
|
|||||||
self.assertIs(retain, False)
|
self.assertIs(retain, False)
|
||||||
|
|
||||||
def test_router_commands_are_qos_one_nonretained_plain_strings(self):
|
def test_router_commands_are_qos_one_nonretained_plain_strings(self):
|
||||||
|
self.adapter.start()
|
||||||
self.adapter._on_connect(self.client, None, None, 0)
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
self.events.get_nowait()
|
self.events.get_nowait()
|
||||||
receipt = self.adapter.publish_router_mode("video")
|
receipt = self.adapter.publish_router_mode("video")
|
||||||
@@ -149,7 +236,75 @@ class MqttAdapterTests(unittest.TestCase):
|
|||||||
("audio/player/mode/set", "video", 1, False),
|
("audio/player/mode/set", "video", 1, False),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_router_self_echo_is_marked_only_while_publish_is_in_flight(self):
|
||||||
|
self.adapter.start()
|
||||||
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
|
self.events.get_nowait()
|
||||||
|
|
||||||
|
def echo(topic, payload, qos, retain):
|
||||||
|
self.adapter._on_message(
|
||||||
|
self.client,
|
||||||
|
None,
|
||||||
|
SimpleNamespace(
|
||||||
|
topic=topic,
|
||||||
|
payload=payload.encode("utf-8"),
|
||||||
|
qos=qos,
|
||||||
|
retain=retain,
|
||||||
|
dup=False,
|
||||||
|
mid=31,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.client.publish_hook = echo
|
||||||
|
self.assertTrue(self.adapter.publish_router_mode("video").accepted)
|
||||||
|
self.assertTrue(self.events.get_nowait().controller_echo)
|
||||||
|
|
||||||
|
self.client.publish_hook = None
|
||||||
|
self.adapter._on_message(
|
||||||
|
self.client,
|
||||||
|
None,
|
||||||
|
SimpleNamespace(
|
||||||
|
topic="audio/player/mode/set",
|
||||||
|
payload=b"video",
|
||||||
|
qos=1,
|
||||||
|
retain=False,
|
||||||
|
dup=False,
|
||||||
|
mid=32,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.assertFalse(self.events.get_nowait().controller_echo)
|
||||||
|
|
||||||
|
def test_retained_router_set_never_consumes_controller_echo_marker(self):
|
||||||
|
self.adapter.start()
|
||||||
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
|
self.events.get_nowait()
|
||||||
|
|
||||||
|
def retained_then_live_echo(topic, payload, qos, retain):
|
||||||
|
for message_retain, mid in ((True, 41), (False, 42)):
|
||||||
|
self.adapter._on_message(
|
||||||
|
self.client,
|
||||||
|
None,
|
||||||
|
SimpleNamespace(
|
||||||
|
topic=topic,
|
||||||
|
payload=payload.encode("utf-8"),
|
||||||
|
qos=qos,
|
||||||
|
retain=message_retain,
|
||||||
|
dup=False,
|
||||||
|
mid=mid,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.client.publish_hook = retained_then_live_echo
|
||||||
|
self.assertTrue(self.adapter.publish_router_mode("video").accepted)
|
||||||
|
retained_event = self.events.get_nowait()
|
||||||
|
live_event = self.events.get_nowait()
|
||||||
|
self.assertTrue(retained_event.retain)
|
||||||
|
self.assertFalse(retained_event.controller_echo)
|
||||||
|
self.assertFalse(live_event.retain)
|
||||||
|
self.assertTrue(live_event.controller_echo)
|
||||||
|
|
||||||
def test_qos_message_is_acked_only_after_consumer_calls_acknowledge(self):
|
def test_qos_message_is_acked_only_after_consumer_calls_acknowledge(self):
|
||||||
|
self.adapter.start()
|
||||||
self.adapter._on_connect(self.client, None, None, 0)
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
self.events.get_nowait()
|
self.events.get_nowait()
|
||||||
message = SimpleNamespace(
|
message = SimpleNamespace(
|
||||||
@@ -167,6 +322,7 @@ class MqttAdapterTests(unittest.TestCase):
|
|||||||
self.assertEqual(self.client.acks, [(42, 1)])
|
self.assertEqual(self.client.acks, [(42, 1)])
|
||||||
|
|
||||||
def test_stale_mid_from_previous_connection_is_not_acked(self):
|
def test_stale_mid_from_previous_connection_is_not_acked(self):
|
||||||
|
self.adapter.start()
|
||||||
self.adapter._on_connect(self.client, None, None, 0)
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
self.events.get_nowait()
|
self.events.get_nowait()
|
||||||
message = SimpleNamespace(
|
message = SimpleNamespace(
|
||||||
@@ -184,6 +340,7 @@ class MqttAdapterTests(unittest.TestCase):
|
|||||||
self.assertEqual(self.client.acks, [])
|
self.assertEqual(self.client.acks, [])
|
||||||
|
|
||||||
def test_retained_flag_is_preserved_for_coordinator_rejection(self):
|
def test_retained_flag_is_preserved_for_coordinator_rejection(self):
|
||||||
|
self.adapter.start()
|
||||||
self.adapter._on_connect(self.client, None, None, 0)
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
self.events.get_nowait()
|
self.events.get_nowait()
|
||||||
message = SimpleNamespace(
|
message = SimpleNamespace(
|
||||||
@@ -196,6 +353,82 @@ class MqttAdapterTests(unittest.TestCase):
|
|||||||
self.adapter._on_message(self.client, None, message)
|
self.adapter._on_message(self.client, None, message)
|
||||||
self.assertIs(self.events.get_nowait().retain, True)
|
self.assertIs(self.events.get_nowait().retain, True)
|
||||||
|
|
||||||
|
def test_duplicate_flag_is_preserved_for_coordinator_deduplication(self):
|
||||||
|
self.adapter.start()
|
||||||
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
|
self.events.get_nowait()
|
||||||
|
message = SimpleNamespace(
|
||||||
|
topic="youtube/controller/cmd",
|
||||||
|
payload=b'{"action":"toggle"}',
|
||||||
|
qos=1,
|
||||||
|
retain=False,
|
||||||
|
dup=True,
|
||||||
|
mid=9,
|
||||||
|
)
|
||||||
|
self.adapter._on_message(self.client, None, message)
|
||||||
|
self.assertIs(self.events.get_nowait().duplicate, True)
|
||||||
|
|
||||||
|
def test_missing_puback_is_uncertain_and_invalidates_connection(self):
|
||||||
|
self.adapter.start()
|
||||||
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
|
self.events.get_nowait()
|
||||||
|
self.client.publish_acknowledged = False
|
||||||
|
receipt = self.adapter.publish_player_command({"action": "stop"})
|
||||||
|
self.assertFalse(receipt.accepted)
|
||||||
|
self.assertTrue(receipt.uncertain)
|
||||||
|
self.assertIn("no broker PUBACK", receipt.reason)
|
||||||
|
self.assertFalse(self.adapter.connected)
|
||||||
|
|
||||||
|
def test_optional_qos_event_puback_failure_does_not_invalidate_transport(self):
|
||||||
|
self.adapter.start()
|
||||||
|
self.adapter._on_connect(self.client, None, None, 0)
|
||||||
|
self.events.get_nowait()
|
||||||
|
self.client.publish_acknowledged = False
|
||||||
|
receipt = self.adapter.publish_json(
|
||||||
|
"youtube/controller/v2/events",
|
||||||
|
{"version": 2, "type": "warning"},
|
||||||
|
qos=1,
|
||||||
|
retain=False,
|
||||||
|
)
|
||||||
|
self.assertFalse(receipt.accepted)
|
||||||
|
self.assertTrue(receipt.uncertain)
|
||||||
|
self.assertTrue(self.adapter.connected)
|
||||||
|
|
||||||
|
def test_discarded_transport_releases_callback_blocked_on_full_queue(self):
|
||||||
|
events = queue.Queue(maxsize=1)
|
||||||
|
client = FakeClient()
|
||||||
|
adapter = MqttAdapter(self.config, client=client, events=events)
|
||||||
|
try:
|
||||||
|
adapter.start()
|
||||||
|
adapter._on_connect(client, None, None, 0)
|
||||||
|
self.assertTrue(events.full())
|
||||||
|
callback = threading.Thread(
|
||||||
|
target=adapter._on_message,
|
||||||
|
args=(
|
||||||
|
client,
|
||||||
|
None,
|
||||||
|
SimpleNamespace(
|
||||||
|
topic="youtube/controller/cmd",
|
||||||
|
payload=b'{"action":"stop"}',
|
||||||
|
qos=1,
|
||||||
|
retain=False,
|
||||||
|
dup=False,
|
||||||
|
mid=99,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
callback.start()
|
||||||
|
callback.join(timeout=0.05)
|
||||||
|
self.assertTrue(callback.is_alive())
|
||||||
|
|
||||||
|
adapter._discard_unacknowledged_transport("test transport discard")
|
||||||
|
callback.join(timeout=1)
|
||||||
|
self.assertFalse(callback.is_alive())
|
||||||
|
self.assertEqual(events.qsize(), 1)
|
||||||
|
self.assertEqual(events.get_nowait().kind, "connected")
|
||||||
|
finally:
|
||||||
|
adapter.stop()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -230,6 +230,154 @@ class PlaybackReducerTests(unittest.TestCase):
|
|||||||
effects = self.reducer.handle_status(payload)
|
effects = self.reducer.handle_status(payload)
|
||||||
self.assertEqual(effects[0].kind, "warning")
|
self.assertEqual(effects[0].kind, "warning")
|
||||||
|
|
||||||
|
def test_contradictory_slot_and_path_cannot_confirm_play(self):
|
||||||
|
expected = str(self.config.storage.media_base / "slot1" / "playlist.m3u")
|
||||||
|
self.reducer.begin_operation("play", slot=1, expected_path=expected)
|
||||||
|
effects = self.reducer.handle_status(
|
||||||
|
{
|
||||||
|
"state": "playing",
|
||||||
|
"slot": 1,
|
||||||
|
"path": str(
|
||||||
|
self.config.storage.media_base / "slot2" / "playlist.m3u"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertEqual(effects[0].kind, "warning")
|
||||||
|
self.assertIsNotNone(self.reducer.pending)
|
||||||
|
self.assertEqual(self.store.player_observation().phase, "unknown")
|
||||||
|
|
||||||
|
def test_mismatched_active_evidence_fails_intent_and_is_not_overwritten(self):
|
||||||
|
self.reducer.begin_operation(
|
||||||
|
"play",
|
||||||
|
slot=1,
|
||||||
|
expected_path="/mnt/Media/YTPlayer/slot1/playlist.m3u",
|
||||||
|
)
|
||||||
|
effects = self.reducer.handle_status(
|
||||||
|
{
|
||||||
|
"state": "playing",
|
||||||
|
"slot": 2,
|
||||||
|
"path": "/mnt/Media/YTPlayer/slot2/playlist.m3u",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertIn("operation_failed", [effect.kind for effect in effects])
|
||||||
|
self.assertIsNone(self.reducer.pending)
|
||||||
|
self.monotonic.value += 100
|
||||||
|
self.assertIsNone(self.reducer.check_timeout())
|
||||||
|
observation = self.store.player_observation()
|
||||||
|
self.assertEqual(observation.active_slot, 2)
|
||||||
|
self.assertEqual(observation.phase, "playing")
|
||||||
|
self.assertEqual(self.store.list_live_guards()[0].slot, 2)
|
||||||
|
|
||||||
|
def test_bare_idle_does_not_clear_live_playback(self):
|
||||||
|
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
|
||||||
|
self.reducer.handle_status({"state": "playing", "slot": 1, "path": path})
|
||||||
|
effects = self.reducer.handle_status({"state": "idle", "slot": 1})
|
||||||
|
self.assertEqual(effects[0].kind, "warning")
|
||||||
|
self.assertEqual(self.store.player_observation().phase, "playing")
|
||||||
|
self.assertEqual(self.store.list_live_guards()[0].slot, 1)
|
||||||
|
|
||||||
|
def test_path_only_evidence_derives_and_protects_managed_slot(self):
|
||||||
|
path = str(self.config.storage.media_base / "slot2" / "playlist.m3u")
|
||||||
|
self.reducer.handle_telemetry(
|
||||||
|
{"state": "telemetry", "slot": None, "path": path, "paused": False}
|
||||||
|
)
|
||||||
|
observation = self.store.player_observation()
|
||||||
|
self.assertEqual(observation.active_slot, 2)
|
||||||
|
self.assertEqual(self.store.list_live_guards()[0].slot, 2)
|
||||||
|
|
||||||
|
def test_duplicate_offline_evidence_does_not_postpone_review(self):
|
||||||
|
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
|
||||||
|
self.reducer.handle_status({"state": "playing", "slot": 1, "path": path})
|
||||||
|
self.reducer.handle_status({"state": "offline"})
|
||||||
|
first_deadline = self.store.list_live_guards()[0].review_after
|
||||||
|
self.wall.value += 20
|
||||||
|
self.reducer.handle_status({"state": "offline"})
|
||||||
|
self.assertEqual(
|
||||||
|
self.store.list_live_guards()[0].review_after,
|
||||||
|
first_deadline,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_repeated_active_telemetry_reuses_guard_without_history_churn(self):
|
||||||
|
path = str(self.config.storage.media_base / "slot1" / "playlist.m3u")
|
||||||
|
payload = {
|
||||||
|
"state": "telemetry",
|
||||||
|
"slot": 1,
|
||||||
|
"path": path,
|
||||||
|
"paused": False,
|
||||||
|
}
|
||||||
|
self.reducer.handle_telemetry(payload)
|
||||||
|
first_guard = self.store.list_live_guards()[0]
|
||||||
|
for _ in range(20):
|
||||||
|
self.reducer.handle_telemetry(payload)
|
||||||
|
self.assertEqual(self.store.list_live_guards()[0].guard_id, first_guard.guard_id)
|
||||||
|
count = self.store.connection.execute(
|
||||||
|
"SELECT COUNT(*) FROM slot_guards"
|
||||||
|
).fetchone()[0]
|
||||||
|
self.assertEqual(count, 1)
|
||||||
|
|
||||||
|
def test_external_play_timeout_transitions_to_review_without_assuming_stop(self):
|
||||||
|
path = "/srv/operator-playlists/special-event.m3u"
|
||||||
|
self.reducer.begin_operation("play", expected_path=path)
|
||||||
|
self.monotonic.value += 6
|
||||||
|
timeout = self.reducer.check_timeout()
|
||||||
|
self.assertEqual(timeout.kind, "operation_timed_out")
|
||||||
|
uncertain = self.store.player_observation()
|
||||||
|
self.assertEqual(uncertain.active_path, path)
|
||||||
|
self.assertEqual(uncertain.reconciliation_state, "awaiting_evidence")
|
||||||
|
self.assertIsNotNone(uncertain.reconciliation_id)
|
||||||
|
|
||||||
|
self.wall.value += 31
|
||||||
|
effects = self.reducer.mark_due_reconciliation()
|
||||||
|
self.assertEqual(len(effects), 1)
|
||||||
|
self.assertEqual(effects[0].kind, "operator_review")
|
||||||
|
self.assertEqual(effects[0].operation_id, uncertain.reconciliation_id)
|
||||||
|
review = self.store.player_observation()
|
||||||
|
self.assertEqual(review.reconciliation_state, "operator_review")
|
||||||
|
self.assertEqual(review.active_path, path)
|
||||||
|
|
||||||
|
def test_external_pause_does_not_bleed_to_a_different_path(self):
|
||||||
|
first_path = "/srv/operator-playlists/first.m3u"
|
||||||
|
second_path = "/srv/operator-playlists/second.m3u"
|
||||||
|
self.reducer.handle_telemetry(
|
||||||
|
{
|
||||||
|
"state": "telemetry",
|
||||||
|
"path": first_path,
|
||||||
|
"paused": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertIs(self.store.player_observation().paused, True)
|
||||||
|
|
||||||
|
self.reducer.handle_status(
|
||||||
|
{"state": "playing", "path": second_path}
|
||||||
|
)
|
||||||
|
observation = self.store.player_observation()
|
||||||
|
self.assertEqual(observation.active_path, second_path)
|
||||||
|
self.assertIsNone(observation.paused)
|
||||||
|
self.assertEqual(observation.phase, "playing")
|
||||||
|
|
||||||
|
def test_stop_timeout_preserves_existing_operator_review_and_token(self):
|
||||||
|
path = str(self.config.storage.media_base / "slot1" / "playlist.m3u")
|
||||||
|
self.reducer.handle_status(
|
||||||
|
{"state": "playing", "slot": 1, "path": path}
|
||||||
|
)
|
||||||
|
self.reducer.handle_status({"state": "offline"})
|
||||||
|
self.wall.value += 31
|
||||||
|
self.reducer.mark_due_reconciliation()
|
||||||
|
before = self.store.player_observation()
|
||||||
|
guard_before = self.store.list_live_guards()[0]
|
||||||
|
self.assertEqual(before.reconciliation_state, "operator_review")
|
||||||
|
|
||||||
|
self.reducer.begin_operation("stop", slot=1, expected_path=path)
|
||||||
|
self.monotonic.value += 6
|
||||||
|
effect = self.reducer.check_timeout()
|
||||||
|
self.assertEqual(effect.kind, "operation_timed_out")
|
||||||
|
after = self.store.player_observation()
|
||||||
|
guard_after = self.store.list_live_guards()[0]
|
||||||
|
self.assertEqual(after.reconciliation_state, "operator_review")
|
||||||
|
self.assertEqual(after.reconciliation_id, before.reconciliation_id)
|
||||||
|
self.assertEqual(guard_after.guard_id, guard_before.guard_id)
|
||||||
|
self.assertEqual(guard_after.state, "review_required")
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -0,0 +1,316 @@
|
|||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from dataclasses import replace
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from controller.config import ControllerConfig, RouterConfig, StorageConfig
|
||||||
|
from controller.mqtt_adapter import PublishReceipt
|
||||||
|
from controller.router import AudioRouterCoordinator
|
||||||
|
from controller.store import ControllerStore
|
||||||
|
|
||||||
|
|
||||||
|
class Clock:
|
||||||
|
def __init__(self, value=1000.0):
|
||||||
|
self.value = value
|
||||||
|
|
||||||
|
def __call__(self):
|
||||||
|
return self.value
|
||||||
|
|
||||||
|
|
||||||
|
class ModePublisher:
|
||||||
|
def __init__(self):
|
||||||
|
self.modes = []
|
||||||
|
self.accepted = True
|
||||||
|
self.reason = None
|
||||||
|
|
||||||
|
def __call__(self, mode):
|
||||||
|
self.modes.append(mode)
|
||||||
|
return PublishReceipt(self.accepted, message_id=len(self.modes), reason=self.reason)
|
||||||
|
|
||||||
|
|
||||||
|
class AudioRouterCoordinatorTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tempdir = tempfile.TemporaryDirectory()
|
||||||
|
base = Path(self.tempdir.name)
|
||||||
|
self.clock = Clock()
|
||||||
|
self.base_config = ControllerConfig(
|
||||||
|
storage=StorageConfig(
|
||||||
|
media_base=base / "media",
|
||||||
|
data_base=base / "data",
|
||||||
|
),
|
||||||
|
router=RouterConfig(
|
||||||
|
required_mode="video",
|
||||||
|
confirmation_timeout_seconds=3,
|
||||||
|
when_state_unknown="attempt",
|
||||||
|
continue_on_confirmation_failure=True,
|
||||||
|
fallback_restore_mode=None,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self.store = ControllerStore(self.base_config.storage, now=self.clock).open()
|
||||||
|
self.publisher = ModePublisher()
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self.store.close()
|
||||||
|
self.tempdir.cleanup()
|
||||||
|
|
||||||
|
def coordinator(self, router_config=None):
|
||||||
|
config = self.base_config
|
||||||
|
if router_config is not None:
|
||||||
|
config = replace(config, router=router_config)
|
||||||
|
return AudioRouterCoordinator(
|
||||||
|
config,
|
||||||
|
self.store,
|
||||||
|
self.publisher,
|
||||||
|
clock=self.clock,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_known_mode_is_confirmed_before_play_and_restored_afterward(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
decision = router.acquire("play-one")
|
||||||
|
self.assertEqual(decision.action, "waiting")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video"])
|
||||||
|
self.assertEqual(router.observe_mode("video").action, "ready")
|
||||||
|
lease = self.store.get_audio_lease(decision.lease_id)
|
||||||
|
self.assertEqual(lease.state, "held")
|
||||||
|
self.assertEqual(lease.previous_mode, "service")
|
||||||
|
restore = router.release(decision.lease_id)
|
||||||
|
self.assertEqual(restore.action, "restore_waiting")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video", "service"])
|
||||||
|
self.assertEqual(router.observe_mode("service").action, "restored")
|
||||||
|
self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "released")
|
||||||
|
|
||||||
|
def test_default_unknown_state_timeout_continues_with_warning_not_hang(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
decision = router.acquire("play-timeout")
|
||||||
|
self.assertEqual(decision.action, "waiting")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video"])
|
||||||
|
self.clock.value += 2.9
|
||||||
|
self.assertIsNone(router.check_deadline())
|
||||||
|
self.clock.value += 0.2
|
||||||
|
decision = router.check_deadline()
|
||||||
|
self.assertEqual(decision.action, "ready")
|
||||||
|
self.assertTrue(decision.continued)
|
||||||
|
self.assertIn("did not confirm", decision.warning)
|
||||||
|
lease = self.store.get_audio_lease(decision.lease_id)
|
||||||
|
self.assertEqual(lease.state, "unconfirmed")
|
||||||
|
self.assertEqual(router.release(lease.lease_id).action, "released")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video"])
|
||||||
|
|
||||||
|
def test_timeout_can_reject_when_continue_policy_is_disabled(self):
|
||||||
|
router = self.coordinator(
|
||||||
|
replace(
|
||||||
|
self.base_config.router,
|
||||||
|
continue_on_confirmation_failure=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
decision = router.acquire("play-reject")
|
||||||
|
self.clock.value += 4
|
||||||
|
decision = router.check_deadline()
|
||||||
|
self.assertEqual(decision.action, "rejected")
|
||||||
|
self.assertFalse(decision.continued)
|
||||||
|
self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "abandoned")
|
||||||
|
|
||||||
|
def test_unknown_state_continue_policy_does_not_wait_or_gain_late_ownership(self):
|
||||||
|
router = self.coordinator(
|
||||||
|
replace(self.base_config.router, when_state_unknown="continue")
|
||||||
|
)
|
||||||
|
decision = router.acquire("play-immediate")
|
||||||
|
self.assertEqual(decision.action, "ready")
|
||||||
|
self.assertTrue(decision.continued)
|
||||||
|
self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "unconfirmed")
|
||||||
|
self.assertEqual(router.observe_mode("video").action, "observed")
|
||||||
|
self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "unconfirmed")
|
||||||
|
|
||||||
|
def test_unknown_state_reject_policy_does_not_publish(self):
|
||||||
|
router = self.coordinator(
|
||||||
|
replace(self.base_config.router, when_state_unknown="reject")
|
||||||
|
)
|
||||||
|
decision = router.acquire("play-reject")
|
||||||
|
self.assertEqual(decision.action, "rejected")
|
||||||
|
self.assertEqual(self.publisher.modes, [])
|
||||||
|
|
||||||
|
def test_already_video_requires_no_change_and_no_restore(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("video")
|
||||||
|
decision = router.acquire("play-video")
|
||||||
|
self.assertEqual(decision.action, "ready")
|
||||||
|
self.assertEqual(self.publisher.modes, [])
|
||||||
|
self.assertEqual(router.release(decision.lease_id).action, "released")
|
||||||
|
self.assertEqual(self.publisher.modes, [])
|
||||||
|
|
||||||
|
def test_confirmed_unknown_acquisition_can_use_configured_fallback(self):
|
||||||
|
router = self.coordinator(
|
||||||
|
replace(self.base_config.router, fallback_restore_mode="service")
|
||||||
|
)
|
||||||
|
decision = router.acquire("play-fallback")
|
||||||
|
self.assertEqual(router.observe_mode("video").action, "ready")
|
||||||
|
lease = self.store.get_audio_lease(decision.lease_id)
|
||||||
|
self.assertEqual(lease.restoration_source, "configured_fallback")
|
||||||
|
self.assertEqual(router.release(decision.lease_id).action, "restore_waiting")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video", "service"])
|
||||||
|
|
||||||
|
def test_operator_change_is_sticky_even_if_mode_later_returns_to_video(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
decision = router.acquire("play-operator")
|
||||||
|
router.observe_mode("video")
|
||||||
|
lost = router.observe_mode("announcements")
|
||||||
|
self.assertEqual(lost.action, "ownership_lost")
|
||||||
|
router.observe_mode("video")
|
||||||
|
self.assertEqual(router.release(decision.lease_id).action, "released")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video"])
|
||||||
|
self.assertEqual(
|
||||||
|
self.store.get_audio_lease(decision.lease_id).state,
|
||||||
|
"ownership_lost",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_connection_loss_relinquishes_confirmed_lease(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
decision = router.acquire("play-disconnect")
|
||||||
|
router.observe_mode("video")
|
||||||
|
lost = router.connection_lost()
|
||||||
|
self.assertEqual(lost.action, "ownership_lost")
|
||||||
|
self.assertEqual(router.release(decision.lease_id).action, "released")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video"])
|
||||||
|
|
||||||
|
def test_publish_failure_uses_confirmation_failure_policy(self):
|
||||||
|
self.publisher.accepted = False
|
||||||
|
self.publisher.reason = "MQTT disconnected"
|
||||||
|
router = self.coordinator()
|
||||||
|
decision = router.acquire("play-publish-fail")
|
||||||
|
self.assertEqual(decision.action, "ready")
|
||||||
|
self.assertTrue(decision.continued)
|
||||||
|
self.assertIn("MQTT disconnected", decision.warning)
|
||||||
|
|
||||||
|
def test_restoration_timeout_is_bounded_and_not_retried(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
decision = router.acquire("play-restore-timeout")
|
||||||
|
router.observe_mode("video")
|
||||||
|
router.release(decision.lease_id)
|
||||||
|
self.clock.value += 4
|
||||||
|
timeout = router.check_deadline()
|
||||||
|
self.assertEqual(timeout.action, "restore_failed")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video", "service"])
|
||||||
|
self.assertIsNone(router.check_deadline())
|
||||||
|
|
||||||
|
def test_repeated_previous_state_during_acquisition_waits_for_deadline(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
waiting = router.acquire("play-repeated-state")
|
||||||
|
self.assertEqual(router.observe_mode("service").action, "observed")
|
||||||
|
self.assertEqual(
|
||||||
|
self.store.get_audio_lease(waiting.lease_id).state,
|
||||||
|
"acquiring",
|
||||||
|
)
|
||||||
|
self.clock.value += 4
|
||||||
|
decision = router.check_deadline()
|
||||||
|
self.assertEqual(decision.action, "ready")
|
||||||
|
self.assertTrue(decision.continued)
|
||||||
|
self.assertEqual(
|
||||||
|
self.store.get_audio_lease(waiting.lease_id).state,
|
||||||
|
"unconfirmed",
|
||||||
|
)
|
||||||
|
router.release(waiting.lease_id)
|
||||||
|
self.assertEqual(self.publisher.modes, ["video"])
|
||||||
|
|
||||||
|
def test_repeated_target_state_is_observation_not_operator_takeover(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
decision = router.acquire("play-repeated-state")
|
||||||
|
self.assertEqual(router.observe_mode("video").action, "ready")
|
||||||
|
self.assertEqual(router.observe_mode("video").action, "observed")
|
||||||
|
self.assertEqual(
|
||||||
|
self.store.get_audio_lease(decision.lease_id).state,
|
||||||
|
"held",
|
||||||
|
)
|
||||||
|
self.assertEqual(router.release(decision.lease_id).action, "restore_waiting")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video", "service"])
|
||||||
|
|
||||||
|
def test_controller_mode_set_echo_does_not_relinquish_ownership(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
decision = router.acquire("play-self-echo")
|
||||||
|
self.assertEqual(
|
||||||
|
router.observe_mode_command("video", controller_echo=True).action,
|
||||||
|
"self_echo",
|
||||||
|
)
|
||||||
|
self.assertEqual(router.observe_mode("video").action, "ready")
|
||||||
|
self.assertEqual(router.release(decision.lease_id).action, "restore_waiting")
|
||||||
|
self.assertEqual(
|
||||||
|
router.observe_mode_command("service", controller_echo=True).action,
|
||||||
|
"self_echo",
|
||||||
|
)
|
||||||
|
self.assertEqual(router.observe_mode("service").action, "restored")
|
||||||
|
|
||||||
|
def test_manual_mode_set_during_acquisition_uses_continue_policy(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
waiting = router.acquire("play-manual-acquire")
|
||||||
|
decision = router.observe_mode_command("announcements")
|
||||||
|
self.assertEqual(decision.action, "ready")
|
||||||
|
self.assertTrue(decision.continued)
|
||||||
|
self.assertEqual(
|
||||||
|
self.store.get_audio_lease(waiting.lease_id).state,
|
||||||
|
"unconfirmed",
|
||||||
|
)
|
||||||
|
self.assertIsNone(self.store.get_audio_lease(waiting.lease_id).confirmation_deadline)
|
||||||
|
|
||||||
|
def test_manual_mode_set_during_acquisition_can_reject(self):
|
||||||
|
router = self.coordinator(
|
||||||
|
replace(
|
||||||
|
self.base_config.router,
|
||||||
|
continue_on_confirmation_failure=False,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
router.observe_mode("service")
|
||||||
|
waiting = router.acquire("play-manual-reject")
|
||||||
|
decision = router.observe_mode_command("announcements")
|
||||||
|
self.assertEqual(decision.action, "rejected")
|
||||||
|
self.assertFalse(decision.continued)
|
||||||
|
self.assertEqual(
|
||||||
|
self.store.get_audio_lease(waiting.lease_id).state,
|
||||||
|
"abandoned",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_manual_mode_set_while_held_relinquishes_restoration(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
decision = router.acquire("play-manual-held")
|
||||||
|
router.observe_mode("video")
|
||||||
|
lost = router.observe_mode_command("video")
|
||||||
|
self.assertEqual(lost.action, "ownership_lost")
|
||||||
|
self.assertEqual(router.release(decision.lease_id).action, "released")
|
||||||
|
self.assertEqual(self.publisher.modes, ["video"])
|
||||||
|
|
||||||
|
def test_manual_mode_set_while_restoring_relinquishes_restoration(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
decision = router.acquire("play-manual-restore")
|
||||||
|
router.observe_mode("video")
|
||||||
|
self.assertEqual(router.release(decision.lease_id).action, "restore_waiting")
|
||||||
|
lost = router.observe_mode_command("announcements")
|
||||||
|
self.assertEqual(lost.action, "ownership_lost")
|
||||||
|
self.assertEqual(
|
||||||
|
self.store.get_audio_lease(decision.lease_id).state,
|
||||||
|
"ownership_lost",
|
||||||
|
)
|
||||||
|
self.assertEqual(self.publisher.modes, ["video", "service"])
|
||||||
|
|
||||||
|
def test_retained_mode_set_does_not_claim_live_operator_intent(self):
|
||||||
|
router = self.coordinator()
|
||||||
|
router.observe_mode("service")
|
||||||
|
decision = router.acquire("play-retained-set")
|
||||||
|
router.observe_mode("video")
|
||||||
|
ignored = router.observe_mode_command("announcements", retained=True)
|
||||||
|
self.assertEqual(ignored.action, "retained_ignored")
|
||||||
|
self.assertEqual(
|
||||||
|
self.store.get_audio_lease(decision.lease_id).state,
|
||||||
|
"held",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,742 @@
|
|||||||
|
import json
|
||||||
|
import queue
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import unittest
|
||||||
|
from dataclasses import replace
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from controller.config import (
|
||||||
|
ControllerConfig,
|
||||||
|
HomeAssistantConfig,
|
||||||
|
RouterConfig,
|
||||||
|
StorageConfig,
|
||||||
|
)
|
||||||
|
from controller.downloads import canonical_playlist_path
|
||||||
|
from controller.mqtt_adapter import MqttEvent, PublishReceipt
|
||||||
|
from controller.service import ControllerService
|
||||||
|
from controller.store import ControllerStore
|
||||||
|
|
||||||
|
|
||||||
|
class MutableClock:
|
||||||
|
def __init__(self, value):
|
||||||
|
self.value = value
|
||||||
|
|
||||||
|
def __call__(self):
|
||||||
|
return self.value
|
||||||
|
|
||||||
|
|
||||||
|
class FakeMqttAdapter:
|
||||||
|
"""In-memory transport boundary; it never starts a network client."""
|
||||||
|
|
||||||
|
def __init__(self, *, connected=True):
|
||||||
|
self.connected = connected
|
||||||
|
self.started = False
|
||||||
|
self.player_commands = []
|
||||||
|
self.router_modes = []
|
||||||
|
self.json_messages = []
|
||||||
|
self.legacy_messages = []
|
||||||
|
self.acknowledged = []
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
self.started = True
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.started = False
|
||||||
|
|
||||||
|
def _receipt(self):
|
||||||
|
if self.connected:
|
||||||
|
return PublishReceipt(True, message_id=1)
|
||||||
|
return PublishReceipt(False, reason="MQTT is disconnected")
|
||||||
|
|
||||||
|
def publish_player_command(self, payload):
|
||||||
|
receipt = self._receipt()
|
||||||
|
if receipt.accepted:
|
||||||
|
self.player_commands.append(dict(payload))
|
||||||
|
return receipt
|
||||||
|
|
||||||
|
def publish_router_mode(self, mode):
|
||||||
|
receipt = self._receipt()
|
||||||
|
if receipt.accepted:
|
||||||
|
self.router_modes.append(mode)
|
||||||
|
return receipt
|
||||||
|
|
||||||
|
def publish_json(self, topic, payload, *, qos=0, retain=False):
|
||||||
|
receipt = self._receipt()
|
||||||
|
if receipt.accepted:
|
||||||
|
self.json_messages.append((topic, dict(payload), qos, retain))
|
||||||
|
return receipt
|
||||||
|
|
||||||
|
def publish_legacy(self, topic, payload, *, retain=True):
|
||||||
|
receipt = self._receipt()
|
||||||
|
if receipt.accepted:
|
||||||
|
self.legacy_messages.append((topic, payload, retain))
|
||||||
|
return receipt
|
||||||
|
|
||||||
|
def acknowledge(self, event):
|
||||||
|
self.acknowledged.append(event)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class FakeDownloadManager:
|
||||||
|
def __init__(self):
|
||||||
|
self.recovered = False
|
||||||
|
self.stopped = False
|
||||||
|
|
||||||
|
def recover_pending_deletions(self):
|
||||||
|
self.recovered = True
|
||||||
|
|
||||||
|
def expire_due_slots(self):
|
||||||
|
return []
|
||||||
|
|
||||||
|
def shutdown(self, *, wait):
|
||||||
|
self.stopped = wait
|
||||||
|
|
||||||
|
|
||||||
|
class ControllerServiceTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tempdir = tempfile.TemporaryDirectory()
|
||||||
|
base = Path(self.tempdir.name)
|
||||||
|
self.wall = MutableClock(1_700_000_000.0)
|
||||||
|
self.monotonic = MutableClock(100.0)
|
||||||
|
self.config = ControllerConfig(
|
||||||
|
storage=StorageConfig(
|
||||||
|
media_base=base / "media",
|
||||||
|
data_base=base / "data",
|
||||||
|
retention_seconds=3600,
|
||||||
|
expiry_interval_seconds=3600,
|
||||||
|
uncertain_review_after_seconds=30,
|
||||||
|
),
|
||||||
|
router=RouterConfig(
|
||||||
|
required_mode="video",
|
||||||
|
confirmation_timeout_seconds=3,
|
||||||
|
when_state_unknown="continue",
|
||||||
|
continue_on_confirmation_failure=True,
|
||||||
|
fallback_restore_mode=None,
|
||||||
|
),
|
||||||
|
homeassistant=HomeAssistantConfig(enabled=False),
|
||||||
|
)
|
||||||
|
self.events = queue.Queue()
|
||||||
|
self.store = None
|
||||||
|
self.mqtt = None
|
||||||
|
self.downloads = None
|
||||||
|
self.service = None
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
if self.service is not None and self.service.started:
|
||||||
|
self.service.stop()
|
||||||
|
self.tempdir.cleanup()
|
||||||
|
|
||||||
|
def start_service(self, *, config=None, mqtt=None):
|
||||||
|
config = config or self.config
|
||||||
|
self.events = queue.Queue()
|
||||||
|
self.store = ControllerStore(config.storage, now=self.wall)
|
||||||
|
self.mqtt = mqtt or FakeMqttAdapter()
|
||||||
|
self.downloads = FakeDownloadManager()
|
||||||
|
self.service = ControllerService(
|
||||||
|
config,
|
||||||
|
store=self.store,
|
||||||
|
mqtt_adapter=self.mqtt,
|
||||||
|
event_queue=self.events,
|
||||||
|
download_manager=self.downloads,
|
||||||
|
monotonic=self.monotonic,
|
||||||
|
wall_clock=self.wall,
|
||||||
|
)
|
||||||
|
self.service.start()
|
||||||
|
self.assertTrue(self.downloads.recovered)
|
||||||
|
return self.service
|
||||||
|
|
||||||
|
def ready_slot(self, slot=1):
|
||||||
|
reservation = self.store.reserve_download(
|
||||||
|
f"https://example.test/{slot}",
|
||||||
|
"tester",
|
||||||
|
"service test",
|
||||||
|
job_id=f"ready-{slot}",
|
||||||
|
)
|
||||||
|
self.assertEqual(reservation.slot, slot)
|
||||||
|
self.assertTrue(
|
||||||
|
self.store.mark_download_running(
|
||||||
|
reservation.job_id,
|
||||||
|
reservation.slot,
|
||||||
|
reservation.generation,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
self.store.complete_download(
|
||||||
|
reservation.job_id,
|
||||||
|
reservation.slot,
|
||||||
|
reservation.generation,
|
||||||
|
title=f"Slot {slot}",
|
||||||
|
items=(("One", "one.mp4"),),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
playlist = canonical_playlist_path(self.service.config.storage.media_base, slot)
|
||||||
|
playlist.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
playlist.write_text("one.mp4\n", encoding="utf-8")
|
||||||
|
return playlist
|
||||||
|
|
||||||
|
def send_message(
|
||||||
|
self,
|
||||||
|
topic,
|
||||||
|
payload,
|
||||||
|
*,
|
||||||
|
qos=0,
|
||||||
|
retain=False,
|
||||||
|
message_id=None,
|
||||||
|
duplicate=False,
|
||||||
|
controller_echo=False,
|
||||||
|
):
|
||||||
|
if not isinstance(payload, bytes):
|
||||||
|
payload = (
|
||||||
|
payload.encode("utf-8")
|
||||||
|
if isinstance(payload, str)
|
||||||
|
else json.dumps(payload).encode("utf-8")
|
||||||
|
)
|
||||||
|
self.events.put(
|
||||||
|
MqttEvent(
|
||||||
|
kind="message",
|
||||||
|
topic=topic,
|
||||||
|
payload=payload,
|
||||||
|
qos=qos,
|
||||||
|
retain=retain,
|
||||||
|
duplicate=duplicate,
|
||||||
|
message_id=message_id,
|
||||||
|
connection_generation=1,
|
||||||
|
controller_echo=controller_echo,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def wait_for(self, predicate, message):
|
||||||
|
deadline = time.monotonic() + 2.0
|
||||||
|
while time.monotonic() < deadline:
|
||||||
|
if predicate():
|
||||||
|
return
|
||||||
|
time.sleep(0.01)
|
||||||
|
self.fail(message)
|
||||||
|
|
||||||
|
def result_events(self):
|
||||||
|
return [
|
||||||
|
payload
|
||||||
|
for topic, payload, qos, retain in self.mqtt.json_messages
|
||||||
|
if topic == self.service.config.topics.controller_result
|
||||||
|
]
|
||||||
|
|
||||||
|
def test_play_waits_for_router_confirmation_and_player_evidence(self):
|
||||||
|
config = replace(
|
||||||
|
self.config,
|
||||||
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
||||||
|
)
|
||||||
|
self.start_service(config=config)
|
||||||
|
playlist = self.ready_slot(1)
|
||||||
|
|
||||||
|
self.send_message(config.topics.router_mode_state, "service", qos=1)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.store.router_observation().mode == "service",
|
||||||
|
"Router state was not consumed",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
||||||
|
self.assertEqual(result["status"], "pending")
|
||||||
|
self.assertEqual(result["router"], "waiting")
|
||||||
|
self.assertEqual(self.mqtt.router_modes, ["video"])
|
||||||
|
self.assertEqual(self.mqtt.player_commands, [])
|
||||||
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
||||||
|
|
||||||
|
self.send_message(
|
||||||
|
config.topics.router_mode_set,
|
||||||
|
"video",
|
||||||
|
qos=1,
|
||||||
|
controller_echo=True,
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: len(self.mqtt.acknowledged) == 2,
|
||||||
|
"Controller Router command echo was not consumed",
|
||||||
|
)
|
||||||
|
self.assertEqual(self.store.live_audio_lease().state, "acquiring")
|
||||||
|
self.assertEqual(self.mqtt.player_commands, [])
|
||||||
|
|
||||||
|
self.send_message(config.topics.router_mode_state, "video", qos=1)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: len(self.mqtt.player_commands) == 1,
|
||||||
|
"Player command was not released after Router confirmation",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
self.mqtt.player_commands,
|
||||||
|
[{"action": "play", "slot": 1, "label": "Slot 1"}],
|
||||||
|
)
|
||||||
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
||||||
|
|
||||||
|
self.send_message(
|
||||||
|
config.topics.player_status,
|
||||||
|
{"state": "playing", "slot": 1, "path": str(playlist)},
|
||||||
|
qos=1,
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.store.player_observation().phase == "playing",
|
||||||
|
"authoritative Player status was not reduced",
|
||||||
|
)
|
||||||
|
self.assertEqual(self.service.playback.projected_slot_state(1), "playing")
|
||||||
|
self.assertIn(
|
||||||
|
"player_operation_confirmed",
|
||||||
|
{event["type"] for event in self.result_events()},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_manual_router_set_during_acquisition_resolves_by_failure_policy(self):
|
||||||
|
config = replace(
|
||||||
|
self.config,
|
||||||
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
||||||
|
)
|
||||||
|
self.start_service(config=config)
|
||||||
|
self.ready_slot(1)
|
||||||
|
self.send_message(config.topics.router_mode_state, "service", qos=1)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.store.router_observation().mode == "service",
|
||||||
|
"Router state was not consumed",
|
||||||
|
)
|
||||||
|
|
||||||
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
||||||
|
self.assertEqual(result["router"], "waiting")
|
||||||
|
self.send_message(config.topics.router_mode_set, "announcements", qos=1)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: len(self.mqtt.player_commands) == 1,
|
||||||
|
"manual Router intent did not resolve acquisition",
|
||||||
|
)
|
||||||
|
lease = self.store.get_audio_lease(self.service._session_lease_id)
|
||||||
|
self.assertIsNotNone(lease)
|
||||||
|
self.assertEqual(lease.state, "unconfirmed")
|
||||||
|
warnings = [
|
||||||
|
event
|
||||||
|
for event in self.result_events()
|
||||||
|
if event["type"] == "audio_router_warning"
|
||||||
|
]
|
||||||
|
self.assertTrue(warnings)
|
||||||
|
self.assertIs(warnings[-1]["continued"], True)
|
||||||
|
|
||||||
|
def test_pause_state_remains_telemetry_authoritative(self):
|
||||||
|
self.start_service()
|
||||||
|
playlist = self.ready_slot(1)
|
||||||
|
self.service.submit_control({"action": "play", "slot": 1})
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.mqtt.player_commands == [
|
||||||
|
{"action": "play", "slot": 1, "label": "Slot 1"}
|
||||||
|
],
|
||||||
|
"play command was not published",
|
||||||
|
)
|
||||||
|
self.send_message(
|
||||||
|
self.config.topics.player_status,
|
||||||
|
{"state": "playing", "slot": 1, "path": str(playlist)},
|
||||||
|
qos=1,
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.service.playback.pending is None,
|
||||||
|
"play operation was not confirmed",
|
||||||
|
)
|
||||||
|
playing_generation = self.store.player_observation().generation
|
||||||
|
|
||||||
|
self.service.submit_control({"action": "pause"})
|
||||||
|
self.send_message(
|
||||||
|
self.config.topics.player_status,
|
||||||
|
{"state": "paused", "slot": 1, "path": str(playlist)},
|
||||||
|
qos=1,
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.store.player_observation().generation > playing_generation,
|
||||||
|
"pause status was not consumed",
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(self.service.playback.pending)
|
||||||
|
self.assertEqual(self.service.playback.projected_slot_state(1), "playing")
|
||||||
|
|
||||||
|
self.send_message(
|
||||||
|
self.config.topics.player_telemetry,
|
||||||
|
{
|
||||||
|
"state": "telemetry",
|
||||||
|
"slot": 1,
|
||||||
|
"path": str(playlist),
|
||||||
|
"paused": True,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.service.playback.pending is None,
|
||||||
|
"pause was not confirmed by telemetry",
|
||||||
|
)
|
||||||
|
self.assertEqual(self.store.player_observation().paused, True)
|
||||||
|
self.assertEqual(self.service.playback.projected_slot_state(1), "paused")
|
||||||
|
self.assertEqual(self.mqtt.player_commands[-1], {"action": "pause"})
|
||||||
|
|
||||||
|
def test_retained_controller_command_is_rejected_then_acked_and_reported(self):
|
||||||
|
self.start_service()
|
||||||
|
self.send_message(
|
||||||
|
self.config.topics.controller_command,
|
||||||
|
{"action": "play", "path": "/must/not/run"},
|
||||||
|
qos=1,
|
||||||
|
retain=True,
|
||||||
|
message_id=42,
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: len(self.mqtt.acknowledged) == 1,
|
||||||
|
"retained command was not acknowledged after rejection",
|
||||||
|
)
|
||||||
|
self.assertEqual(self.mqtt.player_commands, [])
|
||||||
|
event = self.result_events()[-1]
|
||||||
|
self.assertEqual(event["version"], 2)
|
||||||
|
self.assertEqual(event["type"], "retained_command_rejected")
|
||||||
|
topic, _, qos, retain = self.mqtt.json_messages[-1]
|
||||||
|
self.assertEqual(topic, "youtube/controller/v2/events")
|
||||||
|
self.assertEqual((qos, retain), (1, False))
|
||||||
|
self.assertEqual(self.mqtt.acknowledged[0].message_id, 42)
|
||||||
|
|
||||||
|
def test_optional_result_topic_can_be_disabled_without_affecting_ack(self):
|
||||||
|
config = replace(
|
||||||
|
self.config,
|
||||||
|
topics=replace(self.config.topics, controller_result=None),
|
||||||
|
)
|
||||||
|
self.start_service(config=config)
|
||||||
|
self.send_message(
|
||||||
|
config.topics.controller_command,
|
||||||
|
{"action": "delete_slot", "slot": 1},
|
||||||
|
qos=1,
|
||||||
|
retain=True,
|
||||||
|
message_id=7,
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: len(self.mqtt.acknowledged) == 1,
|
||||||
|
"command was not acknowledged with result topic disabled",
|
||||||
|
)
|
||||||
|
self.assertEqual(self.mqtt.json_messages, [])
|
||||||
|
|
||||||
|
def test_broker_unavailable_does_not_prevent_service_startup(self):
|
||||||
|
mqtt = FakeMqttAdapter(connected=False)
|
||||||
|
self.start_service(mqtt=mqtt)
|
||||||
|
health = self.service.health()
|
||||||
|
self.assertTrue(self.service.started)
|
||||||
|
self.assertTrue(mqtt.started)
|
||||||
|
self.assertTrue(health["coordinator_alive"])
|
||||||
|
self.assertEqual(health["status"], "degraded")
|
||||||
|
self.assertIn("mqtt_disconnected", health["degraded_reasons"])
|
||||||
|
|
||||||
|
def test_administrator_can_clear_exact_uncertain_playback_review(self):
|
||||||
|
self.start_service()
|
||||||
|
playlist = self.ready_slot(1)
|
||||||
|
self.store.set_player_observation(
|
||||||
|
availability="unknown",
|
||||||
|
phase="unknown",
|
||||||
|
active_slot=1,
|
||||||
|
active_path=str(playlist),
|
||||||
|
paused=None,
|
||||||
|
reconciliation_state="operator_review",
|
||||||
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id="review-token",
|
||||||
|
)
|
||||||
|
guard = self.store.protect_slot(
|
||||||
|
1,
|
||||||
|
"restart_uncertain",
|
||||||
|
guard_id="review-token",
|
||||||
|
)
|
||||||
|
review = self.service.health()["reconciliation_reviews"]
|
||||||
|
self.assertEqual(review[0]["review_id"], "review-token")
|
||||||
|
|
||||||
|
result = self.service.submit_control(
|
||||||
|
{
|
||||||
|
"action": "reconcile_playback",
|
||||||
|
"decision": "clear",
|
||||||
|
"confirm": True,
|
||||||
|
"slot": 1,
|
||||||
|
"review_id": review[0]["review_id"],
|
||||||
|
"actor": "operator",
|
||||||
|
"reason": "Player host was replaced",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
result,
|
||||||
|
{"status": "cleared", "slot": 1, "review_id": "review-token"},
|
||||||
|
)
|
||||||
|
self.assertEqual(self.store.list_live_guards(), [])
|
||||||
|
observation = self.store.player_observation()
|
||||||
|
self.assertEqual(observation.reconciliation_state, "admin_cleared")
|
||||||
|
self.assertIsNone(observation.active_slot)
|
||||||
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
||||||
|
self.assertEqual(self.result_events()[-1]["type"], "playback_reconciled")
|
||||||
|
|
||||||
|
def test_busy_player_keeps_video_route_until_actual_playback_finishes(self):
|
||||||
|
config = replace(
|
||||||
|
self.config,
|
||||||
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
||||||
|
)
|
||||||
|
self.start_service(config=config)
|
||||||
|
self.ready_slot(1)
|
||||||
|
second_playlist = self.ready_slot(2)
|
||||||
|
self.send_message(config.topics.router_mode_state, "service", qos=1)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.store.router_observation().mode == "service",
|
||||||
|
"Router service mode was not observed",
|
||||||
|
)
|
||||||
|
self.service.submit_control({"action": "play", "slot": 1})
|
||||||
|
self.send_message(config.topics.router_mode_state, "video", qos=1)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: len(self.mqtt.player_commands) == 1,
|
||||||
|
"play command was not released",
|
||||||
|
)
|
||||||
|
self.send_message(
|
||||||
|
config.topics.player_status,
|
||||||
|
{
|
||||||
|
"state": "busy",
|
||||||
|
"slot": 2,
|
||||||
|
"path": str(second_playlist),
|
||||||
|
"message": "mpv already playing; stop first",
|
||||||
|
},
|
||||||
|
qos=1,
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.service.playback.pending is None,
|
||||||
|
"busy response was not reduced",
|
||||||
|
)
|
||||||
|
self.assertEqual(self.store.player_observation().active_slot, 2)
|
||||||
|
self.assertEqual(self.mqtt.router_modes, ["video"])
|
||||||
|
lease = self.store.live_audio_lease()
|
||||||
|
self.assertIsNotNone(lease)
|
||||||
|
self.assertEqual(lease.state, "held")
|
||||||
|
|
||||||
|
def test_disconnected_synchronous_play_failure_is_not_reported_pending(self):
|
||||||
|
self.start_service(mqtt=FakeMqttAdapter(connected=False))
|
||||||
|
self.ready_slot(1)
|
||||||
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
||||||
|
self.assertEqual(result["status"], "unavailable")
|
||||||
|
self.assertIsNone(self.service.playback.pending)
|
||||||
|
self.assertEqual(self.store.list_live_guards(), [])
|
||||||
|
|
||||||
|
def test_router_timeout_uses_monotonic_clock_and_continues_with_warning(self):
|
||||||
|
config = replace(
|
||||||
|
self.config,
|
||||||
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
||||||
|
)
|
||||||
|
self.start_service(config=config)
|
||||||
|
self.ready_slot(1)
|
||||||
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
||||||
|
self.assertEqual(result["router"], "waiting")
|
||||||
|
self.wall.value -= 10_000
|
||||||
|
self.monotonic.value += 4
|
||||||
|
self.wait_for(
|
||||||
|
lambda: len(self.mqtt.player_commands) == 1,
|
||||||
|
"Router timeout did not release play within its finite deadline",
|
||||||
|
)
|
||||||
|
warnings = [
|
||||||
|
event
|
||||||
|
for event in self.result_events()
|
||||||
|
if event["type"] == "audio_router_warning"
|
||||||
|
]
|
||||||
|
self.assertTrue(warnings)
|
||||||
|
self.assertIs(warnings[-1]["continued"], True)
|
||||||
|
|
||||||
|
def test_duplicate_toggle_is_suppressed_and_translated_idempotently(self):
|
||||||
|
self.start_service()
|
||||||
|
playlist = self.ready_slot(1)
|
||||||
|
self.send_message(
|
||||||
|
self.config.topics.player_telemetry,
|
||||||
|
{
|
||||||
|
"state": "telemetry",
|
||||||
|
"slot": 1,
|
||||||
|
"path": str(playlist),
|
||||||
|
"paused": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: self.store.player_observation().active_slot == 1,
|
||||||
|
"active telemetry was not consumed",
|
||||||
|
)
|
||||||
|
command = {"action": "toggle"}
|
||||||
|
self.send_message(
|
||||||
|
self.config.topics.controller_command,
|
||||||
|
command,
|
||||||
|
qos=1,
|
||||||
|
message_id=71,
|
||||||
|
)
|
||||||
|
self.send_message(
|
||||||
|
self.config.topics.controller_command,
|
||||||
|
command,
|
||||||
|
qos=1,
|
||||||
|
message_id=71,
|
||||||
|
duplicate=True,
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: sum(
|
||||||
|
event.message_id == 71 for event in self.mqtt.acknowledged
|
||||||
|
)
|
||||||
|
== 2,
|
||||||
|
"duplicate command packets were not both acknowledged",
|
||||||
|
)
|
||||||
|
self.assertEqual(self.mqtt.player_commands, [{"action": "pause"}])
|
||||||
|
|
||||||
|
def test_path_only_managed_play_adds_stable_slot_identity(self):
|
||||||
|
self.start_service()
|
||||||
|
playlist = self.ready_slot(1)
|
||||||
|
result = self.service.submit_control(
|
||||||
|
{"action": "play", "playlist": str(playlist)}
|
||||||
|
)
|
||||||
|
self.assertEqual(result["status"], "pending")
|
||||||
|
self.assertEqual(
|
||||||
|
self.mqtt.player_commands,
|
||||||
|
[{"action": "play", "playlist": str(playlist), "slot": 1}],
|
||||||
|
)
|
||||||
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
||||||
|
|
||||||
|
def test_explicit_slot_cannot_be_paired_with_another_managed_slot_path(self):
|
||||||
|
self.start_service()
|
||||||
|
self.ready_slot(1)
|
||||||
|
second_playlist = self.ready_slot(2)
|
||||||
|
with self.assertRaisesRegex(ValueError, "conflicts"):
|
||||||
|
self.service.submit_control(
|
||||||
|
{
|
||||||
|
"action": "play",
|
||||||
|
"slot": 1,
|
||||||
|
"playlist": str(second_playlist),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertEqual(self.mqtt.player_commands, [])
|
||||||
|
self.assertEqual(self.store.list_live_guards(), [])
|
||||||
|
|
||||||
|
def test_administrator_can_clear_external_playback_review_without_slot(self):
|
||||||
|
config = replace(
|
||||||
|
self.config,
|
||||||
|
topics=replace(self.config.topics, controller_result=None),
|
||||||
|
)
|
||||||
|
self.start_service(config=config)
|
||||||
|
observation = self.store.mark_external_uncertain(
|
||||||
|
"/srv/operator-playlists/special-event.m3u",
|
||||||
|
review_after=self.wall.value,
|
||||||
|
details={"reason": "restart"},
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(observation.reconciliation_id)
|
||||||
|
reviews = self.service.health()["reconciliation_reviews"]
|
||||||
|
self.assertEqual(len(reviews), 1)
|
||||||
|
review_id = reviews[0]["review_id"]
|
||||||
|
|
||||||
|
result = self.service.submit_control(
|
||||||
|
{
|
||||||
|
"action": "reconcile_playback",
|
||||||
|
"decision": "clear",
|
||||||
|
"confirm": True,
|
||||||
|
"review_id": review_id,
|
||||||
|
"actor": "operator",
|
||||||
|
"reason": "Player appliance was replaced",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
result,
|
||||||
|
{"status": "cleared", "review_id": review_id},
|
||||||
|
)
|
||||||
|
cleared = self.store.player_observation()
|
||||||
|
self.assertIsNone(cleared.active_path)
|
||||||
|
self.assertEqual(cleared.reconciliation_state, "admin_cleared")
|
||||||
|
|
||||||
|
def test_external_play_intent_is_durable_immediately_after_puback(self):
|
||||||
|
self.start_service()
|
||||||
|
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
|
||||||
|
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
|
||||||
|
result = self.service.submit_control(
|
||||||
|
{"action": "play", "playlist": str(playlist)}
|
||||||
|
)
|
||||||
|
self.assertEqual(result["status"], "pending")
|
||||||
|
self.assertEqual(
|
||||||
|
self.mqtt.player_commands,
|
||||||
|
[{"action": "play", "playlist": str(playlist)}],
|
||||||
|
)
|
||||||
|
reviews = self.service.health()["reconciliation_reviews"]
|
||||||
|
self.assertEqual(len(reviews), 1)
|
||||||
|
review_id = reviews[0]["review_id"]
|
||||||
|
self.assertIs(reviews[0]["clearable"], False)
|
||||||
|
|
||||||
|
self.service.stop()
|
||||||
|
self.service = None
|
||||||
|
with ControllerStore(self.config.storage, now=self.wall).open() as reopened:
|
||||||
|
reopened.mark_restart_uncertain()
|
||||||
|
observation = reopened.player_observation()
|
||||||
|
self.assertEqual(observation.active_path, str(playlist))
|
||||||
|
self.assertEqual(observation.reconciliation_id, review_id)
|
||||||
|
|
||||||
|
def test_live_external_play_fence_cannot_be_admin_cleared(self):
|
||||||
|
self.start_service()
|
||||||
|
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
|
||||||
|
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
|
||||||
|
self.service.submit_control(
|
||||||
|
{"action": "play", "playlist": str(playlist)}
|
||||||
|
)
|
||||||
|
review = self.service.health()["reconciliation_reviews"][0]
|
||||||
|
self.assertIs(review["clearable"], False)
|
||||||
|
|
||||||
|
with self.assertRaisesRegex(ValueError, "Player operation is pending"):
|
||||||
|
self.service.submit_control(
|
||||||
|
{
|
||||||
|
"action": "reconcile_playback",
|
||||||
|
"decision": "clear",
|
||||||
|
"confirm": True,
|
||||||
|
"review_id": review["review_id"],
|
||||||
|
"actor": "operator",
|
||||||
|
"reason": "clear requested too early",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertIsNotNone(self.service.playback.pending)
|
||||||
|
observation = self.store.player_observation()
|
||||||
|
self.assertEqual(observation.reconciliation_id, review["review_id"])
|
||||||
|
self.assertEqual(observation.active_path, str(playlist))
|
||||||
|
|
||||||
|
def test_definite_external_publish_failure_clears_prepublication_guard(self):
|
||||||
|
self.start_service(mqtt=FakeMqttAdapter(connected=False))
|
||||||
|
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
|
||||||
|
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
|
||||||
|
result = self.service.submit_control(
|
||||||
|
{"action": "play", "playlist": str(playlist)}
|
||||||
|
)
|
||||||
|
self.assertEqual(result["status"], "unavailable")
|
||||||
|
self.assertEqual(self.service.health()["reconciliation_reviews"], [])
|
||||||
|
observation = self.store.player_observation()
|
||||||
|
self.assertIsNone(observation.active_path)
|
||||||
|
self.assertIsNone(observation.reconciliation_id)
|
||||||
|
|
||||||
|
def test_unexpected_reducer_failure_is_not_acked_or_deduplicated(self):
|
||||||
|
self.start_service()
|
||||||
|
original = self.service.playback.handle_status
|
||||||
|
failure_seen = threading.Event()
|
||||||
|
attempts = 0
|
||||||
|
|
||||||
|
def fail_once(payload):
|
||||||
|
nonlocal attempts
|
||||||
|
attempts += 1
|
||||||
|
if attempts == 1:
|
||||||
|
failure_seen.set()
|
||||||
|
raise RuntimeError("transient reducer failure")
|
||||||
|
return original(payload)
|
||||||
|
|
||||||
|
self.service.playback.handle_status = fail_once
|
||||||
|
payload = {
|
||||||
|
"state": "playing",
|
||||||
|
"slot": 1,
|
||||||
|
"path": str(self.config.storage.media_base / "slot1" / "playlist.m3u"),
|
||||||
|
}
|
||||||
|
self.send_message(
|
||||||
|
self.config.topics.player_status,
|
||||||
|
payload,
|
||||||
|
qos=1,
|
||||||
|
message_id=88,
|
||||||
|
)
|
||||||
|
self.assertTrue(failure_seen.wait(timeout=1))
|
||||||
|
time.sleep(0.05)
|
||||||
|
self.assertEqual(self.mqtt.acknowledged, [])
|
||||||
|
|
||||||
|
self.send_message(
|
||||||
|
self.config.topics.player_status,
|
||||||
|
payload,
|
||||||
|
qos=1,
|
||||||
|
message_id=88,
|
||||||
|
duplicate=True,
|
||||||
|
)
|
||||||
|
self.wait_for(
|
||||||
|
lambda: len(self.mqtt.acknowledged) == 1,
|
||||||
|
"retried status was not processed and acknowledged",
|
||||||
|
)
|
||||||
|
self.assertEqual(attempts, 2)
|
||||||
|
self.assertEqual(self.store.player_observation().phase, "playing")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
import concurrent.futures
|
import concurrent.futures
|
||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
|
import sqlite3
|
||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -49,6 +50,37 @@ class FreshStoreTests(StoreTestCase):
|
|||||||
self.assertTrue(all(slot.media_state == "empty" for slot in slots))
|
self.assertTrue(all(slot.media_state == "empty" for slot in slots))
|
||||||
self.assertEqual(store.player_observation().phase, "unknown")
|
self.assertEqual(store.player_observation().phase, "unknown")
|
||||||
|
|
||||||
|
def test_schema_one_database_upgrades_external_reconciliation_atomically(self):
|
||||||
|
with self.open_store() as store:
|
||||||
|
database_path = store.path
|
||||||
|
connection = sqlite3.connect(database_path)
|
||||||
|
try:
|
||||||
|
connection.execute(
|
||||||
|
"ALTER TABLE player_observation DROP COLUMN reconciliation_id"
|
||||||
|
)
|
||||||
|
connection.execute("DELETE FROM schema_migrations WHERE version = 2")
|
||||||
|
connection.execute("PRAGMA user_version = 1")
|
||||||
|
connection.commit()
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
with self.open_store() as upgraded:
|
||||||
|
self.assertEqual(
|
||||||
|
upgraded.connection.execute("PRAGMA user_version").fetchone()[0],
|
||||||
|
2,
|
||||||
|
)
|
||||||
|
columns = {
|
||||||
|
row["name"]
|
||||||
|
for row in upgraded.connection.execute(
|
||||||
|
"PRAGMA table_info(player_observation)"
|
||||||
|
).fetchall()
|
||||||
|
}
|
||||||
|
self.assertIn("reconciliation_id", columns)
|
||||||
|
migration = upgraded.connection.execute(
|
||||||
|
"SELECT name FROM schema_migrations WHERE version = 2"
|
||||||
|
).fetchone()
|
||||||
|
self.assertEqual(migration["name"], "external_playback_reconciliation")
|
||||||
|
|
||||||
def test_slot_reservation_is_committed_before_worker_transition(self):
|
def test_slot_reservation_is_committed_before_worker_transition(self):
|
||||||
with self.open_store() as store:
|
with self.open_store() as store:
|
||||||
reservation = store.reserve_download(
|
reservation = store.reserve_download(
|
||||||
@@ -127,6 +159,25 @@ class FreshStoreTests(StoreTestCase):
|
|||||||
self.assertEqual(store.get_slot(2).slot, 2)
|
self.assertEqual(store.get_slot(2).slot, 2)
|
||||||
self.assertEqual(store.get_slot(2).url, "https://example.test/two")
|
self.assertEqual(store.get_slot(2).url, "https://example.test/two")
|
||||||
|
|
||||||
|
def test_concurrent_first_open_installs_only_one_valid_database(self):
|
||||||
|
stores = [ControllerStore(self.config, now=self.clock) for _ in range(2)]
|
||||||
|
|
||||||
|
def open_store(store):
|
||||||
|
store.open()
|
||||||
|
return [slot.slot for slot in store.list_slots()]
|
||||||
|
|
||||||
|
try:
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||||
|
results = list(pool.map(open_store, stores))
|
||||||
|
self.assertEqual(results, [list(range(1, 11)), list(range(1, 11))])
|
||||||
|
markers = stores[0].connection.execute(
|
||||||
|
"SELECT name FROM data_migrations ORDER BY name"
|
||||||
|
).fetchall()
|
||||||
|
self.assertEqual([row["name"] for row in markers], ["fresh_database_v1"])
|
||||||
|
finally:
|
||||||
|
for store in stores:
|
||||||
|
store.close()
|
||||||
|
|
||||||
|
|
||||||
class GuardAndExpiryTests(StoreTestCase):
|
class GuardAndExpiryTests(StoreTestCase):
|
||||||
def _ready_slot(self, store, *, slot_expected=1):
|
def _ready_slot(self, store, *, slot_expected=1):
|
||||||
@@ -196,6 +247,7 @@ class GuardAndExpiryTests(StoreTestCase):
|
|||||||
paused=False,
|
paused=False,
|
||||||
reconciliation_state="confirmed",
|
reconciliation_state="confirmed",
|
||||||
reconciliation_due_at=None,
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id=None,
|
||||||
)
|
)
|
||||||
guard = store.mark_restart_uncertain()
|
guard = store.mark_restart_uncertain()
|
||||||
self.assertEqual(guard.slot, 1)
|
self.assertEqual(guard.slot, 1)
|
||||||
@@ -204,6 +256,150 @@ class GuardAndExpiryTests(StoreTestCase):
|
|||||||
self.assertEqual(observation.reconciliation_state, "awaiting_evidence")
|
self.assertEqual(observation.reconciliation_state, "awaiting_evidence")
|
||||||
self.assertEqual(observation.active_slot, 1)
|
self.assertEqual(observation.active_slot, 1)
|
||||||
|
|
||||||
|
def test_external_playback_restart_gets_stable_review_token_and_exact_clear(self):
|
||||||
|
external_path = "/srv/operator-playlists/live.m3u"
|
||||||
|
with self.open_store() as store:
|
||||||
|
store.set_player_observation(
|
||||||
|
availability="online",
|
||||||
|
phase="playing",
|
||||||
|
active_slot=None,
|
||||||
|
active_path=external_path,
|
||||||
|
paused=False,
|
||||||
|
reconciliation_state="confirmed",
|
||||||
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id=None,
|
||||||
|
)
|
||||||
|
self.assertIsNone(store.mark_restart_uncertain())
|
||||||
|
first = store.player_observation()
|
||||||
|
self.assertEqual(first.reconciliation_state, "awaiting_evidence")
|
||||||
|
self.assertIsNotNone(first.reconciliation_id)
|
||||||
|
self.assertEqual(first.reconciliation_due_at, self.clock.value + 30)
|
||||||
|
|
||||||
|
self.clock.value += 10
|
||||||
|
self.assertIsNone(store.mark_restart_uncertain())
|
||||||
|
repeated = store.player_observation()
|
||||||
|
self.assertEqual(repeated.reconciliation_id, first.reconciliation_id)
|
||||||
|
self.assertEqual(repeated.reconciliation_due_at, first.reconciliation_due_at)
|
||||||
|
|
||||||
|
self.clock.value = first.reconciliation_due_at + 1
|
||||||
|
due = store.mark_external_reconciliation_due()
|
||||||
|
self.assertEqual(due, (first.reconciliation_id, external_path))
|
||||||
|
self.assertIsNone(store.mark_external_reconciliation_due())
|
||||||
|
review = store.player_observation()
|
||||||
|
self.assertEqual(review.reconciliation_state, "operator_review")
|
||||||
|
self.assertEqual(review.active_path, external_path)
|
||||||
|
|
||||||
|
self.assertFalse(
|
||||||
|
store.clear_external_reconciliation(
|
||||||
|
"stale-review",
|
||||||
|
actor="operator",
|
||||||
|
reason="Player was replaced",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
store.clear_external_reconciliation(
|
||||||
|
first.reconciliation_id,
|
||||||
|
actor="operator",
|
||||||
|
reason="Player was replaced",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
cleared = store.player_observation()
|
||||||
|
self.assertEqual(cleared.reconciliation_state, "admin_cleared")
|
||||||
|
self.assertIsNone(cleared.active_path)
|
||||||
|
self.assertIsNone(cleared.reconciliation_id)
|
||||||
|
|
||||||
|
def test_restart_converts_existing_active_guard_and_preserves_deadline(self):
|
||||||
|
with self.open_store() as store:
|
||||||
|
self._ready_slot(store)
|
||||||
|
store.protect_slot(1, "player_active", guard_id="active-before-restart")
|
||||||
|
store.set_player_observation(
|
||||||
|
availability="online",
|
||||||
|
phase="playing",
|
||||||
|
active_slot=1,
|
||||||
|
active_path="/mnt/Media/YTPlayer/slot1/playlist.m3u",
|
||||||
|
paused=False,
|
||||||
|
reconciliation_state="confirmed",
|
||||||
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id=None,
|
||||||
|
)
|
||||||
|
guard = store.mark_restart_uncertain()
|
||||||
|
self.assertEqual(guard.guard_id, "active-before-restart")
|
||||||
|
self.assertEqual(guard.kind, "restart_uncertain")
|
||||||
|
first_deadline = guard.review_after
|
||||||
|
self.clock.value += 10
|
||||||
|
repeated = store.mark_restart_uncertain()
|
||||||
|
self.assertEqual(repeated.guard_id, guard.guard_id)
|
||||||
|
self.assertEqual(repeated.review_after, first_deadline)
|
||||||
|
|
||||||
|
def test_restart_converts_pre_evidence_pending_guard_for_admin_review(self):
|
||||||
|
with self.open_store() as store:
|
||||||
|
self._ready_slot(store)
|
||||||
|
store.protect_slot(1, "pending_play", guard_id="published-before-crash")
|
||||||
|
guard = store.mark_restart_uncertain()
|
||||||
|
self.assertEqual(guard.guard_id, "published-before-crash")
|
||||||
|
self.assertEqual(guard.kind, "restart_uncertain")
|
||||||
|
self.assertEqual(store.player_observation().active_slot, 1)
|
||||||
|
self.assertTrue(
|
||||||
|
store.clear_guard(
|
||||||
|
1,
|
||||||
|
guard.guard_id,
|
||||||
|
actor="operator",
|
||||||
|
reason="Player replaced",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_guard_review_transition_is_reported_only_once(self):
|
||||||
|
with self.open_store() as store:
|
||||||
|
self._ready_slot(store)
|
||||||
|
store.protect_slot(
|
||||||
|
1,
|
||||||
|
"restart_uncertain",
|
||||||
|
guard_id="one-warning",
|
||||||
|
review_after=self.clock.value,
|
||||||
|
)
|
||||||
|
self.assertEqual(len(store.mark_due_guards_for_review()), 1)
|
||||||
|
self.assertEqual(store.mark_due_guards_for_review(), [])
|
||||||
|
|
||||||
|
def test_clearing_one_legacy_guard_does_not_erase_other_observation(self):
|
||||||
|
with self.open_store() as store:
|
||||||
|
self._ready_slot(store)
|
||||||
|
second = store.reserve_download(
|
||||||
|
"https://example.test/second", None, None, job_id="second"
|
||||||
|
)
|
||||||
|
store.mark_download_running("second", 2, second.generation)
|
||||||
|
store.complete_download(
|
||||||
|
"second",
|
||||||
|
2,
|
||||||
|
second.generation,
|
||||||
|
title="Second",
|
||||||
|
items=(("Second", "second.mp4"),),
|
||||||
|
)
|
||||||
|
first_guard = store.protect_slot(
|
||||||
|
1, "legacy_uncertain", guard_id="legacy-one"
|
||||||
|
)
|
||||||
|
store.protect_slot(2, "player_active", guard_id="active-two")
|
||||||
|
store.set_player_observation(
|
||||||
|
availability="online",
|
||||||
|
phase="playing",
|
||||||
|
active_slot=2,
|
||||||
|
active_path="/mnt/Media/YTPlayer/slot2/playlist.m3u",
|
||||||
|
paused=False,
|
||||||
|
reconciliation_state="confirmed",
|
||||||
|
reconciliation_due_at=None,
|
||||||
|
reconciliation_id=None,
|
||||||
|
)
|
||||||
|
self.assertTrue(
|
||||||
|
store.clear_guard(
|
||||||
|
1,
|
||||||
|
first_guard.guard_id,
|
||||||
|
actor="operator",
|
||||||
|
reason="legacy evidence resolved",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
observation = store.player_observation()
|
||||||
|
self.assertEqual(observation.active_slot, 2)
|
||||||
|
self.assertEqual(observation.phase, "playing")
|
||||||
|
|
||||||
|
|
||||||
class LegacyMigrationTests(StoreTestCase):
|
class LegacyMigrationTests(StoreTestCase):
|
||||||
def write_legacy(self, data):
|
def write_legacy(self, data):
|
||||||
|
|||||||
Reference in New Issue
Block a user