diff --git a/config.yaml b/config.yaml index 0a9515f..6288778 100644 --- a/config.yaml +++ b/config.yaml @@ -8,6 +8,8 @@ mqtt: reconnect_min_seconds: 1 reconnect_max_seconds: 30 command_queue_size: 512 + # QoS 1 is accepted only after the broker PUBACKs within this bound. + publish_ack_timeout_seconds: 2 topics: player_command: youtube/player/cmd @@ -19,8 +21,9 @@ topics: playback_base: youtube/media/playback router_mode_set: audio/player/mode/set router_mode_state: audio/player/mode/state - # Optional additive result topic. Existing consumers do not depend on it. - controller_result: null + # Optional additive result/warning topic. Set null to disable it. + # Existing consumers never depend on this topic. + controller_result: youtube/controller/v2/events storage: media_base: /mnt/Media/YTPlayer diff --git a/controller/config.py b/controller/config.py index 714be34..2a69c1a 100644 --- a/controller/config.py +++ b/controller/config.py @@ -31,6 +31,7 @@ class MqttConfig: reconnect_min_seconds: int = 1 reconnect_max_seconds: int = 30 command_queue_size: int = 512 + publish_ack_timeout_seconds: float = 2.0 @dataclass(frozen=True) @@ -195,6 +196,11 @@ def load_config( reconnect_min_seconds=_integer(mqtt_raw.get("reconnect_min_seconds", 1), "mqtt.reconnect_min_seconds"), reconnect_max_seconds=_integer(mqtt_raw.get("reconnect_max_seconds", 30), "mqtt.reconnect_max_seconds"), command_queue_size=_integer(mqtt_raw.get("command_queue_size", 512), "mqtt.command_queue_size"), + publish_ack_timeout_seconds=_number( + mqtt_raw.get("publish_ack_timeout_seconds", 2.0), + "mqtt.publish_ack_timeout_seconds", + minimum=0.1, + ), ) if mqtt.reconnect_max_seconds < mqtt.reconnect_min_seconds: raise ConfigError("mqtt.reconnect_max_seconds must be >= reconnect_min_seconds") @@ -211,7 +217,11 @@ def load_config( playback_base=_text(topics_raw.get("playback_base", "youtube/media/playback"), "topics.playback_base"), router_mode_set=_text(topics_raw.get("router_mode_set", "audio/player/mode/set"), "topics.router_mode_set"), router_mode_state=_text(topics_raw.get("router_mode_state", "audio/player/mode/state"), "topics.router_mode_state"), - controller_result=_text(topics_raw.get("controller_result"), "topics.controller_result", allow_none=True), + controller_result=_text( + topics_raw.get("controller_result", "youtube/controller/v2/events"), + "topics.controller_result", + allow_none=True, + ), ) configured_slots = _integer(storage_raw.get("slot_count", 10), "storage.slot_count") diff --git a/controller/contracts.py b/controller/contracts.py index 131f56f..cc1f33f 100644 --- a/controller/contracts.py +++ b/controller/contracts.py @@ -25,7 +25,7 @@ class Topics: playback_base: str = "youtube/media/playback" router_mode_set: str = "audio/player/mode/set" router_mode_state: str = "audio/player/mode/state" - controller_result: str | None = None + controller_result: str | None = "youtube/controller/v2/events" def slot_base(self, slot: int) -> str: if slot not in SLOT_IDS: diff --git a/controller/downloads.py b/controller/downloads.py index 2d20ddb..9ab02dd 100644 --- a/controller/downloads.py +++ b/controller/downloads.py @@ -4,9 +4,11 @@ from __future__ import annotations import logging import os +import signal import shutil import subprocess import threading +import time import uuid from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass @@ -32,6 +34,10 @@ class DownloadEvent: message: str | None = None job_id: str | None = None generation: int | None = None + url: str | None = None + submitter: str | None = None + note: str | None = None + reused_slot: bool = False EventCallback = Callable[[DownloadEvent], None] @@ -66,7 +72,7 @@ class DownloadManager: store: ControllerStore, *, on_event: EventCallback | None = None, - runner: Runner = subprocess.run, + runner: Runner | None = None, executor: ThreadPoolExecutor | None = None, ): self.config = config @@ -80,6 +86,7 @@ class DownloadManager: self._owns_executor = executor is None self._lock = threading.Lock() self._futures: set[Future[Any]] = set() + self._processes: set[subprocess.Popen[str]] = set() self._stopping = False def enqueue( @@ -106,6 +113,10 @@ class DownloadManager: state="queued", job_id=reservation.job_id, generation=reservation.generation, + url=url, + submitter=submitter, + note=note, + reused_slot=reservation.reused_slot, ) ) try: @@ -184,7 +195,7 @@ class DownloadManager: url, ] try: - result = self.runner( + result = self._run_command( arguments, check=True, capture_output=True, @@ -246,7 +257,7 @@ class DownloadManager: ), url, ] - self.runner( + self._run_command( arguments, check=True, text=True, @@ -322,6 +333,90 @@ class DownloadManager: return f"yt-dlp failed with exit code {exc.returncode}" return f"download failed: {exc}" + def _run_command( + self, + arguments: list[str], + *, + check: bool, + text: bool, + timeout: float, + capture_output: bool = False, + ) -> subprocess.CompletedProcess[str]: + if self.runner is not None: + return self.runner( + arguments, + check=check, + text=text, + timeout=timeout, + capture_output=capture_output, + ) + + with self._lock: + if self._stopping: + raise RuntimeError("Controller is shutting down") + process = subprocess.Popen( + arguments, + stdout=subprocess.PIPE if capture_output else None, + stderr=subprocess.PIPE if capture_output else None, + text=text, + start_new_session=True, + ) + self._processes.add(process) + try: + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + self._terminate_process(process, force=False) + try: + stdout, stderr = process.communicate(timeout=2.0) + except subprocess.TimeoutExpired: + self._terminate_process(process, force=True) + stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired( + arguments, + timeout, + output=stdout, + stderr=stderr, + ) + completed = subprocess.CompletedProcess( + arguments, + process.returncode, + stdout=stdout, + stderr=stderr, + ) + if check and process.returncode: + raise subprocess.CalledProcessError( + process.returncode, + arguments, + output=stdout, + stderr=stderr, + ) + return completed + finally: + with self._lock: + self._processes.discard(process) + + @staticmethod + def _terminate_process( + process: subprocess.Popen[str], + *, + force: bool, + ) -> None: + if process.poll() is not None: + return + try: + os.killpg( + process.pid, + signal.SIGKILL if force else signal.SIGTERM, + ) + except ProcessLookupError: + return + except OSError: + try: + process.kill() if force else process.terminate() + except ProcessLookupError: + pass + @staticmethod def _fsync_directory(path: Path) -> None: descriptor = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0)) @@ -387,6 +482,17 @@ class DownloadManager: def shutdown(self, *, wait: bool = True) -> None: with self._lock: self._stopping = True + processes = tuple(self._processes) + for process in processes: + self._terminate_process(process, force=False) + deadline = time.monotonic() + 2.0 + for process in processes: + if process.poll() is not None: + continue + try: + process.wait(timeout=max(0.0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + self._terminate_process(process, force=True) if self._owns_executor: self._executor.shutdown(wait=wait, cancel_futures=True) diff --git a/controller/models.py b/controller/models.py index c06467f..a174b07 100644 --- a/controller/models.py +++ b/controller/models.py @@ -84,10 +84,38 @@ class PlayerObservation: generation: int reconciliation_state: str reconciliation_due_at: float | None + reconciliation_id: str | None last_status_at: float | None last_telemetry_at: float | None +@dataclass(frozen=True) +class RouterObservation: + mode: str | None + freshness: str + observed_at: float | None + last_warning: str | None + + +@dataclass(frozen=True) +class AudioLeaseRecord: + lease_id: str + operation_id: str | None + playback_generation: int | None + state: str + previous_mode: str | None + target_mode: str + restoration_mode: str | None + restoration_source: str + controller_changed: bool + confirmation_deadline: float | None + created_at: float + acquired_at: float | None + released_at: float | None + warning: str | None + error: str | None + + @dataclass(frozen=True) class MigrationReport: migrated: bool diff --git a/controller/mqtt_adapter.py b/controller/mqtt_adapter.py index 5882c53..96f238c 100644 --- a/controller/mqtt_adapter.py +++ b/controller/mqtt_adapter.py @@ -6,6 +6,7 @@ import json import logging import queue import threading +import uuid from dataclasses import dataclass from typing import Any, Mapping @@ -33,9 +34,11 @@ class MqttEvent: payload: bytes = b"" qos: int = 0 retain: bool = False + duplicate: bool = False message_id: int | None = None connection_generation: int | None = None reason: str | None = None + controller_echo: bool = False @dataclass(frozen=True) @@ -43,6 +46,7 @@ class PublishReceipt: accepted: bool message_id: int | None = None reason: str | None = None + uncertain: bool = False class MqttAdapter: @@ -59,11 +63,8 @@ class MqttAdapter: self.events: "queue.Queue[MqttEvent]" = events or queue.Queue( maxsize=config.mqtt.command_queue_size ) - self.client = client or mqtt.Client( - mqtt.CallbackAPIVersion.VERSION2, - client_id=config.mqtt.client_id, - clean_session=True, - ) + self._injected_client = client is not None + self.client = client or self._new_client() self._generation_lock = threading.Lock() self._connection_generation = 0 self._connected = threading.Event() @@ -72,9 +73,18 @@ class MqttAdapter: self._started = False self._loop_started = False self._bootstrap_thread: threading.Thread | None = None + self._router_echo_lock = threading.Lock() + self._pending_router_echoes: dict[str, bytes] = {} self.last_error: str | None = None self._configure_client() + def _new_client(self): + return mqtt.Client( + mqtt.CallbackAPIVersion.VERSION2, + client_id=self.config.mqtt.client_id, + clean_session=True, + ) + def _configure_client(self) -> None: if self.config.mqtt.username is not None: self.client.username_pw_set( @@ -116,20 +126,32 @@ class MqttAdapter: self._stop_event.clear() try: self._begin_async_connection() - except (OSError, mqtt.WebsocketConnectionError) as exc: + except (OSError, mqtt.WebsocketConnectionError, RuntimeError) as exc: self.last_error = f"initial MQTT setup failed: {exc}" LOG.warning("%s; retrying in background", self.last_error) self._start_bootstrap_retry() def _begin_async_connection(self) -> None: - self.client.connect_async( + with self._lifecycle_lock: + if not self._started or self._stop_event.is_set(): + return + client = self.client + result = client.connect_async( self.config.mqtt.host, self.config.mqtt.port, keepalive=self.config.mqtt.keepalive_seconds, ) + if result != mqtt.MQTT_ERR_SUCCESS: + raise RuntimeError(f"failed to configure MQTT connection: {result}") with self._lifecycle_lock: + if ( + not self._started + or self._stop_event.is_set() + or client is not self.client + ): + return if not self._loop_started: - result = self.client.loop_start() + result = client.loop_start() if result != mqtt.MQTT_ERR_SUCCESS: raise RuntimeError(f"failed to start MQTT network loop: {result}") self._loop_started = True @@ -159,17 +181,21 @@ class MqttAdapter: delay = min(maximum, max(delay * 2, 1.0)) except RuntimeError as exc: self.last_error = str(exc) - LOG.error("%s", self.last_error) - return + LOG.warning("%s", self.last_error) + delay = min(maximum, max(delay * 2, 1.0)) def stop(self) -> None: + # Signal callbacks/retry work before waiting for the lifecycle lock. + # A callback already inside subscribe/connect then observes shutdown + # and cannot publish a post-stop connected state. + self._stop_event.set() with self._lifecycle_lock: if not self._started: + self._connected.clear() return self._started = False loop_started = self._loop_started self._loop_started = False - self._stop_event.set() self._connected.clear() if loop_started: try: @@ -190,24 +216,43 @@ class MqttAdapter: def _on_connect(self, client, userdata, flags, reason_code, properties=None) -> None: try: - if not self._reason_success(reason_code): - self.last_error = f"MQTT connect failed: {reason_code}" - LOG.warning("%s", self.last_error) + with self._lifecycle_lock: + if ( + client is not self.client + or not self._started + or self._stop_event.is_set() + ): + return + if not self._reason_success(reason_code): + self.last_error = f"MQTT connect failed: {reason_code}" + LOG.warning("%s", self.last_error) + return + if self._stop_event.is_set(): + return + with self._generation_lock: + self._connection_generation += 1 + generation = self._connection_generation + self._connected.set() + self.last_error = None + subscriptions = ( + (self.config.topics.player_status, PLAYER_STATUS_QOS), + (self.config.topics.player_telemetry, PLAYER_TELEMETRY_QOS), + (self.config.topics.player_state, PLAYER_STATE_QOS), + (self.config.topics.controller_command, 1), + (self.config.topics.router_mode_set, ROUTER_COMMAND_QOS), + (self.config.topics.router_mode_state, 1), + ) + for topic, qos in subscriptions: + if self._stop_event.is_set(): + self._connected.clear() + return + client.subscribe(topic, qos=qos) + if self._stop_event.is_set(): + self._connected.clear() + return + if self._stop_event.is_set(): + self._connected.clear() return - with self._generation_lock: - self._connection_generation += 1 - generation = self._connection_generation - self._connected.set() - self.last_error = None - subscriptions = ( - (self.config.topics.player_status, PLAYER_STATUS_QOS), - (self.config.topics.player_telemetry, PLAYER_TELEMETRY_QOS), - (self.config.topics.player_state, PLAYER_STATE_QOS), - (self.config.topics.controller_command, 1), - (self.config.topics.router_mode_state, 1), - ) - for topic, qos in subscriptions: - client.subscribe(topic, qos=qos) self._enqueue( MqttEvent( kind="connected", @@ -231,6 +276,8 @@ class MqttAdapter: reason_code, properties=None, ) -> None: + if client is not self.client: + return self._connected.clear() try: self._enqueue( @@ -245,15 +292,33 @@ class MqttAdapter: def _on_message(self, client, userdata, message) -> None: try: + # Capture client identity and generation atomically with session + # replacement. A callback that was already running when an + # uncertain command discards the transport must be able to leave a + # full queue so loop_stop() cannot deadlock waiting for it. + with self._lifecycle_lock: + if client is not self.client: + return + with self._generation_lock: + generation = self._connection_generation + topic = str(message.topic) + payload = bytes(message.payload) + retained = bool(getattr(message, "retain", False)) self._enqueue( MqttEvent( kind="message", - topic=str(message.topic), - payload=bytes(message.payload), + topic=topic, + payload=payload, qos=int(getattr(message, "qos", 0)), - retain=bool(getattr(message, "retain", False)), + retain=retained, + duplicate=bool(getattr(message, "dup", False)), message_id=getattr(message, "mid", None), - connection_generation=self.connection_generation, + connection_generation=generation, + controller_echo=( + topic == self.config.topics.router_mode_set + and not retained + and self._claim_router_echo(payload) + ), ) ) except BaseException: @@ -261,6 +326,11 @@ class MqttAdapter: def _enqueue(self, event: MqttEvent) -> bool: while not self._stop_event.is_set(): + if ( + event.connection_generation is not None + and event.connection_generation != self.connection_generation + ): + return False try: self.events.put(event, timeout=0.2) return True @@ -293,8 +363,9 @@ class MqttAdapter: qos: int, retain: bool, require_connection: bool, + invalidate_transport_on_uncertain: bool = False, ) -> PublishReceipt: - if require_connection and not self.connected: + if require_connection and (not self.started or not self.connected): return PublishReceipt(False, reason="MQTT is disconnected") try: info = self.client.publish(topic, payload, qos=qos, retain=retain) @@ -303,8 +374,79 @@ class MqttAdapter: return PublishReceipt(False, reason=str(exc)) if info.rc != mqtt.MQTT_ERR_SUCCESS: return PublishReceipt(False, message_id=getattr(info, "mid", None), reason=f"MQTT rc={info.rc}") + if qos > 0 and hasattr(info, "wait_for_publish"): + try: + info.wait_for_publish( + timeout=self.config.mqtt.publish_ack_timeout_seconds + ) + acknowledged = bool(info.is_published()) + except BaseException as exc: + acknowledged = False + detail = str(exc) or exc.__class__.__name__ + else: + detail = ( + f"no broker PUBACK within " + f"{self.config.mqtt.publish_ack_timeout_seconds:g} seconds" + ) + if not acknowledged: + reason = f"MQTT QoS 1 publication is uncertain: {detail}" + if invalidate_transport_on_uncertain: + self._discard_unacknowledged_transport(reason) + return PublishReceipt( + False, + message_id=getattr(info, "mid", None), + reason=reason, + uncertain=True, + ) return PublishReceipt(True, message_id=getattr(info, "mid", None)) + def _discard_unacknowledged_transport(self, reason: str) -> None: + """Drop the Paho session so stale QoS 1 commands cannot replay.""" + + if self._injected_client: + # Test transports do not own a recreatable Paho session. Production + # always follows the replacement path below. + with self._generation_lock: + self._connection_generation += 1 + self._connected.clear() + return + with self._lifecycle_lock: + old_client = self.client + old_loop_started = self._loop_started + self._loop_started = False + should_restart = self._started and not self._stop_event.is_set() + self._connected.clear() + with self._generation_lock: + previous_generation = self._connection_generation + self._connection_generation += 1 + self.client = self._new_client() + self._configure_client() + try: + old_client.disconnect() + except BaseException: + LOG.exception("failed to disconnect uncertain MQTT transport") + if old_loop_started: + try: + old_client.loop_stop() + except BaseException: + LOG.exception("failed to stop uncertain MQTT transport") + self.last_error = reason + try: + self.events.put_nowait( + MqttEvent( + kind="disconnected", + connection_generation=previous_generation, + reason=reason, + ) + ) + except queue.Full: + # This method can run on the sole coordinator thread; blocking on + # its own queue would deadlock. The uncertain PublishReceipt still + # drives the operation-specific conservative transition. + LOG.error("MQTT event queue was full while discarding uncertain transport") + if should_restart: + self._start_bootstrap_retry() + @staticmethod def _json_payload(payload: Mapping[str, Any]) -> str: return json.dumps(payload, allow_nan=False, separators=(",", ":")) @@ -316,16 +458,41 @@ class MqttAdapter: qos=PLAYER_COMMAND_QOS, retain=PLAYER_COMMAND_RETAIN, require_connection=True, + invalidate_transport_on_uncertain=True, ) def publish_router_mode(self, mode: str) -> PublishReceipt: - return self._publish( - self.config.topics.router_mode_set, - mode, - qos=ROUTER_COMMAND_QOS, - retain=ROUTER_COMMAND_RETAIN, - require_connection=True, - ) + # A broker normally sends our publication back through the matching + # subscription. Register the exact payload before publishing because + # Paho's network thread may receive that echo before publish() returns. + # The marker is removed as soon as the QoS 1 publish call completes; + # this intentionally prefers a false ownership loss over mistaking a + # later same-value operator command for a Controller echo. + echo_token = uuid.uuid4().hex + with self._router_echo_lock: + self._pending_router_echoes[echo_token] = mode.encode("utf-8") + try: + return self._publish( + self.config.topics.router_mode_set, + mode, + qos=ROUTER_COMMAND_QOS, + retain=ROUTER_COMMAND_RETAIN, + require_connection=True, + invalidate_transport_on_uncertain=True, + ) + finally: + with self._router_echo_lock: + self._pending_router_echoes.pop(echo_token, None) + + def _claim_router_echo(self, payload: bytes) -> bool: + """Claim one exact mode-set echo from an in-flight local publish.""" + + with self._router_echo_lock: + for token, expected in tuple(self._pending_router_echoes.items()): + if expected == payload: + del self._pending_router_echoes[token] + return True + return False def publish_legacy(self, topic: str, payload: str, *, retain: bool = True) -> PublishReceipt: return self._publish( diff --git a/controller/playback.py b/controller/playback.py index 4b67010..3b6356a 100644 --- a/controller/playback.py +++ b/controller/playback.py @@ -3,6 +3,7 @@ from __future__ import annotations import math +import os import time import uuid from dataclasses import dataclass, replace @@ -28,6 +29,7 @@ class PlayerOperation: slot: int | None expected_path: str | None deadline: float + reconciliation_id: str | None = None state: str = "awaiting_player" message: str | None = None @@ -71,6 +73,7 @@ class PlaybackReducer: *, slot: int | None = None, expected_path: str | None = None, + operation_id: str | None = None, ) -> PlayerOperation: if action not in CONTROL_ACTIONS: raise ValueError(f"unsupported Player action {action!r}") @@ -82,14 +85,13 @@ class PlaybackReducer: ) self._operation_generation += 1 operation = PlayerOperation( - operation_id=uuid.uuid4().hex, + operation_id=operation_id or uuid.uuid4().hex, generation=self._operation_generation, action=action, slot=slot, expected_path=expected_path, deadline=self._monotonic() + self.operation_timeout_seconds, ) - self.pending = operation if action == "play" and slot is not None: self.store.protect_slot( slot, @@ -97,11 +99,40 @@ class PlaybackReducer: guard_id=f"operation-{operation.operation_id}", details={"operation_id": operation.operation_id}, ) + elif action == "play" and expected_path is not None: + due = None + if self.config.storage.uncertain_review_after_seconds > 0: + due = ( + self._wall_clock() + + self.config.storage.uncertain_review_after_seconds + ) + observation = self.store.mark_external_uncertain( + expected_path, + review_after=due, + details={ + "operation_id": operation.operation_id, + "reason": "play command pending Player evidence", + }, + ) + operation = replace( + operation, + reconciliation_id=observation.reconciliation_id, + ) + self.pending = operation return operation def publish_failed(self, reason: str) -> PlaybackEffect | None: if self.pending is None: return None + if ( + self.pending.action == "play" + and self.pending.slot is None + and self.pending.reconciliation_id is not None + ): + self.store.cancel_external_intent( + self.pending.reconciliation_id, + reason=reason, + ) return self._finish_operation("failed", reason) def _finish_operation(self, state: str, message: str | None = None) -> PlaybackEffect: @@ -142,11 +173,64 @@ class PlaybackReducer: operation = self.pending if operation is None: return False - slot_matches = operation.slot is None or operation.slot == slot - path_matches = operation.expected_path is None or operation.expected_path == path - if operation.slot is not None and operation.expected_path is not None: - return slot_matches or path_matches - return slot_matches and path_matches + positive_match = False + if operation.slot is not None and slot is not None: + if operation.slot != slot: + return False + positive_match = True + if operation.expected_path is not None and path is not None: + if os.path.abspath(operation.expected_path) != os.path.abspath(path): + return False + positive_match = True + return positive_match + + def _canonical_slot_for_path(self, path: str | None) -> int | None: + if path is None: + return None + candidate = os.path.abspath(path) + for slot in SLOT_IDS: + slot_directory = os.path.abspath( + self.config.storage.media_base / f"slot{slot}" + ) + try: + if os.path.commonpath((candidate, slot_directory)) == slot_directory: + return slot + except ValueError: + continue + return None + + def _normalized_identity( + self, + payload: Mapping[str, Any], + ) -> tuple[int | None, str | None, str | None]: + slot = self._slot(payload.get("slot")) + path = self._path(payload.get("path")) + derived = self._canonical_slot_for_path(path) + if slot is not None and derived is not None and slot != derived: + return ( + slot, + path, + f"Player identity conflicts: slot {slot} does not contain path {path!r}", + ) + return (derived if slot is None else slot), path, None + + @staticmethod + def _same_playback_identity( + previous: PlayerObservation, + slot: int | None, + path: str | None, + ) -> bool: + if previous.active_slot != slot: + return False + if slot is None: + return ( + previous.active_path is not None + and path is not None + and os.path.abspath(previous.active_path) == os.path.abspath(path) + ) + if previous.active_path is None or path is None: + return True + return os.path.abspath(previous.active_path) == os.path.abspath(path) def _active_evidence( self, @@ -158,11 +242,11 @@ class PlaybackReducer: telemetry: Mapping[str, Any] | None = None, ) -> PlayerObservation: previous = self.store.player_observation() - if slot is None: + if slot is None and (path is None or path == previous.active_path): slot = previous.active_slot - if path is None: + if path is None and (slot is None or slot == previous.active_slot): path = previous.active_path - if paused is None and previous.active_slot == slot: + if paused is None and self._same_playback_identity(previous, slot, path): paused = previous.paused phase = "paused" if paused is True else "playing" if slot is not None: @@ -171,6 +255,8 @@ class PlaybackReducer: kind="player_active", details={"path": path}, ) + elif path is not None: + self.store.reconcile_external_active() return self.store.set_player_observation( availability="online", phase=phase, @@ -179,6 +265,7 @@ class PlaybackReducer: paused=paused, reconciliation_state="confirmed", reconciliation_due_at=None, + reconciliation_id=None, status_payload=status, telemetry_payload=telemetry, ) @@ -199,6 +286,7 @@ class PlaybackReducer: paused=False, reconciliation_state="clear", reconciliation_due_at=None, + reconciliation_id=None, status_payload=status, telemetry_payload=telemetry, ) @@ -212,36 +300,63 @@ class PlaybackReducer: if not isinstance(state, str) or not state: return [PlaybackEffect("warning", message="Player status is missing state")] state = state.strip() - slot = self._slot(payload.get("slot")) - path = self._path(payload.get("path")) + slot, path, identity_warning = self._normalized_identity(payload) + if identity_warning is not None: + return [PlaybackEffect("warning", slot=slot, message=identity_warning)] effects: list[PlaybackEffect] = [] if state == "offline": previous = self.store.player_observation() due = None - if ( - previous.active_slot is not None - and self.config.storage.uncertain_review_after_seconds > 0 - ): - due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds - self.store.reconcile_active_guard( + active_known = ( + previous.active_slot is not None or previous.active_path is not None + ) + if previous.active_slot is not None: + if self.config.storage.uncertain_review_after_seconds > 0: + due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds + guard = self.store.mark_slot_uncertain( previous.active_slot, - kind="restart_uncertain", review_after=due, details={"reason": "Player offline"}, ) - self.store.set_player_observation( - availability="offline", - phase="unknown" if previous.active_slot is not None else previous.phase, - active_slot=previous.active_slot, - active_path=previous.active_path, - paused=previous.paused, - reconciliation_state=( - "awaiting_evidence" if previous.active_slot is not None else "unknown" - ), - reconciliation_due_at=due, - status_payload=payload, - ) + due = guard.review_after + self.store.set_player_observation( + availability="offline", + phase="unknown", + active_slot=previous.active_slot, + active_path=previous.active_path, + paused=previous.paused, + reconciliation_state=( + "operator_review" + if guard.state == "review_required" + else "awaiting_evidence" + ), + reconciliation_due_at=due, + reconciliation_id=guard.guard_id, + status_payload=payload, + ) + elif previous.active_path is not None: + if self.config.storage.uncertain_review_after_seconds > 0: + due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds + self.store.mark_external_uncertain( + previous.active_path, + review_after=due, + availability="offline", + details={"reason": "Player offline"}, + status_payload=payload, + ) + else: + self.store.set_player_observation( + availability="offline", + phase=previous.phase, + active_slot=None, + active_path=None, + paused=previous.paused, + reconciliation_state="unknown", + reconciliation_due_at=None, + reconciliation_id=None, + status_payload=payload, + ) effects.append( PlaybackEffect( "player_offline", @@ -261,15 +376,56 @@ class PlaybackReducer: message="Reconnect idle announcement carried an active path", ) ) + if self.pending is not None and self.pending.action == "play": + if self._matches_pending(slot, path): + effects.append( + self._finish_operation( + "confirmed", "matching reconnect evidence" + ) + ) + else: + effects.append( + self._finish_operation( + "failed", + "Player reconnected with different active playback", + ) + ) else: self._idle_evidence(status=payload) effects.append(PlaybackEffect("player_idle")) - if self.pending is not None and self.pending.action == "stop": - effects.append(self._finish_operation("confirmed")) + if self.pending is not None: + if self.pending.action == "stop": + effects.append(self._finish_operation("confirmed")) + elif self.pending.action == "play": + effects.append( + self._finish_operation( + "failed", "Player reconnected without active playback" + ) + ) return effects if state == "idle": reason = payload.get("reason") + if reason not in TERMINAL_REASONS: + previous = self.store.player_observation() + self.store.set_player_observation( + availability="online", + phase=previous.phase, + active_slot=previous.active_slot, + active_path=previous.active_path, + paused=previous.paused, + reconciliation_state=previous.reconciliation_state, + reconciliation_due_at=previous.reconciliation_due_at, + reconciliation_id=previous.reconciliation_id, + status_payload=payload, + ) + return [ + PlaybackEffect( + "warning", + slot=previous.active_slot, + message="ambiguous Player idle status was not treated as terminal", + ) + ] self._idle_evidence(status=payload) effects.append(PlaybackEffect("player_idle", slot=slot, message=str(reason or "idle"))) if self.pending is not None: @@ -280,29 +436,33 @@ class PlaybackReducer: return effects if state == "playing": - previous = self.store.player_observation() - actual_paused = previous.paused if previous.active_slot == slot else None self._active_evidence( slot=slot, path=path, - paused=actual_paused, + paused=None, status=payload, ) effects.append(PlaybackEffect("player_active", slot=slot)) if ( self.pending is not None and self.pending.action == "play" - and self._matches_pending(slot, path) ): - effects.append(self._finish_operation("confirmed")) + if self._matches_pending(slot, path): + effects.append(self._finish_operation("confirmed")) + else: + effects.append( + self._finish_operation( + "failed", + "Player became active with a different slot or path", + ) + ) return effects if state in {"paused", "toggled"}: - previous = self.store.player_observation() self._active_evidence( slot=slot, path=path, - paused=previous.paused if previous.active_slot == slot else None, + paused=None, status=payload, ) effects.append( @@ -312,15 +472,24 @@ class PlaybackReducer: message=f"Player issued {state}; awaiting telemetry", ) ) + if self.pending is not None and self.pending.action == "play": + if self._matches_pending(slot, path): + effects.append(self._finish_operation("confirmed", state)) + else: + effects.append( + self._finish_operation( + "failed", + "Player control status identified different playback", + ) + ) return effects if state == "busy": if slot is not None or path is not None: - previous = self.store.player_observation() self._active_evidence( slot=slot, path=path, - paused=previous.paused if previous.active_slot == slot else None, + paused=None, status=payload, ) effects.append(PlaybackEffect("player_busy", slot=slot, message=payload.get("message"))) @@ -338,14 +507,29 @@ class PlaybackReducer: kind="player_active", details={"error": payload.get("message")}, ) + elif active_path is not None: + self.store.reconcile_external_active() self.store.set_player_observation( availability="online", phase="error", active_slot=active_slot, active_path=active_path, - paused=previous.paused, - reconciliation_state=("confirmed" if active_slot is not None else "unknown"), + paused=( + previous.paused + if self._same_playback_identity( + previous, + active_slot, + active_path, + ) + else None + ), + reconciliation_state=( + "confirmed" + if active_slot is not None or active_path is not None + else "unknown" + ), reconciliation_due_at=None, + reconciliation_id=None, status_payload=payload, ) message = str(payload.get("message") or "Player error") @@ -360,8 +544,9 @@ class PlaybackReducer: def handle_telemetry(self, payload: Any) -> list[PlaybackEffect]: if not isinstance(payload, Mapping) or payload.get("state") != "telemetry": return [PlaybackEffect("warning", message="invalid Player telemetry envelope")] - slot = self._slot(payload.get("slot")) - path = self._path(payload.get("path")) + slot, path, identity_warning = self._normalized_identity(payload) + if identity_warning is not None: + return [PlaybackEffect("warning", slot=slot, message=identity_warning)] effects: list[PlaybackEffect] = [] if path is None: self._idle_evidence(telemetry=payload) @@ -383,7 +568,15 @@ class PlaybackReducer: ) effects.append(PlaybackEffect("player_telemetry", slot=observation.active_slot)) operation = self.pending - if operation is None or not self._matches_pending(slot, path): + if operation is None: + return effects + if not self._matches_pending(slot, path): + effects.append( + self._finish_operation( + "failed", + "Player telemetry identified a different slot or path", + ) + ) return effects if operation.action == "play": effects.append(self._finish_operation("confirmed", "matching telemetry")) @@ -399,48 +592,165 @@ class PlaybackReducer: operation = self.pending if operation is None or self._monotonic() < operation.deadline: return None - if operation.action == "play" and operation.slot is not None: - due = None - if self.config.storage.uncertain_review_after_seconds > 0: - due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds - self.store.reconcile_active_guard( - operation.slot, - kind="restart_uncertain", - review_after=due, - details={"operation_id": operation.operation_id, "reason": "play timeout"}, - ) - self.store.set_player_observation( - availability="unknown", - phase="unknown", - active_slot=operation.slot, - active_path=operation.expected_path, - paused=None, - reconciliation_state="awaiting_evidence", - reconciliation_due_at=due, + if operation.action == "play": + previous = self.store.player_observation() + has_confirmed_active = ( + previous.reconciliation_state == "confirmed" + and (previous.active_slot is not None or previous.active_path is not None) ) + if has_confirmed_active and self._matches_pending( + previous.active_slot, + previous.active_path, + ): + return self._finish_operation( + "confirmed", "matching authoritative Player evidence" + ) + if not has_confirmed_active: + self._mark_play_uncertain(operation, "play timeout") elif operation.action == "stop": previous = self.store.player_observation() if previous.active_slot is not None: due = None if self.config.storage.uncertain_review_after_seconds > 0: due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds - self.store.reconcile_active_guard( + guard = self.store.mark_slot_uncertain( previous.active_slot, - kind="restart_uncertain", review_after=due, details={"operation_id": operation.operation_id, "reason": "stop timeout"}, ) + due = guard.review_after self.store.set_player_observation( availability="unknown", phase="unknown", active_slot=previous.active_slot, active_path=previous.active_path, paused=previous.paused, - reconciliation_state="awaiting_evidence", + reconciliation_state=( + "operator_review" + if guard.state == "review_required" + else "awaiting_evidence" + ), reconciliation_due_at=due, + reconciliation_id=guard.guard_id, + ) + elif previous.active_path is not None: + due = None + if self.config.storage.uncertain_review_after_seconds > 0: + due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds + self.store.mark_external_uncertain( + previous.active_path, + review_after=due, + details={ + "operation_id": operation.operation_id, + "reason": "stop timeout", + }, ) return self._finish_operation("timed_out", "no authoritative Player confirmation") + def _mark_play_uncertain(self, operation: PlayerOperation, reason: str) -> None: + due = None + if self.config.storage.uncertain_review_after_seconds > 0: + due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds + slot = operation.slot or self._canonical_slot_for_path(operation.expected_path) + if slot is not None: + guard = self.store.mark_slot_uncertain( + slot, + review_after=due, + details={"operation_id": operation.operation_id, "reason": reason}, + ) + due = guard.review_after + self.store.set_player_observation( + availability="unknown", + phase="unknown", + active_slot=slot, + active_path=operation.expected_path, + paused=None, + reconciliation_state=( + "operator_review" + if guard.state == "review_required" + else "awaiting_evidence" + ), + reconciliation_due_at=due, + reconciliation_id=guard.guard_id, + ) + elif operation.expected_path is not None: + self.store.mark_external_uncertain( + operation.expected_path, + review_after=due, + details={"operation_id": operation.operation_id, "reason": reason}, + ) + + def publication_uncertain(self, reason: str) -> PlaybackEffect | None: + if self.pending is None: + return None + operation = self.pending + if operation.action == "play": + self._mark_play_uncertain(operation, reason) + return self._finish_operation("uncertain", reason) + + def mark_transport_uncertain(self, reason: str) -> PlaybackEffect: + previous = self.store.player_observation() + pending_play = ( + self.pending + if self.pending is not None and self.pending.action == "play" + else None + ) + active_slot = previous.active_slot + active_path = previous.active_path + if active_slot is None and active_path is None and pending_play is not None: + active_slot = pending_play.slot or self._canonical_slot_for_path( + pending_play.expected_path + ) + active_path = pending_play.expected_path + due = None + if active_slot is not None: + if self.config.storage.uncertain_review_after_seconds > 0: + due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds + guard = self.store.mark_slot_uncertain( + active_slot, + review_after=due, + details={"reason": reason}, + ) + due = guard.review_after + self.store.set_player_observation( + availability="unknown", + phase="unknown", + active_slot=active_slot, + active_path=active_path, + paused=previous.paused, + reconciliation_state=( + "operator_review" + if guard.state == "review_required" + else "awaiting_evidence" + ), + reconciliation_due_at=due, + reconciliation_id=guard.guard_id, + ) + elif active_path is not None: + if self.config.storage.uncertain_review_after_seconds > 0: + due = self._wall_clock() + self.config.storage.uncertain_review_after_seconds + self.store.mark_external_uncertain( + active_path, + review_after=due, + details={"reason": reason}, + ) + else: + self.store.set_player_observation( + availability="unknown", + phase=previous.phase, + active_slot=None, + active_path=None, + paused=previous.paused, + reconciliation_state="unknown", + reconciliation_due_at=None, + reconciliation_id=None, + ) + return PlaybackEffect( + "transport_uncertain", + slot=active_slot, + message=reason, + ) + def projected_slot_state(self, slot: int) -> str: record = self.store.get_slot(slot) if record.media_state != "ready": @@ -466,9 +776,9 @@ class PlaybackReducer: return "ready" def mark_due_reconciliation(self) -> list[PlaybackEffect]: - due = self.store.mark_due_guards_for_review(self._wall_clock()) - if not due: - return [] + now = self._wall_clock() + due = self.store.mark_due_guards_for_review(now) + external_due = self.store.mark_external_reconciliation_due(now) observation = self.store.player_observation() if observation.active_slot is not None and any( guard.slot == observation.active_slot for guard in due @@ -481,15 +791,32 @@ class PlaybackReducer: paused=observation.paused, reconciliation_state="operator_review", reconciliation_due_at=observation.reconciliation_due_at, + reconciliation_id=observation.reconciliation_id, ) - return [ + effects = [ PlaybackEffect( "operator_review", slot=guard.slot, + operation_id=guard.guard_id, message=( "Playback remains unknown and protected; fresh Player evidence " - "or an administrative clear is required" + "or an administrative clear is required " + f"(review_id={guard.guard_id})" ), ) for guard in due ] + if external_due is not None: + review_id, path = external_due + effects.append( + PlaybackEffect( + "operator_review", + operation_id=review_id, + message=( + f"Playback at {path!r} remains unknown and protected; " + "fresh Player evidence or an administrative clear is required " + f"(review_id={review_id})" + ), + ) + ) + return effects diff --git a/controller/router.py b/controller/router.py new file mode 100644 index 0000000..ec7ccf7 --- /dev/null +++ b/controller/router.py @@ -0,0 +1,422 @@ +"""Timeout-bounded Audio Router acquisition and restoration leases.""" + +from __future__ import annotations + +import time +import uuid +from dataclasses import dataclass +from typing import Callable + +from .config import ControllerConfig +from .models import AudioLeaseRecord +from .mqtt_adapter import PublishReceipt +from .store import ControllerStore + + +ModePublisher = Callable[[str], PublishReceipt] + + +@dataclass(frozen=True) +class RouterDecision: + action: str + lease_id: str | None = None + warning: str | None = None + continued: bool = False + + +class AudioRouterCoordinator: + """Own one conservative Router lease for one playback session.""" + + def __init__( + self, + config: ControllerConfig, + store: ControllerStore, + publish_mode: ModePublisher, + *, + clock=time.monotonic, + ): + self.config = config + self.store = store + self.publish_mode = publish_mode + self._clock = clock + + def observe_mode(self, mode: str) -> RouterDecision: + mode = mode.strip() + if not mode: + warning = "Audio Router published an empty mode" + self._record_warning(warning) + return RouterDecision("warning", warning=warning) + self.store.set_router_observation(mode=mode, freshness="current") + lease = self.store.live_audio_lease() + if lease is None: + return RouterDecision("observed") + + if lease.state == "acquiring": + if mode == lease.target_mode: + self.store.update_audio_lease( + lease.lease_id, + state="held", + confirmation_deadline=None, + acquired=True, + ) + return RouterDecision("ready", lease_id=lease.lease_id) + # State is evidence of the Router's current output, not evidence + # that an operator issued a command. It may simply be a periodic + # repeat of the pre-acquisition state, so keep waiting for the + # finite confirmation deadline. + return RouterDecision("observed", lease_id=lease.lease_id) + + if lease.state == "held": + if mode != lease.target_mode: + warning = ( + f"Audio Router changed from Controller-owned {lease.target_mode!r} " + f"to {mode!r}; automatic restoration was relinquished" + ) + self.store.update_audio_lease( + lease.lease_id, + state="ownership_lost", + confirmation_deadline=None, + warning=warning, + released=True, + ) + self._record_warning(warning) + return RouterDecision("ownership_lost", lease.lease_id, warning) + return RouterDecision("observed", lease_id=lease.lease_id) + + if lease.state == "unconfirmed": + # A late state event must never resurrect a lease that already + # timed out or deliberately continued without confirmation. + return RouterDecision("observed", lease_id=lease.lease_id) + + if lease.state == "restore_pending": + if mode == lease.restoration_mode: + self.store.update_audio_lease( + lease.lease_id, + state="released", + confirmation_deadline=None, + released=True, + ) + return RouterDecision("restored", lease_id=lease.lease_id) + # The Router may repeat its current state before applying the + # restoration command. Wait for the bounded deadline; explicit + # mode-set intent is handled separately below. + return RouterDecision("observed", lease_id=lease.lease_id) + return RouterDecision("observed", lease_id=lease.lease_id) + + def observe_mode_command( + self, + mode: str, + *, + retained: bool = False, + controller_echo: bool = False, + ) -> RouterDecision: + """Observe a Router mode-set publication without treating state as intent. + + The MQTT adapter marks an exact echo received while a Controller QoS 1 + publication is in flight. Every other live mode-set publication is + conservatively treated as operator intent. Retained commands are old + broker state and can never establish current operator intent. + """ + + mode = mode.strip() + if not mode: + warning = "Audio Router published an empty mode command" + self._record_warning(warning) + return RouterDecision("warning", warning=warning) + + lease = self.store.live_audio_lease() + lease_id = None if lease is None else lease.lease_id + if retained: + warning = ( + f"Retained Audio Router mode command {mode!r} was ignored as stale intent" + ) + self._record_warning(warning) + return RouterDecision("retained_ignored", lease_id, warning) + if controller_echo: + return RouterDecision("self_echo", lease_id=lease_id) + if lease is None: + return RouterDecision("operator_intent") + + if lease.state == "acquiring": + return self._acquisition_failed( + lease.lease_id, + ( + f"Audio Router received an operator mode command for {mode!r} " + f"while {lease.target_mode!r} confirmation was pending; " + "automatic restoration ownership was relinquished" + ), + ) + + if lease.state in {"held", "unconfirmed", "restore_pending"}: + warning = ( + f"Audio Router received an operator mode command for {mode!r}; " + "automatic restoration was relinquished" + ) + self.store.update_audio_lease( + lease.lease_id, + state="ownership_lost", + confirmation_deadline=None, + warning=warning, + released=True, + ) + self._record_warning(warning) + return RouterDecision("ownership_lost", lease.lease_id, warning) + return RouterDecision("operator_intent", lease_id=lease.lease_id) + + def acquire(self, operation_id: str) -> RouterDecision: + observation = self.store.router_observation() + known = observation.freshness == "current" and observation.mode is not None + target = self.config.router.required_mode + previous = observation.mode if known else None + lease_id = uuid.uuid4().hex + + if known and previous == target: + self.store.create_audio_lease( + lease_id=lease_id, + operation_id=operation_id, + state="held", + previous_mode=previous, + target_mode=target, + restoration_mode=None, + restoration_source="none", + controller_changed=False, + confirmation_deadline=None, + ) + self.store.update_audio_lease( + lease_id, + state="held", + acquired=True, + ) + return RouterDecision("ready", lease_id=lease_id) + + if known: + restoration_mode = previous + restoration_source = "observed_previous" + elif self.config.router.fallback_restore_mode is not None: + restoration_mode = self.config.router.fallback_restore_mode + restoration_source = "configured_fallback" + else: + restoration_mode = None + restoration_source = "none" + + policy = self.config.router.when_state_unknown + if not known and policy == "reject": + warning = "Audio Router state is unavailable; playback was rejected by policy" + self.store.create_audio_lease( + lease_id=lease_id, + operation_id=operation_id, + state="abandoned", + previous_mode=None, + target_mode=target, + restoration_mode=restoration_mode, + restoration_source=restoration_source, + controller_changed=False, + confirmation_deadline=None, + warning=warning, + ) + self._record_warning(warning) + return RouterDecision("rejected", lease_id, warning) + + deadline = self._clock() + self.config.router.confirmation_timeout_seconds + initial_state = "unconfirmed" if not known and policy == "continue" else "acquiring" + self.store.create_audio_lease( + lease_id=lease_id, + operation_id=operation_id, + state=initial_state, + previous_mode=previous, + target_mode=target, + restoration_mode=restoration_mode, + restoration_source=restoration_source, + controller_changed=True, + confirmation_deadline=(deadline if initial_state == "acquiring" else None), + ) + receipt = self.publish_mode(target) + if not receipt.accepted: + return self._acquisition_failed( + lease_id, + f"could not request Audio Router mode {target!r}: {receipt.reason}", + ) + + if initial_state == "unconfirmed": + warning = ( + f"Audio Router state is unavailable; requested {target!r} and " + "continued immediately without restoration ownership" + ) + self.store.update_audio_lease( + lease_id, + state="unconfirmed", + warning=warning, + ) + self._record_warning(warning) + return RouterDecision("ready", lease_id, warning, continued=True) + return RouterDecision("waiting", lease_id=lease_id) + + def check_deadline(self) -> RouterDecision | None: + lease = self.store.live_audio_lease() + if ( + lease is None + or lease.confirmation_deadline is None + or self._clock() < lease.confirmation_deadline + ): + return None + if lease.state == "acquiring": + return self._acquisition_failed( + lease.lease_id, + ( + f"Audio Router did not confirm {lease.target_mode!r} within " + f"{self.config.router.confirmation_timeout_seconds:g} seconds" + ), + ) + if lease.state == "restore_pending": + warning = ( + f"Audio Router did not confirm restoration to {lease.restoration_mode!r}; " + "no further automatic change will be attempted" + ) + self.store.update_audio_lease( + lease.lease_id, + state="abandoned", + confirmation_deadline=None, + warning=warning, + released=True, + ) + self._record_warning(warning) + return RouterDecision("restore_failed", lease.lease_id, warning) + return None + + def _acquisition_failed(self, lease_id: str, warning: str) -> RouterDecision: + continue_playback = self.config.router.continue_on_confirmation_failure + self.store.update_audio_lease( + lease_id, + state="unconfirmed" if continue_playback else "abandoned", + confirmation_deadline=None, + warning=warning, + released=not continue_playback, + ) + self._record_warning(warning) + return RouterDecision( + "ready" if continue_playback else "rejected", + lease_id, + warning, + continued=continue_playback, + ) + + def release(self, lease_id: str | None) -> RouterDecision: + if lease_id is None: + return RouterDecision("released") + lease = self.store.get_audio_lease(lease_id) + if lease is None: + return RouterDecision("released") + if lease.state == "held" and not lease.controller_changed: + self.store.update_audio_lease( + lease_id, + state="released", + confirmation_deadline=None, + released=True, + ) + return RouterDecision("released", lease_id) + if lease.state != "held": + if lease.state in {"unconfirmed", "acquiring"}: + self.store.update_audio_lease( + lease_id, + state="abandoned", + confirmation_deadline=None, + warning="Audio Router was never confirmed; restoration was skipped", + released=True, + ) + return RouterDecision("released", lease_id) + + observation = self.store.router_observation() + if observation.freshness != "current" or observation.mode != lease.target_mode: + warning = "Audio Router ownership is no longer current; restoration was skipped" + self.store.update_audio_lease( + lease_id, + state="ownership_lost", + confirmation_deadline=None, + warning=warning, + released=True, + ) + self._record_warning(warning) + return RouterDecision("ownership_lost", lease_id, warning) + if lease.restoration_mode is None: + self.store.update_audio_lease( + lease_id, + state="released", + confirmation_deadline=None, + released=True, + ) + return RouterDecision("released", lease_id) + + receipt = self.publish_mode(lease.restoration_mode) + if not receipt.accepted: + warning = f"could not request Audio Router restoration: {receipt.reason}" + self.store.update_audio_lease( + lease_id, + state="abandoned", + confirmation_deadline=None, + warning=warning, + released=True, + ) + self._record_warning(warning) + return RouterDecision("restore_failed", lease_id, warning) + self.store.update_audio_lease( + lease_id, + state="restore_pending", + confirmation_deadline=( + self._clock() + self.config.router.confirmation_timeout_seconds + ), + ) + return RouterDecision("restore_waiting", lease_id) + + def connection_lost(self) -> RouterDecision | None: + observation = self.store.router_observation() + self.store.set_router_observation( + mode=observation.mode, + freshness="stale", + warning="MQTT continuity was lost", + ) + lease = self.store.live_audio_lease() + if lease is None: + return None + if lease.state == "acquiring": + return self._acquisition_failed( + lease.lease_id, + "MQTT disconnected while waiting for Audio Router confirmation", + ) + warning = "MQTT continuity was lost; automatic Audio Router restoration was relinquished" + self.store.update_audio_lease( + lease.lease_id, + state="ownership_lost", + confirmation_deadline=None, + warning=warning, + released=True, + ) + self._record_warning(warning) + return RouterDecision("ownership_lost", lease.lease_id, warning) + + def abandon(self, lease_id: str | None, reason: str) -> None: + if lease_id is None: + return + lease = self.store.get_audio_lease(lease_id) + if lease is None or lease.state in {"released", "abandoned", "ownership_lost"}: + return + self.store.update_audio_lease( + lease_id, + state="abandoned", + confirmation_deadline=None, + warning=reason, + released=True, + ) + self._record_warning(reason) + + def _record_warning(self, warning: str) -> None: + observation = self.store.router_observation() + self.store.set_router_observation( + mode=observation.mode, + freshness=observation.freshness, + warning=warning, + ) + self.store.record_audit_event( + "audio_router_warning", + actor="controller", + details={"message": warning}, + ) diff --git a/controller/service.py b/controller/service.py new file mode 100644 index 0000000..6544f99 --- /dev/null +++ b/controller/service.py @@ -0,0 +1,1139 @@ +"""Single-coordinator Controller v2 service.""" + +from __future__ import annotations + +import json +import logging +import os +import queue +import threading +import time +import uuid +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any, Mapping + +from .config import ControllerConfig +from .contracts import SLOT_IDS +from .downloads import ( + DownloadEvent, + DownloadManager, + canonical_playlist_path, +) +from .ha import SlotAnnouncement, discovery_messages, slot_messages +from .models import SlotProtectedError, StaleOperationError, StoreError +from .mqtt_adapter import MqttAdapter, MqttEvent +from .playback import OperationInProgressError, PlaybackEffect, PlaybackReducer +from .router import AudioRouterCoordinator, RouterDecision +from .store import ControllerStore + + +LOG = logging.getLogger(__name__) + + +@dataclass +class _ServiceCall: + action: str + payload: Any + response: "queue.Queue[Any]" + + +@dataclass(frozen=True) +class _InternalEvent: + kind: str + payload: Any + + +@dataclass(frozen=True) +class _PendingPlay: + intent_id: str + command: Mapping[str, Any] + slot: int | None + expected_path: str | None + lease_id: str | None = None + + +class ControllerUnavailableError(RuntimeError): + pass + + +class ControllerService: + """Coordinates MQTT, Player evidence, Router leases, and media inventory.""" + + def __init__( + self, + config: ControllerConfig, + *, + store: ControllerStore | None = None, + mqtt_adapter: MqttAdapter | None = None, + event_queue: "queue.Queue[Any] | None" = None, + download_manager: DownloadManager | None = None, + playback: PlaybackReducer | None = None, + router: AudioRouterCoordinator | None = None, + monotonic=time.monotonic, + wall_clock=time.time, + ): + self.config = config + self.events: "queue.Queue[Any]" = event_queue or queue.Queue( + maxsize=config.mqtt.command_queue_size + ) + self.store = store or ControllerStore(config.storage, now=wall_clock) + self.mqtt = mqtt_adapter or MqttAdapter( + config, + events=self.events, + ) + self.downloads = download_manager + self.playback = playback + self.router = router + self._monotonic = monotonic + self._wall_clock = wall_clock + self._lifecycle_lock = threading.Lock() + self._stop_event = threading.Event() + self._coordinator: threading.Thread | None = None + self._started = False + self._pending_play: _PendingPlay | None = None + self._session_lease_id: str | None = None + self._last_announced: dict[int, tuple[Any, ...]] = {} + self._next_expiry = 0.0 + self._next_reconciliation = 0.0 + self._processed_packets: dict[tuple[int, str, int], float] = {} + + @property + def started(self) -> bool: + with self._lifecycle_lock: + return self._started + + def start(self) -> None: + with self._lifecycle_lock: + if self._started: + return + if self._coordinator is not None and self._coordinator.is_alive(): + raise ControllerUnavailableError( + "previous Controller coordinator has not stopped" + ) + try: + self.store.open() + self.config.storage.media_base.mkdir(parents=True, exist_ok=True) + self.store.abandon_audio_leases_on_restart() + router_observation = self.store.router_observation() + self.store.set_router_observation( + mode=router_observation.mode, + freshness=( + "stale" if router_observation.mode is not None else "unknown" + ), + warning=None, + ) + self.store.mark_restart_uncertain() + if self.playback is None: + self.playback = PlaybackReducer( + self.config, + self.store, + monotonic=self._monotonic, + wall_clock=self._wall_clock, + ) + if self.router is None: + self.router = AudioRouterCoordinator( + self.config, + self.store, + self.mqtt.publish_router_mode, + clock=self._monotonic, + ) + if self.downloads is None: + self.downloads = DownloadManager( + self.config, + self.store, + on_event=self._queue_download_event, + ) + self.downloads.recover_pending_deletions() + self._stop_event.clear() + self._next_expiry = ( + self._monotonic() + + self.config.storage.expiry_interval_seconds + ) + self._next_reconciliation = self._monotonic() + 1.0 + self._coordinator = threading.Thread( + target=self._run, + name="controller-coordinator", + daemon=True, + ) + self._coordinator.start() + self.mqtt.start() + self._started = True + except BaseException: + self._stop_event.set() + try: + self.mqtt.stop() + except BaseException: + LOG.exception("MQTT cleanup failed after Controller startup error") + coordinator = self._coordinator + if coordinator is not None and coordinator.is_alive(): + coordinator.join(timeout=5.0) + if coordinator is None or not coordinator.is_alive(): + self.store.close() + self._coordinator = None + raise + + def stop(self) -> None: + with self._lifecycle_lock: + if not self._started: + coordinator = self._coordinator + if coordinator is not None and not coordinator.is_alive(): + self.store.close() + self._coordinator = None + return + self._started = False + self._stop_event.set() + self.mqtt.stop() + if self.downloads is not None: + self.downloads.shutdown(wait=True) + try: + self.events.put_nowait(_InternalEvent("shutdown", None)) + except queue.Full: + pass + coordinator = self._coordinator + if coordinator is threading.current_thread(): + LOG.error("Controller stop was requested on its coordinator thread") + return + if coordinator is not None: + coordinator.join(timeout=5.0) + if coordinator is not None and coordinator.is_alive(): + LOG.error( + "Controller coordinator did not stop; store remains open for safety" + ) + return + self.store.close() + self._coordinator = None + + def _queue_download_event(self, event: DownloadEvent) -> None: + if threading.current_thread() is self._coordinator: + self._handle_download_event(event) + return + self._put_event(_InternalEvent("download", event)) + + def _put_event(self, event: Any) -> bool: + while not self._stop_event.is_set(): + try: + self.events.put(event, timeout=0.2) + return True + except queue.Full: + continue + return False + + def _call(self, action: str, payload: Any, timeout: float = 10.0) -> Any: + if not self.started: + raise ControllerUnavailableError("Controller service is not started") + if threading.current_thread() is self._coordinator: + return self._handle_call(action, payload) + response: "queue.Queue[Any]" = queue.Queue(maxsize=1) + if not self._put_event(_ServiceCall(action, payload, response)): + raise ControllerUnavailableError("Controller service is stopping") + try: + result = response.get(timeout=timeout) + except queue.Empty as exc: + raise ControllerUnavailableError("Controller coordinator did not respond") from exc + if isinstance(result, BaseException): + raise result + return result + + def enqueue_download( + self, + url: str, + submitter: str | None, + note: str | None, + ) -> dict[str, Any]: + return self._call( + "enqueue_download", + {"url": url, "submitter": submitter, "note": note}, + ) + + def submit_control(self, payload: Mapping[str, Any]) -> dict[str, Any]: + return self._call("control", dict(payload)) + + def health(self) -> dict[str, Any]: + with self._lifecycle_lock: + started = self._started + coordinator_alive = ( + self._coordinator is not None and self._coordinator.is_alive() + ) + router_observation = ( + self.store.router_observation() if started else None + ) + player_observation = ( + self.store.player_observation() if started else None + ) + guards = self.store.list_live_guards() if started else [] + player_operation_pending = bool( + started + and self.playback is not None + and self.playback.pending is not None + ) + mqtt_connected = bool(started and self.mqtt.connected) + degraded: list[str] = [] + if started and not mqtt_connected: + degraded.append("mqtt_disconnected") + if ( + started + and router_observation is not None + and router_observation.freshness != "current" + ): + degraded.append("router_state_unavailable") + reviews = [ + { + "review_id": guard.guard_id, + "slot": guard.slot, + "kind": guard.kind, + "state": ( + "operator_review" + if guard.state == "review_required" + else "awaiting_evidence" + ), + "review_after": guard.review_after, + "clearable": not player_operation_pending, + } + for guard in guards + if guard.kind in {"restart_uncertain", "legacy_uncertain"} + ] + if ( + player_observation is not None + and player_observation.active_slot is None + and player_observation.active_path is not None + and player_observation.reconciliation_id is not None + and player_observation.reconciliation_state + in {"awaiting_evidence", "operator_review"} + ): + reviews.append( + { + "review_id": player_observation.reconciliation_id, + "slot": None, + "path": player_observation.active_path, + "kind": "external_playback", + "state": player_observation.reconciliation_state, + "review_after": player_observation.reconciliation_due_at, + "clearable": not player_operation_pending, + } + ) + return { + "status": ( + "stopped" + if not started + else "degraded" if degraded else "ok" + ), + "started": started, + "coordinator_alive": coordinator_alive, + "mqtt_connected": mqtt_connected, + "router_state": ( + None if router_observation is None else router_observation.mode + ), + "player": ( + None + if player_observation is None + else { + "availability": player_observation.availability, + "phase": player_observation.phase, + "active_slot": player_observation.active_slot, + "active_path": player_observation.active_path, + "paused": player_observation.paused, + "reconciliation_state": player_observation.reconciliation_state, + } + ), + "reconciliation_reviews": reviews, + "degraded_reasons": degraded, + } + + def _run(self) -> None: + while not self._stop_event.is_set(): + try: + event = self.events.get(timeout=0.1) + except queue.Empty: + event = None + if event is not None: + try: + self._handle_event(event) + except BaseException: + LOG.exception("Controller coordinator contained an event failure") + try: + self._run_timers() + except BaseException: + LOG.exception("Controller coordinator contained a timer failure") + + def _handle_event(self, event: Any) -> None: + if isinstance(event, _ServiceCall): + try: + result = self._handle_call(event.action, event.payload) + except BaseException as exc: + event.response.put(exc) + else: + event.response.put(result) + return + if isinstance(event, _InternalEvent): + if event.kind == "download": + self._handle_download_event(event.payload) + return + if isinstance(event, MqttEvent): + self._handle_mqtt_event(event) + + def _handle_call(self, action: str, payload: Any) -> Any: + if action == "enqueue_download": + assert self.downloads is not None + reservation = self.downloads.enqueue( + payload["url"], payload.get("submitter"), payload.get("note") + ) + result: dict[str, Any] = { + "status": "queued" if reservation.status == "reserved" else reservation.status, + "slot": reservation.slot, + "reused_slot": reservation.reused_slot, + } + if reservation.oldest_expires_at is not None: + result["oldest_expires_at"] = reservation.oldest_expires_at + result["oldest_expires_in_seconds"] = max( + 0, + int(reservation.oldest_expires_at - self._wall_clock()), + ) + return result + if action == "control": + return self._handle_control_command(payload) + raise ValueError(f"unknown service call {action!r}") + + def _handle_mqtt_event(self, event: MqttEvent) -> None: + packet_key = self._mqtt_packet_key(event) + handled_successfully = False + try: + if packet_key is not None and self._is_duplicate_packet(event, packet_key): + handled_successfully = True + return + if event.kind == "connected": + self._announce_discovery() + self._sync_slot_states(force=True) + self._announce_reconciliation_reviews() + handled_successfully = True + return + if event.kind == "disconnected": + assert self.playback is not None + assert self.router is not None + playback_effect = self.playback.mark_transport_uncertain( + f"Controller MQTT connection was lost: {event.reason}" + ) + self._handle_playback_effects([playback_effect]) + decision = self.router.connection_lost() + if decision is not None: + self._handle_router_decision(decision) + handled_successfully = True + return + if event.kind != "message" or event.topic is None: + handled_successfully = True + return + if event.topic == self.config.topics.player_status: + payload = self._decode_json(event.payload, "Player status") + if payload is not None: + assert self.playback is not None + self._handle_playback_effects(self.playback.handle_status(payload)) + elif event.topic == self.config.topics.player_telemetry: + payload = self._decode_json(event.payload, "Player telemetry") + if payload is not None: + assert self.playback is not None + self._handle_playback_effects(self.playback.handle_telemetry(payload)) + elif event.topic == self.config.topics.controller_command: + if event.retain: + self._publish_event( + "retained_command_rejected", + "warning", + "Retained Controller commands are rejected to prevent replay", + ) + else: + payload = self._decode_json(event.payload, "Controller command") + if payload is not None: + try: + self._handle_control_command(payload) + except ( + ValueError, + StoreError, + ControllerUnavailableError, + OperationInProgressError, + ) as exc: + self._publish_event( + "controller_command_rejected", + "warning", + str(exc), + ) + elif event.topic in { + self.config.topics.router_mode_set, + self.config.topics.router_mode_state, + }: + try: + mode = event.payload.decode("utf-8").strip() + except UnicodeDecodeError: + self._publish_event( + "router_state_invalid", + "warning", + "Audio Router mode payload was not valid UTF-8", + ) + else: + assert self.router is not None + if event.topic == self.config.topics.router_mode_set: + decision = self.router.observe_mode_command( + mode, + retained=event.retain, + controller_echo=event.controller_echo, + ) + else: + decision = self.router.observe_mode(mode) + self._handle_router_decision(decision) + # Player flat state is intentionally only a Companion/cold-start + # hint. It is never reduced as authoritative playback truth. + handled_successfully = True + finally: + if handled_successfully: + if packet_key is not None: + self._remember_packet(packet_key) + self.mqtt.acknowledge(event) + + @staticmethod + def _mqtt_packet_key(event: MqttEvent) -> tuple[int, str, int] | None: + if ( + event.kind != "message" + or event.qos <= 0 + or event.message_id is None + or event.connection_generation is None + or event.topic is None + ): + return None + return (event.connection_generation, event.topic, event.message_id) + + def _is_duplicate_packet( + self, + event: MqttEvent, + key: tuple[int, str, int], + ) -> bool: + return event.duplicate and key in self._processed_packets + + def _remember_packet(self, key: tuple[int, str, int]) -> None: + now = self._monotonic() + self._processed_packets[key] = now + cutoff = now - 300.0 + if len(self._processed_packets) > 1024: + stale = sorted( + self._processed_packets.items(), key=lambda item: item[1] + )[: len(self._processed_packets) - 1024] + for candidate, _ in stale: + self._processed_packets.pop(candidate, None) + for candidate, processed_at in tuple(self._processed_packets.items()): + if processed_at < cutoff: + self._processed_packets.pop(candidate, None) + + def _decode_json(self, payload: bytes, description: str) -> Any | None: + try: + value = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, ValueError, RecursionError) as exc: + self._publish_event( + "malformed_mqtt_payload", + "warning", + f"{description} was rejected: {exc}", + ) + return None + return value + + @staticmethod + def _parse_slot(value: Any) -> int: + if value is None or isinstance(value, bool): + raise ValueError("slot must be an integer from 1 through 10") + if isinstance(value, float) and not value.is_integer(): + raise ValueError("slot must be an integer from 1 through 10") + try: + slot = int(value) + except (TypeError, ValueError, OverflowError) as exc: + raise ValueError("slot must be an integer from 1 through 10") from exc + if slot not in SLOT_IDS: + raise ValueError("slot must be an integer from 1 through 10") + return slot + + def _handle_control_command(self, payload: Any) -> dict[str, Any]: + if not isinstance(payload, Mapping): + raise ValueError("Controller command must be a JSON object") + action = payload.get("action") + if not isinstance(action, str) or not action: + raise ValueError("Controller command is missing a string action") + if action == "play": + return self._request_play(payload) + if action in {"stop", "pause", "resume", "toggle"}: + return self._request_player_control(action) + if action == "delete_slot": + return self._request_delete(payload) + if action == "reconcile_playback": + return self._request_reconciliation(payload) + raise ValueError(f"unknown Controller action {action!r}") + + def _request_play(self, payload: Mapping[str, Any]) -> dict[str, Any]: + assert self.playback is not None + assert self.router is not None + if self._pending_play is not None or self.playback.pending is not None: + raise ValueError("another Player transition is still pending") + observation = self.store.player_observation() + if ( + observation.phase != "idle" + and ( + observation.active_slot is not None + or observation.active_path is not None + ) + ): + raise ValueError( + f"Player state is {observation.phase} for " + f"{observation.active_path or f'slot {observation.active_slot}'}; stop first" + ) + if self.store.list_live_guards(): + raise SlotProtectedError( + "playback protection is unresolved; reconcile it before starting another slot" + ) + + slot = ( + self._parse_slot(payload.get("slot")) + if payload.get("slot") is not None + else None + ) + expected_path: str | None = None + command: dict[str, Any] = {"action": "play"} + + path_value = payload.get("path") + playlist_value = payload.get("playlist") + explicit_key: str | None = None + explicit_value: Any = None + if path_value is not None: + explicit_key, explicit_value = "path", path_value + elif playlist_value is not None: + explicit_key, explicit_value = "playlist", playlist_value + if explicit_key is not None: + if not isinstance(explicit_value, str) or not explicit_value: + raise ValueError(f"play {explicit_key} must be a non-empty string") + expected_path = os.path.abspath(explicit_value) + if not Path(expected_path).is_file(): + raise ValueError(f"playback path is missing: {expected_path}") + command[explicit_key] = expected_path + + if slot is not None: + record = self.store.get_slot(slot) + if record.media_state != "ready": + raise ValueError(f"slot {slot} is {record.media_state}, not ready") + if expected_path is not None: + path_slot = self._managed_slot_for_path(expected_path) + if path_slot is not None and path_slot != slot: + raise ValueError( + f"slot {slot} conflicts with managed playback path for slot {path_slot}" + ) + if expected_path is None: + playlist = canonical_playlist_path( + self.config.storage.media_base, slot + ) + if not playlist.is_file(): + raise ValueError(f"slot {slot} playlist is missing: {playlist}") + expected_path = os.path.abspath(playlist) + command["slot"] = slot + label = payload.get("label") or record.title + if isinstance(label, str) and label: + command["label"] = label + elif expected_path is not None: + derived_slot = self._managed_slot_for_path(expected_path) + if derived_slot is not None: + record = self.store.get_slot(derived_slot) + if record.media_state != "ready": + raise ValueError( + f"slot {derived_slot} is {record.media_state}, not ready" + ) + slot = derived_slot + command["slot"] = slot + label = payload.get("label") or payload.get("title") + if isinstance(label, str) and label: + command["label"] = label + else: + raise ValueError("play command requires slot, path, or playlist") + + intent_id = uuid.uuid4().hex + if slot is not None: + self.store.protect_slot( + slot, + "pending_play", + guard_id=f"router-{intent_id}", + details={"intent_id": intent_id}, + ) + self._pending_play = _PendingPlay( + intent_id=intent_id, + command=command, + slot=slot, + expected_path=expected_path, + ) + try: + decision = self.router.acquire(intent_id) + except BaseException: + if slot is not None: + self.store.clear_playback_guards( + slot=slot, + kind="pending_play", + reason="Audio Router acquisition failed", + ) + self._pending_play = None + raise + self._pending_play = replace( + self._pending_play, + lease_id=decision.lease_id, + ) + self._handle_router_decision(decision) + if decision.action == "rejected": + return { + "status": "rejected", + "operation_id": intent_id, + "message": decision.warning, + } + if decision.action == "ready": + pending = self.playback.pending + if pending is not None and pending.operation_id == intent_id: + return { + "status": "pending", + "operation_id": intent_id, + "router": decision.action, + "warning": decision.warning, + } + completed = self.playback.last_operation + if completed is not None and completed.operation_id == intent_id: + return { + "status": ( + "unknown" if completed.state == "uncertain" else "unavailable" + ), + "operation_id": intent_id, + "router": decision.action, + "warning": completed.message or decision.warning, + } + return { + "status": "pending", + "operation_id": intent_id, + "router": decision.action, + "warning": decision.warning, + } + + def _managed_slot_for_path(self, path: str) -> int | None: + candidate = os.path.abspath(path) + for slot in SLOT_IDS: + directory = os.path.abspath( + self.config.storage.media_base / f"slot{slot}" + ) + try: + if os.path.commonpath((candidate, directory)) == directory: + return slot + except ValueError: + continue + return None + + def _publish_pending_play(self) -> None: + intent = self._pending_play + if intent is None: + return + assert self.playback is not None + assert self.router is not None + try: + operation = self.playback.begin_operation( + "play", + slot=intent.slot, + expected_path=intent.expected_path, + operation_id=intent.intent_id, + ) + except OperationInProgressError as exc: + self.router.abandon(intent.lease_id, str(exc)) + if intent.slot is not None: + self.store.clear_playback_guards( + slot=intent.slot, + kind="pending_play", + reason="Player operation could not begin", + ) + self._pending_play = None + return + receipt = self.mqtt.publish_player_command(intent.command) + self._pending_play = None + if not receipt.accepted: + reason = receipt.reason or "Player command publish failed" + if receipt.uncertain: + self._session_lease_id = intent.lease_id + effect = self.playback.publication_uncertain(reason) + else: + effect = self.playback.publish_failed(reason) + if effect is not None: + self._handle_playback_effects([effect]) + if not receipt.uncertain: + self._handle_router_decision(self.router.release(intent.lease_id)) + return + self._session_lease_id = intent.lease_id + self._publish_event( + "player_command_pending", + "info", + "Play command was published; awaiting authoritative Player state", + slot=intent.slot, + operation_id=operation.operation_id, + ) + + def _request_player_control(self, action: str) -> dict[str, Any]: + assert self.playback is not None + if self._pending_play is not None or self.playback.pending is not None: + raise ValueError("another Player transition is still pending") + observation = self.store.player_observation() + if ( + action != "stop" + and observation.active_slot is None + and observation.active_path is None + ): + raise ValueError(f"nothing is known to be playing for {action}") + player_action = action + if action == "toggle": + if observation.paused is None: + raise ValueError( + "toggle requires authoritative pause telemetry; use pause or resume" + ) + player_action = "resume" if observation.paused else "pause" + operation = self.playback.begin_operation( + player_action, + slot=observation.active_slot, + expected_path=observation.active_path, + ) + receipt = self.mqtt.publish_player_command({"action": player_action}) + if not receipt.accepted: + reason = receipt.reason or "Player command publish failed" + effects: list[PlaybackEffect] = [] + if receipt.uncertain: + effects.append(self.playback.mark_transport_uncertain(reason)) + uncertain = self.playback.publication_uncertain(reason) + if uncertain is not None: + effects.append(uncertain) + else: + failed = self.playback.publish_failed(reason) + if failed is not None: + effects.append(failed) + self._handle_playback_effects(effects) + if receipt.uncertain: + return { + "status": "unknown", + "operation_id": operation.operation_id, + "requested_action": action, + "player_action": player_action, + "warning": reason, + } + raise ControllerUnavailableError(reason) + self._publish_event( + "player_command_pending", + "info", + f"{player_action} command was published; awaiting authoritative Player state", + slot=observation.active_slot, + operation_id=operation.operation_id, + ) + result = {"status": "pending", "operation_id": operation.operation_id} + if action != player_action: + result.update( + {"requested_action": action, "player_action": player_action} + ) + return result + + def _request_delete(self, payload: Mapping[str, Any]) -> dict[str, Any]: + assert self.downloads is not None + slot = self._parse_slot(payload.get("slot")) + expected_url = payload.get("url") + if expected_url is not None and not isinstance(expected_url, str): + raise ValueError("delete URL guard must be a string") + self.downloads.delete_slot( + slot, + reason="manual", + expected_url=expected_url, + ) + self._announce_slot(slot, force=True) + return {"status": "deleted", "slot": slot} + + def _request_reconciliation(self, payload: Mapping[str, Any]) -> dict[str, Any]: + if payload.get("decision") != "clear" or payload.get("confirm") is not True: + raise ValueError("reconciliation clear requires decision=clear and confirm=true") + slot = ( + self._parse_slot(payload.get("slot")) + if payload.get("slot") is not None + else None + ) + review_id = payload.get("review_id") + actor = payload.get("actor") + reason = payload.get("reason") + if not all(isinstance(value, str) and value.strip() for value in (review_id, actor, reason)): + raise ValueError("reconciliation requires review_id, actor, and reason") + assert self.playback is not None + if self._pending_play is not None or self.playback.pending is not None: + raise ValueError( + "playback reconciliation cannot be cleared while a Player operation is pending" + ) + if slot is None: + cleared = self.store.clear_external_reconciliation( + review_id.strip(), + actor=actor.strip(), + reason=reason.strip(), + ) + else: + guard = next( + ( + candidate + for candidate in self.store.list_live_guards() + if candidate.slot == slot + ), + None, + ) + if guard is None or guard.kind not in { + "restart_uncertain", + "legacy_uncertain", + }: + raise ValueError(f"slot {slot} has no uncertain playback review") + cleared = self.store.clear_guard( + slot, + review_id.strip(), + actor=actor.strip(), + reason=reason.strip(), + ) + if not cleared: + raise StaleOperationError("reconciliation review_id is stale") + assert self.router is not None + self.router.abandon( + self._session_lease_id, + "Administrative playback reconciliation cleared uncertain ownership", + ) + self._session_lease_id = None + if slot is not None: + self._announce_slot(slot, force=True) + self._publish_event( + "playback_reconciled", + "warning", + f"Operator cleared uncertain playback protection: {reason.strip()}", + slot=slot, + operation_id=review_id.strip(), + ) + result: dict[str, Any] = { + "status": "cleared", + "review_id": review_id.strip(), + } + if slot is not None: + result["slot"] = slot + return result + + def _handle_router_decision(self, decision: RouterDecision) -> None: + if decision.warning: + self._publish_event( + "audio_router_warning", + "warning" if decision.action != "rejected" else "error", + decision.warning, + slot=(None if self._pending_play is None else self._pending_play.slot), + operation_id=( + None if self._pending_play is None else self._pending_play.intent_id + ), + continued=decision.continued, + ) + if decision.action == "ready" and self._pending_play is not None: + if decision.lease_id == self._pending_play.lease_id: + self._publish_pending_play() + elif decision.action == "rejected" and self._pending_play is not None: + intent = self._pending_play + if intent.slot is not None: + self.store.clear_playback_guards( + slot=intent.slot, + kind="pending_play", + reason="Audio Router policy rejected playback", + ) + self._pending_play = None + + def _handle_playback_effects(self, effects: list[PlaybackEffect]) -> None: + assert self.router is not None + terminal = any(effect.kind == "player_idle" for effect in effects) + failed_play = ( + any(effect.kind == "operation_failed" for effect in effects) + and self.playback is not None + and self.playback.last_operation is not None + and self.playback.last_operation.action == "play" + ) + observation = self.store.player_observation() + no_active_player = ( + observation.active_slot is None and observation.active_path is None + ) + if terminal or (failed_play and no_active_player): + self._handle_router_decision(self.router.release(self._session_lease_id)) + self._session_lease_id = None + for effect in effects: + if effect.kind in { + "warning", + "player_error", + "player_offline", + "transport_uncertain", + "operation_failed", + "operation_uncertain", + "operation_timed_out", + "operator_review", + }: + self._publish_event( + effect.kind, + "warning" if effect.kind != "player_error" else "error", + effect.message or effect.kind, + slot=effect.slot, + operation_id=effect.operation_id, + ) + elif effect.kind == "operation_confirmed": + self._publish_event( + "player_operation_confirmed", + "info", + "Player confirmed the requested operation", + slot=effect.slot, + operation_id=effect.operation_id, + ) + self._sync_slot_states() + + def _handle_download_event(self, event: DownloadEvent) -> None: + payload: dict[str, Any] = {"slot": event.slot, "state": event.state} + for key in ("title", "count", "message", "url", "submitter", "note"): + value = getattr(event, key) + if value is not None: + payload[key] = value + if event.state == "queued": + payload["reused_slot"] = event.reused_slot + self.mqtt.publish_json( + self.config.topics.ingest_progress, + payload, + qos=0, + retain=False, + ) + self._announce_slot(event.slot) + + def _announce_discovery(self) -> None: + if not self.config.homeassistant.enabled: + return + for topic, payload, _, retain in discovery_messages( + self.config.topics, + self.config.homeassistant, + ): + self.mqtt.publish_legacy(topic, payload, retain=retain) + + def _announce_reconciliation_reviews(self) -> None: + """Re-announce durable review tokens after each broker connection.""" + + for guard in self.store.list_live_guards(): + if guard.kind not in {"restart_uncertain", "legacy_uncertain"}: + continue + self._publish_event( + "playback_reconciliation_required", + "warning", + ( + f"Slot {guard.slot} playback is uncertain and protected " + f"(review_id={guard.guard_id}); fresh Player evidence or an " + "explicit administrative clear is required" + ), + slot=guard.slot, + operation_id=guard.guard_id, + ) + observation = self.store.player_observation() + if ( + observation.active_slot is None + and observation.active_path is not None + and observation.reconciliation_id is not None + and observation.reconciliation_state + in {"awaiting_evidence", "operator_review"} + ): + self._publish_event( + "playback_reconciliation_required", + "warning", + ( + f"Playback at {observation.active_path!r} is uncertain and " + f"protected (review_id={observation.reconciliation_id}); fresh " + "Player evidence or an explicit administrative clear is required" + ), + operation_id=observation.reconciliation_id, + ) + + def _sync_slot_states(self, *, force: bool = False) -> None: + for slot in SLOT_IDS: + self._announce_slot(slot, force=force) + + def _announce_slot(self, slot: int, *, force: bool = False) -> None: + assert self.playback is not None + record = self.store.get_slot(slot) + item_records = self.store.get_slot_items(slot) + item_titles = tuple(title for title, _ in item_records) + effective_state = self.playback.projected_slot_state(slot) + signature = (record.title, effective_state, record.item_count, item_titles) + if not force and self._last_announced.get(slot) == signature: + return + authoritative_items = ( + record.media_state == "empty" or len(item_titles) == record.item_count + ) + clear_through = ( + max(record.published_item_count, len(item_titles)) + if authoritative_items + else 0 + ) + all_accepted = True + for topic, payload, _, retain in slot_messages( + SlotAnnouncement( + slot=slot, + title=record.title, + state=effective_state, + count=record.item_count, + items=item_titles, + ), + self.config.topics, + clear_items_through=clear_through, + ): + receipt = self.mqtt.publish_legacy(topic, payload, retain=retain) + all_accepted = all_accepted and receipt.accepted + if all_accepted: + self._last_announced[slot] = signature + if authoritative_items: + self.store.set_published_item_count(slot, len(item_titles)) + + def _publish_event( + self, + event_type: str, + severity: str, + message: str, + *, + slot: int | None = None, + operation_id: str | None = None, + continued: bool | None = None, + ) -> None: + event_id = uuid.uuid4().hex + payload: dict[str, Any] = { + "version": 2, + "event_id": event_id, + "type": event_type, + "severity": severity, + "message": message, + "ts": self._wall_clock(), + } + if slot is not None: + payload["slot"] = slot + if operation_id is not None: + payload["operation_id"] = operation_id + if continued is not None: + payload["continued"] = continued + self.store.record_audit_event( + event_type, + slot=slot, + actor="controller", + details=payload, + ) + if severity in {"warning", "error"}: + LOG.warning("%s: %s", event_type, message) + else: + LOG.info("%s: %s", event_type, message) + topic = self.config.topics.controller_result + if topic is not None: + self.mqtt.publish_json(topic, payload, qos=1, retain=False) + + def _run_timers(self) -> None: + assert self.playback is not None + assert self.router is not None + assert self.downloads is not None + playback_timeout = self.playback.check_timeout() + if playback_timeout is not None: + self._handle_playback_effects([playback_timeout]) + router_timeout = self.router.check_deadline() + if router_timeout is not None: + self._handle_router_decision(router_timeout) + + now = self._monotonic() + if now >= self._next_reconciliation: + self._handle_playback_effects(self.playback.mark_due_reconciliation()) + self._next_reconciliation = now + 1.0 + if now >= self._next_expiry: + for slot in self.downloads.expire_due_slots(): + self._announce_slot(slot, force=True) + self._next_expiry = now + self.config.storage.expiry_interval_seconds diff --git a/controller/store.py b/controller/store.py index 00c13e0..e315b20 100644 --- a/controller/store.py +++ b/controller/store.py @@ -4,6 +4,7 @@ from __future__ import annotations import contextlib import datetime as dt +import fcntl import hashlib import json import os @@ -20,12 +21,14 @@ import yaml from .config import StorageConfig from .contracts import SLOT_IDS from .models import ( + AudioLeaseRecord, DeleteToken, DownloadJob, DownloadReservation, MigrationError, MigrationReport, PlayerObservation, + RouterObservation, SlotGuard, SlotProtectedError, SlotRecord, @@ -33,7 +36,7 @@ from .models import ( ) -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 LEGACY_MIGRATION_NAME = "legacy_playlists_yaml_v1" MAX_LEGACY_YAML_BYTES = 8 * 1024 * 1024 @@ -130,6 +133,7 @@ CREATE TABLE IF NOT EXISTS player_observation ( reconciliation_state TEXT NOT NULL CHECK (reconciliation_state IN ('clear','confirmed','awaiting_evidence','operator_review','admin_cleared','unknown')), reconciliation_due_at REAL, + reconciliation_id TEXT, last_status_at REAL, last_telemetry_at REAL, last_status_json TEXT, @@ -284,14 +288,29 @@ class ControllerStore: if self._connection is not None: return self self.config.data_base.mkdir(parents=True, exist_ok=True) - if not self.path.exists(): - if self.legacy_path.exists(): - self.migration_report = self._migrate_legacy_yaml() - else: - self._create_new_database() - self._connection = self._connect(self.path) - self._validate_database(self._connection) - self._recover_interrupted_jobs() + lock_path = self.config.data_base / ".controller-store.lock" + with lock_path.open("a+b") as lock_file: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + try: + # Recheck only after taking the process lock. This prevents + # two WSGI processes from both staging and replacing the + # authoritative database during first boot or migration. + if not self.path.exists(): + if self.legacy_path.exists(): + self.migration_report = self._migrate_legacy_yaml() + else: + self._create_new_database() + self._connection = self._connect(self.path) + self._upgrade_database(self._connection) + self._validate_database(self._connection) + self._recover_interrupted_jobs() + except BaseException: + if self._connection is not None: + self._connection.close() + self._connection = None + raise + finally: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) return self def close(self) -> None: @@ -371,12 +390,55 @@ class ControllerStore: connection.execute( """ INSERT OR IGNORE INTO schema_migrations(version, name, applied_at) - VALUES (?, ?, ?) + VALUES (1, 'controller_v2_initial', ?) """, - (SCHEMA_VERSION, "controller_v2_initial", now), + (now,), + ) + connection.execute( + """ + INSERT OR IGNORE INTO schema_migrations(version, name, applied_at) + VALUES (2, 'external_playback_reconciliation', ?) + """, + (now,), ) connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + def _upgrade_database(self, connection: sqlite3.Connection) -> None: + """Apply bounded internal schema upgrades under the process lock.""" + + version = connection.execute("PRAGMA user_version").fetchone()[0] + if version == SCHEMA_VERSION: + return + if version != 1: + raise MigrationError( + f"unsupported Controller database schema {version}; expected 1 or {SCHEMA_VERSION}" + ) + connection.execute("BEGIN IMMEDIATE") + try: + columns = { + row["name"] + for row in connection.execute( + "PRAGMA table_info(player_observation)" + ).fetchall() + } + if "reconciliation_id" not in columns: + connection.execute( + "ALTER TABLE player_observation ADD COLUMN reconciliation_id TEXT" + ) + connection.execute( + """ + INSERT OR IGNORE INTO schema_migrations(version, name, applied_at) + VALUES (2, 'external_playback_reconciliation', ?) + """, + (self._now(),), + ) + connection.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + except BaseException: + connection.rollback() + raise + else: + connection.commit() + def _create_new_database(self) -> None: staging = self._staging_path() try: @@ -511,9 +573,45 @@ class ControllerStore: return normalized, [] def _migrate_legacy_yaml(self) -> MigrationReport: - source = self.legacy_path.read_bytes() + before = self.legacy_path.stat() + with self.legacy_path.open("rb") as stream: + opened = os.fstat(stream.fileno()) + source = stream.read(MAX_LEGACY_YAML_BYTES + 1) + closed = os.fstat(stream.fileno()) if len(source) > MAX_LEGACY_YAML_BYTES: raise MigrationError("legacy YAML exceeds migration size limit") + after = self.legacy_path.stat() + identity_before = ( + before.st_dev, + before.st_ino, + before.st_size, + before.st_mtime_ns, + ) + identity_opened = ( + opened.st_dev, + opened.st_ino, + opened.st_size, + opened.st_mtime_ns, + ) + identity_closed = ( + closed.st_dev, + closed.st_ino, + closed.st_size, + closed.st_mtime_ns, + ) + identity_after = ( + after.st_dev, + after.st_ino, + after.st_size, + after.st_mtime_ns, + ) + if not ( + identity_before + == identity_opened + == identity_closed + == identity_after + ): + raise MigrationError("legacy YAML changed while it was being read") digest = hashlib.sha256(source).hexdigest() backup = self._create_backup(source, digest) slots, warnings = self._parse_legacy(source) @@ -1068,43 +1166,378 @@ class ControllerStore: review_after: float | None = None, details: Mapping[str, Any] | None = None, ) -> SlotGuard: - """Replace all uncertain/pending guards with one authoritative guard.""" + """Reconcile guards to one idempotent authoritative guard. + + A pending play for another slot is preserved until its in-memory + operation is explicitly failed. That closes the brief deletion window + that otherwise exists between contradictory evidence and reduction of + the pending operation. Repeated status/telemetry for the same slot + updates the live row in place instead of appending guard history. + """ if slot not in SLOT_IDS: raise ValueError("slot must be in 1..10") with self._transaction() as connection: now = self._now() + existing = connection.execute( + """ + SELECT * FROM slot_guards + WHERE slot_number = ? AND state IN ('active','review_required') + """, + (slot,), + ).fetchone() connection.execute( """ UPDATE slot_guards SET state = 'cleared', cleared_at = ?, cleared_by = 'player', clear_reason = 'reconciled by fresh active evidence' WHERE state IN ('active','review_required') + AND slot_number <> ? AND kind <> 'pending_play' """, - (now,), - ) - guard_id = uuid.uuid4().hex - connection.execute( - """ - INSERT INTO slot_guards - (guard_id, slot_number, kind, state, created_at, - review_after, details_json) - VALUES (?, ?, ?, 'active', ?, ?, ?) - """, - ( - guard_id, - slot, - kind, - now, - review_after, - json.dumps(details or {}, sort_keys=True), - ), + (now, slot), ) + encoded = json.dumps(details or {}, sort_keys=True) + if existing is None: + guard_id = uuid.uuid4().hex + connection.execute( + """ + INSERT INTO slot_guards + (guard_id, slot_number, kind, state, created_at, + review_after, details_json) + VALUES (?, ?, ?, 'active', ?, ?, ?) + """, + (guard_id, slot, kind, now, review_after, encoded), + ) + else: + guard_id = existing["guard_id"] + connection.execute( + """ + UPDATE slot_guards + SET kind = ?, state = 'active', review_after = ?, + cleared_at = NULL, cleared_by = NULL, + clear_reason = NULL, details_json = ? + WHERE guard_id = ? + """, + (kind, review_after, encoded, guard_id), + ) row = connection.execute( "SELECT * FROM slot_guards WHERE guard_id = ?", (guard_id,) ).fetchone() assert row is not None return self._row_to_guard(row) + def mark_external_uncertain( + self, + path: str, + *, + review_after: float | None, + availability: str = "unknown", + details: Mapping[str, Any] | None = None, + status_payload: Mapping[str, Any] | None = None, + ) -> PlayerObservation: + """Persist an exact review token for path-only playback uncertainty.""" + + if not path: + raise ValueError("external playback reconciliation requires a path") + now = self._now() + with self._transaction() as connection: + row = connection.execute( + "SELECT * FROM player_observation WHERE singleton = 1" + ).fetchone() + assert row is not None + same_review = ( + row["active_slot"] is None + and row["active_path"] == path + and row["reconciliation_id"] is not None + and row["reconciliation_state"] + in {"awaiting_evidence", "operator_review"} + ) + review_id = row["reconciliation_id"] if same_review else uuid.uuid4().hex + deadline = row["reconciliation_due_at"] if same_review else None + if deadline is None: + deadline = review_after + elif review_after is not None: + deadline = min(deadline, review_after) + state = ( + "operator_review" + if same_review and row["reconciliation_state"] == "operator_review" + else "awaiting_evidence" + ) + connection.execute( + """ + UPDATE player_observation + SET availability = ?, phase = 'unknown', active_slot = NULL, + active_path = ?, paused = NULL, generation = generation + 1, + reconciliation_state = ?, reconciliation_due_at = ?, + reconciliation_id = ?, + last_status_at = CASE WHEN ? IS NULL THEN last_status_at ELSE ? END, + last_status_json = COALESCE(?, last_status_json) + WHERE singleton = 1 + """, + ( + availability, + path, + state, + deadline, + review_id, + None if status_payload is None else 1, + now, + None + if status_payload is None + else json.dumps(status_payload, sort_keys=True), + ), + ) + if not same_review: + connection.execute( + """ + INSERT INTO audit_events + (occurred_at, event_type, actor, details_json) + VALUES (?, 'external_playback_uncertain', 'controller', ?) + """, + ( + now, + json.dumps( + { + "path": path, + "review_id": review_id, + **dict(details or {}), + }, + sort_keys=True, + ), + ), + ) + return self.player_observation() + + def mark_external_reconciliation_due( + self, now: float | None = None + ) -> tuple[str, str] | None: + """Move path-only uncertainty to operator review without clearing it.""" + + now = self._now() if now is None else now + with self._transaction() as connection: + row = connection.execute( + """ + SELECT * FROM player_observation + WHERE singleton = 1 AND active_slot IS NULL + AND active_path IS NOT NULL AND reconciliation_id IS NOT NULL + AND reconciliation_state = 'awaiting_evidence' + AND reconciliation_due_at IS NOT NULL + AND reconciliation_due_at <= ? + """, + (now,), + ).fetchone() + if row is None: + return None + connection.execute( + """ + UPDATE player_observation + SET phase = 'unknown', reconciliation_state = 'operator_review', + generation = generation + 1 + WHERE singleton = 1 + """ + ) + connection.execute( + """ + INSERT INTO audit_events + (occurred_at, event_type, actor, details_json) + VALUES (?, 'external_playback_operator_review', 'controller', ?) + """, + ( + now, + json.dumps( + { + "path": row["active_path"], + "review_id": row["reconciliation_id"], + }, + sort_keys=True, + ), + ), + ) + return row["reconciliation_id"], row["active_path"] + + def clear_external_reconciliation( + self, + review_id: str, + *, + actor: str, + reason: str, + ) -> bool: + """Administratively clear one exact path-only uncertainty token.""" + + if not actor.strip() or not reason.strip(): + raise ValueError("administrative reconciliation requires actor and reason") + with self._transaction() as connection: + row = connection.execute( + """ + SELECT * FROM player_observation + WHERE singleton = 1 AND active_slot IS NULL + AND active_path IS NOT NULL AND reconciliation_id = ? + AND reconciliation_state IN ('awaiting_evidence','operator_review') + """, + (review_id,), + ).fetchone() + if row is None: + return False + now = self._now() + connection.execute( + """ + UPDATE player_observation + SET phase = 'unknown', active_path = NULL, paused = NULL, + generation = generation + 1, + reconciliation_state = 'admin_cleared', + reconciliation_due_at = NULL, reconciliation_id = NULL + WHERE singleton = 1 + """ + ) + connection.execute( + """ + INSERT INTO audit_events + (occurred_at, event_type, actor, details_json) + VALUES (?, 'external_playback_admin_cleared', ?, ?) + """, + ( + now, + actor.strip(), + json.dumps( + { + "path": row["active_path"], + "reason": reason.strip(), + "review_id": review_id, + }, + sort_keys=True, + ), + ), + ) + return True + + def cancel_external_intent(self, review_id: str, *, reason: str) -> bool: + """Clear one pre-evidence external intent after a definite failure.""" + + if not reason.strip(): + raise ValueError("external intent cancellation requires a reason") + with self._transaction() as connection: + row = connection.execute( + """ + SELECT * FROM player_observation + WHERE singleton = 1 AND active_slot IS NULL + AND active_path IS NOT NULL AND reconciliation_id = ? + AND reconciliation_state = 'awaiting_evidence' + """, + (review_id,), + ).fetchone() + if row is None: + return False + now = self._now() + connection.execute( + """ + UPDATE player_observation + SET phase = 'unknown', active_path = NULL, paused = NULL, + generation = generation + 1, + reconciliation_state = 'clear', + reconciliation_due_at = NULL, reconciliation_id = NULL + WHERE singleton = 1 + """ + ) + connection.execute( + """ + INSERT INTO audit_events + (occurred_at, event_type, actor, details_json) + VALUES (?, 'external_playback_intent_cancelled', 'controller', ?) + """, + ( + now, + json.dumps( + { + "path": row["active_path"], + "reason": reason.strip(), + "review_id": review_id, + }, + sort_keys=True, + ), + ), + ) + return True + + def reconcile_external_active(self) -> None: + """Clear stale slot ownership when fresh evidence is path-only.""" + + with self._transaction() as connection: + now = self._now() + connection.execute( + """ + UPDATE slot_guards + SET state = 'cleared', cleared_at = ?, cleared_by = 'player', + clear_reason = 'fresh Player evidence is outside managed slots' + WHERE state IN ('active','review_required') + AND kind <> 'pending_play' + """, + (now,), + ) + + def mark_slot_uncertain( + self, + slot: int, + *, + review_after: float | None, + details: Mapping[str, Any] | None = None, + ) -> SlotGuard: + """Conservatively convert one live guard to restart uncertainty. + + Other guarded slots are untouched and an existing review deadline is + never moved later by duplicate offline/disconnect evidence. + """ + + if slot not in SLOT_IDS: + raise ValueError("slot must be in 1..10") + with self._transaction() as connection: + now = self._now() + row = connection.execute( + """ + SELECT * FROM slot_guards + WHERE slot_number = ? AND state IN ('active','review_required') + """, + (slot,), + ).fetchone() + encoded = json.dumps(details or {}, sort_keys=True) + if row is None: + guard_id = uuid.uuid4().hex + connection.execute( + """ + INSERT INTO slot_guards + (guard_id, slot_number, kind, state, created_at, + review_after, details_json) + VALUES (?, ?, 'restart_uncertain', 'active', ?, ?, ?) + """, + (guard_id, slot, now, review_after, encoded), + ) + else: + guard_id = row["guard_id"] + # Legacy uncertainty is already an explicit operator review; + # retain that identity/state. Otherwise make the old active or + # pending guard administratively reconcilable. + kind = ( + "legacy_uncertain" + if row["kind"] == "legacy_uncertain" + else "restart_uncertain" + ) + deadline = row["review_after"] + if deadline is None: + deadline = review_after + elif review_after is not None: + deadline = min(deadline, review_after) + connection.execute( + """ + UPDATE slot_guards + SET kind = ?, review_after = ?, details_json = ? + WHERE guard_id = ? + """, + (kind, deadline, encoded, guard_id), + ) + updated = connection.execute( + "SELECT * FROM slot_guards WHERE guard_id = ?", (guard_id,) + ).fetchone() + assert updated is not None + return self._row_to_guard(updated) + def clear_playback_guards( self, *, @@ -1136,19 +1569,32 @@ class ControllerStore: def mark_due_guards_for_review(self, now: float | None = None) -> list[SlotGuard]: now = self._now() if now is None else now with self._transaction() as connection: + due_rows = connection.execute( + """ + SELECT * FROM slot_guards + WHERE state = 'active' AND review_after IS NOT NULL + AND review_after <= ? + ORDER BY slot_number + """, + (now,), + ).fetchall() + if not due_rows: + return [] + guard_ids = [row["guard_id"] for row in due_rows] + placeholders = ",".join("?" for _ in guard_ids) connection.execute( """ UPDATE slot_guards SET state = 'review_required' - WHERE state = 'active' AND review_after IS NOT NULL - AND review_after <= ? - """, - (now,), + WHERE guard_id IN (""" + + placeholders + + ")", + guard_ids, ) rows = connection.execute( - """ - SELECT * FROM slot_guards - WHERE state = 'review_required' ORDER BY slot_number - """ + "SELECT * FROM slot_guards WHERE guard_id IN (" + + placeholders + + ") ORDER BY slot_number", + guard_ids, ).fetchall() return [self._row_to_guard(row) for row in rows] @@ -1181,9 +1627,12 @@ class ControllerStore: UPDATE player_observation SET phase = 'unknown', active_slot = NULL, active_path = NULL, paused = NULL, reconciliation_state = 'admin_cleared', - reconciliation_due_at = NULL - WHERE singleton = 1 - """ + reconciliation_due_at = NULL, reconciliation_id = NULL, + generation = generation + 1 + WHERE singleton = 1 AND active_slot = ? + AND (reconciliation_id IS NULL OR reconciliation_id = ?) + """, + (slot, guard_id), ) connection.execute( """ @@ -1316,6 +1765,7 @@ class ControllerStore: generation=row["generation"], reconciliation_state=row["reconciliation_state"], reconciliation_due_at=row["reconciliation_due_at"], + reconciliation_id=row["reconciliation_id"], last_status_at=row["last_status_at"], last_telemetry_at=row["last_telemetry_at"], ) @@ -1330,6 +1780,7 @@ class ControllerStore: paused: bool | None, reconciliation_state: str, reconciliation_due_at: float | None, + reconciliation_id: str | None, status_payload: Mapping[str, Any] | None = None, telemetry_payload: Mapping[str, Any] | None = None, ) -> PlayerObservation: @@ -1341,6 +1792,7 @@ class ControllerStore: SET availability = ?, phase = ?, active_slot = ?, active_path = ?, paused = ?, generation = generation + 1, reconciliation_state = ?, reconciliation_due_at = ?, + reconciliation_id = ?, last_status_at = CASE WHEN ? IS NULL THEN last_status_at ELSE ? END, last_telemetry_at = CASE WHEN ? IS NULL THEN last_telemetry_at ELSE ? END, last_status_json = COALESCE(?, last_status_json), @@ -1355,6 +1807,7 @@ class ControllerStore: None if paused is None else int(paused), reconciliation_state, reconciliation_due_at, + reconciliation_id, None if status_payload is None else 1, now, None if telemetry_payload is None else 1, @@ -1366,29 +1819,181 @@ class ControllerStore: return self.player_observation() def mark_restart_uncertain(self) -> SlotGuard | None: - observation = self.player_observation() - if observation.active_slot is None: - return None - due = None - if self.config.uncertain_review_after_seconds > 0: - due = self._now() + self.config.uncertain_review_after_seconds - guard = self.protect_slot( - observation.active_slot, - "restart_uncertain", - review_after=due, - details={"previous_phase": observation.phase}, - ) - due = guard.review_after - self.set_player_observation( - availability="unknown", - phase="unknown", - active_slot=observation.active_slot, - active_path=observation.active_path, - paused=None, - reconciliation_state="awaiting_evidence", - reconciliation_due_at=due, - ) - return guard + """Convert every pre-restart active/pending guard conservatively. + + The operation that owned a pending guard was process-local, so after a + restart both a prior active Player and a command awaiting evidence are + uncertain. Deadlines are filled only when absent and therefore cannot + be postponed by restart churn. + """ + + with self._transaction() as connection: + now = self._now() + requested_due = None + if self.config.uncertain_review_after_seconds > 0: + requested_due = now + self.config.uncertain_review_after_seconds + observation = connection.execute( + "SELECT * FROM player_observation WHERE singleton = 1" + ).fetchone() + assert observation is not None + rows = connection.execute( + """ + SELECT * FROM slot_guards + WHERE state IN ('active','review_required') + AND kind IN ( + 'player_active','pending_play','restart_uncertain','legacy_uncertain' + ) + ORDER BY created_at, slot_number + """ + ).fetchall() + + active_slot = observation["active_slot"] + active_path = observation["active_path"] + if active_slot is not None and not any( + row["slot_number"] == active_slot for row in rows + ): + guard_id = uuid.uuid4().hex + connection.execute( + """ + INSERT INTO slot_guards + (guard_id, slot_number, kind, state, created_at, + review_after, details_json) + VALUES (?, ?, 'restart_uncertain', 'active', ?, ?, ?) + """, + ( + guard_id, + active_slot, + now, + requested_due, + json.dumps( + {"previous_phase": observation["phase"]}, + sort_keys=True, + ), + ), + ) + rows = connection.execute( + "SELECT * FROM slot_guards WHERE guard_id = ?", (guard_id,) + ).fetchall() + rows + + converted: list[sqlite3.Row] = [] + for row in rows: + if row["kind"] == "legacy_uncertain": + converted.append(row) + continue + deadline = row["review_after"] + if deadline is None: + deadline = requested_due + connection.execute( + """ + UPDATE slot_guards + SET kind = 'restart_uncertain', review_after = ? + WHERE guard_id = ? + """, + (deadline, row["guard_id"]), + ) + converted_row = connection.execute( + "SELECT * FROM slot_guards WHERE guard_id = ?", + (row["guard_id"],), + ).fetchone() + assert converted_row is not None + converted.append(converted_row) + + if active_slot is None and active_path is None and len(converted) == 1: + active_slot = converted[0]["slot_number"] + active_path = None + active_guard = next( + ( + row + for row in converted + if row["slot_number"] == active_slot + ), + ( + converted[0] + if active_path is None and len(converted) == 1 + else None + ), + ) + external_review_id = None + external_due = None + external_state = None + if active_slot is None and active_path is not None: + existing_external = ( + observation["reconciliation_id"] is not None + and observation["reconciliation_state"] + in {"awaiting_evidence", "operator_review"} + ) + external_review_id = ( + observation["reconciliation_id"] + if existing_external + else uuid.uuid4().hex + ) + external_due = ( + observation["reconciliation_due_at"] + if existing_external + else requested_due + ) + external_state = ( + "operator_review" + if existing_external + and observation["reconciliation_state"] == "operator_review" + else "awaiting_evidence" + ) + + if active_slot is not None or active_path is not None or converted: + reconciliation_state = "awaiting_evidence" + reconciliation_id = None + reconciliation_due_at = None + if active_guard is not None: + reconciliation_id = active_guard["guard_id"] + reconciliation_due_at = active_guard["review_after"] + if active_guard["state"] == "review_required": + reconciliation_state = "operator_review" + elif external_review_id is not None: + reconciliation_id = external_review_id + reconciliation_due_at = external_due + reconciliation_state = external_state + connection.execute( + """ + UPDATE player_observation + SET availability = 'unknown', phase = 'unknown', + active_slot = ?, active_path = ?, paused = NULL, + generation = generation + 1, + reconciliation_state = ?, + reconciliation_due_at = ?, reconciliation_id = ? + WHERE singleton = 1 + """, + ( + active_slot, + active_path, + reconciliation_state, + reconciliation_due_at, + reconciliation_id, + ), + ) + if external_review_id is not None and not existing_external: + connection.execute( + """ + INSERT INTO audit_events + (occurred_at, event_type, actor, details_json) + VALUES (?, 'external_playback_uncertain', 'controller', ?) + """, + ( + now, + json.dumps( + { + "path": active_path, + "review_id": external_review_id, + "reason": "Controller restart", + }, + sort_keys=True, + ), + ), + ) + return ( + None + if active_guard is None + else self._row_to_guard(active_guard) + ) def set_published_item_count(self, slot: int, count: int) -> None: with self._transaction() as connection: @@ -1399,3 +2004,198 @@ class ControllerStore: """, (count, self._now(), slot), ) + + def router_observation(self) -> RouterObservation: + with self._lock: + row = self.connection.execute( + "SELECT * FROM audio_router_observation WHERE singleton = 1" + ).fetchone() + assert row is not None + return RouterObservation( + mode=row["mode"], + freshness=row["freshness"], + observed_at=row["observed_at"], + last_warning=row["last_warning"], + ) + + def set_router_observation( + self, + *, + mode: str | None, + freshness: str, + warning: str | None = None, + ) -> RouterObservation: + with self._transaction() as connection: + connection.execute( + """ + UPDATE audio_router_observation + SET mode = ?, freshness = ?, observed_at = ?, last_warning = ? + WHERE singleton = 1 + """, + (mode, freshness, self._now(), warning), + ) + return self.router_observation() + + @staticmethod + def _row_to_audio_lease(row: sqlite3.Row) -> AudioLeaseRecord: + return AudioLeaseRecord( + lease_id=row["lease_id"], + operation_id=row["operation_id"], + playback_generation=row["playback_generation"], + state=row["state"], + previous_mode=row["previous_mode"], + target_mode=row["target_mode"], + restoration_mode=row["restoration_mode"], + restoration_source=row["restoration_source"], + controller_changed=bool(row["controller_changed"]), + confirmation_deadline=row["confirmation_deadline"], + created_at=row["created_at"], + acquired_at=row["acquired_at"], + released_at=row["released_at"], + warning=row["warning"], + error=row["error"], + ) + + def create_audio_lease( + self, + *, + lease_id: str, + operation_id: str, + state: str, + previous_mode: str | None, + target_mode: str, + restoration_mode: str | None, + restoration_source: str, + controller_changed: bool, + confirmation_deadline: float | None, + warning: str | None = None, + error: str | None = None, + ) -> AudioLeaseRecord: + with self._transaction() as connection: + now = self._now() + connection.execute( + """ + INSERT INTO audio_leases + (lease_id, operation_id, state, previous_mode, target_mode, + restoration_mode, restoration_source, controller_changed, + confirmation_deadline, created_at, warning, error) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + lease_id, + operation_id, + state, + previous_mode, + target_mode, + restoration_mode, + restoration_source, + int(controller_changed), + confirmation_deadline, + now, + warning, + error, + ), + ) + row = connection.execute( + "SELECT * FROM audio_leases WHERE lease_id = ?", (lease_id,) + ).fetchone() + assert row is not None + return self._row_to_audio_lease(row) + + def update_audio_lease( + self, + lease_id: str, + *, + state: str, + confirmation_deadline: float | None = None, + warning: str | None = None, + error: str | None = None, + acquired: bool = False, + released: bool = False, + ) -> AudioLeaseRecord | None: + with self._transaction() as connection: + now = self._now() + updated = connection.execute( + """ + UPDATE audio_leases + SET state = ?, confirmation_deadline = ?, + warning = COALESCE(?, warning), error = COALESCE(?, error), + acquired_at = CASE WHEN ? THEN COALESCE(acquired_at, ?) ELSE acquired_at END, + released_at = CASE WHEN ? THEN ? ELSE released_at END + WHERE lease_id = ? + """, + ( + state, + confirmation_deadline, + warning, + error, + int(acquired), + now, + int(released), + now, + lease_id, + ), + ).rowcount + if not updated: + return None + row = connection.execute( + "SELECT * FROM audio_leases WHERE lease_id = ?", (lease_id,) + ).fetchone() + assert row is not None + return self._row_to_audio_lease(row) + + def get_audio_lease(self, lease_id: str) -> AudioLeaseRecord | None: + with self._lock: + row = self.connection.execute( + "SELECT * FROM audio_leases WHERE lease_id = ?", (lease_id,) + ).fetchone() + return None if row is None else self._row_to_audio_lease(row) + + def live_audio_lease(self) -> AudioLeaseRecord | None: + with self._lock: + row = self.connection.execute( + """ + SELECT * FROM audio_leases + WHERE state IN ('acquiring','held','unconfirmed','restore_pending') + ORDER BY created_at DESC LIMIT 1 + """ + ).fetchone() + return None if row is None else self._row_to_audio_lease(row) + + def abandon_audio_leases_on_restart(self) -> int: + with self._transaction() as connection: + now = self._now() + return connection.execute( + """ + UPDATE audio_leases + SET state = 'abandoned', released_at = ?, + warning = 'Controller restarted; audio ownership was relinquished' + WHERE state IN ('acquiring','held','unconfirmed','restore_pending') + """, + (now,), + ).rowcount + + def record_audit_event( + self, + event_type: str, + *, + slot: int | None = None, + actor: str | None = None, + details: Mapping[str, Any] | None = None, + ) -> int: + with self._transaction() as connection: + cursor = connection.execute( + """ + INSERT INTO audit_events + (occurred_at, event_type, slot_number, actor, details_json) + VALUES (?, ?, ?, ?, ?) + """, + ( + self._now(), + event_type, + slot, + actor, + json.dumps(details or {}, sort_keys=True), + ), + ) + return int(cursor.lastrowid) diff --git a/tests/test_config_contracts.py b/tests/test_config_contracts.py index 2fa7e43..b061202 100644 --- a/tests/test_config_contracts.py +++ b/tests/test_config_contracts.py @@ -29,6 +29,7 @@ class ContractCharacterizationTests(unittest.TestCase): self.assertEqual(topics.playback_base, "youtube/media/playback") self.assertEqual(topics.router_mode_set, "audio/player/mode/set") self.assertEqual(topics.router_mode_state, "audio/player/mode/state") + self.assertEqual(topics.controller_result, "youtube/controller/v2/events") self.assertEqual(PLAYER_COMMAND_QOS, 1) self.assertIs(PLAYER_COMMAND_RETAIN, False) self.assertEqual(SLOT_IDS, tuple(range(1, 11))) @@ -79,6 +80,7 @@ class ConfigurationTests(unittest.TestCase): self.assertIs(config.router.continue_on_confirmation_failure, True) self.assertIsNone(config.router.fallback_restore_mode) self.assertEqual(config.storage.uncertain_review_after_seconds, 900.0) + self.assertEqual(config.mqtt.publish_ack_timeout_seconds, 2.0) def test_documented_environment_overrides_apply(self): env = { diff --git a/tests/test_downloads.py b/tests/test_downloads.py index fc73b6e..859cf9e 100644 --- a/tests/test_downloads.py +++ b/tests/test_downloads.py @@ -1,6 +1,8 @@ import os import subprocess import tempfile +import threading +import time import unittest from concurrent.futures import Future from pathlib import Path @@ -194,6 +196,42 @@ class DownloadManagerTests(unittest.TestCase): self.assertFalse(slot.exists()) self.assertEqual(self.store.get_slot(1).media_state, "empty") + def test_shutdown_terminates_tracked_process_without_waiting_for_job_timeout(self): + manager = DownloadManager(self.config, self.store) + errors = [] + + def run_long_process(): + try: + manager._run_command( + ["/bin/sh", "-c", "sleep 60"], + check=True, + text=True, + timeout=60, + capture_output=True, + ) + except BaseException as exc: + errors.append(exc) + + worker = threading.Thread(target=run_long_process) + worker.start() + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + with manager._lock: + if manager._processes: + break + time.sleep(0.01) + else: + self.fail("test process did not start") + + started = time.monotonic() + manager.shutdown(wait=False) + worker.join(timeout=3) + self.assertFalse(worker.is_alive()) + self.assertLess(time.monotonic() - started, 3) + self.assertTrue(errors) + with manager._lock: + self.assertEqual(manager._processes, set()) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_mqtt_adapter.py b/tests/test_mqtt_adapter.py index 45fbcc8..a84e981 100644 --- a/tests/test_mqtt_adapter.py +++ b/tests/test_mqtt_adapter.py @@ -1,5 +1,6 @@ import json import queue +import threading import unittest from types import SimpleNamespace @@ -9,6 +10,20 @@ from controller.config import ControllerConfig, MqttConfig from controller.mqtt_adapter import MqttAdapter +class FakeMessageInfo: + def __init__(self, mid, published): + self.rc = mqtt.MQTT_ERR_SUCCESS + self.mid = mid + self.published = published + self.waited = [] + + def wait_for_publish(self, timeout=None): + self.waited.append(timeout) + + def is_published(self): + return self.published + + class FakeClient: def __init__(self): self.calls = [] @@ -17,6 +32,9 @@ class FakeClient: self.acks = [] self.suppress_exceptions = False self.connect_error = None + self.loop_result = mqtt.MQTT_ERR_SUCCESS + self.publish_acknowledged = None + self.publish_hook = None def username_pw_set(self, username, password): self.calls.append(("username_pw_set", username, password)) @@ -42,7 +60,7 @@ class FakeClient: def loop_start(self): self.calls.append(("loop_start",)) - return mqtt.MQTT_ERR_SUCCESS + return self.loop_result def loop_stop(self): self.calls.append(("loop_stop",)) @@ -56,6 +74,12 @@ class FakeClient: def publish(self, topic, payload, qos=0, retain=False): self.published.append((topic, payload, qos, retain)) + if self.publish_hook is not None: + self.publish_hook(topic, payload, qos, retain) + if self.publish_acknowledged is not None: + return FakeMessageInfo( + len(self.published), self.publish_acknowledged + ) return SimpleNamespace(rc=mqtt.MQTT_ERR_SUCCESS, mid=len(self.published)) def ack(self, message_id, qos): @@ -106,6 +130,66 @@ class MqttAdapterTests(unittest.TestCase): self.assertIn("network unreachable", self.adapter.last_error) self.assertNotIn("loop_start", [call[0] for call in self.client.calls]) + def test_loop_start_failure_does_not_escape_startup(self): + self.client.loop_result = mqtt.MQTT_ERR_UNKNOWN + self.adapter.start() + self.assertTrue(self.adapter.started) + self.assertIn("failed to start MQTT network loop", self.adapter.last_error) + + def test_stop_during_blocked_connect_never_starts_network_loop_afterward(self): + entered = threading.Event() + release = threading.Event() + original_connect = self.client.connect_async + + def blocked_connect(*args, **kwargs): + entered.set() + release.wait(timeout=2) + return original_connect(*args, **kwargs) + + self.client.connect_async = blocked_connect + starter = threading.Thread(target=self.adapter.start) + starter.start() + self.assertTrue(entered.wait(timeout=1)) + self.adapter.stop() + release.set() + starter.join(timeout=2) + self.assertFalse(starter.is_alive()) + self.assertNotIn("loop_start", [call[0] for call in self.client.calls]) + self.assertFalse(self.adapter.started) + + def test_stop_wins_against_connect_callback_blocked_during_subscribe(self): + self.adapter.start() + entered = threading.Event() + release = threading.Event() + original_subscribe = self.client.subscribe + + def blocked_subscribe(topic, qos=0): + entered.set() + release.wait(timeout=2) + return original_subscribe(topic, qos=qos) + + self.client.subscribe = blocked_subscribe + callback = threading.Thread( + target=self.adapter._on_connect, + args=(self.client, None, None, 0), + ) + callback.start() + self.assertTrue(entered.wait(timeout=1)) + stopper = threading.Thread(target=self.adapter.stop) + stopper.start() + release.set() + callback.join(timeout=2) + stopper.join(timeout=2) + self.assertFalse(callback.is_alive()) + self.assertFalse(stopper.is_alive()) + self.assertFalse(self.adapter.started) + self.assertFalse(self.adapter.connected) + self.assertLess(len(self.client.subscriptions), 6) + self.assertTrue(self.events.empty()) + self.assertFalse( + self.adapter.publish_player_command({"action": "stop"}).accepted + ) + def test_connect_subscribes_to_all_authoritative_inputs(self): self.adapter.start() self.adapter._on_connect(self.client, None, None, 0) @@ -117,6 +201,7 @@ class MqttAdapterTests(unittest.TestCase): ("youtube/player/telemetry", 0), ("youtube/player/state", 0), ("youtube/controller/cmd", 1), + ("audio/player/mode/set", 1), ("audio/player/mode/state", 1), ], ) @@ -127,6 +212,7 @@ class MqttAdapterTests(unittest.TestCase): def test_player_commands_are_exact_qos_one_and_nonretained(self): disconnected = self.adapter.publish_player_command({"action": "stop"}) self.assertFalse(disconnected.accepted) + self.adapter.start() self.adapter._on_connect(self.client, None, None, 0) self.events.get_nowait() receipt = self.adapter.publish_player_command( @@ -140,6 +226,7 @@ class MqttAdapterTests(unittest.TestCase): self.assertIs(retain, False) def test_router_commands_are_qos_one_nonretained_plain_strings(self): + self.adapter.start() self.adapter._on_connect(self.client, None, None, 0) self.events.get_nowait() receipt = self.adapter.publish_router_mode("video") @@ -149,7 +236,75 @@ class MqttAdapterTests(unittest.TestCase): ("audio/player/mode/set", "video", 1, False), ) + def test_router_self_echo_is_marked_only_while_publish_is_in_flight(self): + self.adapter.start() + self.adapter._on_connect(self.client, None, None, 0) + self.events.get_nowait() + + def echo(topic, payload, qos, retain): + self.adapter._on_message( + self.client, + None, + SimpleNamespace( + topic=topic, + payload=payload.encode("utf-8"), + qos=qos, + retain=retain, + dup=False, + mid=31, + ), + ) + + self.client.publish_hook = echo + self.assertTrue(self.adapter.publish_router_mode("video").accepted) + self.assertTrue(self.events.get_nowait().controller_echo) + + self.client.publish_hook = None + self.adapter._on_message( + self.client, + None, + SimpleNamespace( + topic="audio/player/mode/set", + payload=b"video", + qos=1, + retain=False, + dup=False, + mid=32, + ), + ) + self.assertFalse(self.events.get_nowait().controller_echo) + + def test_retained_router_set_never_consumes_controller_echo_marker(self): + self.adapter.start() + self.adapter._on_connect(self.client, None, None, 0) + self.events.get_nowait() + + def retained_then_live_echo(topic, payload, qos, retain): + for message_retain, mid in ((True, 41), (False, 42)): + self.adapter._on_message( + self.client, + None, + SimpleNamespace( + topic=topic, + payload=payload.encode("utf-8"), + qos=qos, + retain=message_retain, + dup=False, + mid=mid, + ), + ) + + self.client.publish_hook = retained_then_live_echo + self.assertTrue(self.adapter.publish_router_mode("video").accepted) + retained_event = self.events.get_nowait() + live_event = self.events.get_nowait() + self.assertTrue(retained_event.retain) + self.assertFalse(retained_event.controller_echo) + self.assertFalse(live_event.retain) + self.assertTrue(live_event.controller_echo) + def test_qos_message_is_acked_only_after_consumer_calls_acknowledge(self): + self.adapter.start() self.adapter._on_connect(self.client, None, None, 0) self.events.get_nowait() message = SimpleNamespace( @@ -167,6 +322,7 @@ class MqttAdapterTests(unittest.TestCase): self.assertEqual(self.client.acks, [(42, 1)]) def test_stale_mid_from_previous_connection_is_not_acked(self): + self.adapter.start() self.adapter._on_connect(self.client, None, None, 0) self.events.get_nowait() message = SimpleNamespace( @@ -184,6 +340,7 @@ class MqttAdapterTests(unittest.TestCase): self.assertEqual(self.client.acks, []) def test_retained_flag_is_preserved_for_coordinator_rejection(self): + self.adapter.start() self.adapter._on_connect(self.client, None, None, 0) self.events.get_nowait() message = SimpleNamespace( @@ -196,6 +353,82 @@ class MqttAdapterTests(unittest.TestCase): self.adapter._on_message(self.client, None, message) self.assertIs(self.events.get_nowait().retain, True) + def test_duplicate_flag_is_preserved_for_coordinator_deduplication(self): + self.adapter.start() + self.adapter._on_connect(self.client, None, None, 0) + self.events.get_nowait() + message = SimpleNamespace( + topic="youtube/controller/cmd", + payload=b'{"action":"toggle"}', + qos=1, + retain=False, + dup=True, + mid=9, + ) + self.adapter._on_message(self.client, None, message) + self.assertIs(self.events.get_nowait().duplicate, True) + + def test_missing_puback_is_uncertain_and_invalidates_connection(self): + self.adapter.start() + self.adapter._on_connect(self.client, None, None, 0) + self.events.get_nowait() + self.client.publish_acknowledged = False + receipt = self.adapter.publish_player_command({"action": "stop"}) + self.assertFalse(receipt.accepted) + self.assertTrue(receipt.uncertain) + self.assertIn("no broker PUBACK", receipt.reason) + self.assertFalse(self.adapter.connected) + + def test_optional_qos_event_puback_failure_does_not_invalidate_transport(self): + self.adapter.start() + self.adapter._on_connect(self.client, None, None, 0) + self.events.get_nowait() + self.client.publish_acknowledged = False + receipt = self.adapter.publish_json( + "youtube/controller/v2/events", + {"version": 2, "type": "warning"}, + qos=1, + retain=False, + ) + self.assertFalse(receipt.accepted) + self.assertTrue(receipt.uncertain) + self.assertTrue(self.adapter.connected) + + def test_discarded_transport_releases_callback_blocked_on_full_queue(self): + events = queue.Queue(maxsize=1) + client = FakeClient() + adapter = MqttAdapter(self.config, client=client, events=events) + try: + adapter.start() + adapter._on_connect(client, None, None, 0) + self.assertTrue(events.full()) + callback = threading.Thread( + target=adapter._on_message, + args=( + client, + None, + SimpleNamespace( + topic="youtube/controller/cmd", + payload=b'{"action":"stop"}', + qos=1, + retain=False, + dup=False, + mid=99, + ), + ), + ) + callback.start() + callback.join(timeout=0.05) + self.assertTrue(callback.is_alive()) + + adapter._discard_unacknowledged_transport("test transport discard") + callback.join(timeout=1) + self.assertFalse(callback.is_alive()) + self.assertEqual(events.qsize(), 1) + self.assertEqual(events.get_nowait().kind, "connected") + finally: + adapter.stop() + if __name__ == "__main__": unittest.main() diff --git a/tests/test_playback.py b/tests/test_playback.py index ea051ad..f9d66fd 100644 --- a/tests/test_playback.py +++ b/tests/test_playback.py @@ -230,6 +230,154 @@ class PlaybackReducerTests(unittest.TestCase): effects = self.reducer.handle_status(payload) self.assertEqual(effects[0].kind, "warning") + def test_contradictory_slot_and_path_cannot_confirm_play(self): + expected = str(self.config.storage.media_base / "slot1" / "playlist.m3u") + self.reducer.begin_operation("play", slot=1, expected_path=expected) + effects = self.reducer.handle_status( + { + "state": "playing", + "slot": 1, + "path": str( + self.config.storage.media_base / "slot2" / "playlist.m3u" + ), + } + ) + self.assertEqual(effects[0].kind, "warning") + self.assertIsNotNone(self.reducer.pending) + self.assertEqual(self.store.player_observation().phase, "unknown") + + def test_mismatched_active_evidence_fails_intent_and_is_not_overwritten(self): + self.reducer.begin_operation( + "play", + slot=1, + expected_path="/mnt/Media/YTPlayer/slot1/playlist.m3u", + ) + effects = self.reducer.handle_status( + { + "state": "playing", + "slot": 2, + "path": "/mnt/Media/YTPlayer/slot2/playlist.m3u", + } + ) + self.assertIn("operation_failed", [effect.kind for effect in effects]) + self.assertIsNone(self.reducer.pending) + self.monotonic.value += 100 + self.assertIsNone(self.reducer.check_timeout()) + observation = self.store.player_observation() + self.assertEqual(observation.active_slot, 2) + self.assertEqual(observation.phase, "playing") + self.assertEqual(self.store.list_live_guards()[0].slot, 2) + + def test_bare_idle_does_not_clear_live_playback(self): + path = "/mnt/Media/YTPlayer/slot1/playlist.m3u" + self.reducer.handle_status({"state": "playing", "slot": 1, "path": path}) + effects = self.reducer.handle_status({"state": "idle", "slot": 1}) + self.assertEqual(effects[0].kind, "warning") + self.assertEqual(self.store.player_observation().phase, "playing") + self.assertEqual(self.store.list_live_guards()[0].slot, 1) + + def test_path_only_evidence_derives_and_protects_managed_slot(self): + path = str(self.config.storage.media_base / "slot2" / "playlist.m3u") + self.reducer.handle_telemetry( + {"state": "telemetry", "slot": None, "path": path, "paused": False} + ) + observation = self.store.player_observation() + self.assertEqual(observation.active_slot, 2) + self.assertEqual(self.store.list_live_guards()[0].slot, 2) + + def test_duplicate_offline_evidence_does_not_postpone_review(self): + path = "/mnt/Media/YTPlayer/slot1/playlist.m3u" + self.reducer.handle_status({"state": "playing", "slot": 1, "path": path}) + self.reducer.handle_status({"state": "offline"}) + first_deadline = self.store.list_live_guards()[0].review_after + self.wall.value += 20 + self.reducer.handle_status({"state": "offline"}) + self.assertEqual( + self.store.list_live_guards()[0].review_after, + first_deadline, + ) + + def test_repeated_active_telemetry_reuses_guard_without_history_churn(self): + path = str(self.config.storage.media_base / "slot1" / "playlist.m3u") + payload = { + "state": "telemetry", + "slot": 1, + "path": path, + "paused": False, + } + self.reducer.handle_telemetry(payload) + first_guard = self.store.list_live_guards()[0] + for _ in range(20): + self.reducer.handle_telemetry(payload) + self.assertEqual(self.store.list_live_guards()[0].guard_id, first_guard.guard_id) + count = self.store.connection.execute( + "SELECT COUNT(*) FROM slot_guards" + ).fetchone()[0] + self.assertEqual(count, 1) + + def test_external_play_timeout_transitions_to_review_without_assuming_stop(self): + path = "/srv/operator-playlists/special-event.m3u" + self.reducer.begin_operation("play", expected_path=path) + self.monotonic.value += 6 + timeout = self.reducer.check_timeout() + self.assertEqual(timeout.kind, "operation_timed_out") + uncertain = self.store.player_observation() + self.assertEqual(uncertain.active_path, path) + self.assertEqual(uncertain.reconciliation_state, "awaiting_evidence") + self.assertIsNotNone(uncertain.reconciliation_id) + + self.wall.value += 31 + effects = self.reducer.mark_due_reconciliation() + self.assertEqual(len(effects), 1) + self.assertEqual(effects[0].kind, "operator_review") + self.assertEqual(effects[0].operation_id, uncertain.reconciliation_id) + review = self.store.player_observation() + self.assertEqual(review.reconciliation_state, "operator_review") + self.assertEqual(review.active_path, path) + + def test_external_pause_does_not_bleed_to_a_different_path(self): + first_path = "/srv/operator-playlists/first.m3u" + second_path = "/srv/operator-playlists/second.m3u" + self.reducer.handle_telemetry( + { + "state": "telemetry", + "path": first_path, + "paused": True, + } + ) + self.assertIs(self.store.player_observation().paused, True) + + self.reducer.handle_status( + {"state": "playing", "path": second_path} + ) + observation = self.store.player_observation() + self.assertEqual(observation.active_path, second_path) + self.assertIsNone(observation.paused) + self.assertEqual(observation.phase, "playing") + + def test_stop_timeout_preserves_existing_operator_review_and_token(self): + path = str(self.config.storage.media_base / "slot1" / "playlist.m3u") + self.reducer.handle_status( + {"state": "playing", "slot": 1, "path": path} + ) + self.reducer.handle_status({"state": "offline"}) + self.wall.value += 31 + self.reducer.mark_due_reconciliation() + before = self.store.player_observation() + guard_before = self.store.list_live_guards()[0] + self.assertEqual(before.reconciliation_state, "operator_review") + + self.reducer.begin_operation("stop", slot=1, expected_path=path) + self.monotonic.value += 6 + effect = self.reducer.check_timeout() + self.assertEqual(effect.kind, "operation_timed_out") + after = self.store.player_observation() + guard_after = self.store.list_live_guards()[0] + self.assertEqual(after.reconciliation_state, "operator_review") + self.assertEqual(after.reconciliation_id, before.reconciliation_id) + self.assertEqual(guard_after.guard_id, guard_before.guard_id) + self.assertEqual(guard_after.state, "review_required") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_router.py b/tests/test_router.py new file mode 100644 index 0000000..9291195 --- /dev/null +++ b/tests/test_router.py @@ -0,0 +1,316 @@ +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path + +from controller.config import ControllerConfig, RouterConfig, StorageConfig +from controller.mqtt_adapter import PublishReceipt +from controller.router import AudioRouterCoordinator +from controller.store import ControllerStore + + +class Clock: + def __init__(self, value=1000.0): + self.value = value + + def __call__(self): + return self.value + + +class ModePublisher: + def __init__(self): + self.modes = [] + self.accepted = True + self.reason = None + + def __call__(self, mode): + self.modes.append(mode) + return PublishReceipt(self.accepted, message_id=len(self.modes), reason=self.reason) + + +class AudioRouterCoordinatorTests(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + base = Path(self.tempdir.name) + self.clock = Clock() + self.base_config = ControllerConfig( + storage=StorageConfig( + media_base=base / "media", + data_base=base / "data", + ), + router=RouterConfig( + required_mode="video", + confirmation_timeout_seconds=3, + when_state_unknown="attempt", + continue_on_confirmation_failure=True, + fallback_restore_mode=None, + ), + ) + self.store = ControllerStore(self.base_config.storage, now=self.clock).open() + self.publisher = ModePublisher() + + def tearDown(self): + self.store.close() + self.tempdir.cleanup() + + def coordinator(self, router_config=None): + config = self.base_config + if router_config is not None: + config = replace(config, router=router_config) + return AudioRouterCoordinator( + config, + self.store, + self.publisher, + clock=self.clock, + ) + + def test_known_mode_is_confirmed_before_play_and_restored_afterward(self): + router = self.coordinator() + router.observe_mode("service") + decision = router.acquire("play-one") + self.assertEqual(decision.action, "waiting") + self.assertEqual(self.publisher.modes, ["video"]) + self.assertEqual(router.observe_mode("video").action, "ready") + lease = self.store.get_audio_lease(decision.lease_id) + self.assertEqual(lease.state, "held") + self.assertEqual(lease.previous_mode, "service") + restore = router.release(decision.lease_id) + self.assertEqual(restore.action, "restore_waiting") + self.assertEqual(self.publisher.modes, ["video", "service"]) + self.assertEqual(router.observe_mode("service").action, "restored") + self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "released") + + def test_default_unknown_state_timeout_continues_with_warning_not_hang(self): + router = self.coordinator() + decision = router.acquire("play-timeout") + self.assertEqual(decision.action, "waiting") + self.assertEqual(self.publisher.modes, ["video"]) + self.clock.value += 2.9 + self.assertIsNone(router.check_deadline()) + self.clock.value += 0.2 + decision = router.check_deadline() + self.assertEqual(decision.action, "ready") + self.assertTrue(decision.continued) + self.assertIn("did not confirm", decision.warning) + lease = self.store.get_audio_lease(decision.lease_id) + self.assertEqual(lease.state, "unconfirmed") + self.assertEqual(router.release(lease.lease_id).action, "released") + self.assertEqual(self.publisher.modes, ["video"]) + + def test_timeout_can_reject_when_continue_policy_is_disabled(self): + router = self.coordinator( + replace( + self.base_config.router, + continue_on_confirmation_failure=False, + ) + ) + decision = router.acquire("play-reject") + self.clock.value += 4 + decision = router.check_deadline() + self.assertEqual(decision.action, "rejected") + self.assertFalse(decision.continued) + self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "abandoned") + + def test_unknown_state_continue_policy_does_not_wait_or_gain_late_ownership(self): + router = self.coordinator( + replace(self.base_config.router, when_state_unknown="continue") + ) + decision = router.acquire("play-immediate") + self.assertEqual(decision.action, "ready") + self.assertTrue(decision.continued) + self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "unconfirmed") + self.assertEqual(router.observe_mode("video").action, "observed") + self.assertEqual(self.store.get_audio_lease(decision.lease_id).state, "unconfirmed") + + def test_unknown_state_reject_policy_does_not_publish(self): + router = self.coordinator( + replace(self.base_config.router, when_state_unknown="reject") + ) + decision = router.acquire("play-reject") + self.assertEqual(decision.action, "rejected") + self.assertEqual(self.publisher.modes, []) + + def test_already_video_requires_no_change_and_no_restore(self): + router = self.coordinator() + router.observe_mode("video") + decision = router.acquire("play-video") + self.assertEqual(decision.action, "ready") + self.assertEqual(self.publisher.modes, []) + self.assertEqual(router.release(decision.lease_id).action, "released") + self.assertEqual(self.publisher.modes, []) + + def test_confirmed_unknown_acquisition_can_use_configured_fallback(self): + router = self.coordinator( + replace(self.base_config.router, fallback_restore_mode="service") + ) + decision = router.acquire("play-fallback") + self.assertEqual(router.observe_mode("video").action, "ready") + lease = self.store.get_audio_lease(decision.lease_id) + self.assertEqual(lease.restoration_source, "configured_fallback") + self.assertEqual(router.release(decision.lease_id).action, "restore_waiting") + self.assertEqual(self.publisher.modes, ["video", "service"]) + + def test_operator_change_is_sticky_even_if_mode_later_returns_to_video(self): + router = self.coordinator() + router.observe_mode("service") + decision = router.acquire("play-operator") + router.observe_mode("video") + lost = router.observe_mode("announcements") + self.assertEqual(lost.action, "ownership_lost") + router.observe_mode("video") + self.assertEqual(router.release(decision.lease_id).action, "released") + self.assertEqual(self.publisher.modes, ["video"]) + self.assertEqual( + self.store.get_audio_lease(decision.lease_id).state, + "ownership_lost", + ) + + def test_connection_loss_relinquishes_confirmed_lease(self): + router = self.coordinator() + router.observe_mode("service") + decision = router.acquire("play-disconnect") + router.observe_mode("video") + lost = router.connection_lost() + self.assertEqual(lost.action, "ownership_lost") + self.assertEqual(router.release(decision.lease_id).action, "released") + self.assertEqual(self.publisher.modes, ["video"]) + + def test_publish_failure_uses_confirmation_failure_policy(self): + self.publisher.accepted = False + self.publisher.reason = "MQTT disconnected" + router = self.coordinator() + decision = router.acquire("play-publish-fail") + self.assertEqual(decision.action, "ready") + self.assertTrue(decision.continued) + self.assertIn("MQTT disconnected", decision.warning) + + def test_restoration_timeout_is_bounded_and_not_retried(self): + router = self.coordinator() + router.observe_mode("service") + decision = router.acquire("play-restore-timeout") + router.observe_mode("video") + router.release(decision.lease_id) + self.clock.value += 4 + timeout = router.check_deadline() + self.assertEqual(timeout.action, "restore_failed") + self.assertEqual(self.publisher.modes, ["video", "service"]) + self.assertIsNone(router.check_deadline()) + + def test_repeated_previous_state_during_acquisition_waits_for_deadline(self): + router = self.coordinator() + router.observe_mode("service") + waiting = router.acquire("play-repeated-state") + self.assertEqual(router.observe_mode("service").action, "observed") + self.assertEqual( + self.store.get_audio_lease(waiting.lease_id).state, + "acquiring", + ) + self.clock.value += 4 + decision = router.check_deadline() + self.assertEqual(decision.action, "ready") + self.assertTrue(decision.continued) + self.assertEqual( + self.store.get_audio_lease(waiting.lease_id).state, + "unconfirmed", + ) + router.release(waiting.lease_id) + self.assertEqual(self.publisher.modes, ["video"]) + + def test_repeated_target_state_is_observation_not_operator_takeover(self): + router = self.coordinator() + router.observe_mode("service") + decision = router.acquire("play-repeated-state") + self.assertEqual(router.observe_mode("video").action, "ready") + self.assertEqual(router.observe_mode("video").action, "observed") + self.assertEqual( + self.store.get_audio_lease(decision.lease_id).state, + "held", + ) + self.assertEqual(router.release(decision.lease_id).action, "restore_waiting") + self.assertEqual(self.publisher.modes, ["video", "service"]) + + def test_controller_mode_set_echo_does_not_relinquish_ownership(self): + router = self.coordinator() + router.observe_mode("service") + decision = router.acquire("play-self-echo") + self.assertEqual( + router.observe_mode_command("video", controller_echo=True).action, + "self_echo", + ) + self.assertEqual(router.observe_mode("video").action, "ready") + self.assertEqual(router.release(decision.lease_id).action, "restore_waiting") + self.assertEqual( + router.observe_mode_command("service", controller_echo=True).action, + "self_echo", + ) + self.assertEqual(router.observe_mode("service").action, "restored") + + def test_manual_mode_set_during_acquisition_uses_continue_policy(self): + router = self.coordinator() + router.observe_mode("service") + waiting = router.acquire("play-manual-acquire") + decision = router.observe_mode_command("announcements") + self.assertEqual(decision.action, "ready") + self.assertTrue(decision.continued) + self.assertEqual( + self.store.get_audio_lease(waiting.lease_id).state, + "unconfirmed", + ) + self.assertIsNone(self.store.get_audio_lease(waiting.lease_id).confirmation_deadline) + + def test_manual_mode_set_during_acquisition_can_reject(self): + router = self.coordinator( + replace( + self.base_config.router, + continue_on_confirmation_failure=False, + ) + ) + router.observe_mode("service") + waiting = router.acquire("play-manual-reject") + decision = router.observe_mode_command("announcements") + self.assertEqual(decision.action, "rejected") + self.assertFalse(decision.continued) + self.assertEqual( + self.store.get_audio_lease(waiting.lease_id).state, + "abandoned", + ) + + def test_manual_mode_set_while_held_relinquishes_restoration(self): + router = self.coordinator() + router.observe_mode("service") + decision = router.acquire("play-manual-held") + router.observe_mode("video") + lost = router.observe_mode_command("video") + self.assertEqual(lost.action, "ownership_lost") + self.assertEqual(router.release(decision.lease_id).action, "released") + self.assertEqual(self.publisher.modes, ["video"]) + + def test_manual_mode_set_while_restoring_relinquishes_restoration(self): + router = self.coordinator() + router.observe_mode("service") + decision = router.acquire("play-manual-restore") + router.observe_mode("video") + self.assertEqual(router.release(decision.lease_id).action, "restore_waiting") + lost = router.observe_mode_command("announcements") + self.assertEqual(lost.action, "ownership_lost") + self.assertEqual( + self.store.get_audio_lease(decision.lease_id).state, + "ownership_lost", + ) + self.assertEqual(self.publisher.modes, ["video", "service"]) + + def test_retained_mode_set_does_not_claim_live_operator_intent(self): + router = self.coordinator() + router.observe_mode("service") + decision = router.acquire("play-retained-set") + router.observe_mode("video") + ignored = router.observe_mode_command("announcements", retained=True) + self.assertEqual(ignored.action, "retained_ignored") + self.assertEqual( + self.store.get_audio_lease(decision.lease_id).state, + "held", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_service.py b/tests/test_service.py new file mode 100644 index 0000000..0b02953 --- /dev/null +++ b/tests/test_service.py @@ -0,0 +1,742 @@ +import json +import queue +import tempfile +import threading +import time +import unittest +from dataclasses import replace +from pathlib import Path + +from controller.config import ( + ControllerConfig, + HomeAssistantConfig, + RouterConfig, + StorageConfig, +) +from controller.downloads import canonical_playlist_path +from controller.mqtt_adapter import MqttEvent, PublishReceipt +from controller.service import ControllerService +from controller.store import ControllerStore + + +class MutableClock: + def __init__(self, value): + self.value = value + + def __call__(self): + return self.value + + +class FakeMqttAdapter: + """In-memory transport boundary; it never starts a network client.""" + + def __init__(self, *, connected=True): + self.connected = connected + self.started = False + self.player_commands = [] + self.router_modes = [] + self.json_messages = [] + self.legacy_messages = [] + self.acknowledged = [] + + def start(self): + self.started = True + + def stop(self): + self.started = False + + def _receipt(self): + if self.connected: + return PublishReceipt(True, message_id=1) + return PublishReceipt(False, reason="MQTT is disconnected") + + def publish_player_command(self, payload): + receipt = self._receipt() + if receipt.accepted: + self.player_commands.append(dict(payload)) + return receipt + + def publish_router_mode(self, mode): + receipt = self._receipt() + if receipt.accepted: + self.router_modes.append(mode) + return receipt + + def publish_json(self, topic, payload, *, qos=0, retain=False): + receipt = self._receipt() + if receipt.accepted: + self.json_messages.append((topic, dict(payload), qos, retain)) + return receipt + + def publish_legacy(self, topic, payload, *, retain=True): + receipt = self._receipt() + if receipt.accepted: + self.legacy_messages.append((topic, payload, retain)) + return receipt + + def acknowledge(self, event): + self.acknowledged.append(event) + return True + + +class FakeDownloadManager: + def __init__(self): + self.recovered = False + self.stopped = False + + def recover_pending_deletions(self): + self.recovered = True + + def expire_due_slots(self): + return [] + + def shutdown(self, *, wait): + self.stopped = wait + + +class ControllerServiceTests(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.TemporaryDirectory() + base = Path(self.tempdir.name) + self.wall = MutableClock(1_700_000_000.0) + self.monotonic = MutableClock(100.0) + self.config = ControllerConfig( + storage=StorageConfig( + media_base=base / "media", + data_base=base / "data", + retention_seconds=3600, + expiry_interval_seconds=3600, + uncertain_review_after_seconds=30, + ), + router=RouterConfig( + required_mode="video", + confirmation_timeout_seconds=3, + when_state_unknown="continue", + continue_on_confirmation_failure=True, + fallback_restore_mode=None, + ), + homeassistant=HomeAssistantConfig(enabled=False), + ) + self.events = queue.Queue() + self.store = None + self.mqtt = None + self.downloads = None + self.service = None + + def tearDown(self): + if self.service is not None and self.service.started: + self.service.stop() + self.tempdir.cleanup() + + def start_service(self, *, config=None, mqtt=None): + config = config or self.config + self.events = queue.Queue() + self.store = ControllerStore(config.storage, now=self.wall) + self.mqtt = mqtt or FakeMqttAdapter() + self.downloads = FakeDownloadManager() + self.service = ControllerService( + config, + store=self.store, + mqtt_adapter=self.mqtt, + event_queue=self.events, + download_manager=self.downloads, + monotonic=self.monotonic, + wall_clock=self.wall, + ) + self.service.start() + self.assertTrue(self.downloads.recovered) + return self.service + + def ready_slot(self, slot=1): + reservation = self.store.reserve_download( + f"https://example.test/{slot}", + "tester", + "service test", + job_id=f"ready-{slot}", + ) + self.assertEqual(reservation.slot, slot) + self.assertTrue( + self.store.mark_download_running( + reservation.job_id, + reservation.slot, + reservation.generation, + ) + ) + self.assertTrue( + self.store.complete_download( + reservation.job_id, + reservation.slot, + reservation.generation, + title=f"Slot {slot}", + items=(("One", "one.mp4"),), + ) + ) + playlist = canonical_playlist_path(self.service.config.storage.media_base, slot) + playlist.parent.mkdir(parents=True, exist_ok=True) + playlist.write_text("one.mp4\n", encoding="utf-8") + return playlist + + def send_message( + self, + topic, + payload, + *, + qos=0, + retain=False, + message_id=None, + duplicate=False, + controller_echo=False, + ): + if not isinstance(payload, bytes): + payload = ( + payload.encode("utf-8") + if isinstance(payload, str) + else json.dumps(payload).encode("utf-8") + ) + self.events.put( + MqttEvent( + kind="message", + topic=topic, + payload=payload, + qos=qos, + retain=retain, + duplicate=duplicate, + message_id=message_id, + connection_generation=1, + controller_echo=controller_echo, + ) + ) + + def wait_for(self, predicate, message): + deadline = time.monotonic() + 2.0 + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.01) + self.fail(message) + + def result_events(self): + return [ + payload + for topic, payload, qos, retain in self.mqtt.json_messages + if topic == self.service.config.topics.controller_result + ] + + def test_play_waits_for_router_confirmation_and_player_evidence(self): + config = replace( + self.config, + router=replace(self.config.router, when_state_unknown="attempt"), + ) + self.start_service(config=config) + playlist = self.ready_slot(1) + + self.send_message(config.topics.router_mode_state, "service", qos=1) + self.wait_for( + lambda: self.store.router_observation().mode == "service", + "Router state was not consumed", + ) + + result = self.service.submit_control({"action": "play", "slot": 1}) + self.assertEqual(result["status"], "pending") + self.assertEqual(result["router"], "waiting") + self.assertEqual(self.mqtt.router_modes, ["video"]) + self.assertEqual(self.mqtt.player_commands, []) + self.assertEqual(self.service.playback.projected_slot_state(1), "ready") + + self.send_message( + config.topics.router_mode_set, + "video", + qos=1, + controller_echo=True, + ) + self.wait_for( + lambda: len(self.mqtt.acknowledged) == 2, + "Controller Router command echo was not consumed", + ) + self.assertEqual(self.store.live_audio_lease().state, "acquiring") + self.assertEqual(self.mqtt.player_commands, []) + + self.send_message(config.topics.router_mode_state, "video", qos=1) + self.wait_for( + lambda: len(self.mqtt.player_commands) == 1, + "Player command was not released after Router confirmation", + ) + self.assertEqual( + self.mqtt.player_commands, + [{"action": "play", "slot": 1, "label": "Slot 1"}], + ) + self.assertEqual(self.service.playback.projected_slot_state(1), "ready") + + self.send_message( + config.topics.player_status, + {"state": "playing", "slot": 1, "path": str(playlist)}, + qos=1, + ) + self.wait_for( + lambda: self.store.player_observation().phase == "playing", + "authoritative Player status was not reduced", + ) + self.assertEqual(self.service.playback.projected_slot_state(1), "playing") + self.assertIn( + "player_operation_confirmed", + {event["type"] for event in self.result_events()}, + ) + + def test_manual_router_set_during_acquisition_resolves_by_failure_policy(self): + config = replace( + self.config, + router=replace(self.config.router, when_state_unknown="attempt"), + ) + self.start_service(config=config) + self.ready_slot(1) + self.send_message(config.topics.router_mode_state, "service", qos=1) + self.wait_for( + lambda: self.store.router_observation().mode == "service", + "Router state was not consumed", + ) + + result = self.service.submit_control({"action": "play", "slot": 1}) + self.assertEqual(result["router"], "waiting") + self.send_message(config.topics.router_mode_set, "announcements", qos=1) + self.wait_for( + lambda: len(self.mqtt.player_commands) == 1, + "manual Router intent did not resolve acquisition", + ) + lease = self.store.get_audio_lease(self.service._session_lease_id) + self.assertIsNotNone(lease) + self.assertEqual(lease.state, "unconfirmed") + warnings = [ + event + for event in self.result_events() + if event["type"] == "audio_router_warning" + ] + self.assertTrue(warnings) + self.assertIs(warnings[-1]["continued"], True) + + def test_pause_state_remains_telemetry_authoritative(self): + self.start_service() + playlist = self.ready_slot(1) + self.service.submit_control({"action": "play", "slot": 1}) + self.wait_for( + lambda: self.mqtt.player_commands == [ + {"action": "play", "slot": 1, "label": "Slot 1"} + ], + "play command was not published", + ) + self.send_message( + self.config.topics.player_status, + {"state": "playing", "slot": 1, "path": str(playlist)}, + qos=1, + ) + self.wait_for( + lambda: self.service.playback.pending is None, + "play operation was not confirmed", + ) + playing_generation = self.store.player_observation().generation + + self.service.submit_control({"action": "pause"}) + self.send_message( + self.config.topics.player_status, + {"state": "paused", "slot": 1, "path": str(playlist)}, + qos=1, + ) + self.wait_for( + lambda: self.store.player_observation().generation > playing_generation, + "pause status was not consumed", + ) + self.assertIsNotNone(self.service.playback.pending) + self.assertEqual(self.service.playback.projected_slot_state(1), "playing") + + self.send_message( + self.config.topics.player_telemetry, + { + "state": "telemetry", + "slot": 1, + "path": str(playlist), + "paused": True, + }, + ) + self.wait_for( + lambda: self.service.playback.pending is None, + "pause was not confirmed by telemetry", + ) + self.assertEqual(self.store.player_observation().paused, True) + self.assertEqual(self.service.playback.projected_slot_state(1), "paused") + self.assertEqual(self.mqtt.player_commands[-1], {"action": "pause"}) + + def test_retained_controller_command_is_rejected_then_acked_and_reported(self): + self.start_service() + self.send_message( + self.config.topics.controller_command, + {"action": "play", "path": "/must/not/run"}, + qos=1, + retain=True, + message_id=42, + ) + self.wait_for( + lambda: len(self.mqtt.acknowledged) == 1, + "retained command was not acknowledged after rejection", + ) + self.assertEqual(self.mqtt.player_commands, []) + event = self.result_events()[-1] + self.assertEqual(event["version"], 2) + self.assertEqual(event["type"], "retained_command_rejected") + topic, _, qos, retain = self.mqtt.json_messages[-1] + self.assertEqual(topic, "youtube/controller/v2/events") + self.assertEqual((qos, retain), (1, False)) + self.assertEqual(self.mqtt.acknowledged[0].message_id, 42) + + def test_optional_result_topic_can_be_disabled_without_affecting_ack(self): + config = replace( + self.config, + topics=replace(self.config.topics, controller_result=None), + ) + self.start_service(config=config) + self.send_message( + config.topics.controller_command, + {"action": "delete_slot", "slot": 1}, + qos=1, + retain=True, + message_id=7, + ) + self.wait_for( + lambda: len(self.mqtt.acknowledged) == 1, + "command was not acknowledged with result topic disabled", + ) + self.assertEqual(self.mqtt.json_messages, []) + + def test_broker_unavailable_does_not_prevent_service_startup(self): + mqtt = FakeMqttAdapter(connected=False) + self.start_service(mqtt=mqtt) + health = self.service.health() + self.assertTrue(self.service.started) + self.assertTrue(mqtt.started) + self.assertTrue(health["coordinator_alive"]) + self.assertEqual(health["status"], "degraded") + self.assertIn("mqtt_disconnected", health["degraded_reasons"]) + + def test_administrator_can_clear_exact_uncertain_playback_review(self): + self.start_service() + playlist = self.ready_slot(1) + self.store.set_player_observation( + availability="unknown", + phase="unknown", + active_slot=1, + active_path=str(playlist), + paused=None, + reconciliation_state="operator_review", + reconciliation_due_at=None, + reconciliation_id="review-token", + ) + guard = self.store.protect_slot( + 1, + "restart_uncertain", + guard_id="review-token", + ) + review = self.service.health()["reconciliation_reviews"] + self.assertEqual(review[0]["review_id"], "review-token") + + result = self.service.submit_control( + { + "action": "reconcile_playback", + "decision": "clear", + "confirm": True, + "slot": 1, + "review_id": review[0]["review_id"], + "actor": "operator", + "reason": "Player host was replaced", + } + ) + self.assertEqual( + result, + {"status": "cleared", "slot": 1, "review_id": "review-token"}, + ) + self.assertEqual(self.store.list_live_guards(), []) + observation = self.store.player_observation() + self.assertEqual(observation.reconciliation_state, "admin_cleared") + self.assertIsNone(observation.active_slot) + self.assertEqual(self.service.playback.projected_slot_state(1), "ready") + self.assertEqual(self.result_events()[-1]["type"], "playback_reconciled") + + def test_busy_player_keeps_video_route_until_actual_playback_finishes(self): + config = replace( + self.config, + router=replace(self.config.router, when_state_unknown="attempt"), + ) + self.start_service(config=config) + self.ready_slot(1) + second_playlist = self.ready_slot(2) + self.send_message(config.topics.router_mode_state, "service", qos=1) + self.wait_for( + lambda: self.store.router_observation().mode == "service", + "Router service mode was not observed", + ) + self.service.submit_control({"action": "play", "slot": 1}) + self.send_message(config.topics.router_mode_state, "video", qos=1) + self.wait_for( + lambda: len(self.mqtt.player_commands) == 1, + "play command was not released", + ) + self.send_message( + config.topics.player_status, + { + "state": "busy", + "slot": 2, + "path": str(second_playlist), + "message": "mpv already playing; stop first", + }, + qos=1, + ) + self.wait_for( + lambda: self.service.playback.pending is None, + "busy response was not reduced", + ) + self.assertEqual(self.store.player_observation().active_slot, 2) + self.assertEqual(self.mqtt.router_modes, ["video"]) + lease = self.store.live_audio_lease() + self.assertIsNotNone(lease) + self.assertEqual(lease.state, "held") + + def test_disconnected_synchronous_play_failure_is_not_reported_pending(self): + self.start_service(mqtt=FakeMqttAdapter(connected=False)) + self.ready_slot(1) + result = self.service.submit_control({"action": "play", "slot": 1}) + self.assertEqual(result["status"], "unavailable") + self.assertIsNone(self.service.playback.pending) + self.assertEqual(self.store.list_live_guards(), []) + + def test_router_timeout_uses_monotonic_clock_and_continues_with_warning(self): + config = replace( + self.config, + router=replace(self.config.router, when_state_unknown="attempt"), + ) + self.start_service(config=config) + self.ready_slot(1) + result = self.service.submit_control({"action": "play", "slot": 1}) + self.assertEqual(result["router"], "waiting") + self.wall.value -= 10_000 + self.monotonic.value += 4 + self.wait_for( + lambda: len(self.mqtt.player_commands) == 1, + "Router timeout did not release play within its finite deadline", + ) + warnings = [ + event + for event in self.result_events() + if event["type"] == "audio_router_warning" + ] + self.assertTrue(warnings) + self.assertIs(warnings[-1]["continued"], True) + + def test_duplicate_toggle_is_suppressed_and_translated_idempotently(self): + self.start_service() + playlist = self.ready_slot(1) + self.send_message( + self.config.topics.player_telemetry, + { + "state": "telemetry", + "slot": 1, + "path": str(playlist), + "paused": False, + }, + ) + self.wait_for( + lambda: self.store.player_observation().active_slot == 1, + "active telemetry was not consumed", + ) + command = {"action": "toggle"} + self.send_message( + self.config.topics.controller_command, + command, + qos=1, + message_id=71, + ) + self.send_message( + self.config.topics.controller_command, + command, + qos=1, + message_id=71, + duplicate=True, + ) + self.wait_for( + lambda: sum( + event.message_id == 71 for event in self.mqtt.acknowledged + ) + == 2, + "duplicate command packets were not both acknowledged", + ) + self.assertEqual(self.mqtt.player_commands, [{"action": "pause"}]) + + def test_path_only_managed_play_adds_stable_slot_identity(self): + self.start_service() + playlist = self.ready_slot(1) + result = self.service.submit_control( + {"action": "play", "playlist": str(playlist)} + ) + self.assertEqual(result["status"], "pending") + self.assertEqual( + self.mqtt.player_commands, + [{"action": "play", "playlist": str(playlist), "slot": 1}], + ) + self.assertEqual(self.service.playback.projected_slot_state(1), "ready") + + def test_explicit_slot_cannot_be_paired_with_another_managed_slot_path(self): + self.start_service() + self.ready_slot(1) + second_playlist = self.ready_slot(2) + with self.assertRaisesRegex(ValueError, "conflicts"): + self.service.submit_control( + { + "action": "play", + "slot": 1, + "playlist": str(second_playlist), + } + ) + self.assertEqual(self.mqtt.player_commands, []) + self.assertEqual(self.store.list_live_guards(), []) + + def test_administrator_can_clear_external_playback_review_without_slot(self): + config = replace( + self.config, + topics=replace(self.config.topics, controller_result=None), + ) + self.start_service(config=config) + observation = self.store.mark_external_uncertain( + "/srv/operator-playlists/special-event.m3u", + review_after=self.wall.value, + details={"reason": "restart"}, + ) + self.assertIsNotNone(observation.reconciliation_id) + reviews = self.service.health()["reconciliation_reviews"] + self.assertEqual(len(reviews), 1) + review_id = reviews[0]["review_id"] + + result = self.service.submit_control( + { + "action": "reconcile_playback", + "decision": "clear", + "confirm": True, + "review_id": review_id, + "actor": "operator", + "reason": "Player appliance was replaced", + } + ) + self.assertEqual( + result, + {"status": "cleared", "review_id": review_id}, + ) + cleared = self.store.player_observation() + self.assertIsNone(cleared.active_path) + self.assertEqual(cleared.reconciliation_state, "admin_cleared") + + def test_external_play_intent_is_durable_immediately_after_puback(self): + self.start_service() + playlist = Path(self.tempdir.name) / "operator-playlist.m3u" + playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8") + result = self.service.submit_control( + {"action": "play", "playlist": str(playlist)} + ) + self.assertEqual(result["status"], "pending") + self.assertEqual( + self.mqtt.player_commands, + [{"action": "play", "playlist": str(playlist)}], + ) + reviews = self.service.health()["reconciliation_reviews"] + self.assertEqual(len(reviews), 1) + review_id = reviews[0]["review_id"] + self.assertIs(reviews[0]["clearable"], False) + + self.service.stop() + self.service = None + with ControllerStore(self.config.storage, now=self.wall).open() as reopened: + reopened.mark_restart_uncertain() + observation = reopened.player_observation() + self.assertEqual(observation.active_path, str(playlist)) + self.assertEqual(observation.reconciliation_id, review_id) + + def test_live_external_play_fence_cannot_be_admin_cleared(self): + self.start_service() + playlist = Path(self.tempdir.name) / "operator-playlist.m3u" + playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8") + self.service.submit_control( + {"action": "play", "playlist": str(playlist)} + ) + review = self.service.health()["reconciliation_reviews"][0] + self.assertIs(review["clearable"], False) + + with self.assertRaisesRegex(ValueError, "Player operation is pending"): + self.service.submit_control( + { + "action": "reconcile_playback", + "decision": "clear", + "confirm": True, + "review_id": review["review_id"], + "actor": "operator", + "reason": "clear requested too early", + } + ) + self.assertIsNotNone(self.service.playback.pending) + observation = self.store.player_observation() + self.assertEqual(observation.reconciliation_id, review["review_id"]) + self.assertEqual(observation.active_path, str(playlist)) + + def test_definite_external_publish_failure_clears_prepublication_guard(self): + self.start_service(mqtt=FakeMqttAdapter(connected=False)) + playlist = Path(self.tempdir.name) / "operator-playlist.m3u" + playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8") + result = self.service.submit_control( + {"action": "play", "playlist": str(playlist)} + ) + self.assertEqual(result["status"], "unavailable") + self.assertEqual(self.service.health()["reconciliation_reviews"], []) + observation = self.store.player_observation() + self.assertIsNone(observation.active_path) + self.assertIsNone(observation.reconciliation_id) + + def test_unexpected_reducer_failure_is_not_acked_or_deduplicated(self): + self.start_service() + original = self.service.playback.handle_status + failure_seen = threading.Event() + attempts = 0 + + def fail_once(payload): + nonlocal attempts + attempts += 1 + if attempts == 1: + failure_seen.set() + raise RuntimeError("transient reducer failure") + return original(payload) + + self.service.playback.handle_status = fail_once + payload = { + "state": "playing", + "slot": 1, + "path": str(self.config.storage.media_base / "slot1" / "playlist.m3u"), + } + self.send_message( + self.config.topics.player_status, + payload, + qos=1, + message_id=88, + ) + self.assertTrue(failure_seen.wait(timeout=1)) + time.sleep(0.05) + self.assertEqual(self.mqtt.acknowledged, []) + + self.send_message( + self.config.topics.player_status, + payload, + qos=1, + message_id=88, + duplicate=True, + ) + self.wait_for( + lambda: len(self.mqtt.acknowledged) == 1, + "retried status was not processed and acknowledged", + ) + self.assertEqual(attempts, 2) + self.assertEqual(self.store.player_observation().phase, "playing") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_store.py b/tests/test_store.py index e898127..1f91263 100644 --- a/tests/test_store.py +++ b/tests/test_store.py @@ -1,6 +1,7 @@ import concurrent.futures import hashlib import os +import sqlite3 import tempfile import unittest from pathlib import Path @@ -49,6 +50,37 @@ class FreshStoreTests(StoreTestCase): self.assertTrue(all(slot.media_state == "empty" for slot in slots)) self.assertEqual(store.player_observation().phase, "unknown") + def test_schema_one_database_upgrades_external_reconciliation_atomically(self): + with self.open_store() as store: + database_path = store.path + connection = sqlite3.connect(database_path) + try: + connection.execute( + "ALTER TABLE player_observation DROP COLUMN reconciliation_id" + ) + connection.execute("DELETE FROM schema_migrations WHERE version = 2") + connection.execute("PRAGMA user_version = 1") + connection.commit() + finally: + connection.close() + + with self.open_store() as upgraded: + self.assertEqual( + upgraded.connection.execute("PRAGMA user_version").fetchone()[0], + 2, + ) + columns = { + row["name"] + for row in upgraded.connection.execute( + "PRAGMA table_info(player_observation)" + ).fetchall() + } + self.assertIn("reconciliation_id", columns) + migration = upgraded.connection.execute( + "SELECT name FROM schema_migrations WHERE version = 2" + ).fetchone() + self.assertEqual(migration["name"], "external_playback_reconciliation") + def test_slot_reservation_is_committed_before_worker_transition(self): with self.open_store() as store: reservation = store.reserve_download( @@ -127,6 +159,25 @@ class FreshStoreTests(StoreTestCase): self.assertEqual(store.get_slot(2).slot, 2) self.assertEqual(store.get_slot(2).url, "https://example.test/two") + def test_concurrent_first_open_installs_only_one_valid_database(self): + stores = [ControllerStore(self.config, now=self.clock) for _ in range(2)] + + def open_store(store): + store.open() + return [slot.slot for slot in store.list_slots()] + + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(open_store, stores)) + self.assertEqual(results, [list(range(1, 11)), list(range(1, 11))]) + markers = stores[0].connection.execute( + "SELECT name FROM data_migrations ORDER BY name" + ).fetchall() + self.assertEqual([row["name"] for row in markers], ["fresh_database_v1"]) + finally: + for store in stores: + store.close() + class GuardAndExpiryTests(StoreTestCase): def _ready_slot(self, store, *, slot_expected=1): @@ -196,6 +247,7 @@ class GuardAndExpiryTests(StoreTestCase): paused=False, reconciliation_state="confirmed", reconciliation_due_at=None, + reconciliation_id=None, ) guard = store.mark_restart_uncertain() self.assertEqual(guard.slot, 1) @@ -204,6 +256,150 @@ class GuardAndExpiryTests(StoreTestCase): self.assertEqual(observation.reconciliation_state, "awaiting_evidence") self.assertEqual(observation.active_slot, 1) + def test_external_playback_restart_gets_stable_review_token_and_exact_clear(self): + external_path = "/srv/operator-playlists/live.m3u" + with self.open_store() as store: + store.set_player_observation( + availability="online", + phase="playing", + active_slot=None, + active_path=external_path, + paused=False, + reconciliation_state="confirmed", + reconciliation_due_at=None, + reconciliation_id=None, + ) + self.assertIsNone(store.mark_restart_uncertain()) + first = store.player_observation() + self.assertEqual(first.reconciliation_state, "awaiting_evidence") + self.assertIsNotNone(first.reconciliation_id) + self.assertEqual(first.reconciliation_due_at, self.clock.value + 30) + + self.clock.value += 10 + self.assertIsNone(store.mark_restart_uncertain()) + repeated = store.player_observation() + self.assertEqual(repeated.reconciliation_id, first.reconciliation_id) + self.assertEqual(repeated.reconciliation_due_at, first.reconciliation_due_at) + + self.clock.value = first.reconciliation_due_at + 1 + due = store.mark_external_reconciliation_due() + self.assertEqual(due, (first.reconciliation_id, external_path)) + self.assertIsNone(store.mark_external_reconciliation_due()) + review = store.player_observation() + self.assertEqual(review.reconciliation_state, "operator_review") + self.assertEqual(review.active_path, external_path) + + self.assertFalse( + store.clear_external_reconciliation( + "stale-review", + actor="operator", + reason="Player was replaced", + ) + ) + self.assertTrue( + store.clear_external_reconciliation( + first.reconciliation_id, + actor="operator", + reason="Player was replaced", + ) + ) + cleared = store.player_observation() + self.assertEqual(cleared.reconciliation_state, "admin_cleared") + self.assertIsNone(cleared.active_path) + self.assertIsNone(cleared.reconciliation_id) + + def test_restart_converts_existing_active_guard_and_preserves_deadline(self): + with self.open_store() as store: + self._ready_slot(store) + store.protect_slot(1, "player_active", guard_id="active-before-restart") + store.set_player_observation( + availability="online", + phase="playing", + active_slot=1, + active_path="/mnt/Media/YTPlayer/slot1/playlist.m3u", + paused=False, + reconciliation_state="confirmed", + reconciliation_due_at=None, + reconciliation_id=None, + ) + guard = store.mark_restart_uncertain() + self.assertEqual(guard.guard_id, "active-before-restart") + self.assertEqual(guard.kind, "restart_uncertain") + first_deadline = guard.review_after + self.clock.value += 10 + repeated = store.mark_restart_uncertain() + self.assertEqual(repeated.guard_id, guard.guard_id) + self.assertEqual(repeated.review_after, first_deadline) + + def test_restart_converts_pre_evidence_pending_guard_for_admin_review(self): + with self.open_store() as store: + self._ready_slot(store) + store.protect_slot(1, "pending_play", guard_id="published-before-crash") + guard = store.mark_restart_uncertain() + self.assertEqual(guard.guard_id, "published-before-crash") + self.assertEqual(guard.kind, "restart_uncertain") + self.assertEqual(store.player_observation().active_slot, 1) + self.assertTrue( + store.clear_guard( + 1, + guard.guard_id, + actor="operator", + reason="Player replaced", + ) + ) + + def test_guard_review_transition_is_reported_only_once(self): + with self.open_store() as store: + self._ready_slot(store) + store.protect_slot( + 1, + "restart_uncertain", + guard_id="one-warning", + review_after=self.clock.value, + ) + self.assertEqual(len(store.mark_due_guards_for_review()), 1) + self.assertEqual(store.mark_due_guards_for_review(), []) + + def test_clearing_one_legacy_guard_does_not_erase_other_observation(self): + with self.open_store() as store: + self._ready_slot(store) + second = store.reserve_download( + "https://example.test/second", None, None, job_id="second" + ) + store.mark_download_running("second", 2, second.generation) + store.complete_download( + "second", + 2, + second.generation, + title="Second", + items=(("Second", "second.mp4"),), + ) + first_guard = store.protect_slot( + 1, "legacy_uncertain", guard_id="legacy-one" + ) + store.protect_slot(2, "player_active", guard_id="active-two") + store.set_player_observation( + availability="online", + phase="playing", + active_slot=2, + active_path="/mnt/Media/YTPlayer/slot2/playlist.m3u", + paused=False, + reconciliation_state="confirmed", + reconciliation_due_at=None, + reconciliation_id=None, + ) + self.assertTrue( + store.clear_guard( + 1, + first_guard.guard_id, + actor="operator", + reason="legacy evidence resolved", + ) + ) + observation = store.player_observation() + self.assertEqual(observation.active_slot, 2) + self.assertEqual(observation.phase, "playing") + class LegacyMigrationTests(StoreTestCase): def write_legacy(self, data):