Media
270ede8b77
The evidence gate on enqueue_download blocked every download, including a URL that could only land in a slot store.reserve_download already proves is empty and unguarded. After a Controller-only restart with the Player still connected and idle, no fresh detailed status ever arrives, so this could deadlock all new ingest indefinitely even though nothing unsafe would occur. Narrow the gate to only require fresh evidence when the URL already occupies a non-empty slot (refresh/reuse). Deletion, expiry, and cleanup/recovery gating are unchanged.
1097 lines
39 KiB
Python
1097 lines
39 KiB
Python
import json
|
|
import queue
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
|
|
from controller.config import (
|
|
ControllerConfig,
|
|
HomeAssistantConfig,
|
|
RouterConfig,
|
|
StorageConfig,
|
|
)
|
|
from controller.downloads import canonical_playlist_path
|
|
from controller.models import SlotProtectedError, StaleOperationError
|
|
from controller.mqtt_adapter import MqttEvent, PublishReceipt
|
|
from controller.service import ControllerService, ControllerUnavailableError
|
|
from controller.store import ControllerStore
|
|
|
|
|
|
class MutableClock:
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
def __call__(self):
|
|
return self.value
|
|
|
|
|
|
class FakeMqttAdapter:
|
|
"""In-memory transport boundary; it never starts a network client."""
|
|
|
|
def __init__(self, *, connected=True):
|
|
self.connected = connected
|
|
self.started = False
|
|
self.player_commands = []
|
|
self.router_modes = []
|
|
self.json_messages = []
|
|
self.legacy_messages = []
|
|
self.acknowledged = []
|
|
|
|
def start(self):
|
|
self.started = True
|
|
|
|
def stop(self):
|
|
self.started = False
|
|
|
|
def _receipt(self):
|
|
if self.connected:
|
|
return PublishReceipt(True, message_id=1)
|
|
return PublishReceipt(False, reason="MQTT is disconnected")
|
|
|
|
def publish_player_command(self, payload):
|
|
receipt = self._receipt()
|
|
if receipt.accepted:
|
|
self.player_commands.append(dict(payload))
|
|
return receipt
|
|
|
|
def publish_router_mode(self, mode):
|
|
receipt = self._receipt()
|
|
if receipt.accepted:
|
|
self.router_modes.append(mode)
|
|
return receipt
|
|
|
|
def publish_json(self, topic, payload, *, qos=0, retain=False):
|
|
receipt = self._receipt()
|
|
if receipt.accepted:
|
|
self.json_messages.append((topic, dict(payload), qos, retain))
|
|
return receipt
|
|
|
|
def publish_legacy(self, topic, payload, *, retain=True):
|
|
receipt = self._receipt()
|
|
if receipt.accepted:
|
|
self.legacy_messages.append((topic, payload, retain))
|
|
return receipt
|
|
|
|
def acknowledge(self, event):
|
|
self.acknowledged.append(event)
|
|
return True
|
|
|
|
|
|
class FakeDownloadManager:
|
|
def __init__(self, store=None):
|
|
self.store = store
|
|
self.recovered = False
|
|
self.recovery_calls = 0
|
|
self.expiry_calls = 0
|
|
self.stopped = False
|
|
self.delete_requests = []
|
|
self.enqueue_calls = []
|
|
|
|
def enqueue(self, url, submitter, note):
|
|
self.enqueue_calls.append((url, submitter, note))
|
|
return self.store.reserve_download(url, submitter, note)
|
|
|
|
def recover_pending_deletions(self):
|
|
self.recovered = True
|
|
|
|
def schedule_pending_deletion_recovery(self):
|
|
self.recovered = True
|
|
self.recovery_calls += 1
|
|
return True
|
|
|
|
def schedule_expired_deletions(self):
|
|
self.expiry_calls += 1
|
|
return []
|
|
|
|
def schedule_delete_slot(self, slot, *, reason, expected_url=None):
|
|
self.delete_requests.append((slot, reason, expected_url))
|
|
if self.store is None:
|
|
return None
|
|
return self.store.begin_delete(
|
|
slot,
|
|
reason=reason,
|
|
expected_url=expected_url,
|
|
)
|
|
|
|
def shutdown(self, *, wait):
|
|
self.stopped = wait
|
|
|
|
|
|
class ControllerServiceTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tempdir = tempfile.TemporaryDirectory()
|
|
base = Path(self.tempdir.name)
|
|
self.wall = MutableClock(1_700_000_000.0)
|
|
self.monotonic = MutableClock(100.0)
|
|
self.config = ControllerConfig(
|
|
storage=StorageConfig(
|
|
media_base=base / "media",
|
|
data_base=base / "data",
|
|
retention_seconds=3600,
|
|
expiry_interval_seconds=3600,
|
|
uncertain_review_after_seconds=30,
|
|
),
|
|
router=RouterConfig(
|
|
required_mode="video",
|
|
confirmation_timeout_seconds=3,
|
|
when_state_unknown="continue",
|
|
continue_on_confirmation_failure=True,
|
|
fallback_restore_mode=None,
|
|
),
|
|
homeassistant=HomeAssistantConfig(enabled=False),
|
|
)
|
|
self.events = queue.Queue()
|
|
self.store = None
|
|
self.mqtt = None
|
|
self.downloads = None
|
|
self.service = None
|
|
|
|
def tearDown(self):
|
|
if self.service is not None and self.service.started:
|
|
self.service.stop()
|
|
self.tempdir.cleanup()
|
|
|
|
def start_service(self, *, config=None, mqtt=None):
|
|
config = config or self.config
|
|
self.events = queue.Queue()
|
|
self.store = ControllerStore(config.storage, now=self.wall)
|
|
self.mqtt = mqtt or FakeMqttAdapter()
|
|
self.downloads = FakeDownloadManager(self.store)
|
|
self.service = ControllerService(
|
|
config,
|
|
store=self.store,
|
|
mqtt_adapter=self.mqtt,
|
|
event_queue=self.events,
|
|
download_manager=self.downloads,
|
|
monotonic=self.monotonic,
|
|
wall_clock=self.wall,
|
|
)
|
|
self.service.start()
|
|
self.assertFalse(self.downloads.recovered)
|
|
return self.service
|
|
|
|
def ready_slot(self, slot=1):
|
|
reservation = self.store.reserve_download(
|
|
f"https://example.test/{slot}",
|
|
"tester",
|
|
"service test",
|
|
job_id=f"ready-{slot}",
|
|
)
|
|
self.assertEqual(reservation.slot, slot)
|
|
self.assertTrue(
|
|
self.store.mark_download_running(
|
|
reservation.job_id,
|
|
reservation.slot,
|
|
reservation.generation,
|
|
)
|
|
)
|
|
self.assertTrue(
|
|
self.store.complete_download(
|
|
reservation.job_id,
|
|
reservation.slot,
|
|
reservation.generation,
|
|
title=f"Slot {slot}",
|
|
items=(("One", "one.mp4"),),
|
|
)
|
|
)
|
|
playlist = canonical_playlist_path(self.service.config.storage.media_base, slot)
|
|
playlist.parent.mkdir(parents=True, exist_ok=True)
|
|
playlist.write_text("one.mp4\n", encoding="utf-8")
|
|
return playlist
|
|
|
|
def send_message(
|
|
self,
|
|
topic,
|
|
payload,
|
|
*,
|
|
qos=0,
|
|
retain=False,
|
|
message_id=None,
|
|
duplicate=False,
|
|
controller_echo=False,
|
|
):
|
|
if not isinstance(payload, bytes):
|
|
payload = (
|
|
payload.encode("utf-8")
|
|
if isinstance(payload, str)
|
|
else json.dumps(payload).encode("utf-8")
|
|
)
|
|
self.events.put(
|
|
MqttEvent(
|
|
kind="message",
|
|
topic=topic,
|
|
payload=payload,
|
|
qos=qos,
|
|
retain=retain,
|
|
duplicate=duplicate,
|
|
message_id=message_id,
|
|
connection_generation=1,
|
|
controller_echo=controller_echo,
|
|
)
|
|
)
|
|
|
|
def wait_for(self, predicate, message):
|
|
deadline = time.monotonic() + 2.0
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return
|
|
time.sleep(0.01)
|
|
self.fail(message)
|
|
|
|
def result_events(self):
|
|
return [
|
|
payload
|
|
for topic, payload, qos, retain in self.mqtt.json_messages
|
|
if topic == self.service.config.topics.controller_result
|
|
]
|
|
|
|
def test_play_waits_for_router_confirmation_and_player_evidence(self):
|
|
config = replace(
|
|
self.config,
|
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
|
)
|
|
self.start_service(config=config)
|
|
playlist = self.ready_slot(1)
|
|
|
|
self.send_message(config.topics.router_mode_state, "service", qos=1)
|
|
self.wait_for(
|
|
lambda: self.store.router_observation().mode == "service",
|
|
"Router state was not consumed",
|
|
)
|
|
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["status"], "pending")
|
|
self.assertEqual(result["router"], "waiting")
|
|
self.assertEqual(self.mqtt.router_modes, ["video"])
|
|
self.assertEqual(self.mqtt.player_commands, [])
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
|
|
|
self.send_message(
|
|
config.topics.router_mode_set,
|
|
"video",
|
|
qos=1,
|
|
controller_echo=True,
|
|
)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.acknowledged) == 2,
|
|
"Controller Router command echo was not consumed",
|
|
)
|
|
self.assertEqual(self.store.live_audio_lease().state, "acquiring")
|
|
self.assertEqual(self.mqtt.player_commands, [])
|
|
|
|
self.send_message(config.topics.router_mode_state, "video", qos=1)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.player_commands) == 1,
|
|
"Player command was not released after Router confirmation",
|
|
)
|
|
self.assertEqual(
|
|
self.mqtt.player_commands,
|
|
[{"action": "play", "slot": 1, "label": "Slot 1"}],
|
|
)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
|
|
|
self.send_message(
|
|
config.topics.player_status,
|
|
{"state": "playing", "slot": 1, "path": str(playlist)},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.store.player_observation().phase == "playing",
|
|
"authoritative Player status was not reduced",
|
|
)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "playing")
|
|
self.assertIn(
|
|
"player_operation_confirmed",
|
|
{event["type"] for event in self.result_events()},
|
|
)
|
|
|
|
def test_manual_router_set_during_acquisition_resolves_by_failure_policy(self):
|
|
config = replace(
|
|
self.config,
|
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
|
)
|
|
self.start_service(config=config)
|
|
self.ready_slot(1)
|
|
self.send_message(config.topics.router_mode_state, "service", qos=1)
|
|
self.wait_for(
|
|
lambda: self.store.router_observation().mode == "service",
|
|
"Router state was not consumed",
|
|
)
|
|
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["router"], "waiting")
|
|
self.send_message(config.topics.router_mode_set, "announcements", qos=1)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.player_commands) == 1,
|
|
"manual Router intent did not resolve acquisition",
|
|
)
|
|
lease = self.store.get_audio_lease(self.service._session_lease_id)
|
|
self.assertIsNotNone(lease)
|
|
self.assertEqual(lease.state, "unconfirmed")
|
|
warnings = [
|
|
event
|
|
for event in self.result_events()
|
|
if event["type"] == "audio_router_warning"
|
|
]
|
|
self.assertTrue(warnings)
|
|
self.assertIs(warnings[-1]["continued"], True)
|
|
|
|
def test_pause_state_remains_telemetry_authoritative(self):
|
|
self.start_service()
|
|
playlist = self.ready_slot(1)
|
|
self.service.submit_control({"action": "play", "slot": 1})
|
|
self.wait_for(
|
|
lambda: self.mqtt.player_commands == [
|
|
{"action": "play", "slot": 1, "label": "Slot 1"}
|
|
],
|
|
"play command was not published",
|
|
)
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "playing", "slot": 1, "path": str(playlist)},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.service.playback.pending is None,
|
|
"play operation was not confirmed",
|
|
)
|
|
playing_generation = self.store.player_observation().generation
|
|
|
|
self.service.submit_control({"action": "pause"})
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "paused", "slot": 1, "path": str(playlist)},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.store.player_observation().generation > playing_generation,
|
|
"pause status was not consumed",
|
|
)
|
|
self.assertIsNotNone(self.service.playback.pending)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "playing")
|
|
|
|
self.send_message(
|
|
self.config.topics.player_telemetry,
|
|
{
|
|
"state": "telemetry",
|
|
"slot": 1,
|
|
"path": str(playlist),
|
|
"paused": True,
|
|
},
|
|
)
|
|
self.wait_for(
|
|
lambda: self.service.playback.pending is None,
|
|
"pause was not confirmed by telemetry",
|
|
)
|
|
self.assertEqual(self.store.player_observation().paused, True)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "paused")
|
|
self.assertEqual(self.mqtt.player_commands[-1], {"action": "pause"})
|
|
|
|
def test_retained_controller_command_is_rejected_then_acked_and_reported(self):
|
|
self.start_service()
|
|
self.send_message(
|
|
self.config.topics.controller_command,
|
|
{"action": "play", "path": "/must/not/run"},
|
|
qos=1,
|
|
retain=True,
|
|
message_id=42,
|
|
)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.acknowledged) == 1,
|
|
"retained command was not acknowledged after rejection",
|
|
)
|
|
self.assertEqual(self.mqtt.player_commands, [])
|
|
event = self.result_events()[-1]
|
|
self.assertEqual(event["version"], 2)
|
|
self.assertEqual(event["type"], "retained_command_rejected")
|
|
topic, _, qos, retain = self.mqtt.json_messages[-1]
|
|
self.assertEqual(topic, "youtube/controller/v2/events")
|
|
self.assertEqual((qos, retain), (1, False))
|
|
self.assertEqual(self.mqtt.acknowledged[0].message_id, 42)
|
|
|
|
def test_retained_detailed_player_evidence_cannot_clear_restart_protection(self):
|
|
self.start_service()
|
|
self.store.protect_slot(
|
|
1,
|
|
"restart_uncertain",
|
|
guard_id="restart-review",
|
|
)
|
|
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "idle", "slot": 1, "reason": "stop"},
|
|
qos=1,
|
|
retain=True,
|
|
message_id=41,
|
|
)
|
|
self.send_message(
|
|
self.config.topics.player_telemetry,
|
|
{
|
|
"state": "telemetry",
|
|
"slot": 1,
|
|
"path": None,
|
|
"paused": False,
|
|
},
|
|
retain=True,
|
|
)
|
|
self.wait_for(
|
|
lambda: any(event.message_id == 41 for event in self.mqtt.acknowledged),
|
|
"retained QoS 1 status was not deliberately acknowledged",
|
|
)
|
|
self.wait_for(
|
|
lambda: len(
|
|
[
|
|
event
|
|
for event in self.result_events()
|
|
if event["type"] == "retained_player_evidence_rejected"
|
|
]
|
|
)
|
|
== 2,
|
|
"retained telemetry rejection was not reported",
|
|
)
|
|
|
|
guards = self.store.list_live_guards()
|
|
self.assertEqual(len(guards), 1)
|
|
self.assertEqual(guards[0].guard_id, "restart-review")
|
|
self.assertIn(
|
|
"awaiting_player_evidence",
|
|
self.service.health()["degraded_reasons"],
|
|
)
|
|
retained_warnings = [
|
|
event
|
|
for event in self.result_events()
|
|
if event["type"] == "retained_player_evidence_rejected"
|
|
]
|
|
self.assertEqual(len(retained_warnings), 2)
|
|
|
|
def test_optional_result_topic_can_be_disabled_without_affecting_ack(self):
|
|
config = replace(
|
|
self.config,
|
|
topics=replace(self.config.topics, controller_result=None),
|
|
)
|
|
self.start_service(config=config)
|
|
self.send_message(
|
|
config.topics.controller_command,
|
|
{"action": "delete_slot", "slot": 1},
|
|
qos=1,
|
|
retain=True,
|
|
message_id=7,
|
|
)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.acknowledged) == 1,
|
|
"command was not acknowledged with result topic disabled",
|
|
)
|
|
self.assertEqual(self.mqtt.json_messages, [])
|
|
|
|
def test_broker_unavailable_does_not_prevent_service_startup(self):
|
|
mqtt = FakeMqttAdapter(connected=False)
|
|
self.start_service(mqtt=mqtt)
|
|
health = self.service.health()
|
|
self.assertTrue(self.service.started)
|
|
self.assertTrue(mqtt.started)
|
|
self.assertTrue(health["coordinator_alive"])
|
|
self.assertEqual(health["status"], "degraded")
|
|
self.assertIn("mqtt_disconnected", health["degraded_reasons"])
|
|
self.assertIn("awaiting_player_evidence", health["degraded_reasons"])
|
|
|
|
def test_cleanup_waits_for_post_start_player_evidence_but_new_ingest_does_not(self):
|
|
self.start_service()
|
|
self.service._next_expiry = self.monotonic()
|
|
time.sleep(0.2)
|
|
|
|
self.assertEqual(self.downloads.expiry_calls, 0)
|
|
self.assertFalse(self.downloads.recovered)
|
|
|
|
# A genuinely new playlist cannot mutate, delete, or replace any
|
|
# occupied slot, so it is not blocked by the missing evidence that
|
|
# still defers cleanup/expiry below.
|
|
result = self.service.enqueue_download(
|
|
"https://example.test/brand-new",
|
|
None,
|
|
None,
|
|
)
|
|
self.assertEqual(result["status"], "queued")
|
|
self.assertFalse(result["reused_slot"])
|
|
|
|
self.send_message(
|
|
self.config.topics.player_state,
|
|
"idle",
|
|
retain=True,
|
|
)
|
|
time.sleep(0.2)
|
|
self.assertFalse(self.downloads.recovered)
|
|
self.assertEqual(self.downloads.expiry_calls, 0)
|
|
self.assertIn(
|
|
"awaiting_player_evidence",
|
|
self.service.health()["degraded_reasons"],
|
|
)
|
|
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "idle", "message": "player online"},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.downloads.recovered,
|
|
"detailed Player idle did not release deferred recovery",
|
|
)
|
|
self.wait_for(
|
|
lambda: self.downloads.expiry_calls > 0,
|
|
"detailed Player idle did not release deferred expiry",
|
|
)
|
|
self.assertNotIn(
|
|
"awaiting_player_evidence",
|
|
self.service.health()["degraded_reasons"],
|
|
)
|
|
|
|
def test_ingest_matching_an_occupied_slot_url_remains_gated_without_evidence(self):
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
|
|
with self.assertRaisesRegex(
|
|
ControllerUnavailableError,
|
|
"fresh Player evidence",
|
|
):
|
|
self.service.enqueue_download("https://example.test/1", None, None)
|
|
|
|
def test_controller_only_restart_with_idle_connected_player_allows_new_ingest(self):
|
|
"""Regression for the too-broad evidence gate deadlock.
|
|
|
|
A Controller-only restart with the Player still connected and idle
|
|
never produces fresh detailed evidence, because the Player has no
|
|
reason to reconnect or emit unsolicited status. New ingest into an
|
|
empty slot must still work; refresh/reuse of already-occupied media,
|
|
expiry, cleanup recovery, and manual deletion must all stay gated
|
|
until fresh evidence or an explicit administrative override arrives.
|
|
"""
|
|
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "idle", "message": "player online"},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: "awaiting_player_evidence"
|
|
not in self.service.health()["degraded_reasons"],
|
|
"evidence did not clear before the simulated restart",
|
|
)
|
|
self.service.stop()
|
|
|
|
# Simulate a Controller-only restart against the same durable store:
|
|
# the Player process itself never restarted, stays connected and
|
|
# idle, and therefore never sends another fresh status.
|
|
self.events = queue.Queue()
|
|
restarted_store = ControllerStore(self.config.storage, now=self.wall)
|
|
restarted_mqtt = FakeMqttAdapter()
|
|
restarted_downloads = FakeDownloadManager(restarted_store)
|
|
restarted_service = ControllerService(
|
|
self.config,
|
|
store=restarted_store,
|
|
mqtt_adapter=restarted_mqtt,
|
|
event_queue=self.events,
|
|
download_manager=restarted_downloads,
|
|
monotonic=self.monotonic,
|
|
wall_clock=self.wall,
|
|
)
|
|
restarted_service.start()
|
|
self.addCleanup(
|
|
lambda: restarted_service.stop() if restarted_service.started else None
|
|
)
|
|
self.assertIn(
|
|
"awaiting_player_evidence",
|
|
restarted_service.health()["degraded_reasons"],
|
|
)
|
|
|
|
# A genuinely new playlist still ingests safely with no evidence.
|
|
result = restarted_service.enqueue_download(
|
|
"https://example.test/after-restart", None, None
|
|
)
|
|
self.assertEqual(result["status"], "queued")
|
|
self.assertFalse(result["reused_slot"])
|
|
self.assertNotEqual(result["slot"], 1)
|
|
|
|
# Refreshing the URL already occupying slot 1 remains gated.
|
|
with self.assertRaisesRegex(
|
|
ControllerUnavailableError,
|
|
"fresh Player evidence",
|
|
):
|
|
restarted_service.enqueue_download("https://example.test/1", None, None)
|
|
|
|
# Expiry and cleanup recovery remain deferred.
|
|
restarted_service._next_expiry = self.monotonic()
|
|
time.sleep(0.2)
|
|
self.assertEqual(restarted_downloads.expiry_calls, 0)
|
|
self.assertFalse(restarted_downloads.recovered)
|
|
|
|
# Manual deletion of occupied slot 1 still requires the explicit
|
|
# confirm_player_unknown override; it is not silently unlocked.
|
|
with self.assertRaisesRegex(
|
|
SlotProtectedError,
|
|
"confirm_player_unknown=true",
|
|
):
|
|
restarted_service.submit_control({"action": "delete_slot", "slot": 1})
|
|
|
|
# Fresh Player evidence still unlocks cleanup normally afterward.
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "idle", "message": "player online"},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: restarted_downloads.recovered,
|
|
"fresh evidence did not unlock deferred recovery after restart",
|
|
)
|
|
|
|
def test_administrator_can_explicitly_unlock_cleanup_without_player_evidence(self):
|
|
self.start_service()
|
|
review = self.service.health()["cleanup_reconciliation"]
|
|
self.assertIsNotNone(review)
|
|
|
|
result = self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"scope": "cleanup",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": review["review_id"],
|
|
"actor": "operator",
|
|
"reason": "Player appliance was independently verified offline",
|
|
}
|
|
)
|
|
|
|
self.assertEqual(
|
|
result,
|
|
{
|
|
"status": "cleared",
|
|
"review_id": review["review_id"],
|
|
"scope": "cleanup",
|
|
},
|
|
)
|
|
self.wait_for(
|
|
lambda: self.downloads.recovered,
|
|
"administrative cleanup override did not schedule recovery",
|
|
)
|
|
health = self.service.health()
|
|
self.assertIsNone(health["cleanup_reconciliation"])
|
|
self.assertNotIn("awaiting_player_evidence", health["degraded_reasons"])
|
|
|
|
def test_cleanup_reconciliation_token_rotates_and_cannot_bypass_a_guard(self):
|
|
self.start_service()
|
|
first = self.service.health()["cleanup_reconciliation"]["review_id"]
|
|
self.events.put(MqttEvent(kind="connected", connection_generation=1))
|
|
self.wait_for(
|
|
lambda: (
|
|
self.service.health()["cleanup_reconciliation"] is not None
|
|
and self.service.health()["cleanup_reconciliation"]["review_id"]
|
|
!= first
|
|
),
|
|
"cleanup reconciliation token did not rotate on MQTT connect",
|
|
)
|
|
current = self.service.health()["cleanup_reconciliation"]["review_id"]
|
|
|
|
with self.assertRaisesRegex(StaleOperationError, "review_id is stale"):
|
|
self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"scope": "cleanup",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": first,
|
|
"actor": "operator",
|
|
"reason": "stale request",
|
|
}
|
|
)
|
|
|
|
self.store.protect_slot(1, "restart_uncertain", guard_id="late-guard")
|
|
with self.assertRaisesRegex(SlotProtectedError, "known playback protection"):
|
|
self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"scope": "cleanup",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": current,
|
|
"actor": "operator",
|
|
"reason": "must not bypass a guard",
|
|
}
|
|
)
|
|
|
|
def test_manual_delete_requires_explicit_override_while_player_is_unknown(self):
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
|
|
with self.assertRaisesRegex(
|
|
SlotProtectedError,
|
|
"confirm_player_unknown=true",
|
|
):
|
|
self.service.submit_control({"action": "delete_slot", "slot": 1})
|
|
|
|
def test_administrator_can_clear_exact_uncertain_playback_review(self):
|
|
self.start_service()
|
|
playlist = self.ready_slot(1)
|
|
self.store.set_player_observation(
|
|
availability="unknown",
|
|
phase="unknown",
|
|
active_slot=1,
|
|
active_path=str(playlist),
|
|
paused=None,
|
|
reconciliation_state="operator_review",
|
|
reconciliation_due_at=None,
|
|
reconciliation_id="review-token",
|
|
)
|
|
guard = self.store.protect_slot(
|
|
1,
|
|
"restart_uncertain",
|
|
guard_id="review-token",
|
|
)
|
|
review = self.service.health()["reconciliation_reviews"]
|
|
self.assertEqual(review[0]["review_id"], "review-token")
|
|
|
|
result = self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"slot": 1,
|
|
"review_id": review[0]["review_id"],
|
|
"actor": "operator",
|
|
"reason": "Player host was replaced",
|
|
}
|
|
)
|
|
self.assertEqual(
|
|
result,
|
|
{"status": "cleared", "slot": 1, "review_id": "review-token"},
|
|
)
|
|
self.assertEqual(self.store.list_live_guards(), [])
|
|
observation = self.store.player_observation()
|
|
self.assertEqual(observation.reconciliation_state, "admin_cleared")
|
|
self.assertIsNone(observation.active_slot)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
|
self.assertEqual(self.result_events()[-1]["type"], "playback_reconciled")
|
|
|
|
def test_busy_player_keeps_video_route_until_actual_playback_finishes(self):
|
|
config = replace(
|
|
self.config,
|
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
|
)
|
|
self.start_service(config=config)
|
|
self.ready_slot(1)
|
|
second_playlist = self.ready_slot(2)
|
|
self.send_message(config.topics.router_mode_state, "service", qos=1)
|
|
self.wait_for(
|
|
lambda: self.store.router_observation().mode == "service",
|
|
"Router service mode was not observed",
|
|
)
|
|
self.service.submit_control({"action": "play", "slot": 1})
|
|
self.send_message(config.topics.router_mode_state, "video", qos=1)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.player_commands) == 1,
|
|
"play command was not released",
|
|
)
|
|
self.send_message(
|
|
config.topics.player_status,
|
|
{
|
|
"state": "busy",
|
|
"message": "mpv already playing; stop first",
|
|
},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.service.playback.pending is None,
|
|
"busy response was not reduced",
|
|
)
|
|
observation = self.store.player_observation()
|
|
self.assertIsNone(observation.active_slot)
|
|
self.assertIsNone(observation.active_path)
|
|
self.assertEqual(observation.phase, "unknown")
|
|
self.assertIn(
|
|
"awaiting_player_evidence",
|
|
self.service.health()["degraded_reasons"],
|
|
)
|
|
self.assertEqual(self.mqtt.router_modes, ["video"])
|
|
lease = self.store.live_audio_lease()
|
|
self.assertIsNotNone(lease)
|
|
self.assertEqual(lease.state, "held")
|
|
|
|
self.send_message(
|
|
config.topics.player_status,
|
|
{
|
|
"state": "playing",
|
|
"slot": 2,
|
|
"path": str(second_playlist),
|
|
},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.store.player_observation().active_slot == 2,
|
|
"Player did not reassert its actual playback identity",
|
|
)
|
|
self.assertEqual(self.store.live_audio_lease().state, "held")
|
|
|
|
def test_disconnected_synchronous_play_failure_is_not_reported_pending(self):
|
|
self.start_service(mqtt=FakeMqttAdapter(connected=False))
|
|
self.ready_slot(1)
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["status"], "unavailable")
|
|
self.assertIsNone(self.service.playback.pending)
|
|
self.assertEqual(self.store.list_live_guards(), [])
|
|
|
|
def test_router_timeout_uses_monotonic_clock_and_continues_with_warning(self):
|
|
config = replace(
|
|
self.config,
|
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
|
)
|
|
self.start_service(config=config)
|
|
self.ready_slot(1)
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["router"], "waiting")
|
|
self.wall.value -= 10_000
|
|
self.monotonic.value += 4
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.player_commands) == 1,
|
|
"Router timeout did not release play within its finite deadline",
|
|
)
|
|
warnings = [
|
|
event
|
|
for event in self.result_events()
|
|
if event["type"] == "audio_router_warning"
|
|
]
|
|
self.assertTrue(warnings)
|
|
self.assertIs(warnings[-1]["continued"], True)
|
|
|
|
def test_duplicate_toggle_is_suppressed_and_translated_idempotently(self):
|
|
self.start_service()
|
|
playlist = self.ready_slot(1)
|
|
self.send_message(
|
|
self.config.topics.player_telemetry,
|
|
{
|
|
"state": "telemetry",
|
|
"slot": 1,
|
|
"path": str(playlist),
|
|
"paused": False,
|
|
},
|
|
)
|
|
self.wait_for(
|
|
lambda: self.store.player_observation().active_slot == 1,
|
|
"active telemetry was not consumed",
|
|
)
|
|
command = {"action": "toggle"}
|
|
self.send_message(
|
|
self.config.topics.controller_command,
|
|
command,
|
|
qos=1,
|
|
message_id=71,
|
|
)
|
|
self.send_message(
|
|
self.config.topics.controller_command,
|
|
command,
|
|
qos=1,
|
|
message_id=71,
|
|
duplicate=True,
|
|
)
|
|
self.wait_for(
|
|
lambda: sum(
|
|
event.message_id == 71 for event in self.mqtt.acknowledged
|
|
)
|
|
== 2,
|
|
"duplicate command packets were not both acknowledged",
|
|
)
|
|
self.assertEqual(self.mqtt.player_commands, [{"action": "pause"}])
|
|
|
|
def test_path_only_managed_play_adds_stable_slot_identity(self):
|
|
self.start_service()
|
|
playlist = self.ready_slot(1)
|
|
result = self.service.submit_control(
|
|
{"action": "play", "playlist": str(playlist)}
|
|
)
|
|
self.assertEqual(result["status"], "pending")
|
|
self.assertEqual(
|
|
self.mqtt.player_commands,
|
|
[{"action": "play", "playlist": str(playlist), "slot": 1}],
|
|
)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
|
|
|
def test_explicit_slot_cannot_be_paired_with_another_managed_slot_path(self):
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
second_playlist = self.ready_slot(2)
|
|
with self.assertRaisesRegex(ValueError, "conflicts"):
|
|
self.service.submit_control(
|
|
{
|
|
"action": "play",
|
|
"slot": 1,
|
|
"playlist": str(second_playlist),
|
|
}
|
|
)
|
|
self.assertEqual(self.mqtt.player_commands, [])
|
|
self.assertEqual(self.store.list_live_guards(), [])
|
|
|
|
def test_administrator_can_clear_external_playback_review_without_slot(self):
|
|
config = replace(
|
|
self.config,
|
|
topics=replace(self.config.topics, controller_result=None),
|
|
)
|
|
self.start_service(config=config)
|
|
observation = self.store.mark_external_uncertain(
|
|
"/srv/operator-playlists/special-event.m3u",
|
|
review_after=self.wall.value,
|
|
details={"reason": "restart"},
|
|
)
|
|
self.assertIsNotNone(observation.reconciliation_id)
|
|
reviews = self.service.health()["reconciliation_reviews"]
|
|
self.assertEqual(len(reviews), 1)
|
|
review_id = reviews[0]["review_id"]
|
|
|
|
result = self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": review_id,
|
|
"actor": "operator",
|
|
"reason": "Player appliance was replaced",
|
|
}
|
|
)
|
|
self.assertEqual(
|
|
result,
|
|
{"status": "cleared", "review_id": review_id},
|
|
)
|
|
cleared = self.store.player_observation()
|
|
self.assertIsNone(cleared.active_path)
|
|
self.assertEqual(cleared.reconciliation_state, "admin_cleared")
|
|
|
|
def test_external_play_intent_is_durable_immediately_after_puback(self):
|
|
self.start_service()
|
|
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
|
|
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
|
|
result = self.service.submit_control(
|
|
{"action": "play", "playlist": str(playlist)}
|
|
)
|
|
self.assertEqual(result["status"], "pending")
|
|
self.assertEqual(
|
|
self.mqtt.player_commands,
|
|
[{"action": "play", "playlist": str(playlist)}],
|
|
)
|
|
reviews = self.service.health()["reconciliation_reviews"]
|
|
self.assertEqual(len(reviews), 1)
|
|
review_id = reviews[0]["review_id"]
|
|
self.assertIs(reviews[0]["clearable"], False)
|
|
|
|
self.service.stop()
|
|
self.service = None
|
|
with ControllerStore(self.config.storage, now=self.wall).open() as reopened:
|
|
reopened.mark_restart_uncertain()
|
|
observation = reopened.player_observation()
|
|
self.assertEqual(observation.active_path, str(playlist))
|
|
self.assertEqual(observation.reconciliation_id, review_id)
|
|
|
|
def test_live_external_play_fence_cannot_be_admin_cleared(self):
|
|
self.start_service()
|
|
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
|
|
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
|
|
self.service.submit_control(
|
|
{"action": "play", "playlist": str(playlist)}
|
|
)
|
|
review = self.service.health()["reconciliation_reviews"][0]
|
|
self.assertIs(review["clearable"], False)
|
|
|
|
with self.assertRaisesRegex(ValueError, "Player operation is pending"):
|
|
self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": review["review_id"],
|
|
"actor": "operator",
|
|
"reason": "clear requested too early",
|
|
}
|
|
)
|
|
self.assertIsNotNone(self.service.playback.pending)
|
|
observation = self.store.player_observation()
|
|
self.assertEqual(observation.reconciliation_id, review["review_id"])
|
|
self.assertEqual(observation.active_path, str(playlist))
|
|
|
|
def test_router_gated_play_marks_all_reconciliation_reviews_not_clearable(self):
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
self.store.set_router_observation(mode="service", freshness="current")
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["router"], "waiting")
|
|
# Model a durable/reconnect review becoming visible while Router
|
|
# acquisition is pending.
|
|
self.store.protect_slot(
|
|
2,
|
|
"restart_uncertain",
|
|
guard_id="review-during-router-acquisition",
|
|
)
|
|
|
|
reviews = self.service.health()["reconciliation_reviews"]
|
|
self.assertEqual(len(reviews), 1)
|
|
self.assertIs(reviews[0]["clearable"], False)
|
|
|
|
def test_definite_external_publish_failure_clears_prepublication_guard(self):
|
|
self.start_service(mqtt=FakeMqttAdapter(connected=False))
|
|
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
|
|
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
|
|
result = self.service.submit_control(
|
|
{"action": "play", "playlist": str(playlist)}
|
|
)
|
|
self.assertEqual(result["status"], "unavailable")
|
|
self.assertEqual(self.service.health()["reconciliation_reviews"], [])
|
|
observation = self.store.player_observation()
|
|
self.assertIsNone(observation.active_path)
|
|
self.assertIsNone(observation.reconciliation_id)
|
|
|
|
def test_unexpected_reducer_failure_is_not_acked_or_deduplicated(self):
|
|
self.start_service()
|
|
original = self.service.playback.handle_status
|
|
failure_seen = threading.Event()
|
|
attempts = 0
|
|
|
|
def fail_once(payload):
|
|
nonlocal attempts
|
|
attempts += 1
|
|
if attempts == 1:
|
|
failure_seen.set()
|
|
raise RuntimeError("transient reducer failure")
|
|
return original(payload)
|
|
|
|
self.service.playback.handle_status = fail_once
|
|
payload = {
|
|
"state": "playing",
|
|
"slot": 1,
|
|
"path": str(self.config.storage.media_base / "slot1" / "playlist.m3u"),
|
|
}
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
payload,
|
|
qos=1,
|
|
message_id=88,
|
|
)
|
|
self.assertTrue(failure_seen.wait(timeout=1))
|
|
time.sleep(0.05)
|
|
self.assertEqual(self.mqtt.acknowledged, [])
|
|
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
payload,
|
|
qos=1,
|
|
message_id=88,
|
|
duplicate=True,
|
|
)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.acknowledged) == 1,
|
|
"retried status was not processed and acknowledged",
|
|
)
|
|
self.assertEqual(attempts, 2)
|
|
self.assertEqual(self.store.player_observation().phase, "playing")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|