1170 lines
42 KiB
Python
1170 lines
42 KiB
Python
import json
|
|
import queue
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unittest
|
|
from dataclasses import replace
|
|
from pathlib import Path
|
|
|
|
from controller.config import (
|
|
ControllerConfig,
|
|
HomeAssistantConfig,
|
|
RouterConfig,
|
|
StorageConfig,
|
|
)
|
|
from controller.downloads import canonical_playlist_path
|
|
from controller.models import SlotProtectedError, StaleOperationError
|
|
from controller.mqtt_adapter import MqttEvent, PublishReceipt
|
|
from controller.service import ControllerService, ControllerUnavailableError
|
|
from controller.store import ControllerStore
|
|
|
|
|
|
class MutableClock:
|
|
def __init__(self, value):
|
|
self.value = value
|
|
|
|
def __call__(self):
|
|
return self.value
|
|
|
|
|
|
class FakeMqttAdapter:
|
|
"""In-memory transport boundary; it never starts a network client."""
|
|
|
|
def __init__(self, *, connected=True):
|
|
self.connected = connected
|
|
self.started = False
|
|
self.player_commands = []
|
|
self.router_modes = []
|
|
self.json_messages = []
|
|
self.legacy_messages = []
|
|
self.acknowledged = []
|
|
|
|
def start(self):
|
|
self.started = True
|
|
|
|
def stop(self):
|
|
self.started = False
|
|
|
|
def _receipt(self):
|
|
if self.connected:
|
|
return PublishReceipt(True, message_id=1)
|
|
return PublishReceipt(False, reason="MQTT is disconnected")
|
|
|
|
def publish_player_command(self, payload):
|
|
receipt = self._receipt()
|
|
if receipt.accepted:
|
|
self.player_commands.append(dict(payload))
|
|
return receipt
|
|
|
|
def publish_router_mode(self, mode):
|
|
receipt = self._receipt()
|
|
if receipt.accepted:
|
|
self.router_modes.append(mode)
|
|
return receipt
|
|
|
|
def publish_json(self, topic, payload, *, qos=0, retain=False):
|
|
receipt = self._receipt()
|
|
if receipt.accepted:
|
|
self.json_messages.append((topic, dict(payload), qos, retain))
|
|
return receipt
|
|
|
|
def publish_legacy(self, topic, payload, *, retain=True):
|
|
receipt = self._receipt()
|
|
if receipt.accepted:
|
|
self.legacy_messages.append((topic, payload, retain))
|
|
return receipt
|
|
|
|
def acknowledge(self, event):
|
|
self.acknowledged.append(event)
|
|
return True
|
|
|
|
|
|
class FakeDownloadManager:
|
|
def __init__(self, store=None):
|
|
self.store = store
|
|
self.recovered = False
|
|
self.recovery_calls = 0
|
|
self.expiry_calls = 0
|
|
self.stopped = False
|
|
self.delete_requests = []
|
|
self.enqueue_calls = []
|
|
self.export_calls = []
|
|
self.export_reconciled = False
|
|
|
|
def enqueue(self, url, submitter, note):
|
|
self.enqueue_calls.append((url, submitter, note))
|
|
return self.store.reserve_download(url, submitter, note)
|
|
|
|
def enqueue_export(self, url, mode, submitter, note):
|
|
self.export_calls.append((url, mode, submitter, note))
|
|
return self.store.reserve_export_job(
|
|
url,
|
|
mode,
|
|
submitter,
|
|
note,
|
|
max_active=20,
|
|
)
|
|
|
|
def schedule_export_reconciliation(self):
|
|
self.export_reconciled = True
|
|
return True
|
|
|
|
def recover_pending_deletions(self):
|
|
self.recovered = True
|
|
|
|
def schedule_pending_deletion_recovery(self):
|
|
self.recovered = True
|
|
self.recovery_calls += 1
|
|
return True
|
|
|
|
def schedule_expired_deletions(self):
|
|
self.expiry_calls += 1
|
|
return []
|
|
|
|
def schedule_delete_slot(self, slot, *, reason, expected_url=None):
|
|
self.delete_requests.append((slot, reason, expected_url))
|
|
if self.store is None:
|
|
return None
|
|
return self.store.begin_delete(
|
|
slot,
|
|
reason=reason,
|
|
expected_url=expected_url,
|
|
)
|
|
|
|
def shutdown(self, *, wait):
|
|
self.stopped = wait
|
|
|
|
|
|
class ControllerServiceTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tempdir = tempfile.TemporaryDirectory()
|
|
base = Path(self.tempdir.name)
|
|
self.wall = MutableClock(1_700_000_000.0)
|
|
self.monotonic = MutableClock(100.0)
|
|
self.config = ControllerConfig(
|
|
storage=StorageConfig(
|
|
media_base=base / "media",
|
|
data_base=base / "data",
|
|
retention_seconds=3600,
|
|
expiry_interval_seconds=3600,
|
|
uncertain_review_after_seconds=30,
|
|
),
|
|
router=RouterConfig(
|
|
required_mode="video",
|
|
confirmation_timeout_seconds=3,
|
|
when_state_unknown="continue",
|
|
continue_on_confirmation_failure=True,
|
|
fallback_restore_mode=None,
|
|
),
|
|
homeassistant=HomeAssistantConfig(enabled=False),
|
|
)
|
|
self.events = queue.Queue()
|
|
self.store = None
|
|
self.mqtt = None
|
|
self.downloads = None
|
|
self.service = None
|
|
|
|
def tearDown(self):
|
|
if self.service is not None and self.service.started:
|
|
self.service.stop()
|
|
self.tempdir.cleanup()
|
|
|
|
def start_service(self, *, config=None, mqtt=None):
|
|
config = config or self.config
|
|
self.events = queue.Queue()
|
|
self.store = ControllerStore(config.storage, now=self.wall)
|
|
self.mqtt = mqtt or FakeMqttAdapter()
|
|
self.downloads = FakeDownloadManager(self.store)
|
|
self.service = ControllerService(
|
|
config,
|
|
store=self.store,
|
|
mqtt_adapter=self.mqtt,
|
|
event_queue=self.events,
|
|
download_manager=self.downloads,
|
|
monotonic=self.monotonic,
|
|
wall_clock=self.wall,
|
|
)
|
|
self.service.start()
|
|
self.assertFalse(self.downloads.recovered)
|
|
self.assertTrue(self.downloads.export_reconciled)
|
|
return self.service
|
|
|
|
def ready_slot(self, slot=1):
|
|
reservation = self.store.reserve_download(
|
|
f"https://example.test/{slot}",
|
|
"tester",
|
|
"service test",
|
|
job_id=f"ready-{slot}",
|
|
)
|
|
self.assertEqual(reservation.slot, slot)
|
|
self.assertTrue(
|
|
self.store.mark_download_running(
|
|
reservation.job_id,
|
|
reservation.slot,
|
|
reservation.generation,
|
|
)
|
|
)
|
|
self.assertTrue(
|
|
self.store.complete_download(
|
|
reservation.job_id,
|
|
reservation.slot,
|
|
reservation.generation,
|
|
title=f"Slot {slot}",
|
|
items=(("One", "one.mp4"),),
|
|
)
|
|
)
|
|
playlist = canonical_playlist_path(self.service.config.storage.media_base, slot)
|
|
playlist.parent.mkdir(parents=True, exist_ok=True)
|
|
playlist.write_text("one.mp4\n", encoding="utf-8")
|
|
return playlist
|
|
|
|
def send_message(
|
|
self,
|
|
topic,
|
|
payload,
|
|
*,
|
|
qos=0,
|
|
retain=False,
|
|
message_id=None,
|
|
duplicate=False,
|
|
controller_echo=False,
|
|
):
|
|
if not isinstance(payload, bytes):
|
|
payload = (
|
|
payload.encode("utf-8")
|
|
if isinstance(payload, str)
|
|
else json.dumps(payload).encode("utf-8")
|
|
)
|
|
self.events.put(
|
|
MqttEvent(
|
|
kind="message",
|
|
topic=topic,
|
|
payload=payload,
|
|
qos=qos,
|
|
retain=retain,
|
|
duplicate=duplicate,
|
|
message_id=message_id,
|
|
connection_generation=1,
|
|
controller_echo=controller_echo,
|
|
)
|
|
)
|
|
|
|
def wait_for(self, predicate, message):
|
|
deadline = time.monotonic() + 2.0
|
|
while time.monotonic() < deadline:
|
|
if predicate():
|
|
return
|
|
time.sleep(0.01)
|
|
self.fail(message)
|
|
|
|
def result_events(self):
|
|
return [
|
|
payload
|
|
for topic, payload, qos, retain in self.mqtt.json_messages
|
|
if topic == self.service.config.topics.controller_result
|
|
]
|
|
|
|
def test_play_waits_for_router_confirmation_and_player_evidence(self):
|
|
config = replace(
|
|
self.config,
|
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
|
)
|
|
self.start_service(config=config)
|
|
playlist = self.ready_slot(1)
|
|
|
|
self.send_message(config.topics.router_mode_state, "service", qos=1)
|
|
self.wait_for(
|
|
lambda: self.store.router_observation().mode == "service",
|
|
"Router state was not consumed",
|
|
)
|
|
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["status"], "pending")
|
|
self.assertEqual(result["router"], "waiting")
|
|
self.assertEqual(self.mqtt.router_modes, ["video"])
|
|
self.assertEqual(self.mqtt.player_commands, [])
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
|
|
|
self.send_message(
|
|
config.topics.router_mode_set,
|
|
"video",
|
|
qos=1,
|
|
controller_echo=True,
|
|
)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.acknowledged) == 2,
|
|
"Controller Router command echo was not consumed",
|
|
)
|
|
self.assertEqual(self.store.live_audio_lease().state, "acquiring")
|
|
self.assertEqual(self.mqtt.player_commands, [])
|
|
|
|
self.send_message(config.topics.router_mode_state, "video", qos=1)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.player_commands) == 1,
|
|
"Player command was not released after Router confirmation",
|
|
)
|
|
self.assertEqual(
|
|
self.mqtt.player_commands,
|
|
[{"action": "play", "slot": 1, "label": "Slot 1"}],
|
|
)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
|
|
|
self.send_message(
|
|
config.topics.player_status,
|
|
{"state": "playing", "slot": 1, "path": str(playlist)},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.store.player_observation().phase == "playing",
|
|
"authoritative Player status was not reduced",
|
|
)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "playing")
|
|
self.assertIn(
|
|
"player_operation_confirmed",
|
|
{event["type"] for event in self.result_events()},
|
|
)
|
|
|
|
def test_manual_router_set_during_acquisition_resolves_by_failure_policy(self):
|
|
config = replace(
|
|
self.config,
|
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
|
)
|
|
self.start_service(config=config)
|
|
self.ready_slot(1)
|
|
self.send_message(config.topics.router_mode_state, "service", qos=1)
|
|
self.wait_for(
|
|
lambda: self.store.router_observation().mode == "service",
|
|
"Router state was not consumed",
|
|
)
|
|
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["router"], "waiting")
|
|
self.send_message(config.topics.router_mode_set, "announcements", qos=1)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.player_commands) == 1,
|
|
"manual Router intent did not resolve acquisition",
|
|
)
|
|
lease = self.store.get_audio_lease(self.service._session_lease_id)
|
|
self.assertIsNotNone(lease)
|
|
self.assertEqual(lease.state, "unconfirmed")
|
|
warnings = [
|
|
event
|
|
for event in self.result_events()
|
|
if event["type"] == "audio_router_warning"
|
|
]
|
|
self.assertTrue(warnings)
|
|
self.assertIs(warnings[-1]["continued"], True)
|
|
|
|
def test_pause_state_remains_telemetry_authoritative(self):
|
|
self.start_service()
|
|
playlist = self.ready_slot(1)
|
|
self.service.submit_control({"action": "play", "slot": 1})
|
|
self.wait_for(
|
|
lambda: self.mqtt.player_commands == [
|
|
{"action": "play", "slot": 1, "label": "Slot 1"}
|
|
],
|
|
"play command was not published",
|
|
)
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "playing", "slot": 1, "path": str(playlist)},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.service.playback.pending is None,
|
|
"play operation was not confirmed",
|
|
)
|
|
playing_generation = self.store.player_observation().generation
|
|
|
|
self.service.submit_control({"action": "pause"})
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "paused", "slot": 1, "path": str(playlist)},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.store.player_observation().generation > playing_generation,
|
|
"pause status was not consumed",
|
|
)
|
|
self.assertIsNotNone(self.service.playback.pending)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "playing")
|
|
|
|
self.send_message(
|
|
self.config.topics.player_telemetry,
|
|
{
|
|
"state": "telemetry",
|
|
"slot": 1,
|
|
"path": str(playlist),
|
|
"paused": True,
|
|
},
|
|
)
|
|
self.wait_for(
|
|
lambda: self.service.playback.pending is None,
|
|
"pause was not confirmed by telemetry",
|
|
)
|
|
self.assertEqual(self.store.player_observation().paused, True)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "paused")
|
|
self.assertEqual(self.mqtt.player_commands[-1], {"action": "pause"})
|
|
|
|
def test_retained_controller_command_is_rejected_then_acked_and_reported(self):
|
|
self.start_service()
|
|
self.send_message(
|
|
self.config.topics.controller_command,
|
|
{"action": "play", "path": "/must/not/run"},
|
|
qos=1,
|
|
retain=True,
|
|
message_id=42,
|
|
)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.acknowledged) == 1,
|
|
"retained command was not acknowledged after rejection",
|
|
)
|
|
self.assertEqual(self.mqtt.player_commands, [])
|
|
event = self.result_events()[-1]
|
|
self.assertEqual(event["version"], 2)
|
|
self.assertEqual(event["type"], "retained_command_rejected")
|
|
topic, _, qos, retain = self.mqtt.json_messages[-1]
|
|
self.assertEqual(topic, "youtube/controller/v2/events")
|
|
self.assertEqual((qos, retain), (1, False))
|
|
self.assertEqual(self.mqtt.acknowledged[0].message_id, 42)
|
|
|
|
def test_retained_detailed_player_evidence_cannot_clear_restart_protection(self):
|
|
self.start_service()
|
|
self.store.protect_slot(
|
|
1,
|
|
"restart_uncertain",
|
|
guard_id="restart-review",
|
|
)
|
|
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "idle", "slot": 1, "reason": "stop"},
|
|
qos=1,
|
|
retain=True,
|
|
message_id=41,
|
|
)
|
|
self.send_message(
|
|
self.config.topics.player_telemetry,
|
|
{
|
|
"state": "telemetry",
|
|
"slot": 1,
|
|
"path": None,
|
|
"paused": False,
|
|
},
|
|
retain=True,
|
|
)
|
|
self.wait_for(
|
|
lambda: any(event.message_id == 41 for event in self.mqtt.acknowledged),
|
|
"retained QoS 1 status was not deliberately acknowledged",
|
|
)
|
|
self.wait_for(
|
|
lambda: len(
|
|
[
|
|
event
|
|
for event in self.result_events()
|
|
if event["type"] == "retained_player_evidence_rejected"
|
|
]
|
|
)
|
|
== 2,
|
|
"retained telemetry rejection was not reported",
|
|
)
|
|
|
|
guards = self.store.list_live_guards()
|
|
self.assertEqual(len(guards), 1)
|
|
self.assertEqual(guards[0].guard_id, "restart-review")
|
|
self.assertIn(
|
|
"awaiting_player_evidence",
|
|
self.service.health()["degraded_reasons"],
|
|
)
|
|
retained_warnings = [
|
|
event
|
|
for event in self.result_events()
|
|
if event["type"] == "retained_player_evidence_rejected"
|
|
]
|
|
self.assertEqual(len(retained_warnings), 2)
|
|
|
|
def test_optional_result_topic_can_be_disabled_without_affecting_ack(self):
|
|
config = replace(
|
|
self.config,
|
|
topics=replace(self.config.topics, controller_result=None),
|
|
)
|
|
self.start_service(config=config)
|
|
self.send_message(
|
|
config.topics.controller_command,
|
|
{"action": "delete_slot", "slot": 1},
|
|
qos=1,
|
|
retain=True,
|
|
message_id=7,
|
|
)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.acknowledged) == 1,
|
|
"command was not acknowledged with result topic disabled",
|
|
)
|
|
self.assertEqual(self.mqtt.json_messages, [])
|
|
|
|
def test_broker_unavailable_does_not_prevent_service_startup(self):
|
|
mqtt = FakeMqttAdapter(connected=False)
|
|
self.start_service(mqtt=mqtt)
|
|
health = self.service.health()
|
|
self.assertTrue(self.service.started)
|
|
self.assertTrue(mqtt.started)
|
|
self.assertTrue(health["coordinator_alive"])
|
|
self.assertEqual(health["status"], "degraded")
|
|
self.assertIn("mqtt_disconnected", health["degraded_reasons"])
|
|
self.assertIn("awaiting_player_evidence", health["degraded_reasons"])
|
|
|
|
def test_utility_export_is_slot_free_and_not_gated_by_player_evidence(self):
|
|
self.start_service()
|
|
self.assertIn(
|
|
"awaiting_player_evidence",
|
|
self.service.health()["degraded_reasons"],
|
|
)
|
|
|
|
result = self.service.enqueue_export(
|
|
"https://www.youtube.com/playlist?list=utility",
|
|
"mp4",
|
|
"Alice",
|
|
"ProPresenter",
|
|
)
|
|
|
|
self.assertEqual(result["kind"], "export")
|
|
self.assertEqual(result["status"], "queued")
|
|
self.assertEqual(result["mode"], "mp4")
|
|
self.assertIsNotNone(result["job_id"])
|
|
self.assertEqual(
|
|
self.downloads.export_calls,
|
|
[
|
|
(
|
|
"https://www.youtube.com/playlist?list=utility",
|
|
"mp4",
|
|
"Alice",
|
|
"ProPresenter",
|
|
)
|
|
],
|
|
)
|
|
self.assertTrue(
|
|
all(slot.media_state == "empty" for slot in self.store.list_slots())
|
|
)
|
|
self.assertEqual(self.mqtt.json_messages, [])
|
|
self.assertEqual(self.mqtt.legacy_messages, [])
|
|
jobs = self.service.list_export_jobs(limit=10)
|
|
self.assertEqual(len(jobs), 1)
|
|
self.assertEqual(jobs[0]["job_id"], result["job_id"])
|
|
self.assertEqual(jobs[0]["output_paths"], [])
|
|
|
|
def test_invalid_utility_mode_is_rejected_without_player_side_effects(self):
|
|
self.start_service()
|
|
|
|
with self.assertRaisesRegex(ValueError, "unsupported utility export mode"):
|
|
self.service.enqueue_export(
|
|
"https://www.youtube.com/watch?v=one",
|
|
"slot1",
|
|
None,
|
|
None,
|
|
)
|
|
|
|
self.assertTrue(
|
|
all(slot.media_state == "empty" for slot in self.store.list_slots())
|
|
)
|
|
self.assertEqual(self.mqtt.json_messages, [])
|
|
self.assertEqual(self.mqtt.legacy_messages, [])
|
|
|
|
def test_cleanup_waits_for_post_start_player_evidence_but_new_ingest_does_not(self):
|
|
self.start_service()
|
|
self.service._next_expiry = self.monotonic()
|
|
time.sleep(0.2)
|
|
|
|
self.assertEqual(self.downloads.expiry_calls, 0)
|
|
self.assertFalse(self.downloads.recovered)
|
|
|
|
# A genuinely new playlist cannot mutate, delete, or replace any
|
|
# occupied slot, so it is not blocked by the missing evidence that
|
|
# still defers cleanup/expiry below.
|
|
result = self.service.enqueue_download(
|
|
"https://example.test/brand-new",
|
|
None,
|
|
None,
|
|
)
|
|
self.assertEqual(result["status"], "queued")
|
|
self.assertFalse(result["reused_slot"])
|
|
|
|
self.send_message(
|
|
self.config.topics.player_state,
|
|
"idle",
|
|
retain=True,
|
|
)
|
|
time.sleep(0.2)
|
|
self.assertFalse(self.downloads.recovered)
|
|
self.assertEqual(self.downloads.expiry_calls, 0)
|
|
self.assertIn(
|
|
"awaiting_player_evidence",
|
|
self.service.health()["degraded_reasons"],
|
|
)
|
|
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "idle", "message": "player online"},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.downloads.recovered,
|
|
"detailed Player idle did not release deferred recovery",
|
|
)
|
|
self.wait_for(
|
|
lambda: self.downloads.expiry_calls > 0,
|
|
"detailed Player idle did not release deferred expiry",
|
|
)
|
|
self.assertNotIn(
|
|
"awaiting_player_evidence",
|
|
self.service.health()["degraded_reasons"],
|
|
)
|
|
|
|
def test_ingest_matching_an_occupied_slot_url_remains_gated_without_evidence(self):
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
|
|
with self.assertRaisesRegex(
|
|
ControllerUnavailableError,
|
|
"fresh Player evidence",
|
|
):
|
|
self.service.enqueue_download("https://example.test/1", None, None)
|
|
|
|
def test_controller_only_restart_with_idle_connected_player_allows_new_ingest(self):
|
|
"""Regression for the too-broad evidence gate deadlock.
|
|
|
|
A Controller-only restart with the Player still connected and idle
|
|
never produces fresh detailed evidence, because the Player has no
|
|
reason to reconnect or emit unsolicited status. New ingest into an
|
|
empty slot must still work; refresh/reuse of already-occupied media,
|
|
expiry, cleanup recovery, and manual deletion must all stay gated
|
|
until fresh evidence or an explicit administrative override arrives.
|
|
"""
|
|
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "idle", "message": "player online"},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: "awaiting_player_evidence"
|
|
not in self.service.health()["degraded_reasons"],
|
|
"evidence did not clear before the simulated restart",
|
|
)
|
|
self.service.stop()
|
|
|
|
# Simulate a Controller-only restart against the same durable store:
|
|
# the Player process itself never restarted, stays connected and
|
|
# idle, and therefore never sends another fresh status.
|
|
self.events = queue.Queue()
|
|
restarted_store = ControllerStore(self.config.storage, now=self.wall)
|
|
restarted_mqtt = FakeMqttAdapter()
|
|
restarted_downloads = FakeDownloadManager(restarted_store)
|
|
restarted_service = ControllerService(
|
|
self.config,
|
|
store=restarted_store,
|
|
mqtt_adapter=restarted_mqtt,
|
|
event_queue=self.events,
|
|
download_manager=restarted_downloads,
|
|
monotonic=self.monotonic,
|
|
wall_clock=self.wall,
|
|
)
|
|
restarted_service.start()
|
|
self.addCleanup(
|
|
lambda: restarted_service.stop() if restarted_service.started else None
|
|
)
|
|
self.assertIn(
|
|
"awaiting_player_evidence",
|
|
restarted_service.health()["degraded_reasons"],
|
|
)
|
|
|
|
# A genuinely new playlist still ingests safely with no evidence.
|
|
result = restarted_service.enqueue_download(
|
|
"https://example.test/after-restart", None, None
|
|
)
|
|
self.assertEqual(result["status"], "queued")
|
|
self.assertFalse(result["reused_slot"])
|
|
self.assertNotEqual(result["slot"], 1)
|
|
|
|
# Refreshing the URL already occupying slot 1 remains gated.
|
|
with self.assertRaisesRegex(
|
|
ControllerUnavailableError,
|
|
"fresh Player evidence",
|
|
):
|
|
restarted_service.enqueue_download("https://example.test/1", None, None)
|
|
|
|
# Expiry and cleanup recovery remain deferred.
|
|
restarted_service._next_expiry = self.monotonic()
|
|
time.sleep(0.2)
|
|
self.assertEqual(restarted_downloads.expiry_calls, 0)
|
|
self.assertFalse(restarted_downloads.recovered)
|
|
|
|
# Manual deletion of occupied slot 1 still requires the explicit
|
|
# confirm_player_unknown override; it is not silently unlocked.
|
|
with self.assertRaisesRegex(
|
|
SlotProtectedError,
|
|
"confirm_player_unknown=true",
|
|
):
|
|
restarted_service.submit_control({"action": "delete_slot", "slot": 1})
|
|
|
|
# Fresh Player evidence still unlocks cleanup normally afterward.
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
{"state": "idle", "message": "player online"},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: restarted_downloads.recovered,
|
|
"fresh evidence did not unlock deferred recovery after restart",
|
|
)
|
|
|
|
def test_administrator_can_explicitly_unlock_cleanup_without_player_evidence(self):
|
|
self.start_service()
|
|
review = self.service.health()["cleanup_reconciliation"]
|
|
self.assertIsNotNone(review)
|
|
|
|
result = self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"scope": "cleanup",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": review["review_id"],
|
|
"actor": "operator",
|
|
"reason": "Player appliance was independently verified offline",
|
|
}
|
|
)
|
|
|
|
self.assertEqual(
|
|
result,
|
|
{
|
|
"status": "cleared",
|
|
"review_id": review["review_id"],
|
|
"scope": "cleanup",
|
|
},
|
|
)
|
|
self.wait_for(
|
|
lambda: self.downloads.recovered,
|
|
"administrative cleanup override did not schedule recovery",
|
|
)
|
|
health = self.service.health()
|
|
self.assertIsNone(health["cleanup_reconciliation"])
|
|
self.assertNotIn("awaiting_player_evidence", health["degraded_reasons"])
|
|
|
|
def test_cleanup_reconciliation_token_rotates_and_cannot_bypass_a_guard(self):
|
|
self.start_service()
|
|
first = self.service.health()["cleanup_reconciliation"]["review_id"]
|
|
self.events.put(MqttEvent(kind="connected", connection_generation=1))
|
|
self.wait_for(
|
|
lambda: (
|
|
self.service.health()["cleanup_reconciliation"] is not None
|
|
and self.service.health()["cleanup_reconciliation"]["review_id"]
|
|
!= first
|
|
),
|
|
"cleanup reconciliation token did not rotate on MQTT connect",
|
|
)
|
|
current = self.service.health()["cleanup_reconciliation"]["review_id"]
|
|
|
|
with self.assertRaisesRegex(StaleOperationError, "review_id is stale"):
|
|
self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"scope": "cleanup",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": first,
|
|
"actor": "operator",
|
|
"reason": "stale request",
|
|
}
|
|
)
|
|
|
|
self.store.protect_slot(1, "restart_uncertain", guard_id="late-guard")
|
|
with self.assertRaisesRegex(SlotProtectedError, "known playback protection"):
|
|
self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"scope": "cleanup",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": current,
|
|
"actor": "operator",
|
|
"reason": "must not bypass a guard",
|
|
}
|
|
)
|
|
|
|
def test_manual_delete_requires_explicit_override_while_player_is_unknown(self):
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
|
|
with self.assertRaisesRegex(
|
|
SlotProtectedError,
|
|
"confirm_player_unknown=true",
|
|
):
|
|
self.service.submit_control({"action": "delete_slot", "slot": 1})
|
|
|
|
def test_administrator_can_clear_exact_uncertain_playback_review(self):
|
|
self.start_service()
|
|
playlist = self.ready_slot(1)
|
|
self.store.set_player_observation(
|
|
availability="unknown",
|
|
phase="unknown",
|
|
active_slot=1,
|
|
active_path=str(playlist),
|
|
paused=None,
|
|
reconciliation_state="operator_review",
|
|
reconciliation_due_at=None,
|
|
reconciliation_id="review-token",
|
|
)
|
|
guard = self.store.protect_slot(
|
|
1,
|
|
"restart_uncertain",
|
|
guard_id="review-token",
|
|
)
|
|
review = self.service.health()["reconciliation_reviews"]
|
|
self.assertEqual(review[0]["review_id"], "review-token")
|
|
|
|
result = self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"slot": 1,
|
|
"review_id": review[0]["review_id"],
|
|
"actor": "operator",
|
|
"reason": "Player host was replaced",
|
|
}
|
|
)
|
|
self.assertEqual(
|
|
result,
|
|
{"status": "cleared", "slot": 1, "review_id": "review-token"},
|
|
)
|
|
self.assertEqual(self.store.list_live_guards(), [])
|
|
observation = self.store.player_observation()
|
|
self.assertEqual(observation.reconciliation_state, "admin_cleared")
|
|
self.assertIsNone(observation.active_slot)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
|
self.assertEqual(self.result_events()[-1]["type"], "playback_reconciled")
|
|
|
|
def test_busy_player_keeps_video_route_until_actual_playback_finishes(self):
|
|
config = replace(
|
|
self.config,
|
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
|
)
|
|
self.start_service(config=config)
|
|
self.ready_slot(1)
|
|
second_playlist = self.ready_slot(2)
|
|
self.send_message(config.topics.router_mode_state, "service", qos=1)
|
|
self.wait_for(
|
|
lambda: self.store.router_observation().mode == "service",
|
|
"Router service mode was not observed",
|
|
)
|
|
self.service.submit_control({"action": "play", "slot": 1})
|
|
self.send_message(config.topics.router_mode_state, "video", qos=1)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.player_commands) == 1,
|
|
"play command was not released",
|
|
)
|
|
self.send_message(
|
|
config.topics.player_status,
|
|
{
|
|
"state": "busy",
|
|
"message": "mpv already playing; stop first",
|
|
},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.service.playback.pending is None,
|
|
"busy response was not reduced",
|
|
)
|
|
observation = self.store.player_observation()
|
|
self.assertIsNone(observation.active_slot)
|
|
self.assertIsNone(observation.active_path)
|
|
self.assertEqual(observation.phase, "unknown")
|
|
self.assertIn(
|
|
"awaiting_player_evidence",
|
|
self.service.health()["degraded_reasons"],
|
|
)
|
|
self.assertEqual(self.mqtt.router_modes, ["video"])
|
|
lease = self.store.live_audio_lease()
|
|
self.assertIsNotNone(lease)
|
|
self.assertEqual(lease.state, "held")
|
|
|
|
self.send_message(
|
|
config.topics.player_status,
|
|
{
|
|
"state": "playing",
|
|
"slot": 2,
|
|
"path": str(second_playlist),
|
|
},
|
|
qos=1,
|
|
)
|
|
self.wait_for(
|
|
lambda: self.store.player_observation().active_slot == 2,
|
|
"Player did not reassert its actual playback identity",
|
|
)
|
|
self.assertEqual(self.store.live_audio_lease().state, "held")
|
|
|
|
def test_disconnected_synchronous_play_failure_is_not_reported_pending(self):
|
|
self.start_service(mqtt=FakeMqttAdapter(connected=False))
|
|
self.ready_slot(1)
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["status"], "unavailable")
|
|
self.assertIsNone(self.service.playback.pending)
|
|
self.assertEqual(self.store.list_live_guards(), [])
|
|
|
|
def test_router_timeout_uses_monotonic_clock_and_continues_with_warning(self):
|
|
config = replace(
|
|
self.config,
|
|
router=replace(self.config.router, when_state_unknown="attempt"),
|
|
)
|
|
self.start_service(config=config)
|
|
self.ready_slot(1)
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["router"], "waiting")
|
|
self.wall.value -= 10_000
|
|
self.monotonic.value += 4
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.player_commands) == 1,
|
|
"Router timeout did not release play within its finite deadline",
|
|
)
|
|
warnings = [
|
|
event
|
|
for event in self.result_events()
|
|
if event["type"] == "audio_router_warning"
|
|
]
|
|
self.assertTrue(warnings)
|
|
self.assertIs(warnings[-1]["continued"], True)
|
|
|
|
def test_duplicate_toggle_is_suppressed_and_translated_idempotently(self):
|
|
self.start_service()
|
|
playlist = self.ready_slot(1)
|
|
self.send_message(
|
|
self.config.topics.player_telemetry,
|
|
{
|
|
"state": "telemetry",
|
|
"slot": 1,
|
|
"path": str(playlist),
|
|
"paused": False,
|
|
},
|
|
)
|
|
self.wait_for(
|
|
lambda: self.store.player_observation().active_slot == 1,
|
|
"active telemetry was not consumed",
|
|
)
|
|
command = {"action": "toggle"}
|
|
self.send_message(
|
|
self.config.topics.controller_command,
|
|
command,
|
|
qos=1,
|
|
message_id=71,
|
|
)
|
|
self.send_message(
|
|
self.config.topics.controller_command,
|
|
command,
|
|
qos=1,
|
|
message_id=71,
|
|
duplicate=True,
|
|
)
|
|
self.wait_for(
|
|
lambda: sum(
|
|
event.message_id == 71 for event in self.mqtt.acknowledged
|
|
)
|
|
== 2,
|
|
"duplicate command packets were not both acknowledged",
|
|
)
|
|
self.assertEqual(self.mqtt.player_commands, [{"action": "pause"}])
|
|
|
|
def test_path_only_managed_play_adds_stable_slot_identity(self):
|
|
self.start_service()
|
|
playlist = self.ready_slot(1)
|
|
result = self.service.submit_control(
|
|
{"action": "play", "playlist": str(playlist)}
|
|
)
|
|
self.assertEqual(result["status"], "pending")
|
|
self.assertEqual(
|
|
self.mqtt.player_commands,
|
|
[{"action": "play", "playlist": str(playlist), "slot": 1}],
|
|
)
|
|
self.assertEqual(self.service.playback.projected_slot_state(1), "ready")
|
|
|
|
def test_explicit_slot_cannot_be_paired_with_another_managed_slot_path(self):
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
second_playlist = self.ready_slot(2)
|
|
with self.assertRaisesRegex(ValueError, "conflicts"):
|
|
self.service.submit_control(
|
|
{
|
|
"action": "play",
|
|
"slot": 1,
|
|
"playlist": str(second_playlist),
|
|
}
|
|
)
|
|
self.assertEqual(self.mqtt.player_commands, [])
|
|
self.assertEqual(self.store.list_live_guards(), [])
|
|
|
|
def test_administrator_can_clear_external_playback_review_without_slot(self):
|
|
config = replace(
|
|
self.config,
|
|
topics=replace(self.config.topics, controller_result=None),
|
|
)
|
|
self.start_service(config=config)
|
|
observation = self.store.mark_external_uncertain(
|
|
"/srv/operator-playlists/special-event.m3u",
|
|
review_after=self.wall.value,
|
|
details={"reason": "restart"},
|
|
)
|
|
self.assertIsNotNone(observation.reconciliation_id)
|
|
reviews = self.service.health()["reconciliation_reviews"]
|
|
self.assertEqual(len(reviews), 1)
|
|
review_id = reviews[0]["review_id"]
|
|
|
|
result = self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": review_id,
|
|
"actor": "operator",
|
|
"reason": "Player appliance was replaced",
|
|
}
|
|
)
|
|
self.assertEqual(
|
|
result,
|
|
{"status": "cleared", "review_id": review_id},
|
|
)
|
|
cleared = self.store.player_observation()
|
|
self.assertIsNone(cleared.active_path)
|
|
self.assertEqual(cleared.reconciliation_state, "admin_cleared")
|
|
|
|
def test_external_play_intent_is_durable_immediately_after_puback(self):
|
|
self.start_service()
|
|
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
|
|
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
|
|
result = self.service.submit_control(
|
|
{"action": "play", "playlist": str(playlist)}
|
|
)
|
|
self.assertEqual(result["status"], "pending")
|
|
self.assertEqual(
|
|
self.mqtt.player_commands,
|
|
[{"action": "play", "playlist": str(playlist)}],
|
|
)
|
|
reviews = self.service.health()["reconciliation_reviews"]
|
|
self.assertEqual(len(reviews), 1)
|
|
review_id = reviews[0]["review_id"]
|
|
self.assertIs(reviews[0]["clearable"], False)
|
|
|
|
self.service.stop()
|
|
self.service = None
|
|
with ControllerStore(self.config.storage, now=self.wall).open() as reopened:
|
|
reopened.mark_restart_uncertain()
|
|
observation = reopened.player_observation()
|
|
self.assertEqual(observation.active_path, str(playlist))
|
|
self.assertEqual(observation.reconciliation_id, review_id)
|
|
|
|
def test_live_external_play_fence_cannot_be_admin_cleared(self):
|
|
self.start_service()
|
|
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
|
|
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
|
|
self.service.submit_control(
|
|
{"action": "play", "playlist": str(playlist)}
|
|
)
|
|
review = self.service.health()["reconciliation_reviews"][0]
|
|
self.assertIs(review["clearable"], False)
|
|
|
|
with self.assertRaisesRegex(ValueError, "Player operation is pending"):
|
|
self.service.submit_control(
|
|
{
|
|
"action": "reconcile_playback",
|
|
"decision": "clear",
|
|
"confirm": True,
|
|
"review_id": review["review_id"],
|
|
"actor": "operator",
|
|
"reason": "clear requested too early",
|
|
}
|
|
)
|
|
self.assertIsNotNone(self.service.playback.pending)
|
|
observation = self.store.player_observation()
|
|
self.assertEqual(observation.reconciliation_id, review["review_id"])
|
|
self.assertEqual(observation.active_path, str(playlist))
|
|
|
|
def test_router_gated_play_marks_all_reconciliation_reviews_not_clearable(self):
|
|
self.start_service()
|
|
self.ready_slot(1)
|
|
self.store.set_router_observation(mode="service", freshness="current")
|
|
result = self.service.submit_control({"action": "play", "slot": 1})
|
|
self.assertEqual(result["router"], "waiting")
|
|
# Model a durable/reconnect review becoming visible while Router
|
|
# acquisition is pending.
|
|
self.store.protect_slot(
|
|
2,
|
|
"restart_uncertain",
|
|
guard_id="review-during-router-acquisition",
|
|
)
|
|
|
|
reviews = self.service.health()["reconciliation_reviews"]
|
|
self.assertEqual(len(reviews), 1)
|
|
self.assertIs(reviews[0]["clearable"], False)
|
|
|
|
def test_definite_external_publish_failure_clears_prepublication_guard(self):
|
|
self.start_service(mqtt=FakeMqttAdapter(connected=False))
|
|
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
|
|
playlist.write_text("/srv/media/opening.mp4\n", encoding="utf-8")
|
|
result = self.service.submit_control(
|
|
{"action": "play", "playlist": str(playlist)}
|
|
)
|
|
self.assertEqual(result["status"], "unavailable")
|
|
self.assertEqual(self.service.health()["reconciliation_reviews"], [])
|
|
observation = self.store.player_observation()
|
|
self.assertIsNone(observation.active_path)
|
|
self.assertIsNone(observation.reconciliation_id)
|
|
|
|
def test_unexpected_reducer_failure_is_not_acked_or_deduplicated(self):
|
|
self.start_service()
|
|
original = self.service.playback.handle_status
|
|
failure_seen = threading.Event()
|
|
attempts = 0
|
|
|
|
def fail_once(payload):
|
|
nonlocal attempts
|
|
attempts += 1
|
|
if attempts == 1:
|
|
failure_seen.set()
|
|
raise RuntimeError("transient reducer failure")
|
|
return original(payload)
|
|
|
|
self.service.playback.handle_status = fail_once
|
|
payload = {
|
|
"state": "playing",
|
|
"slot": 1,
|
|
"path": str(self.config.storage.media_base / "slot1" / "playlist.m3u"),
|
|
}
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
payload,
|
|
qos=1,
|
|
message_id=88,
|
|
)
|
|
self.assertTrue(failure_seen.wait(timeout=1))
|
|
time.sleep(0.05)
|
|
self.assertEqual(self.mqtt.acknowledged, [])
|
|
|
|
self.send_message(
|
|
self.config.topics.player_status,
|
|
payload,
|
|
qos=1,
|
|
message_id=88,
|
|
duplicate=True,
|
|
)
|
|
self.wait_for(
|
|
lambda: len(self.mqtt.acknowledged) == 1,
|
|
"retried status was not processed and acknowledged",
|
|
)
|
|
self.assertEqual(attempts, 2)
|
|
self.assertEqual(self.store.player_observation().phase, "playing")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|