feat: overhaul video player controller

This commit is contained in:
Media
2026-08-06 20:08:12 -05:00
parent 72cc66df92
commit 5e81403be3
10 changed files with 2019 additions and 122 deletions
+167
View File
@@ -24,6 +24,24 @@ class ImmediateExecutor:
return future
class QueuedExecutor:
def __init__(self):
self.calls = []
def submit(self, function, *args, **kwargs):
future = Future()
self.calls.append((future, function, args, kwargs))
return future
def run_next(self):
future, function, args, kwargs = self.calls.pop(0)
try:
future.set_result(function(*args, **kwargs))
except BaseException as exc:
future.set_exception(exc)
return future
class FakeYtDlp:
def __init__(
self,
@@ -267,6 +285,155 @@ class DownloadManagerTests(unittest.TestCase):
self.assertEqual(manager.recover_pending_deletions(), [1])
self.assertFalse(slot.exists())
def test_async_deletion_failure_remains_durable_and_is_retryable(self):
reservation = self.store.reserve_download(
"https://example.test/retry-delete",
None,
None,
job_id="retry-delete",
)
self.store.fail_download(
"retry-delete",
1,
reservation.generation,
"failed",
)
slot = self.media / "slot1"
slot.mkdir(parents=True)
(slot / "retry.mp4").write_bytes(b"retry")
events = []
manager = self.manager(FakeYtDlp(self.media), events)
original_recover = manager._recover_deletion
attempts = 0
def fail_once(token):
nonlocal attempts
attempts += 1
if attempts == 1:
raise OSError("transient media failure")
return original_recover(token)
manager._recover_deletion = fail_once
token = manager.schedule_delete_slot(1, reason="manual")
self.assertEqual(token.slot, 1)
self.assertEqual(self.store.get_slot(1).media_state, "deleting")
self.assertEqual(events[-1].state, "delete_error")
self.assertEqual(len(self.store.pending_deletions()), 1)
self.assertTrue(manager.schedule_pending_deletion_recovery())
self.assertEqual(attempts, 2)
self.assertEqual(events[-1].state, "deleted")
self.assertEqual(self.store.get_slot(1).media_state, "empty")
self.assertFalse(slot.exists())
def test_async_pending_recovery_retries_after_guard_is_cleared(self):
reservation = self.store.reserve_download(
"https://example.test/deferred-async-delete",
None,
None,
job_id="deferred-async-delete",
)
self.store.fail_download(
"deferred-async-delete",
1,
reservation.generation,
"failed",
)
self.store.begin_delete(1, reason="manual")
slot = self.media / "slot1"
slot.mkdir(parents=True)
(slot / "active.mp4").write_bytes(b"active")
self.store.protect_slot(1, "player_active", guard_id="active-player")
events = []
manager = self.manager(FakeYtDlp(self.media), events)
self.assertTrue(manager.schedule_pending_deletion_recovery())
self.assertEqual(events[-1].state, "delete_deferred")
self.assertEqual(self.store.get_slot(1).media_state, "deleting")
self.assertTrue(slot.exists())
self.store.clear_playback_guards(reason="fresh terminal idle evidence")
self.assertTrue(manager.schedule_pending_deletion_recovery())
self.assertEqual(events[-1].state, "deleted")
self.assertEqual(self.store.get_slot(1).media_state, "empty")
self.assertFalse(slot.exists())
def test_guarded_recovery_restores_crash_staged_trash_to_canonical_slot(self):
reservation = self.store.reserve_download(
"https://example.test/restore-trash",
None,
None,
job_id="restore-trash",
)
self.store.fail_download(
"restore-trash",
1,
reservation.generation,
"failed",
)
token = self.store.begin_delete(1, reason="expired")
trash = (
self.media
/ ".controller-trash"
/ f"slot1.generation{token.generation}.crash"
)
trash.mkdir(parents=True)
(trash / "still-active.mp4").write_bytes(b"active")
self.store.protect_slot(1, "player_active", guard_id="fresh-active")
events = []
manager = self.manager(FakeYtDlp(self.media), events)
self.assertTrue(manager.schedule_pending_deletion_recovery())
canonical = self.media / "slot1"
self.assertEqual(events[-1].state, "delete_deferred")
self.assertTrue((canonical / "still-active.mp4").exists())
self.assertFalse(trash.exists())
self.assertEqual(self.store.get_slot(1).media_state, "deleting")
def test_queued_deletion_rechecks_guard_before_filesystem_mutation(self):
reservation = self.store.reserve_download(
"https://example.test/queued-delete-guard",
None,
None,
job_id="queued-delete-guard",
)
self.store.fail_download(
"queued-delete-guard",
1,
reservation.generation,
"failed",
)
self.store.begin_delete(1, reason="manual")
slot = self.media / "slot1"
slot.mkdir(parents=True)
(slot / "must-survive.mp4").write_bytes(b"active")
events = []
executor = QueuedExecutor()
manager = DownloadManager(
self.config,
self.store,
runner=FakeYtDlp(self.media),
executor=executor,
on_event=events.append,
)
self.assertTrue(manager.schedule_pending_deletion_recovery())
self.store.protect_slot(1, "player_active", guard_id="late-player")
executor.run_next()
self.assertEqual(events[-1].state, "delete_deferred")
self.assertEqual(self.store.get_slot(1).media_state, "deleting")
self.assertTrue((slot / "must-survive.mp4").exists())
self.store.clear_playback_guards(reason="fresh terminal idle evidence")
self.assertTrue(manager.schedule_pending_deletion_recovery())
executor.run_next()
self.assertEqual(events[-1].state, "deleted")
self.assertEqual(self.store.get_slot(1).media_state, "empty")
self.assertFalse(slot.exists())
def test_shutdown_terminates_tracked_process_without_waiting_for_job_timeout(self):
manager = DownloadManager(self.config, self.store)
errors = []
+135 -3
View File
@@ -156,8 +156,6 @@ class PlaybackReducerTests(unittest.TestCase):
effects = self.reducer.handle_status(
{
"state": "busy",
"slot": 1,
"path": "/mnt/Media/YTPlayer/slot1/playlist.m3u",
"message": "mpv already playing; stop first",
}
)
@@ -166,6 +164,25 @@ class PlaybackReducerTests(unittest.TestCase):
self.assertEqual(self.reducer.projected_slot_state(1), "playing")
self.assertEqual(self.reducer.projected_slot_state(2), "ready")
def test_real_pathless_busy_without_prior_identity_becomes_unknown(self):
self.reducer.begin_operation(
"play",
slot=2,
expected_path="/mnt/Media/YTPlayer/slot2/playlist.m3u",
)
effects = self.reducer.handle_status(
{
"state": "busy",
"message": "mpv already playing; stop first",
}
)
self.assertIn("operation_failed", [effect.kind for effect in effects])
observation = self.store.player_observation()
self.assertEqual(observation.phase, "unknown")
self.assertIsNone(observation.active_slot)
self.assertIsNone(observation.active_path)
def test_pause_status_does_not_override_actual_telemetry(self):
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
self.reducer.handle_telemetry(
@@ -206,9 +223,91 @@ class PlaybackReducerTests(unittest.TestCase):
{"state": "error", "slot": 1, "path": path, "message": "cannot stop"}
)
self.assertEqual(self.store.player_observation().phase, "error")
self.assertEqual(self.store.list_live_guards()[0].slot, 1)
guard = self.store.list_live_guards()[0]
self.assertEqual(guard.slot, 1)
self.assertEqual(guard.kind, "restart_uncertain")
self.assertEqual(self.reducer.projected_slot_state(1), "unknown")
def test_pathless_error_converts_previous_active_slot_to_reviewable_unknown(self):
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
self.reducer.handle_status({"state": "playing", "slot": 1, "path": path})
self.reducer.handle_status(
{"state": "error", "message": "nothing playing to pause"}
)
observation = self.store.player_observation()
guard = self.store.list_live_guards()[0]
self.assertEqual(observation.active_slot, 1)
self.assertEqual(observation.reconciliation_id, guard.guard_id)
self.assertEqual(observation.reconciliation_state, "awaiting_evidence")
self.assertEqual(guard.kind, "restart_uncertain")
def test_terminal_startup_error_clears_identity_even_if_clear_telemetry_was_lost(self):
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
self.reducer.begin_operation("play", slot=1, expected_path=path)
effects = self.reducer.handle_status(
{
"state": "error",
"slot": 1,
"path": path,
"reason": "startup_error",
"message": "failed to start mpv",
}
)
self.assertIn("player_idle", [effect.kind for effect in effects])
self.assertIn("operation_failed", [effect.kind for effect in effects])
observation = self.store.player_observation()
self.assertEqual(observation.phase, "idle")
self.assertIsNone(observation.active_slot)
self.assertIsNone(observation.active_path)
self.assertEqual(self.store.list_live_guards(), [])
def test_terminal_worker_error_after_cleared_telemetry_remains_idle(self):
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
self.reducer.begin_operation("play", slot=1, expected_path=path)
self.reducer.handle_telemetry(
{"state": "telemetry", "slot": 1, "path": None, "paused": False}
)
effects = self.reducer.handle_status(
{
"state": "error",
"slot": 1,
"path": path,
"reason": "worker_start_error",
"message": "failed to start playback worker",
}
)
self.assertIn("player_idle", [effect.kind for effect in effects])
observation = self.store.player_observation()
self.assertEqual(observation.phase, "idle")
self.assertIsNone(observation.active_path)
self.assertEqual(self.store.list_live_guards(), [])
def test_player_state_takes_precedence_over_deferred_storage_deletion(self):
reservation = self.store.reserve_download(
"https://example.test/late-player",
None,
None,
job_id="late-player",
)
self.store.fail_download(
"late-player",
1,
reservation.generation,
"failed",
)
self.store.begin_delete(1, reason="expired")
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
self.reducer.handle_status(
{"state": "playing", "slot": 1, "path": path}
)
self.assertEqual(self.store.get_slot(1).media_state, "deleting")
self.assertEqual(self.reducer.projected_slot_state(1), "playing")
def test_play_timeout_becomes_unknown_then_operator_review_without_unprotecting(self):
self.reducer.begin_operation(
"play",
@@ -225,6 +324,39 @@ class PlaybackReducerTests(unittest.TestCase):
self.assertEqual(self.reducer.projected_slot_state(1), "operator_review")
self.assertEqual(self.store.list_live_guards()[0].state, "review_required")
def test_pause_timeout_invalidates_stale_playing_projection(self):
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
self.reducer.handle_telemetry(
{"state": "telemetry", "slot": 1, "path": path, "paused": False}
)
self.reducer.begin_operation("pause", slot=1, expected_path=path)
self.monotonic.value += 6
effect = self.reducer.check_timeout()
self.assertEqual(effect.kind, "operation_timed_out")
observation = self.store.player_observation()
self.assertEqual(observation.phase, "unknown")
self.assertIsNone(observation.paused)
self.assertEqual(self.reducer.projected_slot_state(1), "unknown")
self.assertEqual(self.store.list_live_guards()[0].kind, "player_active")
def test_resume_timeout_invalidates_stale_paused_projection(self):
path = "/mnt/Media/YTPlayer/slot1/playlist.m3u"
self.reducer.handle_telemetry(
{"state": "telemetry", "slot": 1, "path": path, "paused": True}
)
self.reducer.begin_operation("resume", slot=1, expected_path=path)
self.monotonic.value += 6
effect = self.reducer.check_timeout()
self.assertEqual(effect.kind, "operation_timed_out")
observation = self.store.player_observation()
self.assertEqual(observation.phase, "unknown")
self.assertIsNone(observation.paused)
self.assertEqual(self.reducer.projected_slot_state(1), "unknown")
def test_malformed_and_unknown_statuses_are_isolated(self):
for payload in (None, [], {}, {"state": "mystery"}):
effects = self.reducer.handle_status(payload)
+201 -9
View File
@@ -14,7 +14,7 @@ from controller.config import (
StorageConfig,
)
from controller.downloads import canonical_playlist_path
from controller.models import SlotProtectedError
from controller.models import SlotProtectedError, StaleOperationError
from controller.mqtt_adapter import MqttEvent, PublishReceipt
from controller.service import ControllerService, ControllerUnavailableError
from controller.store import ControllerStore
@@ -81,11 +81,13 @@ class FakeMqttAdapter:
class FakeDownloadManager:
def __init__(self):
def __init__(self, store=None):
self.store = store
self.recovered = False
self.recovery_calls = 0
self.expiry_calls = 0
self.stopped = False
self.delete_requests = []
def recover_pending_deletions(self):
self.recovered = True
@@ -95,10 +97,20 @@ class FakeDownloadManager:
self.recovery_calls += 1
return True
def expire_due_slots(self):
def schedule_expired_deletions(self):
self.expiry_calls += 1
return []
def schedule_delete_slot(self, slot, *, reason, expected_url=None):
self.delete_requests.append((slot, reason, expected_url))
if self.store is None:
return None
return self.store.begin_delete(
slot,
reason=reason,
expected_url=expected_url,
)
def shutdown(self, *, wait):
self.stopped = wait
@@ -142,7 +154,7 @@ class ControllerServiceTests(unittest.TestCase):
self.events = queue.Queue()
self.store = ControllerStore(config.storage, now=self.wall)
self.mqtt = mqtt or FakeMqttAdapter()
self.downloads = FakeDownloadManager()
self.downloads = FakeDownloadManager(self.store)
self.service = ControllerService(
config,
store=self.store,
@@ -395,6 +407,61 @@ class ControllerServiceTests(unittest.TestCase):
self.assertEqual((qos, retain), (1, False))
self.assertEqual(self.mqtt.acknowledged[0].message_id, 42)
def test_retained_detailed_player_evidence_cannot_clear_restart_protection(self):
self.start_service()
self.store.protect_slot(
1,
"restart_uncertain",
guard_id="restart-review",
)
self.send_message(
self.config.topics.player_status,
{"state": "idle", "slot": 1, "reason": "stop"},
qos=1,
retain=True,
message_id=41,
)
self.send_message(
self.config.topics.player_telemetry,
{
"state": "telemetry",
"slot": 1,
"path": None,
"paused": False,
},
retain=True,
)
self.wait_for(
lambda: any(event.message_id == 41 for event in self.mqtt.acknowledged),
"retained QoS 1 status was not deliberately acknowledged",
)
self.wait_for(
lambda: len(
[
event
for event in self.result_events()
if event["type"] == "retained_player_evidence_rejected"
]
)
== 2,
"retained telemetry rejection was not reported",
)
guards = self.store.list_live_guards()
self.assertEqual(len(guards), 1)
self.assertEqual(guards[0].guard_id, "restart-review")
self.assertIn(
"awaiting_player_evidence",
self.service.health()["degraded_reasons"],
)
retained_warnings = [
event
for event in self.result_events()
if event["type"] == "retained_player_evidence_rejected"
]
self.assertEqual(len(retained_warnings), 2)
def test_optional_result_topic_can_be_disabled_without_affecting_ack(self):
config = replace(
self.config,
@@ -447,19 +514,106 @@ class ControllerServiceTests(unittest.TestCase):
"idle",
retain=True,
)
time.sleep(0.2)
self.assertFalse(self.downloads.recovered)
self.assertEqual(self.downloads.expiry_calls, 0)
self.assertIn(
"awaiting_player_evidence",
self.service.health()["degraded_reasons"],
)
self.send_message(
self.config.topics.player_status,
{"state": "idle", "message": "player online"},
qos=1,
)
self.wait_for(
lambda: self.downloads.recovered,
"retained Player idle did not release deferred recovery",
"detailed Player idle did not release deferred recovery",
)
self.wait_for(
lambda: self.downloads.expiry_calls > 0,
"Player idle did not release deferred expiry",
"detailed Player idle did not release deferred expiry",
)
self.assertNotIn(
"awaiting_player_evidence",
self.service.health()["degraded_reasons"],
)
def test_administrator_can_explicitly_unlock_cleanup_without_player_evidence(self):
self.start_service()
review = self.service.health()["cleanup_reconciliation"]
self.assertIsNotNone(review)
result = self.service.submit_control(
{
"action": "reconcile_playback",
"scope": "cleanup",
"decision": "clear",
"confirm": True,
"review_id": review["review_id"],
"actor": "operator",
"reason": "Player appliance was independently verified offline",
}
)
self.assertEqual(
result,
{
"status": "cleared",
"review_id": review["review_id"],
"scope": "cleanup",
},
)
self.wait_for(
lambda: self.downloads.recovered,
"administrative cleanup override did not schedule recovery",
)
health = self.service.health()
self.assertIsNone(health["cleanup_reconciliation"])
self.assertNotIn("awaiting_player_evidence", health["degraded_reasons"])
def test_cleanup_reconciliation_token_rotates_and_cannot_bypass_a_guard(self):
self.start_service()
first = self.service.health()["cleanup_reconciliation"]["review_id"]
self.events.put(MqttEvent(kind="connected", connection_generation=1))
self.wait_for(
lambda: (
self.service.health()["cleanup_reconciliation"] is not None
and self.service.health()["cleanup_reconciliation"]["review_id"]
!= first
),
"cleanup reconciliation token did not rotate on MQTT connect",
)
current = self.service.health()["cleanup_reconciliation"]["review_id"]
with self.assertRaisesRegex(StaleOperationError, "review_id is stale"):
self.service.submit_control(
{
"action": "reconcile_playback",
"scope": "cleanup",
"decision": "clear",
"confirm": True,
"review_id": first,
"actor": "operator",
"reason": "stale request",
}
)
self.store.protect_slot(1, "restart_uncertain", guard_id="late-guard")
with self.assertRaisesRegex(SlotProtectedError, "known playback protection"):
self.service.submit_control(
{
"action": "reconcile_playback",
"scope": "cleanup",
"decision": "clear",
"confirm": True,
"review_id": current,
"actor": "operator",
"reason": "must not bypass a guard",
}
)
def test_manual_delete_requires_explicit_override_while_player_is_unknown(self):
self.start_service()
self.ready_slot(1)
@@ -536,8 +690,6 @@ class ControllerServiceTests(unittest.TestCase):
config.topics.player_status,
{
"state": "busy",
"slot": 2,
"path": str(second_playlist),
"message": "mpv already playing; stop first",
},
qos=1,
@@ -546,12 +698,34 @@ class ControllerServiceTests(unittest.TestCase):
lambda: self.service.playback.pending is None,
"busy response was not reduced",
)
self.assertEqual(self.store.player_observation().active_slot, 2)
observation = self.store.player_observation()
self.assertIsNone(observation.active_slot)
self.assertIsNone(observation.active_path)
self.assertEqual(observation.phase, "unknown")
self.assertIn(
"awaiting_player_evidence",
self.service.health()["degraded_reasons"],
)
self.assertEqual(self.mqtt.router_modes, ["video"])
lease = self.store.live_audio_lease()
self.assertIsNotNone(lease)
self.assertEqual(lease.state, "held")
self.send_message(
config.topics.player_status,
{
"state": "playing",
"slot": 2,
"path": str(second_playlist),
},
qos=1,
)
self.wait_for(
lambda: self.store.player_observation().active_slot == 2,
"Player did not reassert its actual playback identity",
)
self.assertEqual(self.store.live_audio_lease().state, "held")
def test_disconnected_synchronous_play_failure_is_not_reported_pending(self):
self.start_service(mqtt=FakeMqttAdapter(connected=False))
self.ready_slot(1)
@@ -735,6 +909,24 @@ class ControllerServiceTests(unittest.TestCase):
self.assertEqual(observation.reconciliation_id, review["review_id"])
self.assertEqual(observation.active_path, str(playlist))
def test_router_gated_play_marks_all_reconciliation_reviews_not_clearable(self):
self.start_service()
self.ready_slot(1)
self.store.set_router_observation(mode="service", freshness="current")
result = self.service.submit_control({"action": "play", "slot": 1})
self.assertEqual(result["router"], "waiting")
# Model a durable/reconnect review becoming visible while Router
# acquisition is pending.
self.store.protect_slot(
2,
"restart_uncertain",
guard_id="review-during-router-acquisition",
)
reviews = self.service.health()["reconciliation_reviews"]
self.assertEqual(len(reviews), 1)
self.assertIs(reviews[0]["clearable"], False)
def test_definite_external_publish_failure_clears_prepublication_guard(self):
self.start_service(mqtt=FakeMqttAdapter(connected=False))
playlist = Path(self.tempdir.name) / "operator-playlist.m3u"
+45
View File
@@ -127,6 +127,51 @@ class FreshStoreTests(StoreTestCase):
7,
)
def test_failed_downloads_receive_expiry_and_cannot_exhaust_slots_forever(self):
with self.open_store() as store:
for slot in range(1, 11):
reservation = store.reserve_download(
f"https://example.test/failure-{slot}",
None,
None,
job_id=f"failure-{slot}",
)
self.assertEqual(reservation.slot, slot)
self.assertTrue(
store.fail_download(
reservation.job_id,
slot,
reservation.generation,
"download failed",
)
)
self.assertEqual(
store.get_slot(slot).expires_at,
self.clock.value + self.config.retention_seconds,
)
self.assertEqual(
store.reserve_download(
"https://example.test/temporarily-full",
None,
None,
).status,
"queue_full",
)
self.clock.value += self.config.retention_seconds + 1
tokens = store.begin_expired_deletions()
self.assertEqual([token.slot for token in tokens], list(range(1, 11)))
for token in tokens:
self.assertTrue(store.finish_delete(token))
replacement = store.reserve_download(
"https://example.test/recovered-capacity",
None,
None,
)
self.assertEqual(replacement.slot, 1)
def test_stale_worker_cannot_overwrite_reused_slot(self):
with self.open_store() as store:
first = store.reserve_download("https://example.test/one", None, None, job_id="old")