512 lines
21 KiB
Python
512 lines
21 KiB
Python
import concurrent.futures
|
|
import hashlib
|
|
import os
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
import yaml
|
|
|
|
from controller.config import StorageConfig
|
|
from controller.models import MigrationError, SlotProtectedError
|
|
from controller.store import ControllerStore, LEGACY_MIGRATION_NAME
|
|
|
|
|
|
class MutableClock:
|
|
def __init__(self, value=1_700_000_000.0):
|
|
self.value = value
|
|
|
|
def __call__(self):
|
|
return self.value
|
|
|
|
|
|
class StoreTestCase(unittest.TestCase):
|
|
def setUp(self):
|
|
self.tempdir = tempfile.TemporaryDirectory()
|
|
self.base = Path(self.tempdir.name)
|
|
self.media = self.base / "media"
|
|
self.data = self.base / "data"
|
|
self.clock = MutableClock()
|
|
self.config = StorageConfig(
|
|
media_base=self.media,
|
|
data_base=self.data,
|
|
retention_seconds=100,
|
|
uncertain_review_after_seconds=30,
|
|
)
|
|
|
|
def tearDown(self):
|
|
self.tempdir.cleanup()
|
|
|
|
def open_store(self):
|
|
return ControllerStore(self.config, now=self.clock).open()
|
|
|
|
|
|
class FreshStoreTests(StoreTestCase):
|
|
def test_fresh_database_seeds_exact_stable_slots(self):
|
|
with self.open_store() as store:
|
|
slots = store.list_slots()
|
|
self.assertEqual([slot.slot for slot in slots], list(range(1, 11)))
|
|
self.assertTrue(all(slot.media_state == "empty" for slot in slots))
|
|
self.assertEqual(store.player_observation().phase, "unknown")
|
|
|
|
def test_schema_one_database_upgrades_external_reconciliation_atomically(self):
|
|
with self.open_store() as store:
|
|
database_path = store.path
|
|
connection = sqlite3.connect(database_path)
|
|
try:
|
|
connection.execute(
|
|
"ALTER TABLE player_observation DROP COLUMN reconciliation_id"
|
|
)
|
|
connection.execute("DELETE FROM schema_migrations WHERE version = 2")
|
|
connection.execute("PRAGMA user_version = 1")
|
|
connection.commit()
|
|
finally:
|
|
connection.close()
|
|
|
|
with self.open_store() as upgraded:
|
|
self.assertEqual(
|
|
upgraded.connection.execute("PRAGMA user_version").fetchone()[0],
|
|
2,
|
|
)
|
|
columns = {
|
|
row["name"]
|
|
for row in upgraded.connection.execute(
|
|
"PRAGMA table_info(player_observation)"
|
|
).fetchall()
|
|
}
|
|
self.assertIn("reconciliation_id", columns)
|
|
migration = upgraded.connection.execute(
|
|
"SELECT name FROM schema_migrations WHERE version = 2"
|
|
).fetchone()
|
|
self.assertEqual(migration["name"], "external_playback_reconciliation")
|
|
|
|
def test_slot_reservation_is_committed_before_worker_transition(self):
|
|
with self.open_store() as store:
|
|
reservation = store.reserve_download(
|
|
"https://www.youtube.com/playlist?list=one",
|
|
"Alice",
|
|
"Service",
|
|
job_id="job-one",
|
|
)
|
|
self.assertEqual(reservation.status, "reserved")
|
|
self.assertEqual(reservation.slot, 1)
|
|
self.assertEqual(store.get_slot(1).media_state, "queued")
|
|
job = store.get_job("job-one")
|
|
self.assertIsNotNone(job)
|
|
self.assertEqual(job.status, "reserved")
|
|
self.assertTrue(store.mark_download_running("job-one", 1, 1))
|
|
self.assertEqual(store.get_slot(1).media_state, "downloading")
|
|
|
|
def test_concurrent_requests_get_only_one_remaining_slot(self):
|
|
with self.open_store() as store:
|
|
for index in range(1, 10):
|
|
result = store.reserve_download(
|
|
f"https://example.test/{index}",
|
|
None,
|
|
None,
|
|
job_id=f"seed-{index}",
|
|
)
|
|
self.assertEqual(result.slot, index)
|
|
|
|
def reserve(index):
|
|
return store.reserve_download(
|
|
f"https://example.test/concurrent-{index}",
|
|
None,
|
|
None,
|
|
job_id=f"concurrent-{index}",
|
|
)
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
|
results = list(pool.map(reserve, range(8)))
|
|
winners = [result for result in results if result.status == "reserved"]
|
|
self.assertEqual(len(winners), 1)
|
|
self.assertEqual(winners[0].slot, 10)
|
|
self.assertEqual(
|
|
sum(result.status == "queue_full" for result in results),
|
|
7,
|
|
)
|
|
|
|
def test_stale_worker_cannot_overwrite_reused_slot(self):
|
|
with self.open_store() as store:
|
|
first = store.reserve_download("https://example.test/one", None, None, job_id="old")
|
|
store.mark_download_running("old", 1, first.generation)
|
|
store.fail_download("old", 1, first.generation, "failed")
|
|
deletion = store.begin_delete(1, reason="test")
|
|
self.assertTrue(store.finish_delete(deletion))
|
|
second = store.reserve_download("https://example.test/two", None, None, job_id="new")
|
|
self.assertEqual(second.slot, 1)
|
|
self.assertFalse(
|
|
store.complete_download(
|
|
"old",
|
|
1,
|
|
first.generation,
|
|
title="stale",
|
|
items=(("Stale", "stale.mp4"),),
|
|
)
|
|
)
|
|
self.assertEqual(store.get_slot(1).url, "https://example.test/two")
|
|
|
|
def test_deleting_slot_does_not_renumber_other_slots(self):
|
|
with self.open_store() as store:
|
|
first = store.reserve_download("https://example.test/one", None, None, job_id="one")
|
|
second = store.reserve_download("https://example.test/two", None, None, job_id="two")
|
|
store.fail_download("one", 1, first.generation, "failed")
|
|
store.fail_download("two", 2, second.generation, "failed")
|
|
token = store.begin_delete(1, reason="manual")
|
|
store.finish_delete(token)
|
|
self.assertEqual(store.get_slot(1).media_state, "empty")
|
|
self.assertEqual(store.get_slot(2).slot, 2)
|
|
self.assertEqual(store.get_slot(2).url, "https://example.test/two")
|
|
|
|
def test_concurrent_first_open_installs_only_one_valid_database(self):
|
|
stores = [ControllerStore(self.config, now=self.clock) for _ in range(2)]
|
|
|
|
def open_store(store):
|
|
store.open()
|
|
return [slot.slot for slot in store.list_slots()]
|
|
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
|
results = list(pool.map(open_store, stores))
|
|
self.assertEqual(results, [list(range(1, 11)), list(range(1, 11))])
|
|
markers = stores[0].connection.execute(
|
|
"SELECT name FROM data_migrations ORDER BY name"
|
|
).fetchall()
|
|
self.assertEqual([row["name"] for row in markers], ["fresh_database_v1"])
|
|
finally:
|
|
for store in stores:
|
|
store.close()
|
|
|
|
|
|
class GuardAndExpiryTests(StoreTestCase):
|
|
def _ready_slot(self, store, *, slot_expected=1):
|
|
reservation = store.reserve_download("https://example.test/ready", None, None, job_id="ready")
|
|
self.assertEqual(reservation.slot, slot_expected)
|
|
store.mark_download_running("ready", reservation.slot, reservation.generation)
|
|
store.complete_download(
|
|
"ready",
|
|
reservation.slot,
|
|
reservation.generation,
|
|
title="Ready",
|
|
items=(("One", "one.mp4"),),
|
|
)
|
|
return reservation
|
|
|
|
def test_active_and_review_guards_block_delete_and_expiry(self):
|
|
with self.open_store() as store:
|
|
reservation = self._ready_slot(store)
|
|
guard = store.protect_slot(
|
|
1,
|
|
"restart_uncertain",
|
|
guard_id="review-me",
|
|
review_after=self.clock.value + 10,
|
|
)
|
|
self.clock.value += 200
|
|
due = store.mark_due_guards_for_review()
|
|
self.assertEqual(due[0].state, "review_required")
|
|
with self.assertRaises(SlotProtectedError):
|
|
store.begin_delete(1, reason="manual")
|
|
self.assertEqual(store.begin_expired_deletions(), [])
|
|
self.assertTrue(
|
|
store.clear_guard(
|
|
1,
|
|
guard.guard_id,
|
|
actor="operator",
|
|
reason="Player host replaced",
|
|
)
|
|
)
|
|
self.assertEqual(store.player_observation().phase, "unknown")
|
|
expired = store.begin_expired_deletions()
|
|
self.assertEqual([token.slot for token in expired], [reservation.slot])
|
|
|
|
def test_admin_clear_requires_current_token_actor_and_reason(self):
|
|
with self.open_store() as store:
|
|
self._ready_slot(store)
|
|
guard = store.protect_slot(1, "player_active", guard_id="current")
|
|
with self.assertRaises(ValueError):
|
|
store.clear_guard(1, guard.guard_id, actor="", reason="because")
|
|
self.assertFalse(
|
|
store.clear_guard(
|
|
1,
|
|
"stale-token",
|
|
actor="operator",
|
|
reason="confirmed",
|
|
)
|
|
)
|
|
self.assertEqual(store.list_live_guards()[0].guard_id, "current")
|
|
|
|
def test_restart_marks_last_active_slot_uncertain_without_claiming_idle(self):
|
|
with self.open_store() as store:
|
|
self._ready_slot(store)
|
|
store.set_player_observation(
|
|
availability="online",
|
|
phase="playing",
|
|
active_slot=1,
|
|
active_path="/mnt/Media/YTPlayer/slot1/playlist.m3u",
|
|
paused=False,
|
|
reconciliation_state="confirmed",
|
|
reconciliation_due_at=None,
|
|
reconciliation_id=None,
|
|
)
|
|
guard = store.mark_restart_uncertain()
|
|
self.assertEqual(guard.slot, 1)
|
|
observation = store.player_observation()
|
|
self.assertEqual(observation.phase, "unknown")
|
|
self.assertEqual(observation.reconciliation_state, "awaiting_evidence")
|
|
self.assertEqual(observation.active_slot, 1)
|
|
|
|
def test_external_playback_restart_gets_stable_review_token_and_exact_clear(self):
|
|
external_path = "/srv/operator-playlists/live.m3u"
|
|
with self.open_store() as store:
|
|
store.set_player_observation(
|
|
availability="online",
|
|
phase="playing",
|
|
active_slot=None,
|
|
active_path=external_path,
|
|
paused=False,
|
|
reconciliation_state="confirmed",
|
|
reconciliation_due_at=None,
|
|
reconciliation_id=None,
|
|
)
|
|
self.assertIsNone(store.mark_restart_uncertain())
|
|
first = store.player_observation()
|
|
self.assertEqual(first.reconciliation_state, "awaiting_evidence")
|
|
self.assertIsNotNone(first.reconciliation_id)
|
|
self.assertEqual(first.reconciliation_due_at, self.clock.value + 30)
|
|
|
|
self.clock.value += 10
|
|
self.assertIsNone(store.mark_restart_uncertain())
|
|
repeated = store.player_observation()
|
|
self.assertEqual(repeated.reconciliation_id, first.reconciliation_id)
|
|
self.assertEqual(repeated.reconciliation_due_at, first.reconciliation_due_at)
|
|
|
|
self.clock.value = first.reconciliation_due_at + 1
|
|
due = store.mark_external_reconciliation_due()
|
|
self.assertEqual(due, (first.reconciliation_id, external_path))
|
|
self.assertIsNone(store.mark_external_reconciliation_due())
|
|
review = store.player_observation()
|
|
self.assertEqual(review.reconciliation_state, "operator_review")
|
|
self.assertEqual(review.active_path, external_path)
|
|
|
|
self.assertFalse(
|
|
store.clear_external_reconciliation(
|
|
"stale-review",
|
|
actor="operator",
|
|
reason="Player was replaced",
|
|
)
|
|
)
|
|
self.assertTrue(
|
|
store.clear_external_reconciliation(
|
|
first.reconciliation_id,
|
|
actor="operator",
|
|
reason="Player was replaced",
|
|
)
|
|
)
|
|
cleared = store.player_observation()
|
|
self.assertEqual(cleared.reconciliation_state, "admin_cleared")
|
|
self.assertIsNone(cleared.active_path)
|
|
self.assertIsNone(cleared.reconciliation_id)
|
|
|
|
def test_restart_converts_existing_active_guard_and_preserves_deadline(self):
|
|
with self.open_store() as store:
|
|
self._ready_slot(store)
|
|
store.protect_slot(1, "player_active", guard_id="active-before-restart")
|
|
store.set_player_observation(
|
|
availability="online",
|
|
phase="playing",
|
|
active_slot=1,
|
|
active_path="/mnt/Media/YTPlayer/slot1/playlist.m3u",
|
|
paused=False,
|
|
reconciliation_state="confirmed",
|
|
reconciliation_due_at=None,
|
|
reconciliation_id=None,
|
|
)
|
|
guard = store.mark_restart_uncertain()
|
|
self.assertEqual(guard.guard_id, "active-before-restart")
|
|
self.assertEqual(guard.kind, "restart_uncertain")
|
|
first_deadline = guard.review_after
|
|
self.clock.value += 10
|
|
repeated = store.mark_restart_uncertain()
|
|
self.assertEqual(repeated.guard_id, guard.guard_id)
|
|
self.assertEqual(repeated.review_after, first_deadline)
|
|
|
|
def test_restart_converts_pre_evidence_pending_guard_for_admin_review(self):
|
|
with self.open_store() as store:
|
|
self._ready_slot(store)
|
|
store.protect_slot(1, "pending_play", guard_id="published-before-crash")
|
|
guard = store.mark_restart_uncertain()
|
|
self.assertEqual(guard.guard_id, "published-before-crash")
|
|
self.assertEqual(guard.kind, "restart_uncertain")
|
|
self.assertEqual(store.player_observation().active_slot, 1)
|
|
self.assertTrue(
|
|
store.clear_guard(
|
|
1,
|
|
guard.guard_id,
|
|
actor="operator",
|
|
reason="Player replaced",
|
|
)
|
|
)
|
|
|
|
def test_guard_review_transition_is_reported_only_once(self):
|
|
with self.open_store() as store:
|
|
self._ready_slot(store)
|
|
store.protect_slot(
|
|
1,
|
|
"restart_uncertain",
|
|
guard_id="one-warning",
|
|
review_after=self.clock.value,
|
|
)
|
|
self.assertEqual(len(store.mark_due_guards_for_review()), 1)
|
|
self.assertEqual(store.mark_due_guards_for_review(), [])
|
|
|
|
def test_clearing_one_legacy_guard_does_not_erase_other_observation(self):
|
|
with self.open_store() as store:
|
|
self._ready_slot(store)
|
|
second = store.reserve_download(
|
|
"https://example.test/second", None, None, job_id="second"
|
|
)
|
|
store.mark_download_running("second", 2, second.generation)
|
|
store.complete_download(
|
|
"second",
|
|
2,
|
|
second.generation,
|
|
title="Second",
|
|
items=(("Second", "second.mp4"),),
|
|
)
|
|
first_guard = store.protect_slot(
|
|
1, "legacy_uncertain", guard_id="legacy-one"
|
|
)
|
|
store.protect_slot(2, "player_active", guard_id="active-two")
|
|
store.set_player_observation(
|
|
availability="online",
|
|
phase="playing",
|
|
active_slot=2,
|
|
active_path="/mnt/Media/YTPlayer/slot2/playlist.m3u",
|
|
paused=False,
|
|
reconciliation_state="confirmed",
|
|
reconciliation_due_at=None,
|
|
reconciliation_id=None,
|
|
)
|
|
self.assertTrue(
|
|
store.clear_guard(
|
|
1,
|
|
first_guard.guard_id,
|
|
actor="operator",
|
|
reason="legacy evidence resolved",
|
|
)
|
|
)
|
|
observation = store.player_observation()
|
|
self.assertEqual(observation.active_slot, 2)
|
|
self.assertEqual(observation.phase, "playing")
|
|
|
|
|
|
class LegacyMigrationTests(StoreTestCase):
|
|
def write_legacy(self, data):
|
|
self.data.mkdir(parents=True, exist_ok=True)
|
|
raw = yaml.safe_dump(data, sort_keys=True).encode("utf-8")
|
|
self.config.legacy_yaml_path.write_bytes(raw)
|
|
return raw
|
|
|
|
def test_migration_preserves_holes_timestamps_original_and_backup(self):
|
|
raw = self.write_legacy(
|
|
{
|
|
"slots": {
|
|
"2": {
|
|
"slot": 2,
|
|
"url": "https://example.test/two",
|
|
"yt_id": "abc",
|
|
"title": "Two",
|
|
"path": "/unsafe/ignored",
|
|
"m3u": "/unsafe/ignored/playlist.m3u",
|
|
"created_at": "2025-01-02T03:04:05Z",
|
|
"expires_at": "2025-01-04T03:04:05Z",
|
|
"item_count": 3,
|
|
"state": "playing",
|
|
},
|
|
"7": {
|
|
"slot": 7,
|
|
"url": "https://example.test/seven",
|
|
"title": "Seven",
|
|
"path": "/unsafe/seven",
|
|
"m3u": "/unsafe/seven/playlist.m3u",
|
|
"created_at": "2025-02-01T00:00:00Z",
|
|
"expires_at": "2025-02-03T00:00:00Z",
|
|
"item_count": 1,
|
|
"state": "ready",
|
|
},
|
|
}
|
|
}
|
|
)
|
|
digest = hashlib.sha256(raw).hexdigest()
|
|
with self.open_store() as store:
|
|
self.assertTrue(store.migration_report.migrated)
|
|
self.assertEqual(store.migration_report.source_sha256, digest)
|
|
self.assertEqual(store.get_slot(1).media_state, "empty")
|
|
self.assertEqual(store.get_slot(2).title, "Two")
|
|
self.assertEqual(store.get_slot(7).slot, 7)
|
|
self.assertEqual(store.get_slot(2).created_at, 1_735_787_045.0)
|
|
guards = store.list_live_guards()
|
|
self.assertEqual([(guard.slot, guard.state) for guard in guards], [(2, "review_required")])
|
|
marker = store.connection.execute(
|
|
"SELECT * FROM data_migrations WHERE name = ?",
|
|
(LEGACY_MIGRATION_NAME,),
|
|
).fetchone()
|
|
self.assertEqual(marker["source_sha256"], digest)
|
|
backup = Path(marker["backup_path"])
|
|
self.assertEqual(backup.read_bytes(), raw)
|
|
self.assertEqual(self.config.legacy_yaml_path.read_bytes(), raw)
|
|
|
|
def test_completed_migration_is_idempotent_and_ignores_changed_yaml(self):
|
|
original = self.write_legacy(
|
|
{
|
|
"slots": {
|
|
"3": {
|
|
"slot": 3,
|
|
"url": "https://example.test/original",
|
|
"title": "Original",
|
|
"path": "/ignored",
|
|
"m3u": "/ignored/playlist.m3u",
|
|
"created_at": "2025-01-01T00:00:00Z",
|
|
"expires_at": "2025-01-03T00:00:00Z",
|
|
"item_count": 1,
|
|
"state": "ready",
|
|
}
|
|
}
|
|
}
|
|
)
|
|
with self.open_store() as store:
|
|
generation = store.get_slot(3).generation
|
|
backup_path = Path(store.migration_report.backup_path)
|
|
self.config.legacy_yaml_path.write_text("not: [the, original]\n", encoding="utf-8")
|
|
with self.open_store() as store:
|
|
self.assertFalse(store.migration_report.migrated)
|
|
self.assertEqual(store.get_slot(3).title, "Original")
|
|
self.assertEqual(store.get_slot(3).generation, generation)
|
|
self.assertEqual(backup_path.read_bytes(), original)
|
|
|
|
def test_failed_migration_keeps_original_and_verified_backup_recoverable(self):
|
|
self.data.mkdir(parents=True, exist_ok=True)
|
|
raw = b"slots:\n 1: [truncated\n"
|
|
self.config.legacy_yaml_path.write_bytes(raw)
|
|
store = ControllerStore(self.config, now=self.clock)
|
|
with self.assertRaises(MigrationError):
|
|
store.open()
|
|
self.assertFalse(self.config.database_path.exists())
|
|
self.assertEqual(self.config.legacy_yaml_path.read_bytes(), raw)
|
|
backups = list(self.data.glob("playlists.yaml.pre-v2.*.bak"))
|
|
self.assertEqual(len(backups), 1)
|
|
self.assertEqual(backups[0].read_bytes(), raw)
|
|
|
|
def test_normalized_duplicate_slot_keys_fail_all_or_nothing(self):
|
|
self.data.mkdir(parents=True, exist_ok=True)
|
|
raw = b"slots:\n 1:\n slot: 1\n '01':\n slot: 1\n"
|
|
self.config.legacy_yaml_path.write_bytes(raw)
|
|
with self.assertRaisesRegex(MigrationError, "normalize to 1"):
|
|
self.open_store()
|
|
self.assertFalse(self.config.database_path.exists())
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|