build: harden controller v2 runtime

This commit is contained in:
Media
2026-08-06 19:23:02 -05:00
parent 4e185b2e1c
commit 63ea314b18
15 changed files with 818 additions and 103 deletions
+73 -4
View File
@@ -97,6 +97,9 @@ class ControllerService:
self._next_expiry = 0.0
self._next_reconciliation = 0.0
self._processed_packets: dict[tuple[int, str, int], float] = {}
self._player_evidence_ready = False
self._operator_cleanup_override = False
self._recovery_scheduled = False
@property
def started(self) -> bool:
@@ -144,7 +147,11 @@ class ControllerService:
self.store,
on_event=self._queue_download_event,
)
self.downloads.recover_pending_deletions()
# Cleanup is deliberately deferred until fresh Player evidence
# (or an explicit administrative reconciliation) makes it safe.
self._player_evidence_ready = False
self._operator_cleanup_override = False
self._recovery_scheduled = False
self._stop_event.clear()
self._next_expiry = (
self._monotonic()
@@ -271,6 +278,8 @@ class ControllerService:
degraded: list[str] = []
if started and not mqtt_connected:
degraded.append("mqtt_disconnected")
if started and not self._cleanup_allowed():
degraded.append("awaiting_player_evidence")
if (
started
and router_observation is not None
@@ -375,6 +384,11 @@ class ControllerService:
def _handle_call(self, action: str, payload: Any) -> Any:
if action == "enqueue_download":
assert self.downloads is not None
if not self._cleanup_allowed():
raise ControllerUnavailableError(
"downloads are deferred until fresh Player evidence or "
"administrative playback reconciliation"
)
reservation = self.downloads.enqueue(
payload["url"], payload.get("submitter"), payload.get("note")
)
@@ -402,12 +416,16 @@ class ControllerService:
handled_successfully = True
return
if event.kind == "connected":
self._player_evidence_ready = False
self._operator_cleanup_override = False
self._announce_discovery()
self._sync_slot_states(force=True)
self._announce_reconciliation_reviews()
handled_successfully = True
return
if event.kind == "disconnected":
self._player_evidence_ready = False
self._operator_cleanup_override = False
assert self.playback is not None
assert self.router is not None
playback_effect = self.playback.mark_transport_uncertain(
@@ -426,12 +444,27 @@ class ControllerService:
payload = self._decode_json(event.payload, "Player status")
if payload is not None:
assert self.playback is not None
before_generation = self.store.player_observation().generation
self._handle_playback_effects(self.playback.handle_status(payload))
self._refresh_player_evidence(before_generation)
elif event.topic == self.config.topics.player_telemetry:
payload = self._decode_json(event.payload, "Player telemetry")
if payload is not None:
assert self.playback is not None
before_generation = self.store.player_observation().generation
self._handle_playback_effects(self.playback.handle_telemetry(payload))
self._refresh_player_evidence(before_generation)
elif event.topic == self.config.topics.player_state:
try:
flat_state = event.payload.decode("utf-8").strip()
except UnicodeDecodeError:
flat_state = ""
# Retained flat idle is sufficient only as a quiescence gate;
# it never mutates detailed playback state or clears guards.
self._player_evidence_ready = flat_state == "idle"
self._operator_cleanup_override = False
if self._player_evidence_ready:
self._schedule_deletion_recovery()
elif event.topic == self.config.topics.controller_command:
if event.retain:
self._publish_event(
@@ -478,8 +511,8 @@ class ControllerService:
else:
decision = self.router.observe_mode(mode)
self._handle_router_decision(decision)
# Player flat state is intentionally only a Companion/cold-start
# hint. It is never reduced as authoritative playback truth.
# Flat state is only a cold-start cleanup gate. Detailed playback
# truth continues to come exclusively from status and telemetry.
handled_successfully = True
finally:
if handled_successfully:
@@ -828,6 +861,14 @@ class ControllerService:
def _request_delete(self, payload: Mapping[str, Any]) -> dict[str, Any]:
assert self.downloads is not None
if (
not self._cleanup_allowed()
and payload.get("confirm_player_unknown") is not True
):
raise SlotProtectedError(
"delete requires fresh Player evidence or "
"confirm_player_unknown=true"
)
slot = self._parse_slot(payload.get("slot"))
expected_url = payload.get("url")
if expected_url is not None and not isinstance(expected_url, str):
@@ -886,6 +927,14 @@ class ControllerService:
)
if not cleared:
raise StaleOperationError("reconciliation review_id is stale")
observation = self.store.player_observation()
if (
not self.store.list_live_guards()
and observation.active_slot is None
and observation.active_path is None
):
self._operator_cleanup_override = True
self._schedule_deletion_recovery()
assert self.router is not None
self.router.abandon(
self._session_lease_id,
@@ -1118,6 +1167,26 @@ class ControllerService:
if topic is not None:
self.mqtt.publish_json(topic, payload, qos=1, retain=False)
def _cleanup_allowed(self) -> bool:
return self._player_evidence_ready or self._operator_cleanup_override
def _schedule_deletion_recovery(self) -> None:
if self._recovery_scheduled or not self._cleanup_allowed():
return
assert self.downloads is not None
self._recovery_scheduled = (
self.downloads.schedule_pending_deletion_recovery()
)
def _refresh_player_evidence(self, before_generation: int) -> None:
observation = self.store.player_observation()
if observation.generation == before_generation:
return
self._operator_cleanup_override = False
self._player_evidence_ready = observation.availability == "online"
if self._player_evidence_ready and observation.phase == "idle":
self._schedule_deletion_recovery()
def _run_timers(self) -> None:
assert self.playback is not None
assert self.router is not None
@@ -1133,7 +1202,7 @@ class ControllerService:
if now >= self._next_reconciliation:
self._handle_playback_effects(self.playback.mark_due_reconciliation())
self._next_reconciliation = now + 1.0
if now >= self._next_expiry:
if now >= self._next_expiry and self._cleanup_allowed():
for slot in self.downloads.expire_due_slots():
self._announce_slot(slot, force=True)
self._next_expiry = now + self.config.storage.expiry_interval_seconds