build: harden controller v2 runtime
This commit is contained in:
Media
+118
-48
@@ -7,9 +7,12 @@ import os
|
||||
import signal
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -171,37 +174,78 @@ class DownloadManager:
|
||||
except BaseException:
|
||||
LOG.exception("download event callback failed")
|
||||
|
||||
def _cookie_arguments(self) -> list[str]:
|
||||
path = self.config.downloads.cookie_path
|
||||
if path is None:
|
||||
return []
|
||||
if path.is_file():
|
||||
return ["--cookies", str(path)]
|
||||
def _working_cookie_copy(self) -> Path | None:
|
||||
source = self.config.downloads.cookie_path
|
||||
if source is None:
|
||||
return None
|
||||
if source.is_file():
|
||||
descriptor, temporary_name = tempfile.mkstemp(
|
||||
prefix="ytplayer-cookies-",
|
||||
suffix=".txt",
|
||||
)
|
||||
temporary_path = Path(temporary_name)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as destination:
|
||||
with source.open("rb") as source_file:
|
||||
shutil.copyfileobj(source_file, destination)
|
||||
destination.flush()
|
||||
os.fsync(destination.fileno())
|
||||
temporary_path.chmod(0o600)
|
||||
return temporary_path
|
||||
except BaseException:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
raise
|
||||
if self.config.downloads.cookie_required:
|
||||
raise FileNotFoundError(f"required yt-dlp cookie file is missing: {path}")
|
||||
LOG.warning("yt-dlp cookie file is unavailable; continuing without it: %s", path)
|
||||
return []
|
||||
raise FileNotFoundError(
|
||||
f"required yt-dlp cookie file is missing: {source}"
|
||||
)
|
||||
LOG.warning(
|
||||
"yt-dlp cookie file is unavailable; continuing without it: %s",
|
||||
source,
|
||||
)
|
||||
return None
|
||||
|
||||
def _base_yt_dlp_arguments(self) -> list[str]:
|
||||
return [self.config.downloads.yt_dlp_binary, "--ignore-config"] + self._cookie_arguments()
|
||||
@contextmanager
|
||||
def _yt_dlp_arguments(self) -> Iterator[list[str]]:
|
||||
working_cookie = self._working_cookie_copy()
|
||||
arguments = [
|
||||
self.config.downloads.yt_dlp_binary,
|
||||
"--ignore-config",
|
||||
"--no-js-runtimes",
|
||||
"--js-runtimes",
|
||||
"node",
|
||||
"--cache-dir",
|
||||
"/tmp/yt-dlp-cache",
|
||||
]
|
||||
if working_cookie is not None:
|
||||
arguments.extend(("--cookies", str(working_cookie)))
|
||||
try:
|
||||
yield arguments
|
||||
finally:
|
||||
if working_cookie is not None:
|
||||
working_cookie.unlink(missing_ok=True)
|
||||
|
||||
def _get_title(self, url: str) -> tuple[str, str | None]:
|
||||
arguments = self._base_yt_dlp_arguments() + [
|
||||
"--flat-playlist",
|
||||
"--print",
|
||||
"%(playlist_title)s|||%(title)s|||%(id)s",
|
||||
"--playlist-end",
|
||||
"1",
|
||||
url,
|
||||
]
|
||||
try:
|
||||
result = self._run_command(
|
||||
arguments,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=min(120.0, self.config.downloads.process_timeout_seconds),
|
||||
)
|
||||
with self._yt_dlp_arguments() as base_arguments:
|
||||
arguments = base_arguments + [
|
||||
"--flat-playlist",
|
||||
"--print",
|
||||
"%(playlist_title)s|||%(title)s|||%(id)s",
|
||||
"--playlist-end",
|
||||
"1",
|
||||
url,
|
||||
]
|
||||
result = self._run_command(
|
||||
arguments,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=min(
|
||||
120.0,
|
||||
self.config.downloads.process_timeout_seconds,
|
||||
),
|
||||
)
|
||||
line = (result.stdout or "").strip().splitlines()[0]
|
||||
playlist_title, _, remainder = line.partition("|||")
|
||||
video_title, _, yt_id = remainder.partition("|||")
|
||||
@@ -241,28 +285,29 @@ class DownloadManager:
|
||||
if slot_directory.is_symlink():
|
||||
raise RuntimeError(f"refusing to download through symlink: {slot_directory}")
|
||||
title, yt_id = self._get_title(url)
|
||||
arguments = self._base_yt_dlp_arguments() + [
|
||||
"--yes-playlist",
|
||||
"--no-overwrites",
|
||||
"--no-progress",
|
||||
"-o",
|
||||
str(
|
||||
slot_directory
|
||||
/ "%(playlist_index)03d - %(title)s [%(id)s].%(ext)s"
|
||||
),
|
||||
"-f",
|
||||
(
|
||||
f"bv*[height<={self.config.downloads.max_height}]+ba/"
|
||||
f"b[height<={self.config.downloads.max_height}]/b"
|
||||
),
|
||||
url,
|
||||
]
|
||||
self._run_command(
|
||||
arguments,
|
||||
check=True,
|
||||
text=True,
|
||||
timeout=self.config.downloads.process_timeout_seconds,
|
||||
)
|
||||
with self._yt_dlp_arguments() as base_arguments:
|
||||
arguments = base_arguments + [
|
||||
"--yes-playlist",
|
||||
"--no-overwrites",
|
||||
"--no-progress",
|
||||
"-o",
|
||||
str(
|
||||
slot_directory
|
||||
/ "%(playlist_index)03d - %(title)s [%(id)s].%(ext)s"
|
||||
),
|
||||
"-f",
|
||||
(
|
||||
f"bv*[height<={self.config.downloads.max_height}]+ba/"
|
||||
f"b[height<={self.config.downloads.max_height}]/b"
|
||||
),
|
||||
url,
|
||||
]
|
||||
self._run_command(
|
||||
arguments,
|
||||
check=True,
|
||||
text=True,
|
||||
timeout=self.config.downloads.process_timeout_seconds,
|
||||
)
|
||||
media_files = sorted(
|
||||
path
|
||||
for path in slot_directory.iterdir()
|
||||
@@ -467,6 +512,15 @@ class DownloadManager:
|
||||
recovered: list[int] = []
|
||||
trash_root = self.config.storage.media_base / ".controller-trash"
|
||||
for token in self.store.pending_deletions():
|
||||
if any(
|
||||
guard.slot == token.slot
|
||||
for guard in self.store.list_live_guards()
|
||||
):
|
||||
LOG.warning(
|
||||
"deferring interrupted deletion for protected slot %s",
|
||||
token.slot,
|
||||
)
|
||||
continue
|
||||
if trash_root.is_dir():
|
||||
prefix = f"slot{token.slot}.generation{token.generation}."
|
||||
for candidate in trash_root.iterdir():
|
||||
@@ -479,6 +533,22 @@ class DownloadManager:
|
||||
recovered.append(token.slot)
|
||||
return recovered
|
||||
|
||||
def schedule_pending_deletion_recovery(self) -> bool:
|
||||
"""Run recoverable trash cleanup without delaying service startup."""
|
||||
|
||||
with self._lock:
|
||||
if self._stopping:
|
||||
return False
|
||||
try:
|
||||
future = self._executor.submit(self.recover_pending_deletions)
|
||||
except BaseException:
|
||||
LOG.exception("could not schedule interrupted deletion recovery")
|
||||
return False
|
||||
with self._lock:
|
||||
self._futures.add(future)
|
||||
future.add_done_callback(self._forget_future)
|
||||
return True
|
||||
|
||||
def shutdown(self, *, wait: bool = True) -> None:
|
||||
with self._lock:
|
||||
self._stopping = True
|
||||
|
||||
Reference in New Issue
Block a user