Media
ec826bb8f3
The submit form's URL validation accepted any http/https host. Restrict it to an explicit allowlist of official YouTube hosts using normalized, exact hostname comparison (no substring/suffix matching), reject embedded userinfo and malformed URLs, and add server-side length bounds for the URL, submitter, note, and total request body.
389 lines
14 KiB
Python
389 lines
14 KiB
Python
import importlib
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import unittest
|
|
from pathlib import Path
|
|
from unittest import mock
|
|
|
|
from controller.config import ControllerConfig
|
|
from controller.models import StoreError
|
|
from controller.mqtt_adapter import MqttAdapter
|
|
from controller.service import ControllerService, ControllerUnavailableError
|
|
|
|
|
|
class FakeControllerService:
|
|
def __init__(self, *, enqueue_result=None, enqueue_error=None, health=None):
|
|
self.enqueue_result = enqueue_result or {
|
|
"status": "queued",
|
|
"slot": 3,
|
|
"reused_slot": False,
|
|
}
|
|
self.enqueue_error = enqueue_error
|
|
self.health_result = health or {
|
|
"status": "ok",
|
|
"started": True,
|
|
"coordinator_alive": True,
|
|
"mqtt_connected": True,
|
|
"router_state": "video",
|
|
"degraded_reasons": [],
|
|
}
|
|
self.enqueue_calls = []
|
|
|
|
def enqueue_download(self, url, submitter, note):
|
|
self.enqueue_calls.append((url, submitter, note))
|
|
if self.enqueue_error is not None:
|
|
raise self.enqueue_error
|
|
return dict(self.enqueue_result)
|
|
|
|
def health(self):
|
|
return dict(self.health_result)
|
|
|
|
|
|
class FlaskFactoryTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.config = ControllerConfig()
|
|
|
|
@staticmethod
|
|
def _factory():
|
|
module = importlib.import_module("app")
|
|
return module.create_app
|
|
|
|
def _app(self, service):
|
|
return self._factory()(self.config, service, testing=True)
|
|
|
|
def _post(
|
|
self,
|
|
service,
|
|
*,
|
|
url="https://www.youtube.com/playlist?list=PL1234567890",
|
|
**fields,
|
|
):
|
|
app = self._app(service)
|
|
return app.test_client().post(
|
|
"/submit",
|
|
data={"url": url, **fields},
|
|
follow_redirects=True,
|
|
)
|
|
|
|
def test_get_index_preserves_submission_form(self):
|
|
response = self._app(FakeControllerService()).test_client().get("/")
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"Submit YouTube Playlist", response.data)
|
|
self.assertIn(b'action="/submit"', response.data)
|
|
self.assertIn(b'name="url"', response.data)
|
|
self.assertIn(b'name="submitter"', response.data)
|
|
self.assertIn(b'name="note"', response.data)
|
|
|
|
def test_submit_rejects_non_youtube_and_malformed_urls(self):
|
|
invalid_urls = (
|
|
"",
|
|
"http",
|
|
"https:///missing-host",
|
|
"httpfoo://youtube.com/video",
|
|
"ftp://youtube.com/video",
|
|
"javascript:alert(1)",
|
|
"/relative/path",
|
|
# Arbitrary external hosts, including lookalikes and substring
|
|
# tricks, must be rejected by exact normalized hostname match.
|
|
"https://example.com/video",
|
|
"https://notyoutube.com/video",
|
|
"https://youtube.com.evil.com/video",
|
|
"https://evil.com/?u=youtube.com",
|
|
"https://xyoutube.com/watch?v=abc",
|
|
"https://youtube.com.attacker.net/watch?v=abc",
|
|
# Embedded userinfo must be rejected even when the host matches.
|
|
"https://user:[email protected]/watch?v=abc",
|
|
"https://[email protected]/watch?v=abc",
|
|
"https://[email protected]/watch?v=abc",
|
|
# Oversized URL.
|
|
"https://www.youtube.com/watch?v=" + ("a" * 3000),
|
|
)
|
|
for url in invalid_urls:
|
|
with self.subTest(url=url):
|
|
service = FakeControllerService()
|
|
response = self._post(service, url=url)
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"Invalid URL", response.data)
|
|
self.assertEqual(service.enqueue_calls, [])
|
|
|
|
def test_submit_accepts_only_explicit_official_youtube_hosts(self):
|
|
valid_urls = (
|
|
"https://youtube.com/watch?v=abc",
|
|
"https://www.youtube.com/watch?v=abc",
|
|
"http://www.youtube.com/watch?v=abc",
|
|
"https://m.youtube.com/watch?v=abc",
|
|
"https://music.youtube.com/watch?v=abc",
|
|
"https://youtu.be/abc",
|
|
"https://YouTube.com/watch?v=abc",
|
|
"https://WWW.YOUTUBE.COM/watch?v=abc",
|
|
)
|
|
for url in valid_urls:
|
|
with self.subTest(url=url):
|
|
service = FakeControllerService()
|
|
response = self._post(service, url=url)
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(service.enqueue_calls, [(url, None, None)])
|
|
|
|
def test_queued_submission_preserves_message_and_trims_optional_fields(self):
|
|
service = FakeControllerService(
|
|
enqueue_result={"status": "queued", "slot": 3, "reused_slot": False}
|
|
)
|
|
|
|
response = self._post(
|
|
service,
|
|
url="https://www.youtube.com/playlist?list=PL1234567890",
|
|
submitter=" Alice ",
|
|
note=" Before service ",
|
|
)
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"Playlist queued in slot 3", response.data)
|
|
self.assertIn(b"start downloading shortly", response.data)
|
|
self.assertEqual(
|
|
service.enqueue_calls,
|
|
[
|
|
(
|
|
"https://www.youtube.com/playlist?list=PL1234567890",
|
|
"Alice",
|
|
"Before service",
|
|
)
|
|
],
|
|
)
|
|
|
|
def test_submit_rejects_oversized_submitter_and_note(self):
|
|
module = importlib.import_module("app")
|
|
service = FakeControllerService()
|
|
response = self._post(
|
|
service,
|
|
submitter="x" * (module.MAX_SUBMITTER_LENGTH + 1),
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"must be", response.data)
|
|
self.assertEqual(service.enqueue_calls, [])
|
|
|
|
service = FakeControllerService()
|
|
response = self._post(
|
|
service,
|
|
note="x" * (module.MAX_NOTE_LENGTH + 1),
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"must be", response.data)
|
|
self.assertEqual(service.enqueue_calls, [])
|
|
|
|
def test_submit_rejects_oversized_request_body(self):
|
|
service = FakeControllerService()
|
|
app = self._app(service)
|
|
response = app.test_client().post(
|
|
"/submit",
|
|
data={
|
|
"url": "https://www.youtube.com/watch?v=abc",
|
|
"note": "x" * 64_000,
|
|
},
|
|
follow_redirects=True,
|
|
)
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"too large", response.data.lower())
|
|
self.assertEqual(service.enqueue_calls, [])
|
|
|
|
def test_reused_submission_preserves_existing_slot_message(self):
|
|
service = FakeControllerService(
|
|
enqueue_result={"status": "queued", "slot": 7, "reused_slot": True}
|
|
)
|
|
|
|
response = self._post(service)
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"Playlist queued in slot 7", response.data)
|
|
self.assertIn(b"Existing slot", response.data)
|
|
self.assertIn(b"only new videos will be added", response.data)
|
|
|
|
def test_queue_full_has_a_useful_capacity_message(self):
|
|
service = FakeControllerService(
|
|
enqueue_result={
|
|
"status": "queue_full",
|
|
"slot": None,
|
|
"reused_slot": False,
|
|
"oldest_expires_in_seconds": 7200,
|
|
}
|
|
)
|
|
|
|
response = self._post(service)
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"Queue is full", response.data)
|
|
self.assertIn(b"about 2 hour(s)", response.data)
|
|
self.assertNotIn(b"slot None", response.data)
|
|
|
|
def test_shutting_down_has_a_useful_retry_message(self):
|
|
service = FakeControllerService(
|
|
enqueue_result={
|
|
"status": "shutting_down",
|
|
"slot": None,
|
|
"reused_slot": False,
|
|
}
|
|
)
|
|
|
|
response = self._post(service)
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"shutting down", response.data.lower())
|
|
self.assertIn(b"try again", response.data.lower())
|
|
self.assertNotIn(b"slot None", response.data)
|
|
|
|
def test_worker_error_has_a_useful_failure_message(self):
|
|
service = FakeControllerService(
|
|
enqueue_result={
|
|
"status": "worker_error",
|
|
"slot": 4,
|
|
"reused_slot": False,
|
|
}
|
|
)
|
|
|
|
response = self._post(service)
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertIn(b"download worker", response.data.lower())
|
|
self.assertTrue(
|
|
b"could not" in response.data.lower()
|
|
or b"failed" in response.data.lower()
|
|
)
|
|
self.assertNotIn(b"Playlist queued", response.data)
|
|
|
|
def test_controller_unavailable_is_handled_without_a_500(self):
|
|
service = FakeControllerService(
|
|
enqueue_error=ControllerUnavailableError(
|
|
"Controller coordinator did not respond"
|
|
)
|
|
)
|
|
|
|
response = self._post(service)
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertNotIn(b"Internal Server Error", response.data)
|
|
self.assertIn(b"Controller", response.data)
|
|
self.assertIn(b"unavailable", response.data.lower())
|
|
|
|
def test_store_error_is_handled_without_a_500(self):
|
|
service = FakeControllerService(
|
|
enqueue_error=StoreError("Controller database is unavailable")
|
|
)
|
|
|
|
response = self._post(service)
|
|
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertNotIn(b"Internal Server Error", response.data)
|
|
self.assertIn(b"Controller", response.data)
|
|
self.assertIn(b"unavailable", response.data.lower())
|
|
|
|
def test_health_is_healthy_when_ok_or_degraded(self):
|
|
cases = (
|
|
{
|
|
"status": "ok",
|
|
"started": True,
|
|
"coordinator_alive": True,
|
|
"mqtt_connected": True,
|
|
"degraded_reasons": [],
|
|
},
|
|
{
|
|
"status": "degraded",
|
|
"started": True,
|
|
"coordinator_alive": True,
|
|
"mqtt_connected": False,
|
|
"degraded_reasons": ["mqtt_disconnected"],
|
|
},
|
|
)
|
|
for health in cases:
|
|
with self.subTest(status=health["status"]):
|
|
app = self._app(FakeControllerService(health=health))
|
|
response = app.test_client().get("/healthz")
|
|
self.assertEqual(response.status_code, 200)
|
|
self.assertEqual(response.get_json(), health)
|
|
|
|
def test_health_is_unhealthy_when_stopped_or_coordinator_is_dead(self):
|
|
cases = (
|
|
{
|
|
"status": "stopped",
|
|
"started": False,
|
|
"coordinator_alive": False,
|
|
"mqtt_connected": False,
|
|
"degraded_reasons": [],
|
|
},
|
|
{
|
|
"status": "degraded",
|
|
"started": True,
|
|
"coordinator_alive": False,
|
|
"mqtt_connected": False,
|
|
"degraded_reasons": ["coordinator_dead"],
|
|
},
|
|
)
|
|
for health in cases:
|
|
with self.subTest(status=health["status"]):
|
|
app = self._app(FakeControllerService(health=health))
|
|
response = app.test_client().get("/healthz")
|
|
self.assertEqual(response.status_code, 503)
|
|
self.assertEqual(response.get_json(), health)
|
|
|
|
def test_production_factory_requires_flask_secret_key(self):
|
|
with mock.patch.dict(os.environ, {}, clear=True):
|
|
with self.assertRaisesRegex(RuntimeError, "FLASK_SECRET_KEY"):
|
|
self._factory()(self.config, FakeControllerService())
|
|
|
|
def test_testing_factory_can_use_a_test_only_secret(self):
|
|
with mock.patch.dict(os.environ, {}, clear=True):
|
|
app = self._factory()(
|
|
self.config,
|
|
FakeControllerService(),
|
|
testing=True,
|
|
)
|
|
|
|
self.assertIs(app.testing, True)
|
|
self.assertTrue(app.secret_key)
|
|
self.assertNotEqual(app.secret_key, "dev-secret")
|
|
|
|
|
|
class AppImportLifecycleTests(unittest.TestCase):
|
|
def test_import_does_not_construct_or_start_controller_runtime(self):
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
media = Path(directory) / "media"
|
|
data = Path(directory) / "data"
|
|
environment = {
|
|
**os.environ,
|
|
"YTPLAYER_MEDIA_BASE": str(media),
|
|
"YTPLAYER_DATA_BASE": str(data),
|
|
}
|
|
environment.pop("FLASK_SECRET_KEY", None)
|
|
before_threads = {thread.ident for thread in threading.enumerate()}
|
|
sys.modules.pop("app", None)
|
|
|
|
with mock.patch.dict(os.environ, environment, clear=True):
|
|
with mock.patch.object(
|
|
ControllerService,
|
|
"__init__",
|
|
side_effect=AssertionError("import constructed ControllerService"),
|
|
), mock.patch.object(
|
|
ControllerService,
|
|
"start",
|
|
side_effect=AssertionError("import started ControllerService"),
|
|
), mock.patch.object(
|
|
MqttAdapter,
|
|
"start",
|
|
side_effect=AssertionError("import started MQTT"),
|
|
):
|
|
imported = importlib.import_module("app")
|
|
|
|
self.assertTrue(callable(imported.create_app))
|
|
self.assertFalse(media.exists())
|
|
self.assertFalse(data.exists())
|
|
self.assertEqual(
|
|
{thread.ident for thread in threading.enumerate()},
|
|
before_threads,
|
|
)
|
|
sys.modules.pop("app", None)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|