fix: restrict web ingest to official YouTube hosts and bound submission size
The submit form's URL validation accepted any http/https host. Restrict it to an explicit allowlist of official YouTube hosts using normalized, exact hostname comparison (no substring/suffix matching), reject embedded userinfo and malformed URLs, and add server-side length bounds for the URL, submitter, note, and total request body.
This commit is contained in:
@@ -927,8 +927,11 @@ PYTHONDONTWRITEBYTECODE=1 .venv/bin/python -m unittest discover -s tests -v
|
|||||||
|
|
||||||
- The web form is unauthenticated and listens on `0.0.0.0` by default. Restrict
|
- The web form is unauthenticated and listens on `0.0.0.0` by default. Restrict
|
||||||
the published port with host firewall/network ACLs or place it behind an
|
the published port with host firewall/network ACLs or place it behind an
|
||||||
authenticated reverse proxy. URL validation checks only the `http`/`https`
|
authenticated reverse proxy. Submitted URLs must use `http`/`https`, carry no
|
||||||
shape; yt-dlp performs the outbound request.
|
embedded userinfo, and resolve (by exact, normalized hostname match) to an
|
||||||
|
official YouTube host: `youtube.com`, `www.youtube.com`, `m.youtube.com`,
|
||||||
|
`music.youtube.com`, or `youtu.be`. URL, submitter, note, and total request
|
||||||
|
size are also server-side bounded. yt-dlp performs the outbound request.
|
||||||
- MQTT username/password support is available, but application-level TLS is not
|
- MQTT username/password support is available, but application-level TLS is not
|
||||||
configured here. Keep this connection on a trusted/protected network or add a
|
configured here. Keep this connection on a trusted/protected network or add a
|
||||||
separately managed secure tunnel/proxy, and grant the Controller only the
|
separately managed secure tunnel/proxy, and grant the Controller only the
|
||||||
|
|||||||
@@ -224,12 +224,36 @@ HTML_FORM = """
|
|||||||
</html>
|
</html>
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
ALLOWED_YOUTUBE_HOSTS = frozenset(
|
||||||
|
{
|
||||||
|
"youtube.com",
|
||||||
|
"www.youtube.com",
|
||||||
|
"m.youtube.com",
|
||||||
|
"music.youtube.com",
|
||||||
|
"youtu.be",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
MAX_URL_LENGTH = 2048
|
||||||
|
MAX_SUBMITTER_LENGTH = 200
|
||||||
|
MAX_NOTE_LENGTH = 1000
|
||||||
|
MAX_REQUEST_CONTENT_LENGTH = 32 * 1024
|
||||||
|
|
||||||
|
|
||||||
def _valid_media_url(value: str) -> bool:
|
def _valid_media_url(value: str) -> bool:
|
||||||
|
if not value or len(value) > MAX_URL_LENGTH:
|
||||||
|
return False
|
||||||
try:
|
try:
|
||||||
parsed = urlsplit(value)
|
parsed = urlsplit(value)
|
||||||
|
hostname = parsed.hostname
|
||||||
|
has_userinfo = parsed.username is not None or parsed.password is not None
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return False
|
return False
|
||||||
return parsed.scheme.lower() in {"http", "https"} and bool(parsed.netloc)
|
if parsed.scheme.lower() not in {"http", "https"}:
|
||||||
|
return False
|
||||||
|
if has_userinfo or not hostname:
|
||||||
|
return False
|
||||||
|
return hostname.lower() in ALLOWED_YOUTUBE_HOSTS
|
||||||
|
|
||||||
|
|
||||||
def _submission_message(result: dict[str, object]) -> str:
|
def _submission_message(result: dict[str, object]) -> str:
|
||||||
@@ -277,6 +301,7 @@ def create_app(
|
|||||||
web_app.config.update(
|
web_app.config.update(
|
||||||
TESTING=testing,
|
TESTING=testing,
|
||||||
SECRET_KEY=secret_key or "controller-test-only-secret",
|
SECRET_KEY=secret_key or "controller-test-only-secret",
|
||||||
|
MAX_CONTENT_LENGTH=MAX_REQUEST_CONTENT_LENGTH,
|
||||||
)
|
)
|
||||||
web_app.extensions["controller_service"] = service
|
web_app.extensions["controller_service"] = service
|
||||||
web_app.extensions["controller_config"] = config
|
web_app.extensions["controller_config"] = config
|
||||||
@@ -285,6 +310,11 @@ def create_app(
|
|||||||
def index():
|
def index():
|
||||||
return render_template_string(HTML_FORM)
|
return render_template_string(HTML_FORM)
|
||||||
|
|
||||||
|
@web_app.errorhandler(413)
|
||||||
|
def _submission_too_large(_error):
|
||||||
|
flash("The submission was too large.")
|
||||||
|
return redirect(url_for("index"))
|
||||||
|
|
||||||
@web_app.post("/submit")
|
@web_app.post("/submit")
|
||||||
def submit():
|
def submit():
|
||||||
url = request.form.get("url", "").strip()
|
url = request.form.get("url", "").strip()
|
||||||
@@ -292,7 +322,16 @@ def create_app(
|
|||||||
note = request.form.get("note", "").strip() or None
|
note = request.form.get("note", "").strip() or None
|
||||||
|
|
||||||
if not _valid_media_url(url):
|
if not _valid_media_url(url):
|
||||||
flash("Invalid URL. Use a complete http:// or https:// address.")
|
flash(
|
||||||
|
"Invalid URL. Only YouTube links are accepted "
|
||||||
|
"(e.g. https://www.youtube.com/... or https://youtu.be/...)."
|
||||||
|
)
|
||||||
|
return redirect(url_for("index"))
|
||||||
|
if submitter is not None and len(submitter) > MAX_SUBMITTER_LENGTH:
|
||||||
|
flash(f"Your name must be {MAX_SUBMITTER_LENGTH} characters or fewer.")
|
||||||
|
return redirect(url_for("index"))
|
||||||
|
if note is not None and len(note) > MAX_NOTE_LENGTH:
|
||||||
|
flash(f"The note must be {MAX_NOTE_LENGTH} characters or fewer.")
|
||||||
return redirect(url_for("index"))
|
return redirect(url_for("index"))
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
+79
-7
@@ -53,7 +53,13 @@ class FlaskFactoryTests(unittest.TestCase):
|
|||||||
def _app(self, service):
|
def _app(self, service):
|
||||||
return self._factory()(self.config, service, testing=True)
|
return self._factory()(self.config, service, testing=True)
|
||||||
|
|
||||||
def _post(self, service, *, url="https://example.com/playlist", **fields):
|
def _post(
|
||||||
|
self,
|
||||||
|
service,
|
||||||
|
*,
|
||||||
|
url="https://www.youtube.com/playlist?list=PL1234567890",
|
||||||
|
**fields,
|
||||||
|
):
|
||||||
app = self._app(service)
|
app = self._app(service)
|
||||||
return app.test_client().post(
|
return app.test_client().post(
|
||||||
"/submit",
|
"/submit",
|
||||||
@@ -71,15 +77,29 @@ class FlaskFactoryTests(unittest.TestCase):
|
|||||||
self.assertIn(b'name="submitter"', response.data)
|
self.assertIn(b'name="submitter"', response.data)
|
||||||
self.assertIn(b'name="note"', response.data)
|
self.assertIn(b'name="note"', response.data)
|
||||||
|
|
||||||
def test_submit_accepts_only_http_or_https_urls_with_a_network_location(self):
|
def test_submit_rejects_non_youtube_and_malformed_urls(self):
|
||||||
invalid_urls = (
|
invalid_urls = (
|
||||||
"",
|
"",
|
||||||
"http",
|
"http",
|
||||||
"https:///missing-host",
|
"https:///missing-host",
|
||||||
"httpfoo://example.com/video",
|
"httpfoo://youtube.com/video",
|
||||||
"ftp://example.com/video",
|
"ftp://youtube.com/video",
|
||||||
"javascript:alert(1)",
|
"javascript:alert(1)",
|
||||||
"/relative/path",
|
"/relative/path",
|
||||||
|
# Arbitrary external hosts, including lookalikes and substring
|
||||||
|
# tricks, must be rejected by exact normalized hostname match.
|
||||||
|
"https://example.com/video",
|
||||||
|
"https://notyoutube.com/video",
|
||||||
|
"https://youtube.com.evil.com/video",
|
||||||
|
"https://evil.com/?u=youtube.com",
|
||||||
|
"https://xyoutube.com/watch?v=abc",
|
||||||
|
"https://youtube.com.attacker.net/watch?v=abc",
|
||||||
|
# Embedded userinfo must be rejected even when the host matches.
|
||||||
|
"https://user:[email protected]/watch?v=abc",
|
||||||
|
"https://[email protected]/watch?v=abc",
|
||||||
|
"https://[email protected]/watch?v=abc",
|
||||||
|
# Oversized URL.
|
||||||
|
"https://www.youtube.com/watch?v=" + ("a" * 3000),
|
||||||
)
|
)
|
||||||
for url in invalid_urls:
|
for url in invalid_urls:
|
||||||
with self.subTest(url=url):
|
with self.subTest(url=url):
|
||||||
@@ -89,7 +109,18 @@ class FlaskFactoryTests(unittest.TestCase):
|
|||||||
self.assertIn(b"Invalid URL", response.data)
|
self.assertIn(b"Invalid URL", response.data)
|
||||||
self.assertEqual(service.enqueue_calls, [])
|
self.assertEqual(service.enqueue_calls, [])
|
||||||
|
|
||||||
for url in ("http://example.com/video", "https://example.com/video"):
|
def test_submit_accepts_only_explicit_official_youtube_hosts(self):
|
||||||
|
valid_urls = (
|
||||||
|
"https://youtube.com/watch?v=abc",
|
||||||
|
"https://www.youtube.com/watch?v=abc",
|
||||||
|
"http://www.youtube.com/watch?v=abc",
|
||||||
|
"https://m.youtube.com/watch?v=abc",
|
||||||
|
"https://music.youtube.com/watch?v=abc",
|
||||||
|
"https://youtu.be/abc",
|
||||||
|
"https://YouTube.com/watch?v=abc",
|
||||||
|
"https://WWW.YOUTUBE.COM/watch?v=abc",
|
||||||
|
)
|
||||||
|
for url in valid_urls:
|
||||||
with self.subTest(url=url):
|
with self.subTest(url=url):
|
||||||
service = FakeControllerService()
|
service = FakeControllerService()
|
||||||
response = self._post(service, url=url)
|
response = self._post(service, url=url)
|
||||||
@@ -103,7 +134,7 @@ class FlaskFactoryTests(unittest.TestCase):
|
|||||||
|
|
||||||
response = self._post(
|
response = self._post(
|
||||||
service,
|
service,
|
||||||
url="https://example.com/playlist",
|
url="https://www.youtube.com/playlist?list=PL1234567890",
|
||||||
submitter=" Alice ",
|
submitter=" Alice ",
|
||||||
note=" Before service ",
|
note=" Before service ",
|
||||||
)
|
)
|
||||||
@@ -113,8 +144,49 @@ class FlaskFactoryTests(unittest.TestCase):
|
|||||||
self.assertIn(b"start downloading shortly", response.data)
|
self.assertIn(b"start downloading shortly", response.data)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
service.enqueue_calls,
|
service.enqueue_calls,
|
||||||
[("https://example.com/playlist", "Alice", "Before service")],
|
[
|
||||||
|
(
|
||||||
|
"https://www.youtube.com/playlist?list=PL1234567890",
|
||||||
|
"Alice",
|
||||||
|
"Before service",
|
||||||
)
|
)
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_submit_rejects_oversized_submitter_and_note(self):
|
||||||
|
module = importlib.import_module("app")
|
||||||
|
service = FakeControllerService()
|
||||||
|
response = self._post(
|
||||||
|
service,
|
||||||
|
submitter="x" * (module.MAX_SUBMITTER_LENGTH + 1),
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertIn(b"must be", response.data)
|
||||||
|
self.assertEqual(service.enqueue_calls, [])
|
||||||
|
|
||||||
|
service = FakeControllerService()
|
||||||
|
response = self._post(
|
||||||
|
service,
|
||||||
|
note="x" * (module.MAX_NOTE_LENGTH + 1),
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertIn(b"must be", response.data)
|
||||||
|
self.assertEqual(service.enqueue_calls, [])
|
||||||
|
|
||||||
|
def test_submit_rejects_oversized_request_body(self):
|
||||||
|
service = FakeControllerService()
|
||||||
|
app = self._app(service)
|
||||||
|
response = app.test_client().post(
|
||||||
|
"/submit",
|
||||||
|
data={
|
||||||
|
"url": "https://www.youtube.com/watch?v=abc",
|
||||||
|
"note": "x" * 64_000,
|
||||||
|
},
|
||||||
|
follow_redirects=True,
|
||||||
|
)
|
||||||
|
self.assertEqual(response.status_code, 200)
|
||||||
|
self.assertIn(b"too large", response.data.lower())
|
||||||
|
self.assertEqual(service.enqueue_calls, [])
|
||||||
|
|
||||||
def test_reused_submission_preserves_existing_slot_message(self):
|
def test_reused_submission_preserves_existing_slot_message(self):
|
||||||
service = FakeControllerService(
|
service = FakeControllerService(
|
||||||
|
|||||||
Reference in New Issue
Block a user