ekklesiaandClaude Opus 5.5
3855989029
On a 4 GB VM the kernel killed the import service mid-conversion. Measured: LibreOffice needs 0.2 GB for a text deck and 1.4-2.1 GB for decks of many different photographs, for about twenty seconds — a spike, not a leak. What made it a problem was the rest: OOMPolicy=stop took the watcher down with LibreOffice, it restarted and resumed the same conversion with no limit, and the library said "Getting this ready" throughout. The service is capped at 40% of memory, is first in line when memory runs short everywhere, and survives a killed LibreOffice (OOMPolicy=continue). A killed conversion is a failed import of kind "memory"; an import interrupted twice is not tried a third time; the card says the computer needed more memory, to close other programs and choose Try again; an import nobody has worked on for 20 minutes shows the same card; and an exclusive lock keeps it to one import at a time. Lab, the real service and LibreOffice: at a 4 GB-machine ceiling the 2.1 GB deck still imported; with no swap and a 300 MB ceiling LibreOffice was killed, the watcher kept its PID and the card offered Try again, which worked; the whole service killed during two attempts gave up after the second; three decks at once never ran two LibreOffices together. Co-Authored-By: Claude Opus 5.5 <[email protected]>
567 lines
18 KiB
Python
Executable File
567 lines
18 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
"""Watch the teacher's downloads, and keep the library tidy.
|
|
|
|
A teacher downloads a session in ordinary Chrome and then looks for it in
|
|
Presentations. Nothing in between is their job: no file manager, no
|
|
import button, no "where did you save it". This is the in between.
|
|
|
|
classroom-presentation-import run as a service, forever
|
|
classroom-presentation-import --once one pass, for a technician
|
|
classroom-presentation-import --status what the library holds
|
|
|
|
WHAT IT WILL NOT DO.
|
|
|
|
It never leaves a duplicate in Downloads. Once a supported download is
|
|
complete, its one original is moved into the managed library. Generated
|
|
PDFs and page images live beside it and can always be rebuilt from it.
|
|
|
|
It never imports a Wonder Ink worksheet. Wonder Ink downloads into a
|
|
directory of its own inside Downloads and those are worksheets, not
|
|
sessions; the cleanup will not remove one either.
|
|
|
|
It never imports a file that is still arriving. Chrome writes
|
|
`Session 7.pptx.crdownload` and renames it when the transfer is
|
|
done, so a finished name with that sibling beside it is a download in
|
|
progress, not a presentation.
|
|
|
|
It never overwrites a presentation with a later one of the same
|
|
name. Two downloads called "Bible Study.pdf" are two presentations.
|
|
"""
|
|
|
|
import argparse
|
|
import errno
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, "/usr/local/lib/ekklesia-classroom")
|
|
sys.path.insert(
|
|
0,
|
|
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "lib"),
|
|
)
|
|
|
|
import classroom_presentations as library # noqa: E402
|
|
from classroom_config import feature, read_device_config # noqa: E402
|
|
|
|
# How often the downloads directory is looked at.
|
|
#
|
|
# Polling rather than inotify, deliberately. A teacher waits a few
|
|
# seconds either way, this cannot leak a watch descriptor or miss an
|
|
# event across a restart, and it is the kind of code somebody can still
|
|
# read in two years.
|
|
SCAN_SECONDS = 4
|
|
|
|
# How often expired presentations are cleared out.
|
|
CLEANUP_SECONDS = 3600
|
|
|
|
|
|
def say(message):
|
|
print(f"classroom-presentation-import: {message}", flush=True)
|
|
|
|
|
|
def new_directory(root):
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
directory = root / uuid.uuid4().hex
|
|
|
|
directory.mkdir()
|
|
|
|
return directory
|
|
|
|
|
|
def sync_directory(directory):
|
|
"""Best-effort persistence for a rename or unlink transaction."""
|
|
try:
|
|
descriptor = os.open(directory, os.O_RDONLY)
|
|
except OSError:
|
|
return
|
|
|
|
try:
|
|
os.fsync(descriptor)
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def move_download(source, destination):
|
|
"""Move one completed download without risking it on another filesystem.
|
|
|
|
rename is atomic on one filesystem. Across filesystems, copy to a hidden
|
|
temporary file, flush it, publish it atomically, and only then unlink the
|
|
source. If unlinking fails, the published copy is removed and the source
|
|
remains the one original for the next scan.
|
|
"""
|
|
source = os.fspath(source)
|
|
destination = os.fspath(destination)
|
|
temporary = destination + ".moving"
|
|
|
|
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
|
|
|
if os.path.exists(destination) or os.path.exists(temporary):
|
|
raise FileExistsError(destination)
|
|
|
|
try:
|
|
os.replace(source, destination)
|
|
sync_directory(os.path.dirname(destination))
|
|
sync_directory(os.path.dirname(source))
|
|
return
|
|
except OSError as error:
|
|
if error.errno != errno.EXDEV:
|
|
raise
|
|
|
|
try:
|
|
with open(source, "rb") as incoming, open(temporary, "xb") as outgoing:
|
|
shutil.copyfileobj(incoming, outgoing, length=1024 * 1024)
|
|
outgoing.flush()
|
|
os.fsync(outgoing.fileno())
|
|
|
|
shutil.copystat(source, temporary, follow_symlinks=True)
|
|
os.replace(temporary, destination)
|
|
sync_directory(os.path.dirname(destination))
|
|
|
|
try:
|
|
os.unlink(source)
|
|
except OSError:
|
|
# Roll back to the still-complete source rather than leave two
|
|
# originals or make a failed move look successful.
|
|
try:
|
|
os.unlink(destination)
|
|
sync_directory(os.path.dirname(destination))
|
|
except OSError:
|
|
pass
|
|
raise
|
|
|
|
sync_directory(os.path.dirname(source))
|
|
except Exception:
|
|
try:
|
|
os.unlink(temporary)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
|
|
|
|
def forget_presentation(presentation_id, root):
|
|
"""Remove transaction records for one interrupted presentation."""
|
|
data = library.ledger(root)
|
|
changed = False
|
|
|
|
for mark, value in list(data.items()):
|
|
if value == presentation_id:
|
|
data.pop(mark, None)
|
|
changed = True
|
|
|
|
if changed:
|
|
library.write_json(library.ledger_path(root), data)
|
|
|
|
|
|
class NeedsMemory(RuntimeError):
|
|
"""The kernel killed the converter or renderer for want of memory."""
|
|
|
|
|
|
# An import the service was killed in the middle of is resumed at the next
|
|
# start — once. Measured in the lab: a deck that needs more memory than
|
|
# the machine has free is killed again every time, and with no limit the
|
|
# service converted, died and restarted in a loop while the library said
|
|
# "Getting this ready…" for ever.
|
|
MAX_ATTEMPTS = 2
|
|
|
|
|
|
def prepare(presentation):
|
|
"""Build the derived PDF, pages and cover from a managed original."""
|
|
data = presentation.data
|
|
managed = presentation.managed_copy
|
|
|
|
# Counted and written BEFORE the work, so that a kill in the middle of
|
|
# it is still counted when the service comes back.
|
|
data["attempts"] = int(data.get("attempts") or 0) + 1
|
|
data["attempt_started_at"] = library.stamp()
|
|
presentation.save()
|
|
|
|
if managed is None or not managed.is_file():
|
|
raise RuntimeError("the managed original is missing")
|
|
|
|
# A restart may find conversion or rendering output from the interrupted
|
|
# attempt. The managed original is the recovery point; everything else is
|
|
# derived and is rebuilt cleanly.
|
|
if presentation.slides != managed:
|
|
try:
|
|
presentation.slides.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
shutil.rmtree(presentation.pages_directory, ignore_errors=True)
|
|
|
|
try:
|
|
presentation.thumbnail.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
if data["format"] == "pdf":
|
|
shutil.copy2(managed, presentation.slides)
|
|
else:
|
|
converted, error = library.convert_to_pdf(
|
|
managed, presentation.slides)
|
|
|
|
if not converted:
|
|
if error == library.KILLED:
|
|
raise NeedsMemory(error)
|
|
|
|
raise RuntimeError(error or "the file could not be converted")
|
|
|
|
pages, error = library.render_pages(
|
|
presentation.slides, presentation.pages_directory)
|
|
|
|
if not pages:
|
|
if error == library.KILLED:
|
|
raise NeedsMemory(error)
|
|
|
|
raise RuntimeError(error or "the slides could not be rendered")
|
|
|
|
data["pages"] = pages
|
|
data["title"] = library.unique_title(
|
|
library.readable_title(
|
|
data.get("original_name", ""),
|
|
library.first_text(presentation.slides)),
|
|
library.titles_in_use(presentation.directory.parent,
|
|
exclude=presentation.id),
|
|
)
|
|
|
|
library.render_thumbnail(presentation.page_image(1),
|
|
presentation.thumbnail)
|
|
|
|
data["state"] = "ready"
|
|
data["error"] = None
|
|
data["error_kind"] = None
|
|
presentation.save()
|
|
|
|
say(f"imported {data['title']!r} ({pages} slides)")
|
|
|
|
return presentation
|
|
|
|
|
|
def import_one(source, root=None):
|
|
"""Take one finished download into the library.
|
|
|
|
Ordered so that an interruption is always recoverable. Metadata and the
|
|
transaction record are written before the move. Startup either retries
|
|
from Downloads or resumes from the moved managed original. A presentation
|
|
becomes ready only once every derived file it needs is on disk.
|
|
"""
|
|
root = root if root is not None else library.library_root()
|
|
mark = library.fingerprint(source)
|
|
|
|
if mark is None:
|
|
return None
|
|
|
|
directory = new_directory(root)
|
|
managed = directory / "original" / source.name
|
|
|
|
data = {
|
|
"id": directory.name,
|
|
"state": "importing",
|
|
"title": library.readable_title(source.name),
|
|
"original_name": source.name,
|
|
"source_path": str(source),
|
|
"managed_copy": str(managed.relative_to(directory)),
|
|
"format": library.SUPPORTED.get(source.suffix.lower(), ""),
|
|
"imported_at": library.stamp(),
|
|
"last_presented_at": None,
|
|
"times_presented": 0,
|
|
"thumbnail_page": 1,
|
|
"pages": 0,
|
|
"error": None,
|
|
}
|
|
|
|
presentation = library.Presentation(directory, data)
|
|
presentation.save()
|
|
|
|
# Recorded before the work, so that a file which crashes the
|
|
# conversion every time cannot be retried in a loop forever. The
|
|
# startup sweep is what un-records an import interrupted part way.
|
|
library.remember(mark, directory.name, root)
|
|
|
|
say(f"importing {source.name}")
|
|
|
|
try:
|
|
move_download(source, managed)
|
|
except Exception as error: # noqa: BLE001
|
|
# A failed move leaves the source in Downloads. Remove the transaction
|
|
# and retry on the next scan; there is no half-owned file to show.
|
|
forget_presentation(presentation.id, root)
|
|
shutil.rmtree(directory, ignore_errors=True)
|
|
say(f"could not move {source.name}: {error}")
|
|
return None
|
|
|
|
try:
|
|
return prepare(presentation)
|
|
except Exception as error: # noqa: BLE001
|
|
# Conversion and rendering failures keep the managed original. The
|
|
# teacher can remove it, and a technician can recover or inspect it;
|
|
# no successfully downloaded file is lost because a derived asset
|
|
# could not be made.
|
|
fail(presentation, error)
|
|
say(f"could not import {source.name}: {error}")
|
|
|
|
return presentation
|
|
|
|
|
|
def fail(presentation, error, kind=None):
|
|
"""Record a failure a teacher's card can say something useful about."""
|
|
presentation.data["state"] = "failed"
|
|
presentation.data["error"] = str(error)[:300]
|
|
presentation.data["error_kind"] = kind or (
|
|
"memory" if isinstance(error, NeedsMemory) else "unreadable")
|
|
presentation.save()
|
|
|
|
|
|
def attempt(presentation):
|
|
"""prepare(), with any failure recorded rather than raised."""
|
|
try:
|
|
prepare(presentation)
|
|
except Exception as error: # noqa: BLE001
|
|
fail(presentation, error)
|
|
say(f"could not import {presentation.title!r}: {error}")
|
|
|
|
|
|
def retry_requested(root=None):
|
|
"""Presentations a teacher pressed Try again on, taken up now."""
|
|
root = root if root is not None else library.library_root()
|
|
retried = 0
|
|
|
|
if not root.is_dir():
|
|
return 0
|
|
|
|
for presentation in library.library(root):
|
|
if presentation.state != "retry":
|
|
continue
|
|
|
|
managed = presentation.managed_copy
|
|
|
|
if managed is None or not managed.is_file():
|
|
fail(presentation, RuntimeError("the managed original is missing"), "unreadable")
|
|
continue
|
|
|
|
say(f"trying again: {presentation.title!r}")
|
|
attempt(presentation)
|
|
retried += 1
|
|
|
|
return retried
|
|
|
|
|
|
def sweep(root=None):
|
|
"""Recover imports interrupted during a move, conversion or rendering.
|
|
|
|
If Downloads still has the source, discard only the incomplete managed
|
|
copy and let the next scan retry. If the source has already been moved,
|
|
the managed original is the recovery point and its derived assets are
|
|
rebuilt in place. This distinction prevents a crash after the move from
|
|
deleting the only copy of a successful download.
|
|
"""
|
|
root = root if root is not None else library.library_root()
|
|
|
|
if not root.is_dir():
|
|
return 0
|
|
|
|
stale = 0
|
|
|
|
for presentation in library.library(root):
|
|
if presentation.state != "importing":
|
|
continue
|
|
|
|
source_text = presentation.data.get("source_path")
|
|
source = Path(source_text) if source_text else None
|
|
managed = presentation.managed_copy
|
|
|
|
if (managed is not None and managed.is_file()
|
|
and (source is None or not source.exists())):
|
|
attempts = int(presentation.data.get("attempts") or 0)
|
|
|
|
if attempts >= MAX_ATTEMPTS:
|
|
# Killed in the middle of it every time it was tried.
|
|
say(f"giving up on {presentation.title!r} after {attempts} "
|
|
"interrupted attempts; the teacher can try again")
|
|
fail(presentation, NeedsMemory("interrupted every attempt"))
|
|
else:
|
|
say(f"resuming an interrupted import: {presentation.title!r}")
|
|
attempt(presentation)
|
|
|
|
stale += 1
|
|
continue
|
|
|
|
say(f"retrying an interrupted move: {presentation.title!r}")
|
|
forget_presentation(presentation.id, root)
|
|
shutil.rmtree(presentation.directory, ignore_errors=True)
|
|
stale += 1
|
|
|
|
return stale
|
|
|
|
|
|
def scan(root=None, downloads=None):
|
|
"""One look at the downloads directory."""
|
|
root = root if root is not None else library.library_root()
|
|
known = library.ledger(root)
|
|
imported = 0
|
|
|
|
for candidate in library.candidates(downloads):
|
|
mark = library.fingerprint(candidate)
|
|
|
|
if mark is None or mark in known:
|
|
continue
|
|
|
|
if import_one(candidate, root) is not None:
|
|
imported += 1
|
|
known = library.ledger(root)
|
|
|
|
return imported
|
|
|
|
|
|
def cleanup(root=None, moment=None):
|
|
"""Remove presentations whose time is up.
|
|
|
|
Only ever the selected managed directory, including its one original and
|
|
derived assets, and never the presentation on the screen.
|
|
"""
|
|
root = root if root is not None else library.library_root()
|
|
lock = library.presenting_lock()
|
|
showing = ""
|
|
|
|
try:
|
|
showing = lock.read_text().strip()
|
|
except OSError:
|
|
showing = ""
|
|
|
|
removed = 0
|
|
|
|
for presentation in library.library(root):
|
|
if not presentation.expired(moment):
|
|
continue
|
|
|
|
if presentation.id == showing:
|
|
# A retention timer must never interrupt a room. It will
|
|
# still be expired in an hour.
|
|
say(f"{presentation.title!r} is expired but is being presented")
|
|
continue
|
|
|
|
if presentation.last_presented_at is None:
|
|
why = f"never presented, imported {presentation.data['imported_at']}"
|
|
else:
|
|
why = f"last presented {presentation.data['last_presented_at']}"
|
|
|
|
say(f"removing {presentation.title!r} — {why}")
|
|
|
|
if library.remove(presentation):
|
|
removed += 1
|
|
|
|
return removed
|
|
|
|
|
|
def status(root=None):
|
|
presentations = library.library(root)
|
|
|
|
if not presentations:
|
|
print("the library is empty")
|
|
return 0
|
|
|
|
for presentation in presentations:
|
|
expires = presentation.expires_at()
|
|
print(f"{presentation.title}")
|
|
print(f" state {presentation.state}"
|
|
f"{' — ' + presentation.error if presentation.error else ''}")
|
|
print(f" slides {presentation.pages}"
|
|
f" (card shows slide {presentation.thumbnail_page})")
|
|
print(f" imported {presentation.data.get('imported_at')}")
|
|
presented = presentation.data.get("last_presented_at") or "never"
|
|
print(f" presented {presented} "
|
|
f"({presentation.times_presented} times)")
|
|
print(f" expires {library.stamp(expires) if expires else 'unknown'}")
|
|
|
|
return 0
|
|
|
|
|
|
def hold_lock(root=None):
|
|
"""The importer's lock, held for as long as this process lives."""
|
|
import fcntl
|
|
|
|
root = root if root is not None else library.library_root()
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
handle = open(root / ".importer.lock", "a")
|
|
|
|
try:
|
|
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
handle.close()
|
|
return None
|
|
|
|
return handle
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Bring finished downloads into the Presentations library.")
|
|
parser.add_argument("--once", action="store_true",
|
|
help="one pass, then stop")
|
|
parser.add_argument("--status", action="store_true",
|
|
help="what the library holds; changes nothing")
|
|
parser.add_argument("--cleanup", action="store_true",
|
|
help="apply the retention rules now")
|
|
arguments = parser.parse_args()
|
|
|
|
if arguments.status:
|
|
return status()
|
|
|
|
# A device that does not offer Presentations does not run an importer
|
|
# for them. Said out loud for the technician who started it by hand.
|
|
if not feature(read_device_config(), "PRESENTATIONS_ENABLED"):
|
|
say("PRESENTATIONS_ENABLED is false on this device; not starting")
|
|
return 0
|
|
|
|
if arguments.cleanup:
|
|
removed = cleanup()
|
|
say(f"removed {removed} expired presentation(s)")
|
|
return 0
|
|
|
|
# One importer at a time, whoever started it. Conversions are the
|
|
# memory the machine runs short of, so two at once — the service and a
|
|
# technician's --once, say — must not happen.
|
|
held = hold_lock()
|
|
|
|
if held is None:
|
|
say("another importer is already running; not starting a second")
|
|
return 0
|
|
|
|
sweep()
|
|
retry_requested()
|
|
|
|
if arguments.once:
|
|
imported = scan()
|
|
cleanup()
|
|
say(f"imported {imported} presentation(s)")
|
|
return 0
|
|
|
|
say(f"watching {library.downloads_root()}")
|
|
|
|
last_cleanup = 0.0
|
|
|
|
while True:
|
|
try:
|
|
retry_requested()
|
|
scan()
|
|
|
|
if time.monotonic() - last_cleanup > CLEANUP_SECONDS:
|
|
cleanup()
|
|
last_cleanup = time.monotonic()
|
|
except Exception as error: # noqa: BLE001
|
|
# One bad file must not stop the watcher. A teacher whose
|
|
# next download never appears has no way to find out why.
|
|
say(f"scan failed: {error}")
|
|
|
|
time.sleep(SCAN_SECONDS)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|