471 lines
15 KiB
Python
Executable File
471 lines
15 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
"""Watch the teacher's downloads, and keep the library tidy.
|
|
|
|
A teacher downloads a session in ordinary Chrome and then looks for it in
|
|
Presentations. Nothing in between is their job: no file manager, no
|
|
import button, no "where did you save it". This is the in between.
|
|
|
|
classroom-presentation-import run as a service, forever
|
|
classroom-presentation-import --once one pass, for a technician
|
|
classroom-presentation-import --status what the library holds
|
|
|
|
WHAT IT WILL NOT DO.
|
|
|
|
It never leaves a duplicate in Downloads. Once a supported download is
|
|
complete, its one original is moved into the managed library. Generated
|
|
PDFs and page images live beside it and can always be rebuilt from it.
|
|
|
|
It never imports a Wonder Ink worksheet. Wonder Ink downloads into a
|
|
directory of its own inside Downloads and those are worksheets, not
|
|
sessions; the cleanup will not remove one either.
|
|
|
|
It never imports a file that is still arriving. Chrome writes
|
|
`Session 7.pptx.crdownload` and renames it when the transfer is
|
|
done, so a finished name with that sibling beside it is a download in
|
|
progress, not a presentation.
|
|
|
|
It never overwrites a presentation with a later one of the same
|
|
name. Two downloads called "Bible Study.pdf" are two presentations.
|
|
"""
|
|
|
|
import argparse
|
|
import errno
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, "/usr/local/lib/ekklesia-classroom")
|
|
sys.path.insert(
|
|
0,
|
|
os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "lib"),
|
|
)
|
|
|
|
import classroom_presentations as library # noqa: E402
|
|
from classroom_config import feature, read_device_config # noqa: E402
|
|
|
|
# How often the downloads directory is looked at.
|
|
#
|
|
# Polling rather than inotify, deliberately. A teacher waits a few
|
|
# seconds either way, this cannot leak a watch descriptor or miss an
|
|
# event across a restart, and it is the kind of code somebody can still
|
|
# read in two years.
|
|
SCAN_SECONDS = 4
|
|
|
|
# How often expired presentations are cleared out.
|
|
CLEANUP_SECONDS = 3600
|
|
|
|
|
|
def say(message):
|
|
print(f"classroom-presentation-import: {message}", flush=True)
|
|
|
|
|
|
def new_directory(root):
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
directory = root / uuid.uuid4().hex
|
|
|
|
directory.mkdir()
|
|
|
|
return directory
|
|
|
|
|
|
def sync_directory(directory):
|
|
"""Best-effort persistence for a rename or unlink transaction."""
|
|
try:
|
|
descriptor = os.open(directory, os.O_RDONLY)
|
|
except OSError:
|
|
return
|
|
|
|
try:
|
|
os.fsync(descriptor)
|
|
except OSError:
|
|
pass
|
|
finally:
|
|
os.close(descriptor)
|
|
|
|
|
|
def move_download(source, destination):
|
|
"""Move one completed download without risking it on another filesystem.
|
|
|
|
rename is atomic on one filesystem. Across filesystems, copy to a hidden
|
|
temporary file, flush it, publish it atomically, and only then unlink the
|
|
source. If unlinking fails, the published copy is removed and the source
|
|
remains the one original for the next scan.
|
|
"""
|
|
source = os.fspath(source)
|
|
destination = os.fspath(destination)
|
|
temporary = destination + ".moving"
|
|
|
|
os.makedirs(os.path.dirname(destination), exist_ok=True)
|
|
|
|
if os.path.exists(destination) or os.path.exists(temporary):
|
|
raise FileExistsError(destination)
|
|
|
|
try:
|
|
os.replace(source, destination)
|
|
sync_directory(os.path.dirname(destination))
|
|
sync_directory(os.path.dirname(source))
|
|
return
|
|
except OSError as error:
|
|
if error.errno != errno.EXDEV:
|
|
raise
|
|
|
|
try:
|
|
with open(source, "rb") as incoming, open(temporary, "xb") as outgoing:
|
|
shutil.copyfileobj(incoming, outgoing, length=1024 * 1024)
|
|
outgoing.flush()
|
|
os.fsync(outgoing.fileno())
|
|
|
|
shutil.copystat(source, temporary, follow_symlinks=True)
|
|
os.replace(temporary, destination)
|
|
sync_directory(os.path.dirname(destination))
|
|
|
|
try:
|
|
os.unlink(source)
|
|
except OSError:
|
|
# Roll back to the still-complete source rather than leave two
|
|
# originals or make a failed move look successful.
|
|
try:
|
|
os.unlink(destination)
|
|
sync_directory(os.path.dirname(destination))
|
|
except OSError:
|
|
pass
|
|
raise
|
|
|
|
sync_directory(os.path.dirname(source))
|
|
except Exception:
|
|
try:
|
|
os.unlink(temporary)
|
|
except OSError:
|
|
pass
|
|
raise
|
|
|
|
|
|
def forget_presentation(presentation_id, root):
|
|
"""Remove transaction records for one interrupted presentation."""
|
|
data = library.ledger(root)
|
|
changed = False
|
|
|
|
for mark, value in list(data.items()):
|
|
if value == presentation_id:
|
|
data.pop(mark, None)
|
|
changed = True
|
|
|
|
if changed:
|
|
library.write_json(library.ledger_path(root), data)
|
|
|
|
|
|
def prepare(presentation):
|
|
"""Build the derived PDF, pages and cover from a managed original."""
|
|
data = presentation.data
|
|
managed = presentation.managed_copy
|
|
|
|
if managed is None or not managed.is_file():
|
|
raise RuntimeError("the managed original is missing")
|
|
|
|
# A restart may find conversion or rendering output from the interrupted
|
|
# attempt. The managed original is the recovery point; everything else is
|
|
# derived and is rebuilt cleanly.
|
|
if presentation.slides != managed:
|
|
try:
|
|
presentation.slides.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
shutil.rmtree(presentation.pages_directory, ignore_errors=True)
|
|
|
|
try:
|
|
presentation.thumbnail.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
if data["format"] == "pdf":
|
|
shutil.copy2(managed, presentation.slides)
|
|
else:
|
|
converted, error = library.convert_to_pdf(
|
|
managed, presentation.slides)
|
|
|
|
if not converted:
|
|
raise RuntimeError(error or "the file could not be converted")
|
|
|
|
pages, error = library.render_pages(
|
|
presentation.slides, presentation.pages_directory)
|
|
|
|
if not pages:
|
|
raise RuntimeError(error or "the slides could not be rendered")
|
|
|
|
data["pages"] = pages
|
|
data["title"] = library.unique_title(
|
|
library.readable_title(
|
|
data.get("original_name", ""),
|
|
library.first_text(presentation.slides)),
|
|
library.titles_in_use(presentation.directory.parent,
|
|
exclude=presentation.id),
|
|
)
|
|
|
|
library.render_thumbnail(presentation.page_image(1),
|
|
presentation.thumbnail)
|
|
|
|
data["state"] = "ready"
|
|
data["error"] = None
|
|
presentation.save()
|
|
|
|
say(f"imported {data['title']!r} ({pages} slides)")
|
|
|
|
return presentation
|
|
|
|
|
|
def import_one(source, root=None):
|
|
"""Take one finished download into the library.
|
|
|
|
Ordered so that an interruption is always recoverable. Metadata and the
|
|
transaction record are written before the move. Startup either retries
|
|
from Downloads or resumes from the moved managed original. A presentation
|
|
becomes ready only once every derived file it needs is on disk.
|
|
"""
|
|
root = root if root is not None else library.library_root()
|
|
mark = library.fingerprint(source)
|
|
|
|
if mark is None:
|
|
return None
|
|
|
|
directory = new_directory(root)
|
|
managed = directory / "original" / source.name
|
|
|
|
data = {
|
|
"id": directory.name,
|
|
"state": "importing",
|
|
"title": library.readable_title(source.name),
|
|
"original_name": source.name,
|
|
"source_path": str(source),
|
|
"managed_copy": str(managed.relative_to(directory)),
|
|
"format": library.SUPPORTED.get(source.suffix.lower(), ""),
|
|
"imported_at": library.stamp(),
|
|
"last_presented_at": None,
|
|
"times_presented": 0,
|
|
"thumbnail_page": 1,
|
|
"pages": 0,
|
|
"error": None,
|
|
}
|
|
|
|
presentation = library.Presentation(directory, data)
|
|
presentation.save()
|
|
|
|
# Recorded before the work, so that a file which crashes the
|
|
# conversion every time cannot be retried in a loop forever. The
|
|
# startup sweep is what un-records an import interrupted part way.
|
|
library.remember(mark, directory.name, root)
|
|
|
|
say(f"importing {source.name}")
|
|
|
|
try:
|
|
move_download(source, managed)
|
|
except Exception as error: # noqa: BLE001
|
|
# A failed move leaves the source in Downloads. Remove the transaction
|
|
# and retry on the next scan; there is no half-owned file to show.
|
|
forget_presentation(presentation.id, root)
|
|
shutil.rmtree(directory, ignore_errors=True)
|
|
say(f"could not move {source.name}: {error}")
|
|
return None
|
|
|
|
try:
|
|
return prepare(presentation)
|
|
except Exception as error: # noqa: BLE001
|
|
# Conversion and rendering failures keep the managed original. The
|
|
# teacher can remove it, and a technician can recover or inspect it;
|
|
# no successfully downloaded file is lost because a derived asset
|
|
# could not be made.
|
|
data["state"] = "failed"
|
|
data["error"] = str(error)[:300]
|
|
presentation.save()
|
|
|
|
say(f"could not import {source.name}: {error}")
|
|
|
|
return presentation
|
|
|
|
|
|
def sweep(root=None):
|
|
"""Recover imports interrupted during a move, conversion or rendering.
|
|
|
|
If Downloads still has the source, discard only the incomplete managed
|
|
copy and let the next scan retry. If the source has already been moved,
|
|
the managed original is the recovery point and its derived assets are
|
|
rebuilt in place. This distinction prevents a crash after the move from
|
|
deleting the only copy of a successful download.
|
|
"""
|
|
root = root if root is not None else library.library_root()
|
|
|
|
if not root.is_dir():
|
|
return 0
|
|
|
|
stale = 0
|
|
|
|
for presentation in library.library(root):
|
|
if presentation.state != "importing":
|
|
continue
|
|
|
|
source_text = presentation.data.get("source_path")
|
|
source = Path(source_text) if source_text else None
|
|
managed = presentation.managed_copy
|
|
|
|
if (managed is not None and managed.is_file()
|
|
and (source is None or not source.exists())):
|
|
say(f"resuming an interrupted import: {presentation.title!r}")
|
|
try:
|
|
prepare(presentation)
|
|
except Exception as error: # noqa: BLE001
|
|
presentation.data["state"] = "failed"
|
|
presentation.data["error"] = str(error)[:300]
|
|
presentation.save()
|
|
say(f"could not import {presentation.title!r}: {error}")
|
|
stale += 1
|
|
continue
|
|
|
|
say(f"retrying an interrupted move: {presentation.title!r}")
|
|
forget_presentation(presentation.id, root)
|
|
shutil.rmtree(presentation.directory, ignore_errors=True)
|
|
stale += 1
|
|
|
|
return stale
|
|
|
|
|
|
def scan(root=None, downloads=None):
|
|
"""One look at the downloads directory."""
|
|
root = root if root is not None else library.library_root()
|
|
known = library.ledger(root)
|
|
imported = 0
|
|
|
|
for candidate in library.candidates(downloads):
|
|
mark = library.fingerprint(candidate)
|
|
|
|
if mark is None or mark in known:
|
|
continue
|
|
|
|
if import_one(candidate, root) is not None:
|
|
imported += 1
|
|
known = library.ledger(root)
|
|
|
|
return imported
|
|
|
|
|
|
def cleanup(root=None, moment=None):
|
|
"""Remove presentations whose time is up.
|
|
|
|
Only ever the selected managed directory, including its one original and
|
|
derived assets, and never the presentation on the screen.
|
|
"""
|
|
root = root if root is not None else library.library_root()
|
|
lock = library.presenting_lock()
|
|
showing = ""
|
|
|
|
try:
|
|
showing = lock.read_text().strip()
|
|
except OSError:
|
|
showing = ""
|
|
|
|
removed = 0
|
|
|
|
for presentation in library.library(root):
|
|
if not presentation.expired(moment):
|
|
continue
|
|
|
|
if presentation.id == showing:
|
|
# A retention timer must never interrupt a room. It will
|
|
# still be expired in an hour.
|
|
say(f"{presentation.title!r} is expired but is being presented")
|
|
continue
|
|
|
|
if presentation.last_presented_at is None:
|
|
why = f"never presented, imported {presentation.data['imported_at']}"
|
|
else:
|
|
why = f"last presented {presentation.data['last_presented_at']}"
|
|
|
|
say(f"removing {presentation.title!r} — {why}")
|
|
|
|
if library.remove(presentation):
|
|
removed += 1
|
|
|
|
return removed
|
|
|
|
|
|
def status(root=None):
|
|
presentations = library.library(root)
|
|
|
|
if not presentations:
|
|
print("the library is empty")
|
|
return 0
|
|
|
|
for presentation in presentations:
|
|
expires = presentation.expires_at()
|
|
print(f"{presentation.title}")
|
|
print(f" state {presentation.state}"
|
|
f"{' — ' + presentation.error if presentation.error else ''}")
|
|
print(f" slides {presentation.pages}"
|
|
f" (card shows slide {presentation.thumbnail_page})")
|
|
print(f" imported {presentation.data.get('imported_at')}")
|
|
presented = presentation.data.get("last_presented_at") or "never"
|
|
print(f" presented {presented} "
|
|
f"({presentation.times_presented} times)")
|
|
print(f" expires {library.stamp(expires) if expires else 'unknown'}")
|
|
|
|
return 0
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Bring finished downloads into the Presentations library.")
|
|
parser.add_argument("--once", action="store_true",
|
|
help="one pass, then stop")
|
|
parser.add_argument("--status", action="store_true",
|
|
help="what the library holds; changes nothing")
|
|
parser.add_argument("--cleanup", action="store_true",
|
|
help="apply the retention rules now")
|
|
arguments = parser.parse_args()
|
|
|
|
if arguments.status:
|
|
return status()
|
|
|
|
# A device that does not offer Presentations does not run an importer
|
|
# for them. Said out loud for the technician who started it by hand.
|
|
if not feature(read_device_config(), "PRESENTATIONS_ENABLED"):
|
|
say("PRESENTATIONS_ENABLED is false on this device; not starting")
|
|
return 0
|
|
|
|
if arguments.cleanup:
|
|
removed = cleanup()
|
|
say(f"removed {removed} expired presentation(s)")
|
|
return 0
|
|
|
|
sweep()
|
|
|
|
if arguments.once:
|
|
imported = scan()
|
|
cleanup()
|
|
say(f"imported {imported} presentation(s)")
|
|
return 0
|
|
|
|
say(f"watching {library.downloads_root()}")
|
|
|
|
last_cleanup = 0.0
|
|
|
|
while True:
|
|
try:
|
|
scan()
|
|
|
|
if time.monotonic() - last_cleanup > CLEANUP_SECONDS:
|
|
cleanup()
|
|
last_cleanup = time.monotonic()
|
|
except Exception as error: # noqa: BLE001
|
|
# One bad file must not stop the watcher. A teacher whose
|
|
# next download never appears has no way to find out why.
|
|
say(f"scan failed: {error}")
|
|
|
|
time.sleep(SCAN_SECONDS)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|