fix: a presentation that runs out of memory says so, and the service survives it
On a 4 GB VM the kernel killed the import service mid-conversion. Measured: LibreOffice needs 0.2 GB for a text deck and 1.4-2.1 GB for decks of many different photographs, for about twenty seconds — a spike, not a leak. What made it a problem was the rest: OOMPolicy=stop took the watcher down with LibreOffice, it restarted and resumed the same conversion with no limit, and the library said "Getting this ready" throughout. The service is capped at 40% of memory, is first in line when memory runs short everywhere, and survives a killed LibreOffice (OOMPolicy=continue). A killed conversion is a failed import of kind "memory"; an import interrupted twice is not tried a third time; the card says the computer needed more memory, to close other programs and choose Try again; an import nobody has worked on for 20 minutes shows the same card; and an exclusive lock keeps it to one import at a time. Lab, the real service and LibreOffice: at a 4 GB-machine ceiling the 2.1 GB deck still imported; with no swap and a 300 MB ceiling LibreOffice was killed, the watcher kept its PID and the card offered Try again, which worked; the whole service killed during two attempts gave up after the second; three decks at once never ran two LibreOffices together. Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
ekklesia
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
940f63e0e5
commit
3855989029
@@ -158,11 +158,29 @@ def forget_presentation(presentation_id, root):
|
||||
library.write_json(library.ledger_path(root), data)
|
||||
|
||||
|
||||
class NeedsMemory(RuntimeError):
|
||||
"""The kernel killed the converter or renderer for want of memory."""
|
||||
|
||||
|
||||
# An import the service was killed in the middle of is resumed at the next
|
||||
# start — once. Measured in the lab: a deck that needs more memory than
|
||||
# the machine has free is killed again every time, and with no limit the
|
||||
# service converted, died and restarted in a loop while the library said
|
||||
# "Getting this ready…" for ever.
|
||||
MAX_ATTEMPTS = 2
|
||||
|
||||
|
||||
def prepare(presentation):
|
||||
"""Build the derived PDF, pages and cover from a managed original."""
|
||||
data = presentation.data
|
||||
managed = presentation.managed_copy
|
||||
|
||||
# Counted and written BEFORE the work, so that a kill in the middle of
|
||||
# it is still counted when the service comes back.
|
||||
data["attempts"] = int(data.get("attempts") or 0) + 1
|
||||
data["attempt_started_at"] = library.stamp()
|
||||
presentation.save()
|
||||
|
||||
if managed is None or not managed.is_file():
|
||||
raise RuntimeError("the managed original is missing")
|
||||
|
||||
@@ -189,12 +207,18 @@ def prepare(presentation):
|
||||
managed, presentation.slides)
|
||||
|
||||
if not converted:
|
||||
if error == library.KILLED:
|
||||
raise NeedsMemory(error)
|
||||
|
||||
raise RuntimeError(error or "the file could not be converted")
|
||||
|
||||
pages, error = library.render_pages(
|
||||
presentation.slides, presentation.pages_directory)
|
||||
|
||||
if not pages:
|
||||
if error == library.KILLED:
|
||||
raise NeedsMemory(error)
|
||||
|
||||
raise RuntimeError(error or "the slides could not be rendered")
|
||||
|
||||
data["pages"] = pages
|
||||
@@ -211,6 +235,7 @@ def prepare(presentation):
|
||||
|
||||
data["state"] = "ready"
|
||||
data["error"] = None
|
||||
data["error_kind"] = None
|
||||
presentation.save()
|
||||
|
||||
say(f"imported {data['title']!r} ({pages} slides)")
|
||||
@@ -278,15 +303,55 @@ def import_one(source, root=None):
|
||||
# teacher can remove it, and a technician can recover or inspect it;
|
||||
# no successfully downloaded file is lost because a derived asset
|
||||
# could not be made.
|
||||
data["state"] = "failed"
|
||||
data["error"] = str(error)[:300]
|
||||
presentation.save()
|
||||
|
||||
fail(presentation, error)
|
||||
say(f"could not import {source.name}: {error}")
|
||||
|
||||
return presentation
|
||||
|
||||
|
||||
def fail(presentation, error, kind=None):
|
||||
"""Record a failure a teacher's card can say something useful about."""
|
||||
presentation.data["state"] = "failed"
|
||||
presentation.data["error"] = str(error)[:300]
|
||||
presentation.data["error_kind"] = kind or (
|
||||
"memory" if isinstance(error, NeedsMemory) else "unreadable")
|
||||
presentation.save()
|
||||
|
||||
|
||||
def attempt(presentation):
|
||||
"""prepare(), with any failure recorded rather than raised."""
|
||||
try:
|
||||
prepare(presentation)
|
||||
except Exception as error: # noqa: BLE001
|
||||
fail(presentation, error)
|
||||
say(f"could not import {presentation.title!r}: {error}")
|
||||
|
||||
|
||||
def retry_requested(root=None):
|
||||
"""Presentations a teacher pressed Try again on, taken up now."""
|
||||
root = root if root is not None else library.library_root()
|
||||
retried = 0
|
||||
|
||||
if not root.is_dir():
|
||||
return 0
|
||||
|
||||
for presentation in library.library(root):
|
||||
if presentation.state != "retry":
|
||||
continue
|
||||
|
||||
managed = presentation.managed_copy
|
||||
|
||||
if managed is None or not managed.is_file():
|
||||
fail(presentation, RuntimeError("the managed original is missing"), "unreadable")
|
||||
continue
|
||||
|
||||
say(f"trying again: {presentation.title!r}")
|
||||
attempt(presentation)
|
||||
retried += 1
|
||||
|
||||
return retried
|
||||
|
||||
|
||||
def sweep(root=None):
|
||||
"""Recover imports interrupted during a move, conversion or rendering.
|
||||
|
||||
@@ -313,14 +378,17 @@ def sweep(root=None):
|
||||
|
||||
if (managed is not None and managed.is_file()
|
||||
and (source is None or not source.exists())):
|
||||
say(f"resuming an interrupted import: {presentation.title!r}")
|
||||
try:
|
||||
prepare(presentation)
|
||||
except Exception as error: # noqa: BLE001
|
||||
presentation.data["state"] = "failed"
|
||||
presentation.data["error"] = str(error)[:300]
|
||||
presentation.save()
|
||||
say(f"could not import {presentation.title!r}: {error}")
|
||||
attempts = int(presentation.data.get("attempts") or 0)
|
||||
|
||||
if attempts >= MAX_ATTEMPTS:
|
||||
# Killed in the middle of it every time it was tried.
|
||||
say(f"giving up on {presentation.title!r} after {attempts} "
|
||||
"interrupted attempts; the teacher can try again")
|
||||
fail(presentation, NeedsMemory("interrupted every attempt"))
|
||||
else:
|
||||
say(f"resuming an interrupted import: {presentation.title!r}")
|
||||
attempt(presentation)
|
||||
|
||||
stale += 1
|
||||
continue
|
||||
|
||||
@@ -414,6 +482,23 @@ def status(root=None):
|
||||
return 0
|
||||
|
||||
|
||||
def hold_lock(root=None):
|
||||
"""The importer's lock, held for as long as this process lives."""
|
||||
import fcntl
|
||||
|
||||
root = root if root is not None else library.library_root()
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
handle = open(root / ".importer.lock", "a")
|
||||
|
||||
try:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
handle.close()
|
||||
return None
|
||||
|
||||
return handle
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Bring finished downloads into the Presentations library.")
|
||||
@@ -439,7 +524,17 @@ def main():
|
||||
say(f"removed {removed} expired presentation(s)")
|
||||
return 0
|
||||
|
||||
# One importer at a time, whoever started it. Conversions are the
|
||||
# memory the machine runs short of, so two at once — the service and a
|
||||
# technician's --once, say — must not happen.
|
||||
held = hold_lock()
|
||||
|
||||
if held is None:
|
||||
say("another importer is already running; not starting a second")
|
||||
return 0
|
||||
|
||||
sweep()
|
||||
retry_requested()
|
||||
|
||||
if arguments.once:
|
||||
imported = scan()
|
||||
@@ -453,6 +548,7 @@ def main():
|
||||
|
||||
while True:
|
||||
try:
|
||||
retry_requested()
|
||||
scan()
|
||||
|
||||
if time.monotonic() - last_cleanup > CLEANUP_SECONDS:
|
||||
|
||||
@@ -682,11 +682,11 @@ class Library(Gtk.Window):
|
||||
would make the buttons unclickable.
|
||||
"""
|
||||
return tuple(
|
||||
(p.id, p.state, p.pages, p.thumbnail_page, p.title)
|
||||
(p.id, p.state, p.pages, p.thumbnail_page, p.title, p.failure)
|
||||
for p in presentations
|
||||
)
|
||||
|
||||
def _fit_window(self, count):
|
||||
def _fit_window(self, count, tall=False):
|
||||
"""Size to the cards while staying inside the panel's work area."""
|
||||
workarea = monitor_workarea(self)
|
||||
maximum_width = max(420, int(workarea.width * 0.92))
|
||||
@@ -705,7 +705,10 @@ class Library(Gtk.Window):
|
||||
columns = min(count, available, 4)
|
||||
rows = (count + columns - 1) // columns
|
||||
width = min(maximum_width, max(420, 40 + columns * card_with_gap))
|
||||
height = min(maximum_height, 130 + rows * 340)
|
||||
# A card explaining a memory failure carries three lines of
|
||||
# words above its buttons, not one; measured, at 340 its Try
|
||||
# again was cut off below the window's edge.
|
||||
height = min(maximum_height, 130 + rows * (400 if tall else 340))
|
||||
|
||||
self.resize(width, height)
|
||||
|
||||
@@ -735,14 +738,19 @@ class Library(Gtk.Window):
|
||||
# something is "being prepared" when it never will be is how
|
||||
# somebody ends up waiting through a Bible study for it.
|
||||
ready = sum(1 for p in presentations if p.ready)
|
||||
broken = sum(1 for p in presentations if p.failed)
|
||||
working = len(presentations) - ready - broken
|
||||
again = sum(1 for p in presentations if p.failure == "memory")
|
||||
broken = sum(1 for p in presentations
|
||||
if p.failed and p.failure != "memory")
|
||||
working = len(presentations) - ready - broken - again
|
||||
|
||||
parts = [f"{ready} ready to present"]
|
||||
|
||||
if working:
|
||||
parts.append(f"{working} still being prepared")
|
||||
|
||||
if again:
|
||||
parts.append(f"{again} to try again")
|
||||
|
||||
if broken:
|
||||
parts.append(f"{broken} could not be opened")
|
||||
|
||||
@@ -752,7 +760,8 @@ class Library(Gtk.Window):
|
||||
self.grid.add(self.card(presentation))
|
||||
|
||||
self.grid.show_all()
|
||||
self._fit_window(len(presentations))
|
||||
self._fit_window(len(presentations),
|
||||
tall=any(p.failure == "memory" for p in presentations))
|
||||
|
||||
def card(self, presentation):
|
||||
box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10)
|
||||
@@ -802,6 +811,20 @@ class Library(Gtk.Window):
|
||||
look = Gtk.Button(label="Quick Look")
|
||||
look.connect("clicked", lambda *_: self.quick_look(presentation))
|
||||
buttons.pack_start(look, True, True, 0)
|
||||
elif presentation.failure == "memory":
|
||||
# The computer ran short of memory getting it ready, or the
|
||||
# importer stopped part way. Worth another go with less open;
|
||||
# never a converter's message, a path or a terminal.
|
||||
note.set_text("This needed more memory than the computer had "
|
||||
"free. Close other programs, then choose "
|
||||
"Try again.")
|
||||
note.set_line_wrap(True)
|
||||
note.set_max_width_chars(34)
|
||||
|
||||
again = Gtk.Button(label="Try again")
|
||||
again.get_style_context().add_class("primary")
|
||||
again.connect("clicked", lambda *_: self.try_again(presentation))
|
||||
buttons.pack_start(again, True, True, 0)
|
||||
elif presentation.failed:
|
||||
# SHORT, AND IN WORDS A TEACHER CAN ACT ON.
|
||||
#
|
||||
@@ -878,6 +901,12 @@ class Library(Gtk.Window):
|
||||
self._signature = None
|
||||
self.refresh()
|
||||
|
||||
def try_again(self, presentation):
|
||||
# The importer takes it up within a few seconds; the card says
|
||||
# "Getting this ready" again at the next refresh.
|
||||
if library.request_retry(presentation):
|
||||
self.refresh()
|
||||
|
||||
def remove(self, presentation):
|
||||
dialog = Gtk.MessageDialog(
|
||||
transient_for=self,
|
||||
|
||||
Reference in New Issue
Block a user