Initial commit of Roster

2025-06-28 01:12:46 -05:00
commit c23d3a7676
4367 changed files with 890519 additions and 0 deletions
@@ -0,0 +1,43 @@
+from .clustering import cluster_list, cluster_objects, make_cluster_dict  # noqa: F401
+from .generic import to_list  # noqa: F401
+from .geometry import (  # noqa: F401
+    bbox_to_rect,
+    calculate_area,
+    clip_obj,
+    crop_to_bbox,
+    curve_to_edges,
+    filter_edges,
+    get_bbox_overlap,
+    intersects_bbox,
+    line_to_edge,
+    merge_bboxes,
+    move_object,
+    obj_to_bbox,
+    obj_to_edges,
+    objects_to_bbox,
+    objects_to_rect,
+    outside_bbox,
+    rect_to_edges,
+    resize_object,
+    snap_objects,
+    within_bbox,
+)
+from .pdfinternals import (  # noqa: F401
+    decode_psl_list,
+    decode_text,
+    resolve,
+    resolve_all,
+    resolve_and_decode,
+)
+from .text import (  # noqa: F401
+    DEFAULT_X_DENSITY,
+    DEFAULT_X_TOLERANCE,
+    DEFAULT_Y_DENSITY,
+    DEFAULT_Y_TOLERANCE,
+    chars_to_textmap,
+    collate_line,
+    dedupe_chars,
+    extract_text,
+    extract_text_simple,
+    extract_words,
+)
@@ -0,0 +1,66 @@
+import itertools
+from collections.abc import Hashable
+from operator import itemgetter
+from typing import Any, Callable, Dict, Iterable, List, Tuple, TypeVar, Union
+
+from .._typing import T_num, T_obj
+
+
+def cluster_list(xs: List[T_num], tolerance: T_num = 0) -> List[List[T_num]]:
+    if tolerance == 0:
+        return [[x] for x in sorted(xs)]
+    if len(xs) < 2:
+        return [[x] for x in sorted(xs)]
+    groups = []
+    xs = list(sorted(xs))
+    current_group = [xs[0]]
+    last = xs[0]
+    for x in xs[1:]:
+        if x <= (last + tolerance):
+            current_group.append(x)
+        else:
+            groups.append(current_group)
+            current_group = [x]
+        last = x
+    groups.append(current_group)
+    return groups
+
+
+def make_cluster_dict(values: Iterable[T_num], tolerance: T_num) -> Dict[T_num, int]:
+    clusters = cluster_list(list(set(values)), tolerance)
+
+    nested_tuples = [
+        [(val, i) for val in value_cluster] for i, value_cluster in enumerate(clusters)
+    ]
+
+    return dict(itertools.chain(*nested_tuples))
+
+
+Clusterable = TypeVar("Clusterable", T_obj, Tuple[Any, ...])
+
+
+def cluster_objects(
+    xs: List[Clusterable],
+    key_fn: Union[Hashable, Callable[[Clusterable], T_num]],
+    tolerance: T_num,
+    preserve_order: bool = False,
+) -> List[List[Clusterable]]:
+
+    if not callable(key_fn):
+        key_fn = itemgetter(key_fn)
+
+    values = map(key_fn, xs)
+    cluster_dict = make_cluster_dict(values, tolerance)
+
+    get_0, get_1 = itemgetter(0), itemgetter(1)
+
+    if preserve_order:
+        cluster_tuples = [(x, cluster_dict.get(key_fn(x))) for x in xs]
+    else:
+        cluster_tuples = sorted(
+            ((x, cluster_dict.get(key_fn(x))) for x in xs), key=get_1
+        )
+
+    grouped = itertools.groupby(cluster_tuples, key=get_1)
+
+    return [list(map(get_0, v)) for k, v in grouped]
@@ -0,0 +1,6 @@
+class MalformedPDFException(Exception):
+    pass
+
+
+class PdfminerException(Exception):
+    pass
@@ -0,0 +1,21 @@
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, Dict, Hashable, List, Union
+
+from .._typing import T_seq
+
+if TYPE_CHECKING:  # pragma: nocover
+    from pandas.core.frame import DataFrame
+
+
+def to_list(collection: Union[T_seq[Any], "DataFrame"]) -> List[Any]:
+    if isinstance(collection, list):
+        return collection
+    elif isinstance(collection, Sequence):
+        return list(collection)
+    elif hasattr(collection, "to_dict"):
+        res: List[Dict[Hashable, Any]] = collection.to_dict(
+            "records"
+        )  # pragma: nocover
+        return res
+    else:
+        return list(collection)
@@ -0,0 +1,279 @@
+import itertools
+from operator import itemgetter
+from typing import Dict, Iterable, Optional
+
+from .._typing import T_bbox, T_num, T_obj, T_obj_list
+from .clustering import cluster_objects
+
+
+def objects_to_rect(objects: Iterable[T_obj]) -> Dict[str, T_num]:
+    """
+    Given an iterable of objects, return the smallest rectangle (i.e. a
+    dict with "x0", "top", "x1", and "bottom" keys) that contains them
+    all.
+    """
+    return bbox_to_rect(objects_to_bbox(objects))
+
+
+def objects_to_bbox(objects: Iterable[T_obj]) -> T_bbox:
+    """
+    Given an iterable of objects, return the smallest bounding box that
+    contains them all.
+    """
+    return merge_bboxes(map(bbox_getter, objects))
+
+
+bbox_getter = itemgetter("x0", "top", "x1", "bottom")
+
+
+def obj_to_bbox(obj: T_obj) -> T_bbox:
+    """
+    Return the bounding box for an object.
+    """
+    bbox: T_bbox = bbox_getter(obj)
+    return bbox
+
+
+def bbox_to_rect(bbox: T_bbox) -> Dict[str, T_num]:
+    """
+    Return the rectangle (i.e a dict with keys "x0", "top", "x1",
+    "bottom") for an object.
+    """
+    return {"x0": bbox[0], "top": bbox[1], "x1": bbox[2], "bottom": bbox[3]}
+
+
+def merge_bboxes(bboxes: Iterable[T_bbox]) -> T_bbox:
+    """
+    Given an iterable of bounding boxes, return the smallest bounding box
+    that contains them all.
+    """
+    x0, top, x1, bottom = zip(*bboxes)
+    return (min(x0), min(top), max(x1), max(bottom))
+
+
+def get_bbox_overlap(a: T_bbox, b: T_bbox) -> Optional[T_bbox]:
+    a_left, a_top, a_right, a_bottom = a
+    b_left, b_top, b_right, b_bottom = b
+    o_left = max(a_left, b_left)
+    o_right = min(a_right, b_right)
+    o_bottom = min(a_bottom, b_bottom)
+    o_top = max(a_top, b_top)
+    o_width = o_right - o_left
+    o_height = o_bottom - o_top
+    if o_height >= 0 and o_width >= 0 and o_height + o_width > 0:
+        return (o_left, o_top, o_right, o_bottom)
+    else:
+        return None
+
+
+def calculate_area(bbox: T_bbox) -> T_num:
+    left, top, right, bottom = bbox
+    if left > right or top > bottom:
+        raise ValueError(f"{bbox} has a negative width or height.")
+    return (right - left) * (bottom - top)
+
+
+def clip_obj(obj: T_obj, bbox: T_bbox) -> Optional[T_obj]:
+    overlap = get_bbox_overlap(obj_to_bbox(obj), bbox)
+    if overlap is None:
+        return None
+
+    dims = bbox_to_rect(overlap)
+    copy = dict(obj)
+
+    for attr in ["x0", "top", "x1", "bottom"]:
+        copy[attr] = dims[attr]
+
+    diff = dims["top"] - obj["top"]
+    if "doctop" in copy:
+        copy["doctop"] = obj["doctop"] + diff
+    copy["width"] = copy["x1"] - copy["x0"]
+    copy["height"] = copy["bottom"] - copy["top"]
+
+    return copy
+
+
+def intersects_bbox(objs: Iterable[T_obj], bbox: T_bbox) -> T_obj_list:
+    """
+    Filters objs to only those intersecting the bbox
+    """
+    return [obj for obj in objs if get_bbox_overlap(obj_to_bbox(obj), bbox) is not None]
+
+
+def within_bbox(objs: Iterable[T_obj], bbox: T_bbox) -> T_obj_list:
+    """
+    Filters objs to only those fully within the bbox
+    """
+    return [
+        obj
+        for obj in objs
+        if get_bbox_overlap(obj_to_bbox(obj), bbox) == obj_to_bbox(obj)
+    ]
+
+
+def outside_bbox(objs: Iterable[T_obj], bbox: T_bbox) -> T_obj_list:
+    """
+    Filters objs to only those fully outside the bbox
+    """
+    return [obj for obj in objs if get_bbox_overlap(obj_to_bbox(obj), bbox) is None]
+
+
+def crop_to_bbox(objs: Iterable[T_obj], bbox: T_bbox) -> T_obj_list:
+    """
+    Filters objs to only those intersecting the bbox,
+    and crops the extent of the objects to the bbox.
+    """
+    return list(filter(None, (clip_obj(obj, bbox) for obj in objs)))
+
+
+def move_object(obj: T_obj, axis: str, value: T_num) -> T_obj:
+    assert axis in ("h", "v")
+    if axis == "h":
+        new_items = [
+            ("x0", obj["x0"] + value),
+            ("x1", obj["x1"] + value),
+        ]
+    if axis == "v":
+        new_items = [
+            ("top", obj["top"] + value),
+            ("bottom", obj["bottom"] + value),
+        ]
+        if "doctop" in obj:
+            new_items += [("doctop", obj["doctop"] + value)]
+        if "y0" in obj:
+            new_items += [
+                ("y0", obj["y0"] - value),
+                ("y1", obj["y1"] - value),
+            ]
+    return obj.__class__(tuple(obj.items()) + tuple(new_items))
+
+
+def snap_objects(objs: Iterable[T_obj], attr: str, tolerance: T_num) -> T_obj_list:
+    axis = {"x0": "h", "x1": "h", "top": "v", "bottom": "v"}[attr]
+    list_objs = list(objs)
+    clusters = cluster_objects(list_objs, itemgetter(attr), tolerance)
+    avgs = [sum(map(itemgetter(attr), cluster)) / len(cluster) for cluster in clusters]
+    snapped_clusters = [
+        [move_object(obj, axis, avg - obj[attr]) for obj in cluster]
+        for cluster, avg in zip(clusters, avgs)
+    ]
+    return list(itertools.chain(*snapped_clusters))
+
+
+def resize_object(obj: T_obj, key: str, value: T_num) -> T_obj:
+    assert key in ("x0", "x1", "top", "bottom")
+    old_value = obj[key]
+    diff = value - old_value
+    new_items = [
+        (key, value),
+    ]
+    if key == "x0":
+        assert value <= obj["x1"]
+        new_items.append(("width", obj["x1"] - value))
+    elif key == "x1":
+        assert value >= obj["x0"]
+        new_items.append(("width", value - obj["x0"]))
+    elif key == "top":
+        assert value <= obj["bottom"]
+        new_items.append(("doctop", obj["doctop"] + diff))
+        new_items.append(("height", obj["height"] - diff))
+        if "y1" in obj:
+            new_items.append(("y1", obj["y1"] - diff))
+    elif key == "bottom":
+        assert value >= obj["top"]
+        new_items.append(("height", obj["height"] + diff))
+        if "y0" in obj:
+            new_items.append(("y0", obj["y0"] - diff))
+    return obj.__class__(tuple(obj.items()) + tuple(new_items))
+
+
+def curve_to_edges(curve: T_obj) -> T_obj_list:
+    point_pairs = zip(curve["pts"], curve["pts"][1:])
+    return [
+        {
+            "object_type": "curve_edge",
+            "x0": min(p0[0], p1[0]),
+            "x1": max(p0[0], p1[0]),
+            "top": min(p0[1], p1[1]),
+            "doctop": min(p0[1], p1[1]) + (curve["doctop"] - curve["top"]),
+            "bottom": max(p0[1], p1[1]),
+            "width": abs(p0[0] - p1[0]),
+            "height": abs(p0[1] - p1[1]),
+            "orientation": "v" if p0[0] == p1[0] else ("h" if p0[1] == p1[1] else None),
+        }
+        for p0, p1 in point_pairs
+    ]
+
+
+def rect_to_edges(rect: T_obj) -> T_obj_list:
+    top, bottom, left, right = [dict(rect) for x in range(4)]
+    top.update(
+        {
+            "object_type": "rect_edge",
+            "height": 0,
+            "y0": rect["y1"],
+            "bottom": rect["top"],
+            "orientation": "h",
+        }
+    )
+    bottom.update(
+        {
+            "object_type": "rect_edge",
+            "height": 0,
+            "y1": rect["y0"],
+            "top": rect["top"] + rect["height"],
+            "doctop": rect["doctop"] + rect["height"],
+            "orientation": "h",
+        }
+    )
+    left.update(
+        {
+            "object_type": "rect_edge",
+            "width": 0,
+            "x1": rect["x0"],
+            "orientation": "v",
+        }
+    )
+    right.update(
+        {
+            "object_type": "rect_edge",
+            "width": 0,
+            "x0": rect["x1"],
+            "orientation": "v",
+        }
+    )
+    return [top, bottom, left, right]
+
+
+def line_to_edge(line: T_obj) -> T_obj:
+    edge = dict(line)
+    edge["orientation"] = "h" if (line["top"] == line["bottom"]) else "v"
+    return edge
+
+
+def obj_to_edges(obj: T_obj) -> T_obj_list:
+    t = obj["object_type"]
+    if "_edge" in t:
+        return [obj]
+    elif t == "line":
+        return [line_to_edge(obj)]
+    else:
+        return {"rect": rect_to_edges, "curve": curve_to_edges}[t](obj)
+
+
+def filter_edges(
+    edges: Iterable[T_obj],
+    orientation: Optional[str] = None,
+    edge_type: Optional[str] = None,
+    min_length: T_num = 1,
+) -> T_obj_list:
+    if orientation not in ("v", "h", None):
+        raise ValueError("Orientation must be 'v' or 'h'")
+
+    def test(e: T_obj) -> bool:
+        dim = "height" if e["orientation"] == "v" else "width"
+        et_correct = e["object_type"] == edge_type if edge_type is not None else True
+        orient_correct = orientation is None or e["orientation"] == orientation
+        return bool(et_correct and orient_correct and (e[dim] >= min_length))
+
+    return list(filter(test, edges))
@@ -0,0 +1,87 @@
+from typing import Any, List, Optional, Union
+
+from pdfminer.pdftypes import PDFObjRef
+from pdfminer.psparser import PSLiteral
+from pdfminer.utils import PDFDocEncoding
+
+from .exceptions import MalformedPDFException
+
+
+def decode_text(s: Union[bytes, str]) -> str:
+    """
+    Decodes a PDFDocEncoding string to Unicode.
+    Adds py3 compatibility to pdfminer's version.
+    """
+    if isinstance(s, bytes) and s.startswith(b"\xfe\xff"):
+        return str(s[2:], "utf-16be", "ignore")
+    try:
+        ords = (ord(c) if isinstance(c, str) else c for c in s)
+        return "".join(PDFDocEncoding[o] for o in ords)
+    except IndexError:
+        return str(s)
+
+
+def resolve_and_decode(obj: Any) -> Any:
+    """Recursively resolve the metadata values."""
+    if hasattr(obj, "resolve"):
+        obj = obj.resolve()
+    if isinstance(obj, list):
+        return list(map(resolve_and_decode, obj))
+    elif isinstance(obj, PSLiteral):
+        return decode_text(obj.name)
+    elif isinstance(obj, (str, bytes)):
+        return decode_text(obj)
+    elif isinstance(obj, dict):
+        for k, v in obj.items():
+            obj[k] = resolve_and_decode(v)
+        return obj
+
+    return obj
+
+
+def decode_psl_list(_list: List[Union[PSLiteral, str]]) -> List[str]:
+    return [
+        decode_text(value.name) if isinstance(value, PSLiteral) else value
+        for value in _list
+    ]
+
+
+def resolve(x: Any) -> Any:
+    if isinstance(x, PDFObjRef):
+        return x.resolve()
+    else:
+        return x
+
+
+def get_dict_type(d: Any) -> Optional[str]:
+    if not isinstance(d, dict):
+        return None
+    t = d.get("Type")
+    if isinstance(t, PSLiteral):
+        return decode_text(t.name)
+    else:
+        return t
+
+
+def resolve_all(x: Any) -> Any:
+    """
+    Recursively resolves the given object and all the internals.
+    """
+    if isinstance(x, PDFObjRef):
+        resolved = x.resolve()
+
+        # Avoid infinite recursion
+        if get_dict_type(resolved) == "Page":
+            return x
+
+        try:
+            return resolve_all(resolved)
+        except RecursionError as e:
+            raise MalformedPDFException(e)
+    elif isinstance(x, (list, tuple)):
+        return type(x)(resolve_all(v) for v in x)
+    elif isinstance(x, dict):
+        exceptions = ["Parent"] if get_dict_type(x) == "Annot" else []
+        return {k: v if k in exceptions else resolve_all(v) for k, v in x.items()}
+    else:
+        return x
@@ -0,0 +1,819 @@
+import inspect
+import itertools
+import logging
+import re
+import string
+from operator import itemgetter
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Match,
+    Optional,
+    Pattern,
+    Tuple,
+    Union,
+)
+
+from .._typing import T_bbox, T_dir, T_num, T_obj, T_obj_iter, T_obj_list
+from .clustering import cluster_objects
+from .generic import to_list
+from .geometry import objects_to_bbox
+
+logger = logging.getLogger(__name__)
+
+DEFAULT_X_TOLERANCE = 3
+DEFAULT_Y_TOLERANCE = 3
+DEFAULT_X_DENSITY = 7.25
+DEFAULT_Y_DENSITY = 13
+DEFAULT_LINE_DIR: T_dir = "ttb"
+DEFAULT_CHAR_DIR: T_dir = "ltr"
+
+LIGATURES = {
+    "ﬀ": "ff",
+    "ﬃ": "ffi",
+    "ﬄ": "ffl",
+    "ﬁ": "fi",
+    "ﬂ": "fl",
+    "ﬆ": "st",
+    "ﬅ": "st",
+}
+
+
+def get_line_cluster_key(line_dir: T_dir) -> Callable[[T_obj], T_num]:
+    return {
+        "ttb": lambda x: x["top"],
+        "btt": lambda x: -x["bottom"],
+        "ltr": lambda x: x["x0"],
+        "rtl": lambda x: -x["x1"],
+    }[line_dir]
+
+
+def get_char_sort_key(char_dir: T_dir) -> Callable[[T_obj], Tuple[T_num, T_num]]:
+    return {
+        "ttb": lambda x: (x["top"], x["bottom"]),
+        "btt": lambda x: (-(x["top"] + x["height"]), -x["top"]),
+        "ltr": lambda x: (x["x0"], x["x0"]),
+        "rtl": lambda x: (-x["x1"], -x["x0"]),
+    }[char_dir]
+
+
+BBOX_ORIGIN_KEYS = {
+    "ttb": itemgetter(1),
+    "btt": itemgetter(3),
+    "ltr": itemgetter(0),
+    "rtl": itemgetter(2),
+}
+
+POSITION_KEYS = {
+    "ttb": itemgetter("top"),
+    "btt": itemgetter("bottom"),
+    "ltr": itemgetter("x0"),
+    "rtl": itemgetter("x1"),
+}
+
+
+def validate_directions(line_dir: T_dir, char_dir: T_dir, suffix: str = "") -> None:
+    valid_dirs = set(POSITION_KEYS.keys())
+    if line_dir not in valid_dirs:
+        raise ValueError(
+            f"line_dir{suffix} must be one of {valid_dirs}, not {line_dir}"
+        )
+    if char_dir not in valid_dirs:
+        raise ValueError(
+            f"char_dir{suffix} must be one of {valid_dirs}, not {char_dir}"
+        )
+    if set(line_dir) == set(char_dir):
+        raise ValueError(
+            f"line_dir{suffix}={line_dir} is incompatible "
+            f"with char_dir{suffix}={char_dir}"
+        )
+
+
+class TextMap:
+    """
+    A TextMap maps each unicode character in the text to an individual `char`
+    object (or, in the case of layout-implied whitespace, `None`).
+    """
+
+    def __init__(
+        self,
+        tuples: List[Tuple[str, Optional[T_obj]]],
+        line_dir_render: T_dir,
+        char_dir_render: T_dir,
+    ) -> None:
+        validate_directions(line_dir_render, char_dir_render, "_render")
+        self.tuples = tuples
+        self.line_dir_render = line_dir_render
+        self.char_dir_render = char_dir_render
+        self.as_string = self.to_string()
+
+    def to_string(self) -> str:
+        cd = self.char_dir_render
+        ld = self.line_dir_render
+
+        base = "".join(map(itemgetter(0), self.tuples))
+
+        if cd == "ltr" and ld == "ttb":
+            return base
+        else:
+            lines = base.split("\n")
+            if ld in ("btt", "rtl"):
+                lines = list(reversed(lines))
+
+            if cd == "rtl":
+                lines = ["".join(reversed(line)) for line in lines]
+
+            if ld in ("rtl", "ltr"):
+                max_line_length = max(map(len, lines))
+                if cd == "btt":
+                    lines = [
+                        (" " * (max_line_length - len(line))) + line for line in lines
+                    ]
+                else:
+                    lines = [
+                        line + (" " * (max_line_length - len(line))) for line in lines
+                    ]
+                return "\n".join(
+                    "".join(line[i] for line in lines) for i in range(max_line_length)
+                )
+            else:
+                return "\n".join(lines)
+
+    def match_to_dict(
+        self,
+        m: Match[str],
+        main_group: int = 0,
+        return_groups: bool = True,
+        return_chars: bool = True,
+    ) -> Dict[str, Any]:
+        subset = self.tuples[m.start(main_group) : m.end(main_group)]
+        chars = [c for (text, c) in subset if c is not None]
+        x0, top, x1, bottom = objects_to_bbox(chars)
+
+        result = {
+            "text": m.group(main_group),
+            "x0": x0,
+            "top": top,
+            "x1": x1,
+            "bottom": bottom,
+        }
+
+        if return_groups:
+            result["groups"] = m.groups()
+
+        if return_chars:
+            result["chars"] = chars
+
+        return result
+
+    def search(
+        self,
+        pattern: Union[str, Pattern[str]],
+        regex: bool = True,
+        case: bool = True,
+        return_groups: bool = True,
+        return_chars: bool = True,
+        main_group: int = 0,
+    ) -> List[Dict[str, Any]]:
+        if isinstance(pattern, Pattern):
+            if regex is False:
+                raise ValueError(
+                    "Cannot pass a compiled search pattern *and* regex=False together."
+                )
+            if case is False:
+                raise ValueError(
+                    "Cannot pass a compiled search pattern *and* case=False together."
+                )
+            compiled = pattern
+        else:
+            if regex is False:
+                pattern = re.escape(pattern)
+
+            flags = re.I if case is False else 0
+            compiled = re.compile(pattern, flags)
+
+        gen = re.finditer(compiled, self.as_string)
+        # Remove zero-length matches (can happen, e.g., with optional
+        # patterns in regexes) and whitespace-only matches
+        filtered = filter(lambda m: bool(m.group(main_group).strip()), gen)
+        return [
+            self.match_to_dict(
+                m,
+                return_groups=return_groups,
+                return_chars=return_chars,
+                main_group=main_group,
+            )
+            for m in filtered
+        ]
+
+    def extract_text_lines(
+        self, strip: bool = True, return_chars: bool = True
+    ) -> List[Dict[str, Any]]:
+        """
+        `strip` is analogous to Python's `str.strip()` method, and returns
+        `text` attributes without their surrounding whitespace. Only
+        relevant when the relevant TextMap is created with `layout` = True
+
+        Setting `return_chars` to False will exclude the individual
+        character objects from the returned text-line dicts.
+        """
+        if strip:
+            pat = r" *([^\n]+?) *(\n|$)"
+        else:
+            pat = r"([^\n]+)"
+
+        return self.search(
+            pat, main_group=1, return_chars=return_chars, return_groups=False
+        )
+
+
+class WordMap:
+    """
+    A WordMap maps words->chars.
+    """
+
+    def __init__(self, tuples: List[Tuple[T_obj, T_obj_list]]) -> None:
+        self.tuples = tuples
+
+    def to_textmap(
+        self,
+        layout: bool = False,
+        layout_width: T_num = 0,
+        layout_height: T_num = 0,
+        layout_width_chars: int = 0,
+        layout_height_chars: int = 0,
+        layout_bbox: T_bbox = (0, 0, 0, 0),
+        x_density: T_num = DEFAULT_X_DENSITY,
+        y_density: T_num = DEFAULT_Y_DENSITY,
+        x_shift: T_num = 0,
+        y_shift: T_num = 0,
+        y_tolerance: T_num = DEFAULT_Y_TOLERANCE,
+        line_dir: T_dir = DEFAULT_LINE_DIR,
+        char_dir: T_dir = DEFAULT_CHAR_DIR,
+        line_dir_rotated: Optional[T_dir] = None,
+        char_dir_rotated: Optional[T_dir] = None,
+        char_dir_render: Optional[T_dir] = None,
+        line_dir_render: Optional[T_dir] = None,
+        use_text_flow: bool = False,
+        presorted: bool = False,
+        expand_ligatures: bool = True,
+    ) -> TextMap:
+        """
+        Given a list of (word, chars) tuples (i.e., a WordMap), return a list of
+        (char-text, char) tuples (i.e., a TextMap) that can be used to mimic
+        the structural layout of the text on the page(s), using the following
+        approach for top-to-bottom, left-to-right text:
+
+        - Sort the words by (top, x0) if not already sorted.
+
+        - Cluster the words by top (taking `y_tolerance` into account), and
+          iterate through them.
+
+        - For each cluster, divide (top - y_shift) by `y_density` to calculate
+          the minimum number of newlines that should come before this cluster.
+          Append that number of newlines *minus* the number of newlines already
+          appended, with a minimum of one.
+
+        - Then for each cluster, iterate through each word in it. Divide each
+          word's x0, minus `x_shift`, by `x_density` to calculate the minimum
+          number of characters that should come before this cluster.  Append that
+          number of spaces *minus* the number of characters and spaces already
+          appended, with a minimum of one. Then append the word's text.
+
+        - At the termination of each line, add more spaces if necessary to
+          mimic `layout_width`.
+
+        - Finally, add newlines to the end if necessary to mimic to
+          `layout_height`.
+
+        For other line/character directions (e.g., bottom-to-top,
+        right-to-left), these steps are adjusted.
+        """
+        _textmap: List[Tuple[str, Optional[T_obj]]] = []
+
+        if not len(self.tuples):
+            return TextMap(
+                _textmap,
+                line_dir_render=line_dir_render or line_dir,
+                char_dir_render=char_dir_render or char_dir,
+            )
+
+        expansions = LIGATURES if expand_ligatures else {}
+
+        if layout:
+            if layout_width_chars:
+                if layout_width:
+                    raise ValueError(
+                        "`layout_width` and `layout_width_chars` cannot both be set."
+                    )
+            else:
+                layout_width_chars = int(round(layout_width / x_density))
+
+            if layout_height_chars:
+                if layout_height:
+                    raise ValueError(
+                        "`layout_height` and `layout_height_chars` cannot both be set."
+                    )
+            else:
+                layout_height_chars = int(round(layout_height / y_density))
+
+            blank_line = [(" ", None)] * layout_width_chars
+        else:
+            blank_line = []
+
+        num_newlines = 0
+
+        line_cluster_key = get_line_cluster_key(line_dir)
+        char_sort_key = get_char_sort_key(char_dir)
+
+        line_position_key = POSITION_KEYS[line_dir]
+        char_position_key = POSITION_KEYS[char_dir]
+
+        y_origin = BBOX_ORIGIN_KEYS[line_dir](layout_bbox)
+        x_origin = BBOX_ORIGIN_KEYS[char_dir](layout_bbox)
+
+        words_sorted_line_dir = (
+            self.tuples
+            if presorted or use_text_flow
+            else sorted(self.tuples, key=lambda x: line_cluster_key(x[0]))
+        )
+
+        tuples_by_line = cluster_objects(
+            words_sorted_line_dir,
+            lambda x: line_cluster_key(x[0]),
+            y_tolerance,
+            preserve_order=presorted or use_text_flow,
+        )
+
+        for i, line_tuples in enumerate(tuples_by_line):
+            if layout:
+                line_position = line_position_key(line_tuples[0][0])
+                y_dist_raw = line_position - (y_origin + y_shift)
+                adj = -1 if line_dir in ["btt", "rtl"] else 1
+                y_dist = y_dist_raw * adj / y_density
+            else:
+                y_dist = 0
+            num_newlines_prepend = max(
+                # At least one newline, unless this iis the first line
+                int(i > 0),
+                # ... or as many as needed to get the imputed "distance" from the top
+                round(y_dist) - num_newlines,
+            )
+
+            for i in range(num_newlines_prepend):
+                if not len(_textmap) or _textmap[-1][0] == "\n":
+                    _textmap += blank_line
+                _textmap.append(("\n", None))
+
+            num_newlines += num_newlines_prepend
+
+            line_len = 0
+
+            line_tuples_sorted = (
+                line_tuples
+                if presorted or use_text_flow
+                else sorted(line_tuples, key=lambda x: char_sort_key(x[0]))
+            )
+
+            for word, chars in line_tuples_sorted:
+                if layout:
+                    char_position = char_position_key(word)
+                    x_dist_raw = char_position - (x_origin + x_shift)
+                    adj = -1 if char_dir in ["btt", "rtl"] else 1
+                    x_dist = x_dist_raw * adj / x_density
+                else:
+                    x_dist = 0
+
+                num_spaces_prepend = max(min(1, line_len), round(x_dist) - line_len)
+                _textmap += [(" ", None)] * num_spaces_prepend
+                line_len += num_spaces_prepend
+
+                for c in chars:
+                    letters = expansions.get(c["text"], c["text"])
+                    for letter in letters:
+                        _textmap.append((letter, c))
+                        line_len += 1
+
+            # Append spaces at end of line
+            if layout:
+                _textmap += [(" ", None)] * (layout_width_chars - line_len)
+
+        # Append blank lines at end of text
+        if layout:
+            num_newlines_append = layout_height_chars - (num_newlines + 1)
+            for i in range(num_newlines_append):
+                if i > 0:
+                    _textmap += blank_line
+                _textmap.append(("\n", None))
+
+            # Remove terminal newline
+            if _textmap[-1] == ("\n", None):
+                _textmap = _textmap[:-1]
+
+        return TextMap(
+            _textmap,
+            line_dir_render=line_dir_render or line_dir,
+            char_dir_render=char_dir_render or char_dir,
+        )
+
+
+class WordExtractor:
+    def __init__(
+        self,
+        x_tolerance: T_num = DEFAULT_X_TOLERANCE,
+        y_tolerance: T_num = DEFAULT_Y_TOLERANCE,
+        x_tolerance_ratio: Union[int, float, None] = None,
+        y_tolerance_ratio: Union[int, float, None] = None,
+        keep_blank_chars: bool = False,
+        use_text_flow: bool = False,
+        vertical_ttb: bool = True,  # Should vertical words be read top-to-bottom?
+        horizontal_ltr: bool = True,  # Should words be read left-to-right?
+        line_dir: T_dir = DEFAULT_LINE_DIR,
+        char_dir: T_dir = DEFAULT_CHAR_DIR,
+        line_dir_rotated: Optional[T_dir] = None,
+        char_dir_rotated: Optional[T_dir] = None,
+        extra_attrs: Optional[List[str]] = None,
+        split_at_punctuation: Union[bool, str] = False,
+        expand_ligatures: bool = True,
+    ):
+        self.x_tolerance = x_tolerance
+        self.y_tolerance = y_tolerance
+        self.x_tolerance_ratio = x_tolerance_ratio
+        self.y_tolerance_ratio = y_tolerance_ratio
+        self.keep_blank_chars = keep_blank_chars
+        self.use_text_flow = use_text_flow
+        self.horizontal_ltr = horizontal_ltr
+        self.vertical_ttb = vertical_ttb
+        if vertical_ttb is False:
+            logger.warning(
+                "vertical_ttb is deprecated and will be removed;"
+                " use line_dir/char_dir instead."
+            )
+        if horizontal_ltr is False:
+            logger.warning(
+                "horizontal_ltr is deprecated and will be removed;"
+                " use line_dir/char_dir instead."
+            )
+        self.line_dir = line_dir
+        self.char_dir = char_dir
+        # Default is to "flip" the directions for rotated text
+        self.line_dir_rotated = line_dir_rotated or char_dir
+        self.char_dir_rotated = char_dir_rotated or line_dir
+        validate_directions(self.line_dir, self.char_dir)
+        validate_directions(self.line_dir_rotated, self.char_dir_rotated, "_rotated")
+        self.extra_attrs = [] if extra_attrs is None else extra_attrs
+
+        # Note: string.punctuation = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'
+        self.split_at_punctuation = (
+            string.punctuation
+            if split_at_punctuation is True
+            else (split_at_punctuation or "")
+        )
+
+        self.expansions = LIGATURES if expand_ligatures else {}
+
+    def get_char_dir(self, upright: int) -> T_dir:
+        # Note: This can be simplified and reincorporated into .merge_chars and
+        # .iter_chars_to_lines once .vertical_ttb and .horizontal_ltr
+        # deprecation is complete.
+        if not upright and not self.vertical_ttb:
+            return "btt"
+
+        elif upright and not self.horizontal_ltr:
+            return "rtl"
+
+        return self.char_dir if upright else self.char_dir_rotated
+
+    def merge_chars(self, ordered_chars: T_obj_list) -> T_obj:
+        x0, top, x1, bottom = objects_to_bbox(ordered_chars)
+        doctop_adj = ordered_chars[0]["doctop"] - ordered_chars[0]["top"]
+        upright = ordered_chars[0]["upright"]
+        char_dir = self.get_char_dir(upright)
+
+        word = {
+            "text": "".join(
+                self.expansions.get(c["text"], c["text"]) for c in ordered_chars
+            ),
+            "x0": x0,
+            "x1": x1,
+            "top": top,
+            "doctop": top + doctop_adj,
+            "bottom": bottom,
+            "upright": upright,
+            "height": bottom - top,
+            "width": x1 - x0,
+            "direction": char_dir,
+        }
+
+        for key in self.extra_attrs:
+            word[key] = ordered_chars[0][key]
+
+        return word
+
+    def char_begins_new_word(
+        self,
+        prev_char: T_obj,
+        curr_char: T_obj,
+        direction: T_dir,
+        x_tolerance: T_num,
+        y_tolerance: T_num,
+    ) -> bool:
+        """This method takes several factors into account to determine if
+        `curr_char` represents the beginning of a new word:
+
+        - Whether the text is "upright" (i.e., non-rotated)
+        - Whether the user has specified that horizontal text runs
+          left-to-right (default) or right-to-left, as represented by
+          self.horizontal_ltr
+        - Whether the user has specified that vertical text the text runs
+          top-to-bottom (default) or bottom-to-top, as represented by
+          self.vertical_ttb
+        - The x0, top, x1, and bottom attributes of prev_char and
+          curr_char
+        - The self.x_tolerance and self.y_tolerance settings. Note: In
+          this case, x/y refer to those directions for non-rotated text.
+          For vertical text, they are flipped. A more accurate terminology
+          might be "*intra*line character distance tolerance" and
+          "*inter*line character distance tolerance"
+
+        An important note: The *intra*line distance is measured from the
+        *end* of the previous character to the *beginning* of the current
+        character, while the *inter*line distance is measured from the
+        *top* of the previous character to the *top* of the next
+        character. The reasons for this are partly repository-historical,
+        and partly logical, as successive text lines' bounding boxes often
+        overlap slightly (and we don't want that overlap to be interpreted
+        as the two lines being the same line).
+
+        The upright-ness of the character determines the attributes to
+        compare, while horizontal_ltr/vertical_ttb determine the direction
+        of the comparison.
+        """
+        # Note: Due to the grouping step earlier in the process,
+        # curr_char["upright"] will always equal prev_char["upright"].
+        if direction in ("ltr", "rtl"):
+            x = x_tolerance
+            y = y_tolerance
+            ay = prev_char["top"]
+            cy = curr_char["top"]
+            if direction == "ltr":
+                ax = prev_char["x0"]
+                bx = prev_char["x1"]
+                cx = curr_char["x0"]
+            else:
+                ax = -prev_char["x1"]
+                bx = -prev_char["x0"]
+                cx = -curr_char["x1"]
+
+        else:
+            x = y_tolerance
+            y = x_tolerance
+            ay = prev_char["x0"]
+            cy = curr_char["x0"]
+            if direction == "ttb":
+                ax = prev_char["top"]
+                bx = prev_char["bottom"]
+                cx = curr_char["top"]
+            else:
+                ax = -prev_char["bottom"]
+                bx = -prev_char["top"]
+                cx = -curr_char["bottom"]
+
+        return bool(
+            # Intraline test
+            (cx < ax)
+            or (cx > bx + x)
+            # Interline test
+            or abs(cy - ay) > y
+        )
+
+    def iter_chars_to_words(
+        self,
+        ordered_chars: T_obj_iter,
+        direction: T_dir,
+    ) -> Generator[T_obj_list, None, None]:
+        current_word: T_obj_list = []
+
+        def start_next_word(
+            new_char: Optional[T_obj],
+        ) -> Generator[T_obj_list, None, None]:
+            nonlocal current_word
+
+            if current_word:
+                yield current_word
+
+            current_word = [] if new_char is None else [new_char]
+
+        xt = self.x_tolerance
+        xtr = self.x_tolerance_ratio
+        yt = self.y_tolerance
+        ytr = self.y_tolerance_ratio
+
+        for char in ordered_chars:
+            text = char["text"]
+
+            if not self.keep_blank_chars and text.isspace():
+                yield from start_next_word(None)
+
+            elif text in self.split_at_punctuation:
+                yield from start_next_word(char)
+                yield from start_next_word(None)
+
+            elif current_word and self.char_begins_new_word(
+                current_word[-1],
+                char,
+                direction,
+                x_tolerance=(xt if xtr is None else xtr * current_word[-1]["size"]),
+                y_tolerance=(yt if ytr is None else ytr * current_word[-1]["size"]),
+            ):
+                yield from start_next_word(char)
+
+            else:
+                current_word.append(char)
+
+        # Finally, after all chars processed
+        if current_word:
+            yield current_word
+
+    def iter_chars_to_lines(
+        self, chars: T_obj_iter
+    ) -> Generator[Tuple[T_obj_list, T_dir], None, None]:
+        chars = list(chars)
+        upright = chars[0]["upright"]
+        line_dir = self.line_dir if upright else self.line_dir_rotated
+        char_dir = self.get_char_dir(upright)
+
+        line_cluster_key = get_line_cluster_key(line_dir)
+        char_sort_key = get_char_sort_key(char_dir)
+
+        # Cluster by line
+        subclusters = cluster_objects(
+            chars,
+            line_cluster_key,
+            (self.y_tolerance if line_dir in ("ttb", "btt") else self.x_tolerance),
+        )
+
+        for sc in subclusters:
+            # Sort within line
+            chars_sorted = sorted(sc, key=char_sort_key)
+            yield (chars_sorted, char_dir)
+
+    def iter_extract_tuples(
+        self, chars: T_obj_iter
+    ) -> Generator[Tuple[T_obj, T_obj_list], None, None]:
+        grouping_key = itemgetter("upright", *self.extra_attrs)
+        grouped_chars = itertools.groupby(chars, grouping_key)
+
+        for keyvals, char_group in grouped_chars:
+            line_groups = (
+                [(char_group, self.char_dir)]
+                if self.use_text_flow
+                else self.iter_chars_to_lines(char_group)
+            )
+            for line_chars, direction in line_groups:
+                for word_chars in self.iter_chars_to_words(line_chars, direction):
+                    yield (self.merge_chars(word_chars), word_chars)
+
+    def extract_wordmap(self, chars: T_obj_iter) -> WordMap:
+        return WordMap(list(self.iter_extract_tuples(chars)))
+
+    def extract_words(
+        self, chars: T_obj_list, return_chars: bool = False
+    ) -> T_obj_list:
+        if return_chars:
+            return list(
+                {**word, "chars": word_chars}
+                for word, word_chars in self.iter_extract_tuples(chars)
+            )
+        else:
+            return list(word for word, word_chars in self.iter_extract_tuples(chars))
+
+
+def extract_words(
+    chars: T_obj_list, return_chars: bool = False, **kwargs: Any
+) -> T_obj_list:
+    return WordExtractor(**kwargs).extract_words(chars, return_chars)
+
+
+TEXTMAP_KWARGS = inspect.signature(WordMap.to_textmap).parameters.keys()
+WORD_EXTRACTOR_KWARGS = inspect.signature(WordExtractor).parameters.keys()
+
+
+def chars_to_textmap(chars: T_obj_list, **kwargs: Any) -> TextMap:
+    kwargs.update(
+        {
+            "presorted": True,
+            "layout_bbox": kwargs.get("layout_bbox") or objects_to_bbox(chars),
+        }
+    )
+
+    extractor = WordExtractor(
+        **{k: kwargs[k] for k in WORD_EXTRACTOR_KWARGS if k in kwargs}
+    )
+    wordmap = extractor.extract_wordmap(chars)
+    textmap = wordmap.to_textmap(
+        **{k: kwargs[k] for k in TEXTMAP_KWARGS if k in kwargs}
+    )
+    return textmap
+
+
+def extract_text(
+    chars: T_obj_list,
+    line_dir_render: Optional[T_dir] = None,
+    char_dir_render: Optional[T_dir] = None,
+    **kwargs: Any,
+) -> str:
+    chars = to_list(chars)
+    if len(chars) == 0:
+        return ""
+
+    if kwargs.get("layout"):
+        textmap_kwargs = {
+            **kwargs,
+            **{"line_dir_render": line_dir_render, "char_dir_render": char_dir_render},
+        }
+        return chars_to_textmap(chars, **textmap_kwargs).as_string
+    else:
+        extractor = WordExtractor(
+            **{k: kwargs[k] for k in WORD_EXTRACTOR_KWARGS if k in kwargs}
+        )
+        words = extractor.extract_words(chars)
+
+        line_dir_render = line_dir_render or extractor.line_dir
+        char_dir_render = char_dir_render or extractor.char_dir
+
+        line_cluster_key = get_line_cluster_key(extractor.line_dir)
+
+        x_tolerance = kwargs.get("x_tolerance", DEFAULT_X_TOLERANCE)
+        y_tolerance = kwargs.get("y_tolerance", DEFAULT_Y_TOLERANCE)
+
+        lines = cluster_objects(
+            words,
+            line_cluster_key,
+            y_tolerance if line_dir_render in ("ttb", "btt") else x_tolerance,
+        )
+
+        return TextMap(
+            [
+                (char, None)
+                for char in (
+                    "\n".join(" ".join(word["text"] for word in line) for line in lines)
+                )
+            ],
+            line_dir_render=line_dir_render,
+            char_dir_render=char_dir_render,
+        ).as_string
+
+
+def collate_line(
+    line_chars: T_obj_list,
+    tolerance: T_num = DEFAULT_X_TOLERANCE,
+) -> str:
+    coll = ""
+    last_x1 = None
+    for char in sorted(line_chars, key=itemgetter("x0")):
+        if (last_x1 is not None) and (char["x0"] > (last_x1 + tolerance)):
+            coll += " "
+        last_x1 = char["x1"]
+        coll += char["text"]
+    return coll
+
+
+def extract_text_simple(
+    chars: T_obj_list,
+    x_tolerance: T_num = DEFAULT_X_TOLERANCE,
+    y_tolerance: T_num = DEFAULT_Y_TOLERANCE,
+) -> str:
+    clustered = cluster_objects(chars, itemgetter("doctop"), y_tolerance)
+    return "\n".join(collate_line(c, x_tolerance) for c in clustered)
+
+
+def dedupe_chars(
+    chars: T_obj_list,
+    tolerance: T_num = 1,
+    extra_attrs: Optional[Tuple[str, ...]] = ("fontname", "size"),
+) -> T_obj_list:
+    """
+    Removes duplicate chars — those sharing the same text and positioning
+    (within `tolerance`) as other characters in the set. Use extra_args to
+    be more restrictive with the properties shared by the matching chars.
+    """
+    key = itemgetter(*("upright", "text"), *(extra_attrs or tuple()))
+    pos_key = itemgetter("doctop", "x0")
+
+    def yield_unique_chars(chars: T_obj_list) -> Generator[T_obj, None, None]:
+        sorted_chars = sorted(chars, key=key)
+        for grp, grp_chars in itertools.groupby(sorted_chars, key=key):
+            for y_cluster in cluster_objects(
+                list(grp_chars), itemgetter("doctop"), tolerance
+            ):
+                for x_cluster in cluster_objects(
+                    y_cluster, itemgetter("x0"), tolerance
+                ):
+                    yield sorted(x_cluster, key=pos_key)[0]
+
+    deduped = yield_unique_chars(chars)
+    return sorted(deduped, key=chars.index)