Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions skills/deeppapernote/scripts/pdf_captions.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
# PyMuPDF sets this span flag on raised text; footnote marks are short.
SUPERSCRIPT_FLAG = 1
MAX_FOOTNOTE_MARK_CHARS = 3
FOOTNOTE_LETTERS_RE = re.compile(r"[a-z](?:,[a-z])*")

# Used to decide whether a continuation line still belongs to the caption text
# or has already entered the data body of a table. A row of pure tabular data
Expand Down Expand Up @@ -135,12 +136,34 @@ def dominant_font_size(blocks: list[dict]) -> float:
return sizes.most_common(1)[0][0] if sizes else 0.0


def is_acronym_footnote_mark(span: dict, before: str) -> bool:
"""Tell whether a span is a footnote mark raised right after an acronym.

A table caption can mark an abbreviation, as in "GAD-7a", for a note below the
table that spells it out. Only lowercase letters ("a" or "a,b") raised directly
after a token with two or more capitals count, so a raised variable such as the
"l" of "h" + "l" or a unit power stays caption text.
"""
if not int(span.get("flags", 0)) & SUPERSCRIPT_FLAG:
return False
if not FOOTNOTE_LETTERS_RE.fullmatch(str(span.get("text", "")).strip()):
return False
token = re.search(r"[^\s(\[]+$", before)
return bool(token) and sum(char.isupper() for char in token.group()) >= 2


def spans_text(spans: list[dict]) -> str:
"""Join a line's spans, leaving out footnote marks raised after its last sentence.
"""Join a line's spans, leaving out footnote marks.

A table caption can end in a mark such as the "a" of "effects.a" that refers to a
note below the table. It is set as a superscript and is not part of the caption.
note below the table, or mark an acronym inside it, as in "GAD-7a". Both are set
as superscripts and are not part of the caption.
"""
spans = [
span
for index, span in enumerate(spans)
if not is_acronym_footnote_mark(span, "".join(str(s.get("text", "")) for s in spans[:index]))
]
texts = [str(span.get("text", "")) for span in spans]
end = len(spans)
mark_chars = 0
Expand Down
47 changes: 47 additions & 0 deletions tests/test_caption_extraction.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,53 @@ def test_footnote_mark_after_the_caption_sentence_is_not_caption_text(
assert caption_item(find_caption_blocks(doc[0])[0])["caption"] == caption


def caption_with_raised_parts(parts: list[tuple[str, bool]]) -> fitz.Document:
"""Set one caption line from (text, raised) parts; raised parts are smaller and higher."""
doc = fitz.open()
page = doc.new_page(width=595, height=791)
x = 40.0
for text, raised in parts:
size = 7 if raised else 8.5
page.insert_text((x, 397 if raised else 400), text, fontsize=size, fontname="tiro")
x += fitz.get_text_length(text, fontname="tiro", fontsize=size)
page.insert_textbox(fitz.Rect(72, 460, 540, 760), BODY_TEXT, fontsize=10)
return doc


@pytest.mark.parametrize(
("parts", "caption"),
[
(
[("Table 2. Changes in anxiety (GAD-7", False), ("a", True),
("), depression (ODSIS", False), ("b", True), (") by group.", False)],
"Changes in anxiety (GAD-7), depression (ODSIS) by group.",
),
(
[("Table 1. Baseline PHQ-9", False), ("a,b", True), (" scores.", False)],
"Baseline PHQ-9 scores.",
),
(
[("Figure 3. Hidden state h", False), ("l", True), (" at layer l.", False)],
"Hidden state hl at layer l.",
),
(
[("Figure 4. Projection W", False), ("q", True), (" of the query.", False)],
"Projection Wq of the query.",
),
(
[("Figure 5. Area of LAI", False), ("2", True), (" per plot.", False)],
"Area of LAI2 per plot.",
),
],
)
def test_footnote_mark_after_an_acronym_inside_a_caption_is_not_caption_text(
parts: list[tuple[str, bool]], caption: str
) -> None:
doc = caption_with_raised_parts(parts)

assert caption_item(find_caption_blocks(doc[0])[0])["caption"] == caption


def test_table_header_below_an_unfinished_caption_stays_out() -> None:
doc = fitz.open()
page = doc.new_page(width=595, height=791)
Expand Down
Loading