diff --git a/skills/deeppapernote/scripts/pdf_captions.py b/skills/deeppapernote/scripts/pdf_captions.py index 98d0699..048f9b8 100644 --- a/skills/deeppapernote/scripts/pdf_captions.py +++ b/skills/deeppapernote/scripts/pdf_captions.py @@ -33,6 +33,7 @@ # PyMuPDF sets this span flag on raised text; footnote marks are short. SUPERSCRIPT_FLAG = 1 MAX_FOOTNOTE_MARK_CHARS = 3 +FOOTNOTE_LETTERS_RE = re.compile(r"[a-z](?:,[a-z])*") # Used to decide whether a continuation line still belongs to the caption text # or has already entered the data body of a table. A row of pure tabular data @@ -135,12 +136,34 @@ def dominant_font_size(blocks: list[dict]) -> float: return sizes.most_common(1)[0][0] if sizes else 0.0 +def is_acronym_footnote_mark(span: dict, before: str) -> bool: + """Tell whether a span is a footnote mark raised right after an acronym. + + A table caption can mark an abbreviation, as in "GAD-7a", for a note below the + table that spells it out. Only lowercase letters ("a" or "a,b") raised directly + after a token with two or more capitals count, so a raised variable such as the + "l" of "h" + "l" or a unit power stays caption text. + """ + if not int(span.get("flags", 0)) & SUPERSCRIPT_FLAG: + return False + if not FOOTNOTE_LETTERS_RE.fullmatch(str(span.get("text", "")).strip()): + return False + token = re.search(r"[^\s(\[]+$", before) + return bool(token) and sum(char.isupper() for char in token.group()) >= 2 + + def spans_text(spans: list[dict]) -> str: - """Join a line's spans, leaving out footnote marks raised after its last sentence. + """Join a line's spans, leaving out footnote marks. A table caption can end in a mark such as the "a" of "effects.a" that refers to a - note below the table. It is set as a superscript and is not part of the caption. + note below the table, or mark an acronym inside it, as in "GAD-7a". Both are set + as superscripts and are not part of the caption. """ + spans = [ + span + for index, span in enumerate(spans) + if not is_acronym_footnote_mark(span, "".join(str(s.get("text", "")) for s in spans[:index])) + ] texts = [str(span.get("text", "")) for span in spans] end = len(spans) mark_chars = 0 diff --git a/tests/test_caption_extraction.py b/tests/test_caption_extraction.py index 0237615..fb59253 100644 --- a/tests/test_caption_extraction.py +++ b/tests/test_caption_extraction.py @@ -280,6 +280,53 @@ def test_footnote_mark_after_the_caption_sentence_is_not_caption_text( assert caption_item(find_caption_blocks(doc[0])[0])["caption"] == caption +def caption_with_raised_parts(parts: list[tuple[str, bool]]) -> fitz.Document: + """Set one caption line from (text, raised) parts; raised parts are smaller and higher.""" + doc = fitz.open() + page = doc.new_page(width=595, height=791) + x = 40.0 + for text, raised in parts: + size = 7 if raised else 8.5 + page.insert_text((x, 397 if raised else 400), text, fontsize=size, fontname="tiro") + x += fitz.get_text_length(text, fontname="tiro", fontsize=size) + page.insert_textbox(fitz.Rect(72, 460, 540, 760), BODY_TEXT, fontsize=10) + return doc + + +@pytest.mark.parametrize( + ("parts", "caption"), + [ + ( + [("Table 2. Changes in anxiety (GAD-7", False), ("a", True), + ("), depression (ODSIS", False), ("b", True), (") by group.", False)], + "Changes in anxiety (GAD-7), depression (ODSIS) by group.", + ), + ( + [("Table 1. Baseline PHQ-9", False), ("a,b", True), (" scores.", False)], + "Baseline PHQ-9 scores.", + ), + ( + [("Figure 3. Hidden state h", False), ("l", True), (" at layer l.", False)], + "Hidden state hl at layer l.", + ), + ( + [("Figure 4. Projection W", False), ("q", True), (" of the query.", False)], + "Projection Wq of the query.", + ), + ( + [("Figure 5. Area of LAI", False), ("2", True), (" per plot.", False)], + "Area of LAI2 per plot.", + ), + ], +) +def test_footnote_mark_after_an_acronym_inside_a_caption_is_not_caption_text( + parts: list[tuple[str, bool]], caption: str +) -> None: + doc = caption_with_raised_parts(parts) + + assert caption_item(find_caption_blocks(doc[0])[0])["caption"] == caption + + def test_table_header_below_an_unfinished_caption_stays_out() -> None: doc = fitz.open() page = doc.new_page(width=595, height=791)