Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions evals/expectations/jmir-chatmind-rct.json
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,5 @@
],
"crops": {},
"same_visual": [],
"known_gaps": {
"caption:Table 3": "The superscript footnote marker 'a' after the caption's final period is kept as caption text ('effects.a').",
"review:Figure 1": "The caption sits above the figure; the crop takes the page-header strip above the caption, misses the flow diagram, and the quality gate rejects it."
}
"known_gaps": {}
}
57 changes: 57 additions & 0 deletions skills/deeppapernote/scripts/extract_pdf_assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@
RUNNING_HEADER_RATIO = 0.05
RUNNING_HEADER_SEARCH_RATIO = 0.1
SIDE_FIGURE_PANEL_GAP_PT = 24.0
# A figure printed under its caption starts within this distance of it.
CAPTION_FIGURE_GAP_PT = 24.0

_TEXT_TABLE_SEPARATOR_RE = re.compile(r"(?:\t+|\s{2,}|[|;])")
_TEXT_TABLE_PROSE_STARTERS = {
Expand Down Expand Up @@ -811,6 +813,59 @@ def _estimate_figure_bbox_above_caption(
return bbox


def _estimate_figure_bbox_below_caption(
page,
caption_anchor: dict,
caption_anchors: list[dict],
page_rect,
) -> tuple[float, float, float, float] | None:
"""Estimate the figure printed under its caption, as journals such as JMIR set it.

The caller tries this only when no visual sits above the caption. The figure
must start right under the caption, and it must not end at a caption of its
own, which would make it the next figure.
"""
cap_y1 = caption_anchor["bbox"][3]
col_x0, col_x1 = _caption_column(caption_anchor, page_rect)

def in_column(box: tuple[float, ...]) -> bool:
return box[0] < col_x1 and box[2] > col_x0

below = [
a["bbox"]
for a in caption_anchors
if a["bbox"] != caption_anchor["bbox"] and a["bbox"][1] >= cap_y1 and in_column(a["bbox"])
]
prose = [b for b in _find_paragraph_blocks(page) if b[1] >= cap_y1 and in_column(b)]
lower_bound = min([page_rect.y1, *(box[1] for box in below + prose)])
visuals = sorted(
(
r
for r in _page_visual_rects(page)
if r[1] >= cap_y1 - 2.0 and r[3] <= lower_bound + 2.0 and in_column(r)
),
key=lambda r: r[1],
)
if not visuals or visuals[0][1] - cap_y1 > CAPTION_FIGURE_GAP_PT:
return None
x0, top, x1, bottom = visuals[0]
for rect in visuals[1:]:
if rect[1] > bottom + SIDE_FIGURE_PANEL_GAP_PT:
break
x0, x1, bottom = min(x0, rect[0]), max(x1, rect[2]), max(bottom, rect[3])
if any(box[1] >= bottom - 2.0 and box[1] - bottom <= CAPTION_FIGURE_GAP_PT for box in below):
return None

bbox = _grow_bbox_over_figure_labels(
page, (x0, top, x1, bottom), upper_bound=cap_y1, lower_bound=lower_bound
)
bbox = _clip_to_page(bbox, page_rect, padding=2.0)
bbox = (bbox[0], max(bbox[1], cap_y1 + 1.0), bbox[2], bbox[3])
if bbox[2] - bbox[0] < MIN_FIGURE_WIDTH_PT or bbox[3] - bbox[1] < MIN_VISUAL_FIGURE_HEIGHT_PT:
return None
return bbox


def _estimate_figure_bbox_beside_caption(
page,
caption_anchor: dict,
Expand Down Expand Up @@ -1539,6 +1594,8 @@ def extract_figure_regions(
)
else:
bbox = _estimate_figure_bbox_beside_caption(page, anchor, anchors, page_rect)
if bbox is None and not _visuals_above_caption(page, anchor, prev_anchor, anchors):
bbox = _estimate_figure_bbox_below_caption(page, anchor, anchors, page_rect)
if bbox is None:
bbox = _estimate_figure_bbox_above_caption(
page, anchor, prev_anchor, page_rect, anchors
Expand Down
32 changes: 29 additions & 3 deletions skills/deeppapernote/scripts/pdf_captions.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@
CAPTION_SEPARATORS = "::.。|—–-"
CJK_RE = re.compile(r"[ -〿㐀-鿿＀-￯]")
CAPTION_SENTENCE_ENDS = (".", "?", "!", "。", "?", "!")
# PyMuPDF sets this span flag on raised text; footnote marks are short.
SUPERSCRIPT_FLAG = 1
MAX_FOOTNOTE_MARK_CHARS = 3

# Used to decide whether a continuation line still belongs to the caption text
# or has already entered the data body of a table. A row of pure tabular data
Expand Down Expand Up @@ -132,8 +135,31 @@ def dominant_font_size(blocks: list[dict]) -> float:
return sizes.most_common(1)[0][0] if sizes else 0.0


def spans_text(spans: list[dict]) -> str:
"""Join a line's spans, leaving out footnote marks raised after its last sentence.

A table caption can end in a mark such as the "a" of "effects.a" that refers to a
note below the table. It is set as a superscript and is not part of the caption.
"""
texts = [str(span.get("text", "")) for span in spans]
end = len(spans)
mark_chars = 0
while end and (
not texts[end - 1].strip()
or int(spans[end - 1].get("flags", 0)) & SUPERSCRIPT_FLAG
):
mark_chars += len(texts[end - 1].strip())
end -= 1
head = "".join(texts[:end])
if 0 < mark_chars <= MAX_FOOTNOTE_MARK_CHARS and head.rstrip().endswith(
CAPTION_SENTENCE_ENDS
):
return head
return "".join(texts)


def block_lines(block: dict) -> list[str]:
lines = ("".join(s.get("text", "") for s in line.get("spans", [])) for line in block["lines"])
lines = (spans_text(line.get("spans", [])) for line in block["lines"])
return [text for text in (line.strip() for line in lines) if text]


Expand Down Expand Up @@ -259,7 +285,7 @@ def find_caption_blocks(page) -> list[dict]:
]
for block in blocks:
lines = merge_line_fragments(block.get("lines", []))
line_texts = ["".join(s.get("text", "") for s in line.get("spans", [])) for line in lines]
line_texts = [spans_text(line.get("spans", [])) for line in lines]
for line_idx, line in enumerate(lines):
spans = line.get("spans", [])
if not spans:
Expand All @@ -285,7 +311,7 @@ def find_caption_blocks(page) -> list[dict]:
cont_spans = cont_line.get("spans", [])
if not cont_spans:
break
cont_text = "".join(s.get("text", "") for s in cont_spans).strip()
cont_text = spans_text(cont_spans).strip()
if not cont_text:
break
if CAPTION_RE.match(cont_text) and not line_is_wrapped_reference(
Expand Down
33 changes: 32 additions & 1 deletion tests/test_caption_extraction.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

import pytest
from _pdf_text import caption_label_key, extract_caption_lines, normalize_caption_label
from pdf_captions import find_caption_blocks
from pdf_captions import caption_item, find_caption_blocks
from plan_figure_table_decisions import matched_figure_asset
from plan_figures import _match_figure_asset, classify_caption_kind
from test_extract_pdf_assets_quality import EXTRACT_PDF_ASSETS_SCRIPT, write_fetch_input
Expand Down Expand Up @@ -249,6 +249,37 @@ def test_caption_continues_in_a_text_block_right_below() -> None:
assert find_caption_blocks(page)[0]["line_text"].endswith("trained via next-scale prediction.")


def caption_with_superscript(text: str, superscript: str) -> fitz.Document:
"""Set a caption line ending in a raised, smaller mark, as a footnote reference is set."""
doc = fitz.open()
page = doc.new_page(width=595, height=791)
page.insert_text((72, 400), text, fontsize=8.5, fontname="tiro")
width = fitz.get_text_length(text, fontname="tiro", fontsize=8.5)
page.insert_text((72 + width, 397), superscript, fontsize=7, fontname="tiro")
page.insert_textbox(fitz.Rect(72, 460, 540, 760), BODY_TEXT, fontsize=10)
return doc


@pytest.mark.parametrize(
("text", "superscript", "caption"),
[
(
"Table 3. Effect sizes for intention-to-treat intervention effects.",
"a",
"Effect sizes for intention-to-treat intervention effects.",
),
("Table 3. Effect sizes by arm.", "*,†", "Effect sizes by arm."),
("Figure 2. Leaf area per plant in cm", "2", "Leaf area per plant in cm2"),
],
)
def test_footnote_mark_after_the_caption_sentence_is_not_caption_text(
text: str, superscript: str, caption: str
) -> None:
doc = caption_with_superscript(text, superscript)

assert caption_item(find_caption_blocks(doc[0])[0])["caption"] == caption


def test_table_header_below_an_unfinished_caption_stays_out() -> None:
doc = fitz.open()
page = doc.new_page(width=595, height=791)
Expand Down
60 changes: 60 additions & 0 deletions tests/test_extract_pdf_assets_quality.py
Original file line number Diff line number Diff line change
Expand Up @@ -777,6 +777,66 @@ def test_side_caption_asset_passes_the_visual_quality_gate(tmp_path: Path) -> No
doc.close()


def add_bitmap(page, rect: tuple[float, float, float, float]) -> None:
pixmap = fitz.Pixmap(fitz.csRGB, fitz.IRect(0, 0, 200, 300), False)
pixmap.set_rect(pixmap.irect, (40, 120, 200))
page.insert_image(fitz.Rect(*rect), pixmap=pixmap, keep_proportion=False)


def caption_above_figure_page(doc):
"""Lay out a journal page with a running header, then a caption, then its figure."""
page = doc.new_page(width=595.0, height=842.0)
page.insert_text((42.0, 50.0), "JOURNAL OF MENTAL HEALTH", fontsize=10)
page.insert_text((400.0, 50.0), "Author et al", fontsize=10)
page.insert_text(
(42.0, 76.0),
"Figure 1. Consolidated Standards of Reporting Trials flow diagram.",
fontsize=8.5,
)
add_bitmap(page, (75.0, 85.0, 520.0, 770.0))
return page


def test_caption_above_its_figure_crops_the_figure_below(tmp_path: Path) -> None:
if fitz is None:
pytest.skip("PyMuPDF is required for figure bbox tests.")
doc = fitz.open()
try:
page = caption_above_figure_page(doc)

assets = extract_figure_regions(page, 1, tmp_path, find_caption_blocks(page))

assert [asset["label"] for asset in assets] == ["Figure 1"]
x0, y0, x1, y1 = assets[0]["bbox_pt"]
assert x0 <= 75.0 and x1 >= 520.0
assert 76.0 < y0 <= 85.0 and y1 >= 770.0
assert assets[0]["quality_signals"]["visual_quality_status"] == "usable"
finally:
doc.close()


def test_caption_with_no_visual_above_leaves_the_next_captioned_figure_alone(
tmp_path: Path,
) -> None:
if fitz is None:
pytest.skip("PyMuPDF is required for figure bbox tests.")
doc = fitz.open()
try:
page = doc.new_page(width=595.0, height=842.0)
page.insert_textbox(fitz.Rect(72.0, 60.0, 523.0, 200.0), SIDE_PROSE, fontsize=9)
page.insert_text((72.0, 230.0), "Figure 1. A diagram drawn as text.", fontsize=9)
add_bitmap(page, (150.0, 240.0, 450.0, 500.0))
page.insert_text((72.0, 515.0), "Figure 2. The figure above this caption.", fontsize=9)

assets = extract_figure_regions(page, 1, tmp_path, find_caption_blocks(page))
crops = {asset["label"]: asset["bbox_pt"] for asset in assets}

assert crops["Figure 2"][1] < 240.0 and crops["Figure 2"][3] < 515.0
assert "Figure 1" not in crops or crops["Figure 1"][3] < 240.0
finally:
doc.close()


def add_table_rows(page, top: float, name: str, rows: int = 6) -> None:
for row in range(rows):
y = top + row * 12.0
Expand Down
Loading