Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions app/public/cantusdata/helpers/mei_processing/mei_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
ContourType,
Neume,
Syllable,
TextLine,
)

# Mapping from pitch names to integer pitch class where C = 0
Expand Down Expand Up @@ -411,6 +412,22 @@ def parse_mei(self) -> List[Syllable]:
syllables.append(syllable_dict)
return syllables

def parse_text_lines(self) -> List[TextLine]:
"""
Parses the lines of text ('l' elements) in the MEI file, in document
order. Lines without any text are skipped.

:return: A list of text lines
"""
text_lines: List[TextLine] = []
for line in self.mei.iter(f"{self.MEINS}l"):
text = "".join(line.itertext()).strip()
if text:
text_lines.append(
{"text": text, "bounding_box": self._get_element_zone(line)}
)
return text_lines


def get_semitones_between_neume_components(
neume_component_1: NeumeComponentElementData,
Expand Down
56 changes: 56 additions & 0 deletions app/public/cantusdata/helpers/mei_processing/mei_parsing_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,3 +140,59 @@ class NgramDocument(TypedDict):
id: NotRequired[str]
type: NotRequired[Literal["omr_ngram"]]
image_uri: NotRequired[str]


class TextLine(TypedDict):
"""A type for a line of text (an MEI 'l' element), such as a line of
OCR'd text in a manuscript without syllable-aligned lyrics.

text: The text of the line, as encoded.
bounding_box: The zone of the line.
"""

text: str
bounding_box: Zone


class TextLineDocument(TypedDict):
"""
A type for documents indexed into Solr so that a manuscript's lines of
text can be searched.

location_json: A JSON string with the bounding box of the line (see
NgramDocument).
text_line_stored: The text of the line, as encoded, for display.
line_order_i: The position of the line among the lines of its file, so
results can be listed in page order.

Searchable text, normalized by search_utils.normalize_search_text with
syllables joined (fields ending "_t") or separated ("_syl_t"):
text_line_t / text_line_syl_t: the line.
text_next_t / text_next_syl_t: the following line in the file, if any.
text_window_t / text_window_syl_t: the line and the following line
together, so that a search can continue onto the next line.

If there is a following line, it is also stored for display:
next_text_line_stored: its text, as encoded.
next_location_stored: a JSON string with its bounding box.

The following are added when the document is indexed (see NgramDocument):
manuscript_id, folio, image_uri.
"""

location_json: str
text_line_stored: str
line_order_i: int
text_line_t: str
text_line_syl_t: str
text_window_t: str
text_window_syl_t: str
text_next_t: NotRequired[str]
text_next_syl_t: NotRequired[str]
next_text_line_stored: NotRequired[str]
next_location_stored: NotRequired[str]
id: str
type: Literal["mei_text_line"]
manuscript_id: NotRequired[str]
folio: NotRequired[str]
image_uri: NotRequired[str]
47 changes: 47 additions & 0 deletions app/public/cantusdata/helpers/mei_processing/mei_tokenizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,14 @@
import uuid
from typing import List, Tuple, Optional
from cantusdata.helpers.neume_helpers import NeumeName
from cantusdata.helpers.search_utils import normalize_search_text
from .mei_parser import MEIParser
from .mei_parsing_types import (
Neume,
NeumeComponent,
ContourType,
NgramDocument,
TextLineDocument,
Zone,
)
from .bounding_box_utils import combine_bounding_boxes, stringify_bounding_boxes
Expand Down Expand Up @@ -230,3 +232,48 @@ def create_ngram_documents(self) -> List[NgramDocument]:
doc["neume_names"] = "_".join(ngram_neume_names)
ngram_docs.append(doc)
return ngram_docs

def create_text_line_documents(self) -> List[TextLineDocument]:
"""
Create a document for each line of text in the MEI file, so that
text that is not aligned to syllables (e.g. OCR'd text) can be
searched. Lines with no searchable words are skipped.

Each document also carries the following line in the file, so a
search can continue onto it (see TextLineDocument). That is the next
line in the order the MEI encodes them, which is the reading order
only as far as the encoding follows it.

:return: A list of TextLineDocuments.
"""
lines = [
line
for line in self.parse_text_lines()
if normalize_search_text(line["text"])
]
text_line_docs: List[TextLineDocument] = []
for order, line in enumerate(lines):
next_line = lines[order + 1] if order + 1 < len(lines) else None
# The line break is kept so that a hyphen or "¬" ending the line
# joins the word it splits, as it does within a line.
window_text = line["text"] + ("\n" + next_line["text"] if next_line else "")
doc: TextLineDocument = {
"location_json": stringify_bounding_boxes([line["bounding_box"]]),
"text_line_stored": line["text"],
"line_order_i": order,
"text_line_t": normalize_search_text(line["text"]),
"text_line_syl_t": normalize_search_text(line["text"], False),
"text_window_t": normalize_search_text(window_text),
"text_window_syl_t": normalize_search_text(window_text, False),
"id": str(uuid.uuid4()),
"type": "mei_text_line",
}
if next_line:
doc["text_next_t"] = normalize_search_text(next_line["text"])
doc["text_next_syl_t"] = normalize_search_text(next_line["text"], False)
doc["next_text_line_stored"] = next_line["text"]
doc["next_location_stored"] = stringify_bounding_boxes(
[next_line["bounding_box"]]
)
text_line_docs.append(doc)
return text_line_docs
87 changes: 87 additions & 0 deletions app/public/cantusdata/helpers/search_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,51 @@
queries.
"""

import re
import unicodedata
from typing import Optional

from cantusdata.helpers.neume_helpers import NEUME_NAMES

# Volpiano note characters and the pitch names they represent. Uppercase
# letters are liquescent forms of the same pitch; "(" and ")" are the
# liquescent forms of "8" and "9".
VOLPIANO_PITCH_NAMES: dict[str, str] = {
"8": "f",
"9": "g",
"a": "a",
"b": "b",
"c": "c",
"d": "d",
"e": "e",
"f": "f",
"g": "g",
"h": "a",
"j": "b",
"k": "c",
"l": "d",
"m": "e",
"n": "f",
"o": "g",
"p": "a",
"q": "b",
"r": "c",
"s": "d",
}
VOLPIANO_PITCH_NAMES.update(
{
char.upper(): pname
for char, pname in VOLPIANO_PITCH_NAMES.items()
if char.isalpha()
}
)
VOLPIANO_PITCH_NAMES.update({"(": "f", ")": "g"})

# Volpiano characters that are not notes: clefs, spacing, barlines, and
# flat/natural signs. OMR n-grams store pitch names without accidentals,
# so none of these can narrow a search of the MEI.
VOLPIANO_IGNORED_CHARS = set("12-34567 iwxyzIWXYZ")


def validate_intervals_query_word(word: str) -> bool:
"""
Expand Down Expand Up @@ -73,6 +116,28 @@ def get_transpositions(sequence: list[str]) -> list[list[str]]:
return transpositions


def volpiano_to_pitch_names(volpiano: str) -> Optional[list[str]]:
"""
Convert a Volpiano string into the sequence of pitch names it contains,
so that it can be searched against OMR n-grams.

Octave is dropped (OMR n-grams do not record it), and clefs, spacing,
barlines and accidentals are skipped (see VOLPIANO_IGNORED_CHARS).

e.g. volpiano_to_pitch_names("1---f--gh--3") returns ["f", "g", "a"]

:return: the pitch names, or None if the string contains a character
that is not Volpiano or contains no notes
"""
pitch_names: list[str] = []
for char in volpiano:
if char in VOLPIANO_PITCH_NAMES:
pitch_names.append(VOLPIANO_PITCH_NAMES[char])
elif char not in VOLPIANO_IGNORED_CHARS:
return None
return pitch_names or None


def translate_interval_query_direction(query_terms: list[str]) -> list[str]:
"""
Translate the terms of an interval query (alphanumeric strings; e.g. "u3", "d2", "r")
Expand Down Expand Up @@ -101,3 +166,25 @@ def translate_interval_query_direction(query_terms: list[str]) -> list[str]:
magnitude = term[1:]
solr_query.append(f"{direction}{magnitude}")
return solr_query


def normalize_search_text(text: str, join_syllables: bool = True) -> str:
"""
Reduce text to space-separated lowercase ASCII words, so that OCR'd chant
text and a user's query can be compared.

With `join_syllables`, hyphens and line-continuation marks ("¬") join the
syllables around them, even across a line break, so hyphenated chant text
("Mi- se-rére") matches the whole word ("miserere"). Without it, they
separate syllables, so text is matched as printed. Accents are dropped and
"æ"/"œ" are spelled out, since editions and users differ in whether they
write them.

e.g. normalize_search_text("Ps. 50. Mi- se-rére mé-i,") returns
"ps 50 miserere mei", or "ps 50 mi se rere me i" without join_syllables
"""
text = unicodedata.normalize("NFKD", text.lower())
text = "".join(char for char in text if not unicodedata.combining(char))
text = text.replace("æ", "ae").replace("œ", "oe")
text = re.sub(r"[-¬]\s*", "" if join_syllables else " ", text)
return " ".join(re.findall(r"[a-z0-9]+", text))
26 changes: 18 additions & 8 deletions app/public/cantusdata/management/commands/index_manuscript_mei.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import Any, Dict, Optional
from typing import Any, Dict, List, Optional, Union
from os import path, listdir
import re

Expand All @@ -7,6 +7,10 @@
from solr.core import SolrConnection # type: ignore[import-untyped]

from cantusdata.helpers.mei_processing.mei_tokenizer import MEITokenizer
from cantusdata.helpers.mei_processing.mei_parsing_types import (
NgramDocument,
TextLineDocument,
)
from cantusdata.models.folio import Folio

MEI4_DIR = path.join("/code", "production-mei-files")
Expand Down Expand Up @@ -173,14 +177,17 @@ def handle(self, *args: Any, **options: Any) -> None:
min_ngram=options["min_ngram"],
max_ngram=options["max_ngram"],
)
ngram_docs = tokenizer.create_ngram_documents()
for doc in ngram_docs:
docs: List[Union[NgramDocument, TextLineDocument]] = [
*tokenizer.create_ngram_documents(),
*tokenizer.create_text_line_documents(),
]
for doc in docs:
doc["manuscript_id"] = manuscript_id
doc["folio"] = folio_number
# `or ""` because Folio.image_uri is nullable: a None here would
# be dropped by Solr, leaving the document without the field.
doc["image_uri"] = folio_map.get(folio_number) or ""
solr_conn.add_many(ngram_docs)
solr_conn.add_many(docs)
solr_conn.commit()
return None

Expand All @@ -191,13 +198,16 @@ def flush_manuscript_ngrams_from_index(
folio: Optional[str] = None,
) -> None:
"""
Deletes n-gram documents for a given manuscript from the Solr index.
Deletes the documents built from a manuscript's MEI (n-grams and text
lines) from the Solr index.

Deletes every n-gram of the manuscript unless `folio` is given, in which
case only that folio's n-grams are deleted. The folio number is matched
Deletes every such document of the manuscript unless `folio` is given,
in which case only that folio's are deleted. The folio number is matched
exactly: `folio` is a Solr string field.
"""
query = f"type:omr_ngram AND manuscript_id:{manuscript_id}"
query = (
f"(type:omr_ngram OR type:mei_text_line) AND manuscript_id:{manuscript_id}"
)
if folio is not None:
query += f' AND folio:"{escape_solr_phrase(folio)}"'
solr_conn.delete_query(query)
Expand Down
Loading
Loading