diff --git a/app/public/cantusdata/helpers/mei_processing/mei_parser.py b/app/public/cantusdata/helpers/mei_processing/mei_parser.py
index b411bc257..b4b3472b4 100644
--- a/app/public/cantusdata/helpers/mei_processing/mei_parser.py
+++ b/app/public/cantusdata/helpers/mei_processing/mei_parser.py
@@ -26,6 +26,7 @@
ContourType,
Neume,
Syllable,
+ TextLine,
)
# Mapping from pitch names to integer pitch class where C = 0
@@ -411,6 +412,22 @@ def parse_mei(self) -> List[Syllable]:
syllables.append(syllable_dict)
return syllables
+ def parse_text_lines(self) -> List[TextLine]:
+ """
+ Parses the lines of text ('l' elements) in the MEI file, in document
+ order. Lines without any text are skipped.
+
+ :return: A list of text lines
+ """
+ text_lines: List[TextLine] = []
+ for line in self.mei.iter(f"{self.MEINS}l"):
+ text = "".join(line.itertext()).strip()
+ if text:
+ text_lines.append(
+ {"text": text, "bounding_box": self._get_element_zone(line)}
+ )
+ return text_lines
+
def get_semitones_between_neume_components(
neume_component_1: NeumeComponentElementData,
diff --git a/app/public/cantusdata/helpers/mei_processing/mei_parsing_types.py b/app/public/cantusdata/helpers/mei_processing/mei_parsing_types.py
index 50684564a..0202ecf1b 100644
--- a/app/public/cantusdata/helpers/mei_processing/mei_parsing_types.py
+++ b/app/public/cantusdata/helpers/mei_processing/mei_parsing_types.py
@@ -140,3 +140,59 @@ class NgramDocument(TypedDict):
id: NotRequired[str]
type: NotRequired[Literal["omr_ngram"]]
image_uri: NotRequired[str]
+
+
+class TextLine(TypedDict):
+ """A type for a line of text (an MEI 'l' element), such as a line of
+ OCR'd text in a manuscript without syllable-aligned lyrics.
+
+ text: The text of the line, as encoded.
+ bounding_box: The zone of the line.
+ """
+
+ text: str
+ bounding_box: Zone
+
+
+class TextLineDocument(TypedDict):
+ """
+ A type for documents indexed into Solr so that a manuscript's lines of
+ text can be searched.
+
+ location_json: A JSON string with the bounding box of the line (see
+ NgramDocument).
+ text_line_stored: The text of the line, as encoded, for display.
+ line_order_i: The position of the line among the lines of its file, so
+ results can be listed in page order.
+
+ Searchable text, normalized by search_utils.normalize_search_text with
+ syllables joined (fields ending "_t") or separated ("_syl_t"):
+ text_line_t / text_line_syl_t: the line.
+ text_next_t / text_next_syl_t: the following line in the file, if any.
+ text_window_t / text_window_syl_t: the line and the following line
+ together, so that a search can continue onto the next line.
+
+ If there is a following line, it is also stored for display:
+ next_text_line_stored: its text, as encoded.
+ next_location_stored: a JSON string with its bounding box.
+
+ The following are added when the document is indexed (see NgramDocument):
+ manuscript_id, folio, image_uri.
+ """
+
+ location_json: str
+ text_line_stored: str
+ line_order_i: int
+ text_line_t: str
+ text_line_syl_t: str
+ text_window_t: str
+ text_window_syl_t: str
+ text_next_t: NotRequired[str]
+ text_next_syl_t: NotRequired[str]
+ next_text_line_stored: NotRequired[str]
+ next_location_stored: NotRequired[str]
+ id: str
+ type: Literal["mei_text_line"]
+ manuscript_id: NotRequired[str]
+ folio: NotRequired[str]
+ image_uri: NotRequired[str]
diff --git a/app/public/cantusdata/helpers/mei_processing/mei_tokenizer.py b/app/public/cantusdata/helpers/mei_processing/mei_tokenizer.py
index ac87da729..8628366b5 100644
--- a/app/public/cantusdata/helpers/mei_processing/mei_tokenizer.py
+++ b/app/public/cantusdata/helpers/mei_processing/mei_tokenizer.py
@@ -7,12 +7,14 @@
import uuid
from typing import List, Tuple, Optional
from cantusdata.helpers.neume_helpers import NeumeName
+from cantusdata.helpers.search_utils import normalize_search_text
from .mei_parser import MEIParser
from .mei_parsing_types import (
Neume,
NeumeComponent,
ContourType,
NgramDocument,
+ TextLineDocument,
Zone,
)
from .bounding_box_utils import combine_bounding_boxes, stringify_bounding_boxes
@@ -230,3 +232,48 @@ def create_ngram_documents(self) -> List[NgramDocument]:
doc["neume_names"] = "_".join(ngram_neume_names)
ngram_docs.append(doc)
return ngram_docs
+
+ def create_text_line_documents(self) -> List[TextLineDocument]:
+ """
+ Create a document for each line of text in the MEI file, so that
+ text that is not aligned to syllables (e.g. OCR'd text) can be
+ searched. Lines with no searchable words are skipped.
+
+ Each document also carries the following line in the file, so a
+ search can continue onto it (see TextLineDocument). That is the next
+ line in the order the MEI encodes them, which is the reading order
+ only as far as the encoding follows it.
+
+ :return: A list of TextLineDocuments.
+ """
+ lines = [
+ line
+ for line in self.parse_text_lines()
+ if normalize_search_text(line["text"])
+ ]
+ text_line_docs: List[TextLineDocument] = []
+ for order, line in enumerate(lines):
+ next_line = lines[order + 1] if order + 1 < len(lines) else None
+ # The line break is kept so that a hyphen or "¬" ending the line
+ # joins the word it splits, as it does within a line.
+ window_text = line["text"] + ("\n" + next_line["text"] if next_line else "")
+ doc: TextLineDocument = {
+ "location_json": stringify_bounding_boxes([line["bounding_box"]]),
+ "text_line_stored": line["text"],
+ "line_order_i": order,
+ "text_line_t": normalize_search_text(line["text"]),
+ "text_line_syl_t": normalize_search_text(line["text"], False),
+ "text_window_t": normalize_search_text(window_text),
+ "text_window_syl_t": normalize_search_text(window_text, False),
+ "id": str(uuid.uuid4()),
+ "type": "mei_text_line",
+ }
+ if next_line:
+ doc["text_next_t"] = normalize_search_text(next_line["text"])
+ doc["text_next_syl_t"] = normalize_search_text(next_line["text"], False)
+ doc["next_text_line_stored"] = next_line["text"]
+ doc["next_location_stored"] = stringify_bounding_boxes(
+ [next_line["bounding_box"]]
+ )
+ text_line_docs.append(doc)
+ return text_line_docs
diff --git a/app/public/cantusdata/helpers/search_utils.py b/app/public/cantusdata/helpers/search_utils.py
index 6c74897bf..03274e278 100644
--- a/app/public/cantusdata/helpers/search_utils.py
+++ b/app/public/cantusdata/helpers/search_utils.py
@@ -3,8 +3,51 @@
queries.
"""
+import re
+import unicodedata
+from typing import Optional
+
from cantusdata.helpers.neume_helpers import NEUME_NAMES
+# Volpiano note characters and the pitch names they represent. Uppercase
+# letters are liquescent forms of the same pitch; "(" and ")" are the
+# liquescent forms of "8" and "9".
+VOLPIANO_PITCH_NAMES: dict[str, str] = {
+ "8": "f",
+ "9": "g",
+ "a": "a",
+ "b": "b",
+ "c": "c",
+ "d": "d",
+ "e": "e",
+ "f": "f",
+ "g": "g",
+ "h": "a",
+ "j": "b",
+ "k": "c",
+ "l": "d",
+ "m": "e",
+ "n": "f",
+ "o": "g",
+ "p": "a",
+ "q": "b",
+ "r": "c",
+ "s": "d",
+}
+VOLPIANO_PITCH_NAMES.update(
+ {
+ char.upper(): pname
+ for char, pname in VOLPIANO_PITCH_NAMES.items()
+ if char.isalpha()
+ }
+)
+VOLPIANO_PITCH_NAMES.update({"(": "f", ")": "g"})
+
+# Volpiano characters that are not notes: clefs, spacing, barlines, and
+# flat/natural signs. OMR n-grams store pitch names without accidentals,
+# so none of these can narrow a search of the MEI.
+VOLPIANO_IGNORED_CHARS = set("12-34567 iwxyzIWXYZ")
+
def validate_intervals_query_word(word: str) -> bool:
"""
@@ -73,6 +116,28 @@ def get_transpositions(sequence: list[str]) -> list[list[str]]:
return transpositions
+def volpiano_to_pitch_names(volpiano: str) -> Optional[list[str]]:
+ """
+ Convert a Volpiano string into the sequence of pitch names it contains,
+ so that it can be searched against OMR n-grams.
+
+ Octave is dropped (OMR n-grams do not record it), and clefs, spacing,
+ barlines and accidentals are skipped (see VOLPIANO_IGNORED_CHARS).
+
+ e.g. volpiano_to_pitch_names("1---f--gh--3") returns ["f", "g", "a"]
+
+ :return: the pitch names, or None if the string contains a character
+ that is not Volpiano or contains no notes
+ """
+ pitch_names: list[str] = []
+ for char in volpiano:
+ if char in VOLPIANO_PITCH_NAMES:
+ pitch_names.append(VOLPIANO_PITCH_NAMES[char])
+ elif char not in VOLPIANO_IGNORED_CHARS:
+ return None
+ return pitch_names or None
+
+
def translate_interval_query_direction(query_terms: list[str]) -> list[str]:
"""
Translate the terms of an interval query (alphanumeric strings; e.g. "u3", "d2", "r")
@@ -101,3 +166,25 @@ def translate_interval_query_direction(query_terms: list[str]) -> list[str]:
magnitude = term[1:]
solr_query.append(f"{direction}{magnitude}")
return solr_query
+
+
+def normalize_search_text(text: str, join_syllables: bool = True) -> str:
+ """
+ Reduce text to space-separated lowercase ASCII words, so that OCR'd chant
+ text and a user's query can be compared.
+
+ With `join_syllables`, hyphens and line-continuation marks ("¬") join the
+ syllables around them, even across a line break, so hyphenated chant text
+ ("Mi- se-rére") matches the whole word ("miserere"). Without it, they
+ separate syllables, so text is matched as printed. Accents are dropped and
+ "æ"/"œ" are spelled out, since editions and users differ in whether they
+ write them.
+
+ e.g. normalize_search_text("Ps. 50. Mi- se-rére mé-i,") returns
+ "ps 50 miserere mei", or "ps 50 mi se rere me i" without join_syllables
+ """
+ text = unicodedata.normalize("NFKD", text.lower())
+ text = "".join(char for char in text if not unicodedata.combining(char))
+ text = text.replace("æ", "ae").replace("œ", "oe")
+ text = re.sub(r"[-¬]\s*", "" if join_syllables else " ", text)
+ return " ".join(re.findall(r"[a-z0-9]+", text))
diff --git a/app/public/cantusdata/management/commands/index_manuscript_mei.py b/app/public/cantusdata/management/commands/index_manuscript_mei.py
index 1baa60872..78bd93771 100644
--- a/app/public/cantusdata/management/commands/index_manuscript_mei.py
+++ b/app/public/cantusdata/management/commands/index_manuscript_mei.py
@@ -1,4 +1,4 @@
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional, Union
from os import path, listdir
import re
@@ -7,6 +7,10 @@
from solr.core import SolrConnection # type: ignore[import-untyped]
from cantusdata.helpers.mei_processing.mei_tokenizer import MEITokenizer
+from cantusdata.helpers.mei_processing.mei_parsing_types import (
+ NgramDocument,
+ TextLineDocument,
+)
from cantusdata.models.folio import Folio
MEI4_DIR = path.join("/code", "production-mei-files")
@@ -173,14 +177,17 @@ def handle(self, *args: Any, **options: Any) -> None:
min_ngram=options["min_ngram"],
max_ngram=options["max_ngram"],
)
- ngram_docs = tokenizer.create_ngram_documents()
- for doc in ngram_docs:
+ docs: List[Union[NgramDocument, TextLineDocument]] = [
+ *tokenizer.create_ngram_documents(),
+ *tokenizer.create_text_line_documents(),
+ ]
+ for doc in docs:
doc["manuscript_id"] = manuscript_id
doc["folio"] = folio_number
# `or ""` because Folio.image_uri is nullable: a None here would
# be dropped by Solr, leaving the document without the field.
doc["image_uri"] = folio_map.get(folio_number) or ""
- solr_conn.add_many(ngram_docs)
+ solr_conn.add_many(docs)
solr_conn.commit()
return None
@@ -191,13 +198,16 @@ def flush_manuscript_ngrams_from_index(
folio: Optional[str] = None,
) -> None:
"""
- Deletes n-gram documents for a given manuscript from the Solr index.
+ Deletes the documents built from a manuscript's MEI (n-grams and text
+ lines) from the Solr index.
- Deletes every n-gram of the manuscript unless `folio` is given, in which
- case only that folio's n-grams are deleted. The folio number is matched
+ Deletes every such document of the manuscript unless `folio` is given,
+ in which case only that folio's are deleted. The folio number is matched
exactly: `folio` is a Solr string field.
"""
- query = f"type:omr_ngram AND manuscript_id:{manuscript_id}"
+ query = (
+ f"(type:omr_ngram OR type:mei_text_line) AND manuscript_id:{manuscript_id}"
+ )
if folio is not None:
query += f' AND folio:"{escape_solr_phrase(folio)}"'
solr_conn.delete_query(query)
diff --git a/app/public/cantusdata/test/core/helpers/mei_processing/test_mei_files/991961/LU-1961_0123.mei b/app/public/cantusdata/test/core/helpers/mei_processing/test_mei_files/991961/LU-1961_0123.mei
new file mode 100644
index 000000000..d1e84f4cc
--- /dev/null
+++ b/app/public/cantusdata/test/core/helpers/mei_processing/test_mei_files/991961/LU-1961_0123.mei
@@ -0,0 +1,236 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ The Ordinary Chantsof the MassThe Asperges.OUTSIDE PASCHAL TIME. ANT. 7.XIII. c.-SPERGES me," Dómi-ne, hyssópo, et mundá¬bor: lavá- bis me,et super nívem de¬ albá-bor.Ps. 50. Mi- se-rére mé-i, Dé-us, se cúndum mágnam mi¬se-ricórdi-am tú- am. Gló-ri-a Pátri, et Fi-li-o,etSpi-rí-tu-i Sáncto. "“Sic-ut é-rat in princí-pi-o, et nunc,et semper, et in saécu-la saecu-ló-rum. A-men.Repeat the Ant. Aspérges me.¡On the first Sunday of the Passion the Gloria Pátri. is not said, but theAntiphon Aspérges me. is repeated immediately after the Psalm.On the second Sunday of the Passion, or Palm Sunday, the Aspérges is omitted.
+
\ No newline at end of file
diff --git a/app/public/cantusdata/test/core/helpers/mei_processing/test_mei_text_lines.py b/app/public/cantusdata/test/core/helpers/mei_processing/test_mei_text_lines.py
new file mode 100644
index 000000000..1149b9212
--- /dev/null
+++ b/app/public/cantusdata/test/core/helpers/mei_processing/test_mei_text_lines.py
@@ -0,0 +1,67 @@
+import json
+from os import path
+from unittest import TestCase
+
+from django.conf import settings
+
+from cantusdata.helpers.mei_processing.mei_tokenizer import MEITokenizer
+
+# A Liber Usualis page: its text is OCR'd lines ('l' elements), not syllables.
+LU_PAGE_PATH = path.join(settings.TEST_MEI_FILES_PATH, "991961", "LU-1961_0123.mei")
+
+
+class MEITextLinesTestCase(TestCase):
+ def setUp(self) -> None:
+ self.tokenizer = MEITokenizer(LU_PAGE_PATH, min_ngram=1, max_ngram=5)
+
+ def test_parse_text_lines(self) -> None:
+ lines = self.tokenizer.parse_text_lines()
+ self.assertEqual(len(lines), 17)
+ self.assertEqual(lines[0]["text"], "The Ordinary Chants")
+ self.assertEqual(lines[0]["bounding_box"]["coordinates"], (214, 182, 1294, 306))
+ # Accents in these files are decomposed (e + combining accent), so
+ # compare the start of the line, which has none.
+ self.assertTrue(
+ lines[-1]["text"].startswith("On the second Sunday of the Passion")
+ )
+
+ def test_create_text_line_documents(self) -> None:
+ docs = self.tokenizer.create_text_line_documents()
+ self.assertEqual(len(docs), 17)
+ self.assertEqual([doc["line_order_i"] for doc in docs], list(range(17)))
+ self.assertTrue(all(doc["type"] == "mei_text_line" for doc in docs))
+ self.assertEqual(
+ json.loads(docs[0]["location_json"]),
+ [{"ulx": 214, "uly": 182, "width": 1080, "height": 124}],
+ )
+ psalm_line = next(doc for doc in docs if "Ps. 50" in doc["text_line_stored"])
+ self.assertIn("miserere mei deus", psalm_line["text_line_t"])
+ self.assertIn("mi se rere me i de us", psalm_line["text_line_syl_t"])
+
+ def test_text_line_documents_carry_the_next_line(self) -> None:
+ docs = self.tokenizer.create_text_line_documents()
+ psalm_index = next(
+ i for i, doc in enumerate(docs) if "Ps. 50" in doc["text_line_stored"]
+ )
+ psalm_line, next_line = docs[psalm_index], docs[psalm_index + 1]
+ self.assertEqual(
+ psalm_line["next_text_line_stored"], next_line["text_line_stored"]
+ )
+ self.assertEqual(psalm_line["next_location_stored"], next_line["location_json"])
+ self.assertEqual(psalm_line["text_next_t"], next_line["text_line_t"])
+ # "mi¬" ends the line and "se-ricórdi-am" starts the next.
+ self.assertIn("misericordiam", psalm_line["text_window_t"])
+ self.assertIn("mi se ricordi am", psalm_line["text_window_syl_t"])
+ with self.subTest("The last line has no next line"):
+ self.assertNotIn("next_text_line_stored", docs[-1])
+ self.assertEqual(docs[-1]["text_window_t"], docs[-1]["text_line_t"])
+
+ def test_files_without_text_lines(self) -> None:
+ tokenizer = MEITokenizer(
+ path.join(
+ settings.TEST_MEI_FILES_PATH, "123723", "cdn-hsmu-m2149l4_001r.mei"
+ ),
+ min_ngram=1,
+ max_ngram=5,
+ )
+ self.assertEqual(tokenizer.create_text_line_documents(), [])
diff --git a/app/public/cantusdata/test/core/helpers/test_search_utils.py b/app/public/cantusdata/test/core/helpers/test_search_utils.py
index 11b0718a0..eb2101a7a 100644
--- a/app/public/cantusdata/test/core/helpers/test_search_utils.py
+++ b/app/public/cantusdata/test/core/helpers/test_search_utils.py
@@ -4,6 +4,8 @@
validate_query,
get_transpositions,
translate_interval_query_direction,
+ volpiano_to_pitch_names,
+ normalize_search_text,
)
@@ -67,3 +69,52 @@ def test_translate_interval_query_direction(self) -> None:
expected_translated_query = ["2", "-3", "1", "-14", "12"]
translated_query = translate_interval_query_direction(query_elems)
self.assertEqual(translated_query, expected_translated_query)
+
+ def test_volpiano_to_pitch_names(self) -> None:
+ with self.subTest("Clef, spacing and barlines are skipped"):
+ self.assertEqual(
+ volpiano_to_pitch_names("1---f--gh--3---k"), ["f", "g", "a", "c"]
+ )
+ with self.subTest("Octaves map to the same pitch names"):
+ self.assertEqual(
+ volpiano_to_pitch_names("9abcdefghjklmnopqrs"),
+ list("gabcdefgabcdefgabcd"),
+ )
+ self.assertEqual(volpiano_to_pitch_names("8"), ["f"])
+ with self.subTest("Liquescents map to their plain pitch"):
+ self.assertEqual(volpiano_to_pitch_names("fG()H"), list("fgfga"))
+ with self.subTest("Accidentals are skipped"):
+ self.assertEqual(volpiano_to_pitch_names("1-ij-Ij"), ["b", "b"])
+ with self.subTest("Invalid characters"):
+ self.assertIsNone(volpiano_to_pitch_names("1-f-t"))
+ self.assertIsNone(volpiano_to_pitch_names("f g?"))
+ with self.subTest("No notes"):
+ self.assertIsNone(volpiano_to_pitch_names("1---3"))
+ self.assertIsNone(volpiano_to_pitch_names(""))
+
+ def test_normalize_search_text(self) -> None:
+ with self.subTest("Hyphenated syllables join into words"):
+ self.assertEqual(
+ normalize_search_text("Ps. 50. Mi- se-rére mé-i, Dé-us"),
+ "ps 50 miserere mei deus",
+ )
+ with self.subTest("Line-continuation marks join syllables"):
+ self.assertEqual(
+ normalize_search_text("non e-rubé¬ scam: exspé¬ ctant"),
+ "non erubescam exspectant",
+ )
+ with self.subTest("Accents and ligatures are spelled out"):
+ self.assertEqual(
+ normalize_search_text("saécu-la Cǽli cœli Œ"), "saecula caeli coeli oe"
+ )
+ with self.subTest("Without joining, hyphens separate syllables"):
+ self.assertEqual(
+ normalize_search_text("Mi- se-rére mi¬\nse-ricórdi-am", False),
+ "mi se rere mi se ricordi am",
+ )
+ with self.subTest("Joining works across a line break"):
+ self.assertEqual(
+ normalize_search_text("mi¬\nse-ricórdi-am"), "misericordiam"
+ )
+ with self.subTest("Punctuation-only text is empty"):
+ self.assertEqual(normalize_search_text(" ¬ — : "), "")
diff --git a/app/public/cantusdata/test/core/views/test_search_notation_view.py b/app/public/cantusdata/test/core/views/test_search_notation_view.py
index 958275e8d..ae57a6d45 100644
--- a/app/public/cantusdata/test/core/views/test_search_notation_view.py
+++ b/app/public/cantusdata/test/core/views/test_search_notation_view.py
@@ -83,6 +83,16 @@ def test_create_query_string(self) -> None:
)
expected_query_string = "intervals:2_-3_1_12"
self.assertEqual(query_string, expected_query_string)
+ # A volpiano query is converted to pitch names, keeping case so that
+ # liquescents (uppercase) are recognized.
+ with self.subTest("Test volpiano query"):
+ query_string = self.search_notation_view.create_query_string(
+ "1---f--G--h---3", "volpiano"
+ )
+ self.assertEqual(query_string, "pitch_names:f_g_a")
+ with self.subTest("Test invalid volpiano query"):
+ with self.assertRaises(NotationSearchException):
+ self.search_notation_view.create_query_string("1---3", "volpiano")
def test_do_query(self) -> None:
with self.subTest("Test fields returned"):
@@ -165,6 +175,15 @@ def test_get(self) -> None:
response_data = response.json()
self.assertIn("results", response_data)
self.assertIn("numFound", response_data)
+ with self.subTest("Test volpiano finds the same results as pitch"):
+ pitch_response = self.client.get(
+ url, {"q": "f g a", "type": "pitch_names", "manuscript_id": 123723}
+ ).json()
+ volpiano_response = self.client.get(
+ url, {"q": "1---f--g--h", "type": "volpiano", "manuscript_id": 123723}
+ ).json()
+ self.assertGreater(pitch_response["numFound"], 0)
+ self.assertEqual(volpiano_response, pitch_response)
def test_get_result_whose_document_has_no_image_uri(self) -> None:
"""
diff --git a/app/public/cantusdata/test/core/views/test_search_text_view.py b/app/public/cantusdata/test/core/views/test_search_text_view.py
new file mode 100644
index 000000000..694d5aaa4
--- /dev/null
+++ b/app/public/cantusdata/test/core/views/test_search_text_view.py
@@ -0,0 +1,205 @@
+import shutil
+import tempfile
+from os import path
+
+from rest_framework.test import APITestCase
+from django.core.management import call_command
+from django.urls import reverse
+from django.conf import settings
+
+from cantusdata.views.search_text import SearchMEITextView, TextSearchException
+from cantusdata.models import Manuscript, Folio
+
+from solr.core import SolrConnection # type: ignore[import-untyped]
+
+MANUSCRIPT_ID = 991961
+LU_PAGE = path.join(
+ settings.TEST_MEI_FILES_PATH, str(MANUSCRIPT_ID), "LU-1961_0123.mei"
+)
+
+
+class TestSearchMEITextView(APITestCase):
+ search_text_view = SearchMEITextView()
+
+ @classmethod
+ def setUpTestData(cls) -> None:
+ source = Manuscript.objects.create(
+ id=MANUSCRIPT_ID, name="Test Liber Usualis", siglum="LU-TEST"
+ )
+ Folio.objects.create(manuscript=source, number="0123", image_uri="lu_0123.jpg")
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ super().setUpClass()
+ call_command(
+ "index_manuscript_mei",
+ str(MANUSCRIPT_ID),
+ "--mei-dir",
+ settings.TEST_MEI_FILES_PATH,
+ )
+
+ def search(self, q: str, **options: str) -> dict:
+ url = reverse("search-mei-text-view")
+ response = self.client.get(
+ url, {"q": q, "manuscript_id": MANUSCRIPT_ID, **options}
+ )
+ self.assertEqual(response.status_code, 200)
+ return response.json()
+
+ def count_text_line_docs(self) -> int:
+ solr_conn = SolrConnection(settings.SOLR_TEST_SERVER)
+ return solr_conn.query(
+ "*:*",
+ fq=f"type:mei_text_line AND manuscript_id:{MANUSCRIPT_ID}",
+ rows=0,
+ ).numFound
+
+ def test_get_query_words(self) -> None:
+ view = self.search_text_view
+ with self.subTest("Words are normalized"):
+ self.assertEqual(
+ view.get_query_words("Mi-se-rére MEI", True), ["miserere", "mei"]
+ )
+ self.assertEqual(
+ view.get_query_words("Mi-se-rére", False), ["mi", "se", "rere"]
+ )
+ with self.subTest("Long words are cut to the indexed n-gram length"):
+ self.assertEqual(
+ view.get_query_words("abcdefghijklmnopq", True), ["abcdefghijklmno"]
+ )
+ with self.subTest("A query with no words is invalid"):
+ with self.assertRaises(TextSearchException):
+ view.get_query_words(" -¬: ", True)
+
+ def test_create_query_string(self) -> None:
+ view = self.search_text_view
+ with self.subTest("Single line, syllables joined"):
+ self.assertEqual(
+ view.create_query_string(["a", "b"], True, False),
+ "text_line_t:a AND text_line_t:b",
+ )
+ with self.subTest("Single line, syllables separate"):
+ self.assertEqual(
+ view.create_query_string(["a"], False, False), "text_line_syl_t:a"
+ )
+ with self.subTest("Multi-line"):
+ self.assertEqual(
+ view.create_query_string(["a", "b"], True, True),
+ "text_window_t:a AND text_window_t:b AND "
+ "((text_line_t:a AND text_line_t:b) OR "
+ "(*:* -(text_next_t:a AND text_next_t:b)))",
+ )
+
+ def test_get(self) -> None:
+ with self.subTest("Matches hyphenated, accented text"):
+ response = self.search("miserere mei")
+ self.assertEqual(response["numFound"], 1)
+ result = response["results"][0]
+ self.assertEqual(len(result["lines"]), 1)
+ self.assertIn("Ps. 50", result["lines"][0])
+ self.assertEqual(result["boxes"][0]["f"], "0123")
+ self.assertEqual(result["boxes"][0]["p"], "lu_0123.jpg")
+ with self.subTest("Results are in page order"):
+ response = self.search("asperges")
+ texts = [result["lines"][0] for result in response["results"]]
+ self.assertEqual(texts[0], "The Asperges.")
+ ys = [result["boxes"][0]["y"] for result in response["results"]]
+ self.assertEqual(ys, sorted(ys))
+ with self.subTest("No match"):
+ self.assertEqual(self.search("alleluia")["numFound"], 0)
+ with self.subTest("Missing or invalid parameters"):
+ url = reverse("search-mei-text-view")
+ for params in [
+ {"q": "mei"},
+ {"q": "--", "manuscript_id": MANUSCRIPT_ID},
+ {"q": "mei", "manuscript_id": MANUSCRIPT_ID, "hyphens": "x"},
+ {"q": "mei", "manuscript_id": MANUSCRIPT_ID, "multiline": "1"},
+ ]:
+ self.assertEqual(self.client.get(url, params).status_code, 400)
+
+ def test_hyphens_option(self) -> None:
+ # The line reads "se-ricórdi-am tú- am."
+ with self.subTest("Ignored: syllables join into words"):
+ self.assertEqual(self.search("ricordiam")["numFound"], 1)
+ with self.subTest("Kept: text is matched syllable by syllable"):
+ self.assertEqual(self.search("ricordiam", hyphens="keep")["numFound"], 0)
+ self.assertEqual(
+ self.search("se-ricordi-am", hyphens="keep")["numFound"], 1
+ )
+
+ def test_multiline_option(self) -> None:
+ # "... et mundá¬" ends one line and "bor: lavá- bis me," starts the next.
+ with self.subTest("A word split across lines needs multi-line"):
+ self.assertEqual(self.search("mundabor")["numFound"], 0)
+ response = self.search("mundabor", multiline="true")
+ self.assertEqual(response["numFound"], 1)
+ result = response["results"][0]
+ self.assertEqual(len(result["lines"]), 2)
+ self.assertTrue(result["lines"][1].startswith("bor:"))
+ self.assertEqual(len(result["boxes"]), 2)
+ with self.subTest("Joined across lines: 'mi¬' then 'se-ricórdi-am'"):
+ self.assertEqual(self.search("misericordiam")["numFound"], 0)
+ response = self.search("misericordiam", multiline="true")
+ self.assertEqual(response["numFound"], 1)
+ self.assertIn("Ps. 50", response["results"][0]["lines"][0])
+ with self.subTest("A match within one line is listed once, at that line"):
+ single = self.search("miserere mei")
+ multi = self.search("miserere mei", multiline="true")
+ self.assertEqual(multi["numFound"], single["numFound"])
+ self.assertEqual(multi["results"], single["results"])
+
+ def test_reindexing_updates_text(self) -> None:
+ """
+ Text lines are rebuilt whenever a folio's MEI is reindexed, so a
+ correction to the MEI replaces the old text instead of adding to it.
+ """
+ line_count = self.count_text_line_docs()
+ mei_dir = tempfile.mkdtemp()
+ self.addCleanup(shutil.rmtree, mei_dir)
+ # Restore the original index for the other tests.
+ self.addCleanup(
+ call_command,
+ "index_manuscript_mei",
+ str(MANUSCRIPT_ID),
+ "--mei-dir",
+ settings.TEST_MEI_FILES_PATH,
+ "--replace",
+ )
+ manuscript_dir = path.join(mei_dir, str(MANUSCRIPT_ID))
+ shutil.copytree(path.dirname(LU_PAGE), manuscript_dir)
+ corrected_page = path.join(manuscript_dir, path.basename(LU_PAGE))
+ with open(corrected_page, encoding="utf-8") as mei_file:
+ content = mei_file.read()
+ with open(corrected_page, "w", encoding="utf-8") as mei_file:
+ mei_file.write(content.replace("The Ordinary Chants", "Corrected Heading"))
+
+ call_command(
+ "index_manuscript_mei",
+ str(MANUSCRIPT_ID),
+ "--mei-dir",
+ mei_dir,
+ "--folio",
+ "0123",
+ "--replace",
+ )
+
+ self.assertEqual(self.count_text_line_docs(), line_count)
+ self.assertEqual(self.search("corrected heading")["numFound"], 1)
+ self.assertEqual(self.search("ordinary chants")["numFound"], 0)
+
+ def test_flush_index_removes_text(self) -> None:
+ self.addCleanup(
+ call_command,
+ "index_manuscript_mei",
+ str(MANUSCRIPT_ID),
+ "--mei-dir",
+ settings.TEST_MEI_FILES_PATH,
+ )
+ self.assertGreater(self.count_text_line_docs(), 0)
+ call_command("index_manuscript_mei", str(MANUSCRIPT_ID), "--flush-index")
+ self.assertEqual(self.count_text_line_docs(), 0)
+
+ @classmethod
+ def tearDownClass(cls) -> None:
+ call_command("index_manuscript_mei", str(MANUSCRIPT_ID), "--flush-index")
+ super().tearDownClass()
diff --git a/app/public/cantusdata/urls.py b/app/public/cantusdata/urls.py
index b4d63bbbe..50cc7ef12 100644
--- a/app/public/cantusdata/urls.py
+++ b/app/public/cantusdata/urls.py
@@ -10,6 +10,7 @@
from cantusdata.views.search import SearchView
from cantusdata.views.suggestion import SuggestionView
from cantusdata.views.search_notation import SearchNotationView
+from cantusdata.views.search_text import SearchMEITextView
from cantusdata.views.chant_set import (
FolioChantSetView,
ManuscriptChantSetView,
@@ -128,6 +129,12 @@
SearchNotationView.as_view(),
name="search-notation-view",
),
+ # Search of the text lines in MEI files
+ path(
+ "text-search/",
+ SearchMEITextView.as_view(),
+ name="search-mei-text-view",
+ ),
# MEI deposits from external OMR pipelines. POST files one folio's MEI for
# admin review; GET reports a submitter's submission statuses. Token-
# authenticated, see views/mei_submission.py.
diff --git a/app/public/cantusdata/views/search_notation.py b/app/public/cantusdata/views/search_notation.py
index 01051c3b8..359ca2fad 100644
--- a/app/public/cantusdata/views/search_notation.py
+++ b/app/public/cantusdata/views/search_notation.py
@@ -11,6 +11,7 @@
validate_query,
get_transpositions,
translate_interval_query_direction,
+ volpiano_to_pitch_names,
)
RETURNED_FIELDS = [
@@ -84,6 +85,13 @@ def create_query_string(self, q: str, q_type: str) -> str:
"""
Format the query and query type into a string for solr.
"""
+ if q_type == "volpiano":
+ # Volpiano is case-sensitive (uppercase marks liquescents) and is
+ # not space-separated, so it skips the normalization below.
+ pitch_names = volpiano_to_pitch_names(q)
+ if pitch_names is None:
+ raise NotationSearchException("Invalid query.")
+ return f"pitch_names:{'_'.join(pitch_names)}"
normalized_q_elems = q.lower().split()
q_valid = validate_query(normalized_q_elems, q_type)
if not q_valid:
@@ -106,8 +114,9 @@ def create_query_string(self, q: str, q_type: str) -> str:
q_str = "_".join(normalized_q_elems)
return f"{q_type}:{q_str}"
+ @staticmethod
def create_boxes(
- self, locations: list[dict[str, int]], image_uri: str, folio: str
+ locations: list[dict[str, int]], image_uri: str, folio: str
) -> list[dict[str, Union[int, str]]]:
boxes: list[dict[str, Union[int, str]]] = []
for location in locations:
diff --git a/app/public/cantusdata/views/search_text.py b/app/public/cantusdata/views/search_text.py
new file mode 100644
index 000000000..75a582126
--- /dev/null
+++ b/app/public/cantusdata/views/search_text.py
@@ -0,0 +1,191 @@
+import json
+from typing import Any, NamedTuple, TypedDict, Union
+
+import requests
+from django.conf import settings
+from rest_framework.views import APIView
+from rest_framework.response import Response
+from rest_framework.request import Request
+from rest_framework.exceptions import APIException
+
+from cantusdata.helpers.search_utils import normalize_search_text
+from cantusdata.views.search_notation import SearchNotationView
+
+# text_general indexes n-grams of at most 15 characters, so a longer query
+# word can never match. Its first 15 characters still match as a prefix.
+MAX_QUERY_WORD_LENGTH = 15
+
+
+class TextFields(NamedTuple):
+ """The Solr fields searched for one way of normalizing text"""
+
+ line: str
+ next: str
+ window: str
+
+
+# Fields indexed by MEITokenizer.create_text_line_documents, keyed by whether
+# syllables are joined (hyphens ignored) or separated.
+TEXT_FIELDS = {
+ True: TextFields("text_line_t", "text_next_t", "text_window_t"),
+ False: TextFields("text_line_syl_t", "text_next_syl_t", "text_window_syl_t"),
+}
+
+
+class TextSearchResultItem(TypedDict):
+ boxes: list[dict[str, Union[int, str]]]
+ lines: list[str]
+
+
+class TextSearchException(APIException):
+ status_code = 400
+ default_detail = "Text search request invalid."
+
+
+class SearchMEITextView(APIView):
+ """
+ Search the lines of text encoded in a manuscript's MEI files (e.g. OCR'd
+ text), as indexed by index_manuscript_mei. A line matches when it contains
+ every word of the query, compared after normalize_search_text.
+
+ Optional parameters:
+ hyphens: "ignore" (default) joins syllables split by hyphens; "keep"
+ matches them as separate syllables, as printed.
+ multiline: "true" lets the words continue onto the next line, in
+ which case a result may span two lines; "false" (default) does not.
+ """
+
+ def get(self, request: Request, *args: Any, **kwargs: Any) -> Response:
+ q = request.GET.get("q", None)
+ manuscript_param = request.GET.get("manuscript_id", None)
+ rows_param = request.GET.get("rows", "100")
+ start_param = request.GET.get("start", "0")
+ hyphens_param = request.GET.get("hyphens", "ignore")
+ multiline_param = request.GET.get("multiline", "false")
+
+ if not q or not manuscript_param:
+ raise TextSearchException("Missing required parameters.")
+ if not rows_param.isdigit() or not start_param.isdigit():
+ raise TextSearchException("'Rows' and 'Start' must be integers.")
+ if not manuscript_param.isdigit():
+ raise TextSearchException("Manuscript ID must be digits.")
+ if hyphens_param not in ("ignore", "keep"):
+ raise TextSearchException("'Hyphens' must be 'ignore' or 'keep'.")
+ if multiline_param not in ("true", "false"):
+ raise TextSearchException("'Multiline' must be 'true' or 'false'.")
+ join_syllables = hyphens_param == "ignore"
+ multiline = multiline_param == "true"
+
+ words = self.get_query_words(q, join_syllables)
+ results, num_found = self.do_query(
+ int(manuscript_param),
+ words,
+ join_syllables,
+ multiline,
+ int(rows_param),
+ int(start_param),
+ )
+ return Response({"numFound": num_found, "results": results})
+
+ def get_query_words(self, q: str, join_syllables: bool) -> list[str]:
+ """
+ Normalize the user's query into the words to search for. Normalization
+ leaves only [a-z0-9] words, so nothing needs escaping for Solr.
+ """
+ words = [
+ word[:MAX_QUERY_WORD_LENGTH]
+ for word in normalize_search_text(q, join_syllables).split()
+ ]
+ if not words:
+ raise TextSearchException("Invalid query.")
+ return words
+
+ def create_query_string(
+ self, words: list[str], join_syllables: bool, multiline: bool
+ ) -> str:
+ """
+ Build a Solr query requiring every word.
+
+ A multi-line search matches a line together with the next one. To list
+ each match once, a line only matches this way when the next line does
+ not have every word by itself; that match is listed at the next line.
+ """
+ fields = TEXT_FIELDS[join_syllables]
+
+ def all_words_in(field: str) -> str:
+ return " AND ".join(f"{field}:{word}" for word in words)
+
+ if not multiline:
+ return all_words_in(fields.line)
+ return (
+ f"{all_words_in(fields.window)} AND "
+ f"(({all_words_in(fields.line)}) OR (*:* -({all_words_in(fields.next)})))"
+ )
+
+ def do_query(
+ self,
+ manuscript_id: int,
+ words: list[str],
+ join_syllables: bool,
+ multiline: bool,
+ rows: int,
+ start: int,
+ ) -> tuple[list[TextSearchResultItem], int]:
+ line_field = TEXT_FIELDS[join_syllables].line
+ q_str = self.create_query_string(words, join_syllables, multiline)
+ params: dict[str, Union[str, int]] = {
+ "q": "*:*",
+ "fq": f"type:mei_text_line AND manuscript_id:{manuscript_id} AND ({q_str})",
+ "fl": ",".join(
+ [
+ "folio",
+ "image_uri",
+ "location_json",
+ "text_line_stored",
+ "next_text_line_stored",
+ "next_location_stored",
+ line_field,
+ ]
+ ),
+ "sort": "folio asc, line_order_i asc",
+ "rows": rows,
+ "start": start,
+ }
+ response = requests.get(
+ f"{settings.SOLR_SERVER}/select", params=params, timeout=10
+ ).json()
+ results: list[TextSearchResultItem] = []
+ for d in response["response"]["docs"]:
+ image_uri = d.get("image_uri", "")
+ result: TextSearchResultItem = {
+ "boxes": SearchNotationView.create_boxes(
+ d["location_json"], image_uri, d["folio"]
+ ),
+ "lines": [d["text_line_stored"]],
+ }
+ # A multi-line match that the line can't account for by itself
+ # continues onto the next line, so show that line too.
+ if (
+ multiline
+ and "next_text_line_stored" in d
+ and not self.has_all_words(d[line_field], words)
+ ):
+ result["boxes"] += SearchNotationView.create_boxes(
+ json.loads(d["next_location_stored"]), image_uri, d["folio"]
+ )
+ result["lines"].append(d["next_text_line_stored"])
+ results.append(result)
+ return results, response["response"]["numFound"]
+
+ @staticmethod
+ def has_all_words(normalized_text: str, words: list[str]) -> bool:
+ """
+ Whether normalized text matches every query word, the way Solr matches
+ text_general fields here: a query word matches any indexed word
+ containing it.
+ """
+ indexed_words = normalized_text.split()
+ return all(
+ any(word in indexed_word for indexed_word in indexed_words)
+ for word in words
+ )
diff --git a/nginx/app/src/js/collections/SearchMEITextResultCollection.js b/nginx/app/src/js/collections/SearchMEITextResultCollection.js
new file mode 100644
index 000000000..6409d4509
--- /dev/null
+++ b/nginx/app/src/js/collections/SearchMEITextResultCollection.js
@@ -0,0 +1,51 @@
+import _ from 'underscore';
+import Backbone from 'backbone';
+import Qs from 'qs';
+import GlobalVars from 'config/GlobalVars';
+
+/**
+ * Collection for results of searches of the text lines in a manuscript's MEI
+ *
+ * @constructor
+ */
+export default Backbone.Collection.extend({
+ initialize: function (options)
+ {
+ this.updateParameters(options);
+ },
+
+ /**
+ * Update the parameters used to build the URL
+ *
+ * @param options
+ */
+ updateParameters: function (options)
+ {
+ this.parameters = options || {};
+ },
+
+ url: function ()
+ {
+ var queryParams = {
+ q: _.result(this.parameters, 'query'),
+ manuscript_id: _.result(this.parameters, 'manuscript'),
+ hyphens: _.result(this.parameters, 'ignoreHyphens') ? 'ignore' : 'keep',
+ multiline: _.result(this.parameters, 'multiline') ? 'true' : 'false'
+ };
+
+ return GlobalVars.siteUrl + "text-search/?" + Qs.stringify(queryParams);
+ },
+
+ /**
+ * Given a response of the format {numFound: ..., results: ...},
+ * just take the list of results.
+ *
+ * @param response
+ * @returns {Array}
+ */
+ parse: function (response)
+ {
+ this.numFound = response && response.numFound || 0;
+ return response && response.results || [];
+ }
+});
diff --git a/nginx/app/src/js/manuscript-detail/ManuscriptDetailPageView.js b/nginx/app/src/js/manuscript-detail/ManuscriptDetailPageView.js
index 2939a68f5..1b4e10b2d 100644
--- a/nginx/app/src/js/manuscript-detail/ManuscriptDetailPageView.js
+++ b/nginx/app/src/js/manuscript-detail/ManuscriptDetailPageView.js
@@ -5,6 +5,7 @@ import Marionette from 'marionette';
import SearchView from "search/SearchView";
import ChantSearchProvider from "search/chant-search/ChantSearchProvider";
import OMRSearchProvider from "search/omr-search/OMRSearchProvider";
+import MEITextSearchProvider from "search/mei-text-search/MEITextSearchProvider";
import fillViewportHeight from "behaviors/FillViewportHeightBehavior";
import SidenavView from 'ui/SidenavView';
@@ -111,20 +112,43 @@ export default Marionette.View.extend({
});
// Initialize the search view
+ // Volpiano normally searches chant records from Cantus Database. A
+ // manuscript without them can still be searched by Volpiano through
+ // its MEI, which the notation search provider then handles instead.
+ // Volpiano and text normally search chant records from Cantus Database.
+ // A manuscript without them is searched through its MEI instead.
+ var hasChants = this.model.get('chants_loaded');
+ var volpianoFromMei = !hasChants && this.model.isPluginActivated('pitch-search');
+
+ var excludedChantFields = [];
+ if (volpianoFromMei)
+ excludedChantFields.push('volpiano', 'volpiano_literal');
+ if (!hasChants)
+ excludedChantFields.push('all');
+
var chantSearchProvider = new ChantSearchProvider({
- additionalResultFields: ['genre', 'mode', 'feast', 'office', 'position']
+ additionalResultFields: ['genre', 'mode', 'feast', 'office', 'position'],
+ excludedFields: excludedChantFields
});
chantSearchProvider.setRestriction('manuscript_id', '"' + this.model.get("id") + '"');
var notationSearchProvider = new OMRSearchProvider({
- manuscript: this.model
+ manuscript: this.model,
+ volpianoSearch: volpianoFromMei
});
+ var providers = [chantSearchProvider, notationSearchProvider];
+
+ // Listed first so that its text field is the default, as "All Text
+ // Fields" is for manuscripts with chant records
+ if (!hasChants)
+ providers.unshift(new MEITextSearchProvider({ manuscript: this.model }));
+
var searchTerm = manuscriptStateChannel.request('search');
var searchView = new SearchView({
searchTerm: searchTerm,
- providers: [chantSearchProvider, notationSearchProvider]
+ providers: providers
});
// Set the global search state when the search term changes
diff --git a/nginx/app/src/js/search/chant-search/ChantSearchProvider.js b/nginx/app/src/js/search/chant-search/ChantSearchProvider.js
index 47202bee1..6d4548f9a 100644
--- a/nginx/app/src/js/search/chant-search/ChantSearchProvider.js
+++ b/nginx/app/src/js/search/chant-search/ChantSearchProvider.js
@@ -68,6 +68,7 @@ export default Marionette.Object.extend({
* search type is Volpiano) the chant's Volpiano
* - `restriction`: object with field: value mappings of restrictions
* to apply to all queries made by the controller
+ * - `excludedFields`: types of search fields not to offer
*/
initialize: function () {
_.bindAll(this, 'search', 'setRestriction', 'getSearchMetadata');
@@ -75,6 +76,11 @@ export default Marionette.Object.extend({
// Set options
this.restrictions = this.getOption('restrictions') || {};
+ var excludedFields = this.getOption('excludedFields') || [];
+ this.fields = _.reject(this.fields, function (field) {
+ return _.contains(excludedFields, field.type);
+ });
+
// Initialize search input model
this.searchParameters = new SearchInput();
diff --git a/nginx/app/src/js/search/mei-text-search/MEITextSearchProvider.js b/nginx/app/src/js/search/mei-text-search/MEITextSearchProvider.js
new file mode 100644
index 000000000..0c099e148
--- /dev/null
+++ b/nginx/app/src/js/search/mei-text-search/MEITextSearchProvider.js
@@ -0,0 +1,218 @@
+import _ from "underscore";
+import $ from "jquery";
+import Backbone from "backbone";
+import Marionette from "marionette";
+import Radio from "backbone.radio";
+
+import SearchMEITextResultCollection from "collections/SearchMEITextResultCollection";
+import IncrementalClientSideLoader from "utils/IncrementalClientSideLoader";
+
+import SearchResultHeadingView from "../SearchResultHeadingView";
+import InputView from "../omr-search/InputView";
+import ResultView from "./ResultView";
+import template from './text-help.template.html';
+
+var manuscriptChannel = Radio.channel('manuscript');
+
+/** Explain the search and let the user set its options */
+var OptionsView = Marionette.View.extend({
+ template,
+
+ events: {
+ 'change [data-option]': function (event) {
+ this.model.set($(event.target).data('option'), event.target.checked);
+ }
+ }
+});
+
+/**
+ * Provide support for searching the lines of text encoded in a manuscript's
+ * MEI files (e.g. OCR'd text), for manuscripts without chant records. Required
+ * initialization parameters:
+ *
+ * - `manuscript`: the manuscript model to initialize with
+ *
+ * See SearchView for a further description of the contract this class fulfills.
+ */
+export default Marionette.Object.extend({
+ description: 'Page Text Search',
+
+ fields: [
+ {
+ name: 'Text',
+ type: 'mei_text'
+ }
+ ],
+
+ initialize: function (options)
+ {
+ _.bindAll(this, 'resultFetchCallback', 'zoomToResult', 'getSearchMetadata');
+
+ this.manuscript = options.manuscript.get('id');
+
+ this.searchOptions = new Backbone.Model({
+ ignoreHyphens: true,
+ multiline: false
+ });
+
+ this.results = new SearchMEITextResultCollection();
+ this.displayedResults = new SearchMEITextResultCollection();
+ this.resultLoadingHandler = new IncrementalClientSideLoader(this.displayedResults, this.results);
+
+ this.listenTo(this.results, "sync", this.resultFetchCallback);
+
+ // Rerun the current search with the new options
+ this.listenTo(this.searchOptions, 'change', function ()
+ {
+ this.triggerMethod('search', this.query);
+ });
+ },
+
+ onSearch: function (query)
+ {
+ this.query = query;
+
+ this.clearDivaBoxes();
+ this.abortPendingRequest();
+
+ if (!this.query)
+ {
+ this.results.reset(null);
+ }
+ else
+ {
+ this.results.updateParameters(_.extend({
+ query: this.query,
+ manuscript: this.manuscript
+ }, this.searchOptions.toJSON()));
+
+ this.pendingRequest = this.resultLoadingHandler.fetch();
+ }
+ return false;
+ },
+
+ /**
+ * Abort a search still in progress, so that its results can't arrive after,
+ * and replace, those of a newer search. The heading's error state from the
+ * abort is replaced straight away by the newer search's loading state, or
+ * cleared by the reset of an empty search.
+ */
+ abortPendingRequest: function ()
+ {
+ if (this.pendingRequest)
+ this.pendingRequest.abort();
+
+ this.pendingRequest = null;
+ },
+
+ getDivaAdapter: function ()
+ {
+ return manuscriptChannel.request('diva');
+ },
+
+ resultFetchCallback: function ()
+ {
+ var divaAdapter = this.getDivaAdapter();
+ if (!divaAdapter)
+ return;
+
+ divaAdapter.setHighlights(_.flatten(this.results.map(function (model)
+ {
+ return _.map(model.get('boxes'), _.clone);
+ })));
+
+ if (this.results.length > 0)
+ this.zoomToResult(this.results.at(0));
+ },
+
+ /** Stop displaying Diva highlights */
+ clearDivaBoxes: function ()
+ {
+ var divaAdapter = this.getDivaAdapter();
+ if (divaAdapter)
+ divaAdapter.setHighlights([]);
+ },
+
+ zoomToResult: function (model)
+ {
+ var divaAdapter = this.getDivaAdapter();
+ if (!divaAdapter)
+ return;
+
+ // A result spanning two lines has a box for each; frame both. Both
+ // are on the same page.
+ var boxes = model.get('boxes');
+ if (boxes.length === 0)
+ return;
+
+ var left = _.min(_.pluck(boxes, 'x'));
+ var top = _.min(_.pluck(boxes, 'y'));
+ var right = _.max(_.map(boxes, function (box) { return box.x + box.w; }));
+ var bottom = _.max(_.map(boxes, function (box) { return box.y + box.h; }));
+
+ divaAdapter.focusRegion({
+ imageURI: boxes[0].p,
+ x: left,
+ y: top,
+ width: right - left,
+ height: bottom - top
+ });
+ },
+
+ getSearchMetadata: function ()
+ {
+ return {
+ fieldName: this.field.name,
+ query: this.query,
+ displayedQuery: this.query,
+ numFound: this.results.numFound || 0
+ };
+ },
+
+ display: function (field, query, regions)
+ {
+ this.field = field;
+ this.query = query;
+ this.results.reset();
+
+ regions.searchInput.show(new InputView({ initialQuery: this.query }));
+ regions.searchHelper.show(new OptionsView({ model: this.searchOptions }));
+
+ regions.searchResultHeading.show(new SearchResultHeadingView({
+ collection: this.results,
+ showLoading: true,
+ getSearchMetadata: this.getSearchMetadata
+ }));
+
+ var resultView = new ResultView({
+ collection: this.displayedResults
+ });
+
+ this.listenTo(resultView, 'continue:loading', this._continueLoadingResults);
+ this.listenTo(resultView, 'zoomToResult', this.zoomToResult);
+
+ // Clear results from Diva when the results are no longer displayed
+ this.listenTo(resultView, 'destroy', this.clearDivaBoxes);
+
+ regions.searchResults.show(resultView);
+
+ if (this.query)
+ this.triggerMethod('search', query);
+ },
+
+ /**
+ * Display more search results if they are available
+ *
+ * @private
+ */
+ _continueLoadingResults: function ()
+ {
+ this.resultLoadingHandler.continueLoading();
+ },
+
+ onDestroy: function ()
+ {
+ this.abortPendingRequest();
+ this.stopListening();
+ }
+});
diff --git a/nginx/app/src/js/search/mei-text-search/ResultItemView.js b/nginx/app/src/js/search/mei-text-search/ResultItemView.js
new file mode 100644
index 000000000..1434aa14b
--- /dev/null
+++ b/nginx/app/src/js/search/mei-text-search/ResultItemView.js
@@ -0,0 +1,15 @@
+import Marionette from "marionette";
+
+import template from './result-item.template.html';
+
+export default Marionette.View.extend({
+ template,
+ tagName: 'tr',
+
+ events: {
+ 'click .result-target': function (event) {
+ event.preventDefault();
+ this.trigger('showResult', this.model);
+ }
+ }
+});
diff --git a/nginx/app/src/js/search/mei-text-search/ResultView.js b/nginx/app/src/js/search/mei-text-search/ResultView.js
new file mode 100644
index 000000000..eaa3abcbd
--- /dev/null
+++ b/nginx/app/src/js/search/mei-text-search/ResultView.js
@@ -0,0 +1,11 @@
+import NotationResultView from "../omr-search/ResultView";
+
+import ResultItemView from "./ResultItemView";
+
+import template from './result-list.template.html';
+
+/** Notation search's result list, with text lines instead of notation */
+export default NotationResultView.extend({
+ template,
+ childView: ResultItemView
+});
diff --git a/nginx/app/src/js/search/mei-text-search/result-item.template.html b/nginx/app/src/js/search/mei-text-search/result-item.template.html
new file mode 100644
index 000000000..9a59bf8ea
--- /dev/null
+++ b/nginx/app/src/js/search/mei-text-search/result-item.template.html
@@ -0,0 +1,8 @@
+
diff --git a/nginx/app/src/js/search/mei-text-search/result-list.template.html b/nginx/app/src/js/search/mei-text-search/result-list.template.html
new file mode 100644
index 000000000..93ff17b28
--- /dev/null
+++ b/nginx/app/src/js/search/mei-text-search/result-list.template.html
@@ -0,0 +1,11 @@
+
+
+
+
+
Folio
+
Text
+
+
+
+
+
diff --git a/nginx/app/src/js/search/mei-text-search/text-help.template.html b/nginx/app/src/js/search/mei-text-search/text-help.template.html
new file mode 100644
index 000000000..f9a13c02c
--- /dev/null
+++ b/nginx/app/src/js/search/mei-text-search/text-help.template.html
@@ -0,0 +1,21 @@
+
+ This manuscript has no chant records, so its text is searched as encoded on each page,
+ which may include OCR errors. A line matches when it contains every word you enter;
+ accents and capitals are ignored.
+
+
+ >
+
+
+
+ >
+
+
diff --git a/nginx/app/src/js/search/omr-search/OMRSearchProvider.js b/nginx/app/src/js/search/omr-search/OMRSearchProvider.js
index c6ed9e847..91572885f 100644
--- a/nginx/app/src/js/search/omr-search/OMRSearchProvider.js
+++ b/nginx/app/src/js/search/omr-search/OMRSearchProvider.js
@@ -10,6 +10,7 @@ import SearchResultHeadingView from "../SearchResultHeadingView";
import NeumeGalleryView from "./NeumeGalleryView";
import ContourChoiceView from "./ContourChoiceView";
import IntervalChoiceView from "./IntervalChoiceView";
+import VolpianoHelpView from "./VolpianoHelpView";
import InputView from "./InputView";
import ResultView from "./ResultView";
@@ -21,6 +22,10 @@ var manuscriptChannel = Radio.channel('manuscript');
*
* - `manuscript`: the manuscript model to initialize with
*
+ * Optional initialization parameters:
+ *
+ * - `volpianoSearch`: offer a Volpiano field that searches the MEI by pitch
+ *
* See SearchView for a further description of the contract this class fulfills.
*/
export default Marionette.Object.extend({
@@ -82,6 +87,9 @@ export default Marionette.Object.extend({
this.fields.push.apply(this.fields, plugin.fields);
}, this);
+ if (options.volpianoSearch)
+ this.fields.push({ name: 'Volpiano', type: 'volpiano' });
+
this.results = new SearchNotationResultCollection();
this.displayedResults = new SearchNotationResultCollection();
this.resultLoadingHandler = new IncrementalClientSideLoader(this.displayedResults, this.results);
@@ -220,6 +228,10 @@ export default Marionette.Object.extend({
});
regions.searchHelper.show(intervalChoices);
}
+ else if (field.type === 'volpiano')
+ {
+ regions.searchHelper.show(new VolpianoHelpView());
+ }
else
{
regions.searchHelper.empty();
diff --git a/nginx/app/src/js/search/omr-search/VolpianoHelpView.js b/nginx/app/src/js/search/omr-search/VolpianoHelpView.js
new file mode 100644
index 000000000..c8421cf34
--- /dev/null
+++ b/nginx/app/src/js/search/omr-search/VolpianoHelpView.js
@@ -0,0 +1,8 @@
+import Marionette from 'marionette';
+
+import template from './volpiano-help.template.html';
+
+/** Explain how a Volpiano query is matched against the MEI */
+export default Marionette.View.extend({
+ template
+});
diff --git a/nginx/app/src/js/search/omr-search/volpiano-help.template.html b/nginx/app/src/js/search/omr-search/volpiano-help.template.html
new file mode 100644
index 000000000..557feee2e
--- /dev/null
+++ b/nginx/app/src/js/search/omr-search/volpiano-help.template.html
@@ -0,0 +1,6 @@
+
+ This manuscript has no chant records, so Volpiano is searched in its encoded notation.
+ Notes are matched by pitch name only: octave, accidentals, clefs and barlines are ignored.
+ Melodies of up to 5 notes are always found; longer ones are found only where they span
+ at most 5 whole neumes.
+