diff --git a/.gitignore b/.gitignore
index aac88e97..b5524d05 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,7 +56,7 @@ hs_err_pid* # virtual machine crash logs, see http://www.java.com/en/download/he
**/__pycache__
**/dist
**/*.egg-info
-**/.pytype
+**/.mypy_cache
build/**
.coverage
coverage.xml
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f6704270..086438b4 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -77,7 +77,7 @@ This means adding a properly structured module within the CLI package is all tha
pip install -e ".[dev]"
```
-An editable install (`pip install -e .`) is required before running tests or building docs. The package uses `importlib.metadata` for version resolution at runtime, which only works when the package is registered in the environment. You can no longer run `pytest` or `pytype` directly against the source tree without installing first. If you want to avoid pulling in all dependencies, `pip install -e . --no-deps` is sufficient to register the package metadata.
+An editable install (`pip install -e .`) is required before running tests or building docs. The package uses `importlib.metadata` for version resolution at runtime, which only works when the package is registered in the environment. You can no longer run `pytest` or `mypy` directly against the source tree without installing first. If you want to avoid pulling in all dependencies, `pip install -e . --no-deps` is sufficient to register the package metadata.
## Local Development
diff --git a/build-tools/clean.py b/build-tools/clean.py
index 9a35c493..5a3f270b 100644
--- a/build-tools/clean.py
+++ b/build-tools/clean.py
@@ -13,7 +13,7 @@
# Directories to remove
CLEAN_DIRS = [
"build", "dist", "*.egg-info", "mmif_python-*",
- ".pytest_cache", ".pytype", ".hypothesis",
+ ".pytest_cache", ".mypy_cache", ".hypothesis",
"tests/.hypothesis", "htmlcov",
"docs-test",
]
diff --git a/build-tools/test.py b/build-tools/test.py
index fe5455e4..48ae390f 100644
--- a/build-tools/test.py
+++ b/build-tools/test.py
@@ -4,7 +4,7 @@
This script is equivalent to::
pip install -e ".[test]"
- pytype mmif
+ python -m mypy
python -m pytest --cov=mmif --cov-report=xml -m "not slow"
"""
import argparse
@@ -51,10 +51,10 @@ def main():
cwd=project_root,
)
- # Run pytype static analysis
- print("\n--- Running pytype ---")
+ # Run mypy static analysis (configured in pyproject.toml)
+ print("\n--- Running mypy ---")
run_command(
- ["pytype", "mmif"],
+ [sys.executable, "-m", "mypy"],
cwd=project_root,
)
diff --git a/documentation/target-versions.csv b/documentation/target-versions.csv
index f6df6fd0..38ec0d61 100644
--- a/documentation/target-versions.csv
+++ b/documentation/target-versions.csv
@@ -1,4 +1,5 @@
"``mmif-python`` version","Target MMIF Specification"
+`1.7.0 `__,`1.2.0 `__
`1.6.0 `__,`1.2.0 `__
`1.5.4 `__,`1.2.0 `__
`1.5.3 `__,`1.2.0 `__
diff --git a/mmif/serialize/__init__.py b/mmif/serialize/__init__.py
index 06964253..dca6aa24 100644
--- a/mmif/serialize/__init__.py
+++ b/mmif/serialize/__init__.py
@@ -3,10 +3,13 @@
"""
from .annotation import *
-from .annotation import __all__ as anno_all
from .mmif import *
-from .mmif import __all__ as mmif_all
+from .producer import *
from .view import *
-from .view import __all__ as view_all
-__all__ = anno_all + mmif_all + view_all
+# a literal, so that type checkers can follow the star import of this package
+__all__ = [
+ 'Annotation', 'AnnotationProperties', 'Contain', 'Document',
+ 'DocumentProperties', 'EmptyOutputWarning', 'Mmif', 'ProducerContext',
+ 'Text', 'View', 'ViewMetadata', 'canonicalize_prop_name',
+]
diff --git a/mmif/serialize/annotation.py b/mmif/serialize/annotation.py
index 487a2001..a3f8d91d 100644
--- a/mmif/serialize/annotation.py
+++ b/mmif/serialize/annotation.py
@@ -15,21 +15,27 @@
import re
import typing
import warnings
-from typing import Union, Dict, Optional, Iterator, MutableMapping, TypeVar
+from collections.abc import Iterator, MutableMapping
+from typing import TYPE_CHECKING, Any, TypeVar, cast
from urllib.parse import urlparse
-from mmif.vocabulary import ThingTypesBase, DocumentTypesBase
-from .model import MmifObject, PRMTV_TYPES
-from .. import DocumentTypes, AnnotationTypes
import mmif_docloc_http
+from mmif.vocabulary import DocumentTypesBase, ThingTypesBase
+
+from .. import AnnotationTypes, DocumentTypes
+from .model import PRMTV_TYPES, MmifObject
+
+if TYPE_CHECKING:
+ from .view import View
+
__all__ = ['Annotation', 'AnnotationProperties', 'Document', 'DocumentProperties', 'Text', 'canonicalize_prop_name']
T = TypeVar('T')
-LIST_PRMTV = typing.List[PRMTV_TYPES] # list of values (most cases for annotation props)
-LIST_LIST_PRMTV = typing.List[LIST_PRMTV] # list of lists of values (e.g. for coordinates)
-DICT_PRMTV = typing.Dict[str, PRMTV_TYPES] # dict of values (`text` prop of `TextDocument` and other complex props)
-DICT_LIST_PRMTV = typing.Dict[str, LIST_PRMTV] # dict of list of values (even more complex props)
+LIST_PRMTV = list[PRMTV_TYPES] # list of values (most cases for annotation props)
+LIST_LIST_PRMTV = list[LIST_PRMTV] # list of lists of values (e.g. for coordinates)
+DICT_PRMTV = dict[str, PRMTV_TYPES] # dict of values (`text` prop of `TextDocument` and other complex props)
+DICT_LIST_PRMTV = dict[str, LIST_PRMTV] # dict of list of values (even more complex props)
# some built-in document location helpers
@@ -43,7 +49,7 @@
@functools.lru_cache(maxsize=None)
-def _prop_alias_map(shortname: str) -> Dict[str, str]:
+def _prop_alias_map(shortname: str) -> dict[str, str]:
"""
Maps every spelling of an aliased property of a type to the canonical
name. The table is read from the latest version of the type in the
@@ -81,12 +87,17 @@ class Annotation(MmifObject):
MmifObject that represents an annotation in a MMIF view.
"""
- def __init__(self, anno_obj: Optional[Union[bytes, str, dict]] = None, *_) -> None:
+ def __init__(self, anno_obj: bytes | str | dict | None = None, *_) -> None:
self._type: ThingTypesBase = ThingTypesBase('')
- # to store the parent view ID
+ # values that are not stored on the annotation itself but cached at
+ # load time: properties recorded by capital `Annotation`s (on
+ # documents) and computed `start`/`end` (on `targets` annotations)
self._props_ephemeral: AnnotationProperties = AnnotationProperties(None, self)
- self._alignments = {} # to hold alignment information (Alignment anno long_id -> aligned anno long_id)
- self.reserved_names.update(('_props_ephemeral', '_alignments'))
+ # the view holding this annotation; backs `parent` and the lookup of
+ # view-level `contains` defaults
+ self._parent_view: 'View | None' = None
+ self._alignments: dict = {} # to hold alignment information (Alignment anno long_id -> aligned anno long_id)
+ self.reserved_names.update(('_props_ephemeral', '_parent_view', '_alignments'))
if not hasattr(self, 'properties'): # don't overwrite DocumentProperties on super() call
self.properties: AnnotationProperties = AnnotationProperties(None, self)
self._attribute_classes = {'properties': AnnotationProperties}
@@ -115,7 +126,7 @@ def _cache_alignment(self, alignment_ann: 'Annotation', alignedto_ann: 'Annotati
"""
self._alignments[alignment_ann] = alignedto_ann
- def aligned_to_by(self, alignment: 'Annotation') -> Optional['Annotation']:
+ def aligned_to_by(self, alignment: 'Annotation') -> 'Annotation | None':
"""
Retrieves the other side of ``Alignment`` annotation that has this annotation on one side.
@@ -140,22 +151,43 @@ def get_all_aligned(self) -> Iterator['Annotation']:
yield aligned
- def _add_ephemeral_defaults(self, defaults: typing.Mapping[str, typing.Any]) -> None:
+ def _get_view_default(self, prop_name: str):
+ """
+ Looks up a view-level default for this annotation's ``@type`` in the
+ ``contains`` metadata of the parent view, at read time. Deprecated
+ spellings in ``contains`` are matched by their canonical name.
+
+ :param prop_name: a canonical property name
+ :return: the default value
+ :raises KeyError: if there is no parent view or no such default
+ """
+ if self._parent_view is not None:
+ defaults = self._parent_view.metadata.contains.get(self._type)
+ if defaults:
+ for name, value in defaults.items():
+ if canonicalize_prop_name(self._type, name) == prop_name:
+ return value
+ raise KeyError(prop_name)
+
+ def _resolved_properties(self) -> dict[str, typing.Any]:
"""
- Distributes view-level defaults (from the ``contains`` metadata of the
- parent view) to this annotation as ephemeral properties, under their
- canonical names. An annotation-level value takes precedence over a
- view-level default, so a default is skipped when the annotation
- carries the property itself.
+ Returns the properties of this annotation as a reader sees them:
+ view-level defaults from the parent view's ``contains``, overridden by
+ the values stored on the annotation itself. Values cached at load time
+ are not included.
- :param defaults: property defaults for this annotation's ``@type``
+ :return: a new dict of canonical property names to values
"""
- for name, value in defaults.items():
- name = canonicalize_prop_name(self._type, name)
- if name not in self.properties:
- self._props_ephemeral[name] = value
+ resolved: dict = {}
+ if self._parent_view is not None:
+ defaults = self._parent_view.metadata.contains.get(self._type)
+ if defaults:
+ for name, value in defaults.items():
+ resolved[canonicalize_prop_name(self._type, name)] = value
+ resolved.update(self.properties.items())
+ return resolved
- def is_type(self, at_type: Union[str, ThingTypesBase]) -> bool:
+ def is_type(self, at_type: str | ThingTypesBase) -> bool:
"""
Check if the @type of this object matches.
"""
@@ -168,7 +200,7 @@ def at_type(self) -> ThingTypesBase:
return self._type
@at_type.setter
- def at_type(self, at_type: Union[str, ThingTypesBase]) -> None:
+ def at_type(self, at_type: str | ThingTypesBase) -> None:
if isinstance(at_type, str):
self._type = ThingTypesBase.from_str(at_type)
elif isinstance(at_type, type) and issubclass(at_type, ThingTypesBase):
@@ -179,14 +211,22 @@ def at_type(self, at_type: Union[str, ThingTypesBase]) -> None:
# Use ``model_construct()`` to skip Pydantic field validation —
# direct ``__init__`` rejects types with required fields
# (e.g., ``Alignment`` requires ``source``/``target``).
- instance = at_type.model_construct()
- instance.initialized_from = at_type.uri
+ vocab_type: Any = at_type
+ instance = vocab_type.model_construct()
+ instance.initialized_from = vocab_type.uri
self._type = instance
else:
self._type = at_type
@property
def parent(self) -> str:
+ """
+ The ID of the view that holds this annotation. Taken from the view
+ itself once the annotation is added to one, otherwise parsed from the
+ view-prefixed annotation ID.
+ """
+ if self._parent_view is not None:
+ return self._parent_view.id
if self.id_delimiter in self.properties.id:
id_split = self.properties.id.split(self.id_delimiter)
if len(id_split) == 2:
@@ -197,17 +237,15 @@ def parent(self) -> str:
def parent(self, parent_id: str) -> None:
"""
.. deprecated:: 1.1.0
- Will be removed in 2.0.0.
- Setting parent ID is no longer allowed. Instead, the parent ID
- should be prefixed to the annotation ID.
+ Will be removed in 2.0.0. Use a prefixed annotation ID instead.
Sets the parent view ID of this annotation.
:param parent_id:
"""
- warnings.warn(
- "Setting parent ID is deprecated and nothing happened. Use prefixed annotation ID instead.",
- DeprecationWarning
- )
+ warnings.warn('Setting the parent ID is deprecated and will be '
+ 'removed in 2.0.0. Use a prefixed annotation ID '
+ 'instead.',
+ DeprecationWarning)
@property
@@ -220,16 +258,20 @@ def id(self, aid: str) -> None:
@property
def long_id(self) -> str:
- warnings.warn(
- 'long_id is deprecated. Use `id` instead. ', DeprecationWarning
- )
+ """
+ .. deprecated:: 1.1.0
+ Will be removed in 2.0.0. Use :attr:`id` instead.
+ """
+ warnings.warn('long_id is deprecated and will be removed in 2.0.0. '
+ 'Use id instead.',
+ DeprecationWarning)
return self.id
@long_id.setter
def long_id(self, long_id: str) -> None:
- warnings.warn(
- 'long_id is deprecated. Use `id` instead. ', DeprecationWarning
- )
+ warnings.warn('long_id is deprecated and will be removed in 2.0.0. '
+ 'Use id instead.',
+ DeprecationWarning)
self.id = long_id
@property
@@ -244,7 +286,7 @@ def _short_id(self) -> str:
@staticmethod
def check_prop_value_is_simple_enough(
- value: Union[PRMTV_TYPES, LIST_PRMTV, LIST_LIST_PRMTV, DICT_PRMTV, DICT_LIST_PRMTV]) -> bool:
+ value: PRMTV_TYPES | LIST_PRMTV | LIST_LIST_PRMTV | DICT_PRMTV | DICT_LIST_PRMTV) -> bool:
def json_primitives(x):
return isinstance(x, typing.get_args(PRMTV_TYPES))
@@ -261,7 +303,7 @@ def json_primitives_list_of_list(x):
or (isinstance(value, dict) and all(map(lambda x: isinstance(x[0], str) and (json_primitives(x[1]) or json_primitives_list(x[1])), value.items())))
def add_property(self, name: str,
- value: Union[PRMTV_TYPES, LIST_PRMTV, LIST_LIST_PRMTV, DICT_PRMTV, DICT_LIST_PRMTV]) -> None:
+ value: PRMTV_TYPES | LIST_PRMTV | LIST_LIST_PRMTV | DICT_PRMTV | DICT_LIST_PRMTV) -> None:
"""
Adds a property to the annotation's properties. A deprecated spelling
of an aliased property (e.g. ``frameType``) is stored under its
@@ -289,7 +331,9 @@ def __getitem__(self, prop_name: str):
return self.properties[prop_name]
elif prop_name in self._props_ephemeral:
return self._props_ephemeral[prop_name]
- else:
+ try:
+ return self._get_view_default(prop_name)
+ except KeyError:
raise KeyError(f"Property {prop_name} does not exist in this annotation.")
def get(self, prop_name: str, default=None):
@@ -300,9 +344,13 @@ def get(self, prop_name: str, default=None):
or a default value if not found. This method searches in multiple
locations with the following priority:
- 1. Direct properties (in ``annotation.properties``)
- 2. Ephemeral properties (view-level metadata from ``contains``)
- 3. Special fields (``@type``, ``properties``)
+ 1. Special fields (``@type``, ``properties``)
+ 2. Direct properties (in ``annotation.properties``)
+ 3. Values cached at load time (properties recorded by capital
+ ``Annotation`` objects, on documents; computed ``start``/``end``,
+ on ``targets`` annotations)
+ 4. View-level defaults (the parent view's ``contains`` metadata for
+ this annotation's ``@type``)
This allows convenient access to properties without explicitly
checking the ``properties`` object or view-level metadata. A
@@ -363,73 +411,101 @@ class Document(Annotation):
:param document_obj: the JSON data that defines the document
"""
- def __init__(self, doc_obj: Optional[Union[bytes, str, dict]] = None, *_) -> None:
+ #: the spellings of the ``text`` property in :meth:`add_property`
+ _TEXT_PROP_NAMES = frozenset({'text', '@text', '_text'})
+
+ def __init__(self, doc_obj: bytes | str | dict | None = None, *_) -> None:
# see https://github.com/clamsproject/mmif-python/issues/226 for discussion
- # around the use of these three dictionaries
+ # around the property stores of a document
# (names changed since, `existing` >> `ephemeral` and `temporary` >> `pending`)
- self._props_original: DocumentProperties = DocumentProperties(None, self)
- # pending props are never cleared: every `Mmif.serialize()` call
- # converts them into `Annotation`s again, and
- # `Mmif.generate_capital_annotations()` relies on its duplicate check
- # to skip values already written. Calling it more than once (e.g.,
- # early in clams-python's `ClamsApp.annotate()`) is safe only while
- # that check holds.
+ # properties added to a document that are recorded as capital
+ # `Annotation`s. This dict is never cleared. `Mmif.serialize()` writes
+ # the records into its output only, so repeated serialization gives
+ # the same records. `Mmif.generate_capital_annotations()` writes them
+ # into the views, and calling it more than once (e.g., early in
+ # clams-python's `ClamsApp.annotate()`) relies on its duplicate check
+ # to skip values already written.
self._props_pending: AnnotationProperties = AnnotationProperties(None, self)
- self.reserved_names.update(('_props_original', '_props_pending'))
+ self.reserved_names.add('_props_pending')
- self._type: Union[ThingTypesBase, DocumentTypesBase] = ThingTypesBase('')
- self.properties = self._props_original
+ self._type: ThingTypesBase | DocumentTypesBase = ThingTypesBase('')
+ self.properties: DocumentProperties = DocumentProperties(None, self)
self.disallow_additional_properties()
self._attribute_classes = {'properties': DocumentProperties}
super().__init__(doc_obj)
def add_property(self, name: str,
- value: Union[PRMTV_TYPES, LIST_PRMTV]
+ value: PRMTV_TYPES | LIST_PRMTV | LIST_LIST_PRMTV | DICT_PRMTV | DICT_LIST_PRMTV
) -> None:
"""
Adds a property to the document's properties.
- Unlike the parent :class:`Annotation` class, added properties of a
- ``Document`` object can be lost during serialization unless it belongs
- to somewhere in a ``Mmif`` object. This is because we want to keep
- ``Document`` object as "read-only" as possible. Thus, if you want to add
- a property to a ``Document`` object,
-
- * add the document to a ``Mmif`` object (either in the documents list or
- in a view from the views list), or
- * directly write to ``Document.properties`` instead of using this method
- (which is not recommended).
-
- With the former method, the SDK will record the added property as a
- `Annotation` annotation object, separate from the original `Document`
- object. See :meth:`mmif.serialize.mmif.Mmif.generate_capital_annotations` for more.
+ ``Document`` objects stay as "read-only" as possible, so a property
+ added here is not written to the document. It is kept aside, and
+ recorded as a separate ``Annotation`` object when the ``Mmif`` object
+ holding the document is serialized (see
+ :meth:`mmif.serialize.mmif.Mmif.set_producer`). A document that is not
+ in a ``Mmif`` object loses it on serialization. Without a producer,
+ recording also needs a view with no error and no warnings.
A few notes to keep in mind:
- #. You can't overwrite an existing property of a ``Document`` object.
+ #. ``location``, ``text``, and ``mime`` are set once. A different value
+ raises ``AttributeError`` once the document has one, and ``text``
+ is also refused on a document with a ``location``. Setting the same
+ value again changes nothing. To record a later value of any other
+ property, add it here, and it becomes an ``Annotation`` object.
#. A MMIF can have multiple ``Annotation`` objects with the same
property name but different values. When this happens, the SDK will
only keep the latest value (in order of appearances in views list) of
the property, effectively overwriting the previous values.
:param name: the name of the property
- :param value: the property's desired value (note: Document accepts fewer value types than Annotation)
- """
- # we don't checking if this k-v already exists in _original (new props) or _ephemeral (read from existing MMIF)
- # because it is impossible to keep the _original updated when a new annotation is added (via `new_annotation`)
- # without look across other views and top-level documents list. Also see
- # ``mmif.serialize.mmif.Mmif.generate_capital_annotations`` for where the "de-duplication" happens.
- if name == "text":
- self.properties.text = Text(value)
+ :param value: the property's desired value
+ """
+ # whether this k-v is already recorded can't be told from here, because
+ # records live in other views. De-duplication happens when the records
+ # are planned, in `Mmif._plan_capital_annotations()` and
+ # `ProducerContext._plan_capital_annotations()`
+ # TODO: upgrade this set-once check to an ownership-based guardrail:
+ # objects the current run created are writable (last write wins,
+ # set-once keys excepted); documents that already existed are never
+ # changed, and new values become capital `Annotation`s; annotations
+ # that already existed refuse `add_property`; "already existed" is
+ # defined by the producer context
+ if name in self._TEXT_PROP_NAMES:
+ text = Text(cast(str | dict, value))
+ current = self.properties.text
+ if (self.location is not None or current.value) and text != current:
+ raise self._read_only_error('text')
+ self.properties.text = text
elif name == "mime":
- self.properties.mime = str(value)
+ mime = str(value)
+ if self.properties.mime and self.properties.mime != mime:
+ raise self._read_only_error(name)
+ self.properties.mime = mime
elif name == "location":
- self.location = value
- elif name not in self._props_original:
- if self.check_prop_value_is_simple_enough(value):
- self._props_pending[name] = value
- else:
- super().add_property(name, value)
+ location = cast(str, value)
+ requested = DocumentProperties()
+ requested.location = location
+ if self.location is not None and self.location != requested.location:
+ raise self._read_only_error(name)
+ self.location = location
+ elif self.check_prop_value_is_simple_enough(value):
+ self._props_pending[name] = value
+ else:
+ super().add_property(name, value)
+
+ def _read_only_error(self, name: str) -> AttributeError:
+ """
+ Builds the error for an attempt to change a set-once property.
+
+ :param name: the name of the property
+ :return: the error to raise
+ """
+ return AttributeError(f'Property "{name}" of document {self.id} is '
+ f'read-only. Documents keep the source data '
+ f'they were created with.')
def __getitem__(self, prop_name: str):
"""
@@ -439,9 +515,9 @@ def __getitem__(self, prop_name: str):
#. Special fields (``id``, ``location``)
#. Pending properties (added via :meth:`add_property` after creation,
to be serialized as capital ``Annotation`` objects)
- #. Original properties (in ``document.properties``)
- #. Ephemeral properties (from existing ``Annotation`` objects or view
- ``contains`` metadata)
+ #. Stored properties (in ``document.properties``)
+ #. Values recorded by existing ``Annotation`` objects
+ #. View-level defaults (the parent view's ``contains`` metadata)
``Document`` overrides ``__getitem__`` (not only :meth:`get`) so that
the ``in`` operator, subscripting, and :meth:`get` all agree.
@@ -469,9 +545,10 @@ def __getitem__(self, prop_name: str):
return self._props_pending[prop_name]
else:
# Delegates to Annotation.__getitem__(), which checks
- # self.properties (_props_original) before _props_ephemeral.
- # This gives annotation-level properties precedence over
- # view-level contains defaults (MMIF spec >= 1.1.1).
+ # self.properties, then values recorded by capital `Annotation`s
+ # (_props_ephemeral), then the parent view's `contains` defaults.
+ # Annotation-level properties take precedence over view-level
+ # defaults (MMIF spec >= 1.1.1).
return super().__getitem__(prop_name)
def get(self, prop_name, default=None):
@@ -531,7 +608,7 @@ def text_language(self, lang_code: str) -> None:
def text_value(self) -> str:
if self._type == DocumentTypes.TextDocument:
if self.location:
- f = open(self.location_path(nonexist_ok=False), 'r', encoding='utf8')
+ f = open(cast(str, self.location_path(nonexist_ok=False)), 'r', encoding='utf8')
textvalue = f.read()
f.close()
return textvalue
@@ -548,7 +625,7 @@ def text_value(self, text_value: str) -> None:
raise ValueError("Only TextDocument can have `text` field.")
@property
- def location(self) -> Optional[str]:
+ def location(self) -> str | None:
"""
``location`` property must be a legitimate URI. That is, should the document be a local file
then the file:// scheme must be used.
@@ -560,21 +637,21 @@ def location(self) -> Optional[str]:
def location(self, location: str) -> None:
self.properties.location = location
- def location_scheme(self) -> Optional[str]:
+ def location_scheme(self) -> str | None:
"""
Retrieves URI scheme of the document location.
Returns None when no location is set.
"""
return self.properties.location_scheme()
- def location_address(self) -> Optional[str]:
+ def location_address(self) -> str | None:
"""
Retrieves the full address from the document location URI.
Returns None when no location is set.
"""
return self.properties.location_address()
- def location_path(self, nonexist_ok=True) -> Optional[str]:
+ def location_path(self, nonexist_ok=True) -> str | None:
"""
Retrieves a path that's resolved to a pathname in the local file system.
To obtain the original value of the "path" part in the location string
@@ -593,6 +670,7 @@ class AnnotationProperties(MmifObject, MutableMapping[str, T]):
:param mmif_obj: the JSON data that defines the properties
"""
+ _unnamed_attributes: dict
def __delitem__(self, key: str) -> None:
key = self._canonical(key)
@@ -646,8 +724,8 @@ def __setitem__(self, key: str, value) -> None:
DeprecationWarning)
super().__setitem__(canonical, value)
- def __init__(self, mmif_obj: Optional[Union[bytes, str, dict]] = None,
- parent: Optional['Annotation'] = None, *_) -> None:
+ def __init__(self, mmif_obj: bytes | str | dict | None = None,
+ parent: 'Annotation | None' = None, *_) -> None:
self.id: str = ''
# any individual at_type (subclassing this class) can have its own set of required attributes
self._required_attributes = ["id"]
@@ -673,7 +751,7 @@ def _deserialize(self, input_dict: dict) -> None:
"""
canonical_dict = {}
# canonical name -> the deprecated spelling it was read from
- from_alias = {}
+ from_alias: dict = {}
for key, value in input_dict.items():
canonical = self._canonical(key)
if canonical in canonical_dict and (canonical != key or canonical in from_alias):
@@ -697,8 +775,8 @@ class DocumentProperties(AnnotationProperties):
:param mmif_obj: the JSON data that defines the properties
"""
- def __init__(self, mmif_obj: Optional[Union[bytes, str, dict]] = None,
- parent: Optional['Annotation'] = None, *_) -> None:
+ def __init__(self, mmif_obj: bytes | str | dict | None = None,
+ parent: 'Annotation | None' = None, *_) -> None:
self.mime: str = ''
# note the trailing underscore here. I wanted to use the name `location`
# for @property in this class and `Document` class, so had to use a diff
@@ -741,7 +819,7 @@ def text_value(self, s: str) -> None:
self.text.value = s
@property
- def location(self) -> Optional[str]:
+ def location(self) -> str | None:
"""
``location`` property must be a legitimate URI. That is, should the document be a local file
then the file:// scheme must be used.
@@ -757,7 +835,7 @@ def location(self, location: str) -> None:
else:
self.location_ = location
- def location_scheme(self) -> Optional[str]:
+ def location_scheme(self) -> str | None:
"""
Retrieves URI scheme of the document location.
Returns None when no location is set.
@@ -766,7 +844,7 @@ def location_scheme(self) -> Optional[str]:
return None
return urlparse(self.location).scheme
- def location_address(self) -> Optional[str]:
+ def location_address(self) -> str | None:
"""
Retrieves the full address from the document location URI.
Returns None when no location is set.
@@ -779,16 +857,19 @@ def location_address(self) -> Optional[str]:
else:
return "".join((parsed_location.netloc, parsed_location.path))
- def location_path(self) -> Optional[str]:
+ def location_path(self) -> str | None:
"""
.. deprecated:: 1.0.2
- Will be removed in 2.0.0.
- Use :meth:`location_path_resolved` instead.
+ Will be removed in 2.0.0. Use :meth:`location_path_resolved`
+ instead.
"""
- warnings.warn('location_path() is deprecated. Use location_path_resolved() instead.', DeprecationWarning)
+ warnings.warn('DocumentProperties.location_path() is deprecated and '
+ 'will be removed in 2.0.0. Use location_path_resolved()'
+ ' instead.',
+ DeprecationWarning)
return self.location_path_resolved()
- def location_path_resolved(self, nonexist_ok=True) -> Optional[str]:
+ def location_path_resolved(self, nonexist_ok=True) -> str | None:
"""
Retrieves only path name of the document location (hostname is ignored),
and then try to resolve the path name in the local file system.
@@ -812,7 +893,7 @@ def location_path_resolved(self, nonexist_ok=True) -> Optional[str]:
else:
return p
- def location_path_literal(self) -> Optional[str]:
+ def location_path_literal(self) -> str | None:
"""
Retrieves only path name of the document location (hostname is ignored).
Returns None when no location is set.
@@ -824,7 +905,7 @@ def location_path_literal(self) -> Optional[str]:
class Text(MmifObject):
- def __init__(self, text_obj: Optional[Union[bytes, str, dict]] = None, *_) -> None:
+ def __init__(self, text_obj: bytes | str | dict | None = None, *_) -> None:
self._value: str = ''
self._language: str = ''
self.disallow_additional_properties()
@@ -847,4 +928,3 @@ def value(self) -> str:
@value.setter
def value(self, s: str) -> None:
self._value = s
-
diff --git a/mmif/serialize/mmif.py b/mmif/serialize/mmif.py
index 3b4f54a9..632e1c61 100644
--- a/mmif/serialize/mmif.py
+++ b/mmif/serialize/mmif.py
@@ -14,8 +14,9 @@
import math
import warnings
from collections import defaultdict
+from collections.abc import Iterable, Iterator
from datetime import datetime, timezone
-from typing import Any, List, Union, Optional, Dict, cast, Iterable, Iterator
+from typing import cast
from urllib.parse import urlparse
import jsonschema.validators
@@ -23,7 +24,8 @@
import mmif
from mmif import ThingTypesBase
from mmif.serialize.annotation import Annotation, Document
-from mmif.serialize.model import MmifObject, DataList
+from mmif.serialize.model import DataList, MmifObject, MmifObjectEncoder
+from mmif.serialize.producer import ProducerContext
from mmif.serialize.view import View
from mmif.vocabulary import AnnotationTypes, DocumentTypesBase
@@ -41,7 +43,7 @@ class MmifMetadata(MmifObject):
:param metadata_obj: the JSON data
"""
- def __init__(self, metadata_obj: Optional[Union[bytes, str, dict]] = None, *_) -> None:
+ def __init__(self, metadata_obj: bytes | str | dict | None = None, *_) -> None:
# TODO (krim @ 10/7/20): there could be a better name and a better way to give a value to this
self.mmif: str = f"http://mmif.clams.ai/{mmif.__specver__}"
self._required_attributes = ["mmif"]
@@ -53,9 +55,9 @@ class DocumentsList(DataList[Document]):
DocumentsList object that implements :class:`mmif.serialize.model.DataList`
for :class:`mmif.serialize.document.Document`.
"""
- _items: Dict[str, Document]
+ _items: dict[str, Document]
- def _deserialize(self, input_list: list) -> None: # pytype: disable=signature-mismatch
+ def _deserialize(self, input_list: list) -> None: # type: ignore[override]
"""
Extends base ``_deserialize`` method to initialize ``items`` as a dict from
document IDs to :class:`mmif.serialize.document.Document` objects.
@@ -89,14 +91,14 @@ class ViewsList(DataList[View]):
ViewsList object that implements :class:`mmif.serialize.model.DataList`
for :class:`mmif.serialize.view.View`.
"""
- _items: Dict[str, View]
+ _items: dict[str, View]
- def __init__(self, mmif_obj: Optional[Union[bytes, str, list]] = None, parent_mmif=None, *_):
+ def __init__(self, mmif_obj: bytes | str | list | None = None, parent_mmif=None, *_):
self._parent_mmif = parent_mmif
self.reserved_names.update(("_parent_mmif", "_id_counts"))
super().__init__(mmif_obj)
- def _deserialize(self, input_list: list) -> None: # pytype: disable=signature-mismatch
+ def _deserialize(self, input_list: list) -> None: # type: ignore[override]
"""
Extends base ``_deserialize`` method to initialize ``items`` as a dict from
view IDs to :class:`mmif.serialize.view.View` objects.
@@ -125,7 +127,7 @@ def append(self, value: View, overwrite=False) -> None:
"""
super()._append_with_key(value.id, value, overwrite)
- def get_last_contentful_view(self) -> Optional[View]:
+ def get_last_contentful_view(self) -> View | None:
"""
Returns the last view that is contentful, i.e., has at least one
annotation (see :meth:`~mmif.serialize.view.View.has_content`).
@@ -137,15 +139,22 @@ def get_last_contentful_view(self) -> Optional[View]:
return view
return None
- def get_last_view(self) -> Optional[View]:
+ def get_last_view(self) -> View | None:
"""
Returns the last view appended.
"""
if self._items:
return self._items[list(self._items.keys())[-1]]
-
- def get_last(self) -> Optional[View]:
- warnings.warn('get_last() is deprecated, use get_last_contentful_view() instead.', DeprecationWarning)
+ return None
+
+ def get_last(self) -> View | None:
+ """
+ .. deprecated:: 1.0.18
+ Will be removed in 2.0.0. Use :meth:`get_last_contentful_view` instead.
+ """
+ warnings.warn('get_last() is deprecated and will be removed in 2.0.0.'
+ ' Use get_last_contentful_view() instead.',
+ DeprecationWarning)
return self.get_last_contentful_view()
@@ -192,10 +201,13 @@ class Mmif(MmifObject):
'mime', 'location', 'location_', 'text',
})
- def __init__(self, mmif_obj: Optional[Union[bytes, str, dict]] = None, *, validate: bool = True) -> None:
+ def __init__(self, mmif_obj: bytes | str | dict | None = None, *, validate: bool = True) -> None:
self.metadata: MmifMetadata = MmifMetadata()
self.documents: DocumentsList = DocumentsList()
self.views: ViewsList = ViewsList()
+ # the write mode, see `set_producer()`
+ self._producer: ProducerContext | None = None
+ self.reserved_names.add('_producer')
if validate and mmif_obj is not None:
self.validate(mmif_obj)
self.disallow_additional_properties()
@@ -208,7 +220,7 @@ def __init__(self, mmif_obj: Optional[Union[bytes, str, dict]] = None, *, valida
super().__init__(mmif_obj)
@staticmethod
- def validate(json_str: Union[bytes, str, dict]) -> None:
+ def validate(json_str: bytes | str | dict) -> None:
"""
Validates a MMIF JSON object against the MMIF Schema.
Note that this method operates before processing by MmifObject._load_str,
@@ -233,69 +245,220 @@ def serialize(self,
include_context: bool = True,
*,
sanitize: bool = False,
- autogenerate_capital_annotations: bool = True,
- factor_out_shared_properties: bool = True) -> str:
+ autogenerate_capital_annotations: bool | None = None,
+ factor_out_shared_properties: bool | None = None) -> str:
"""
Serializes the MMIF object to a JSON string.
+ The output is built once, and capital ``Annotation`` records,
+ factored properties, and sanitization apply to that output only. The
+ in-memory ``Mmif`` object itself never change, so calling this
+ method repeatedly gives the same records. While a producer is set (see
+ :meth:`set_producer`), only the views of its run are finalized, and
+ ``autogenerate_capital_annotations`` and
+ ``factor_out_shared_properties`` have no effect.
+
:param pretty: If True, returns the string with indentation.
:param include_context: If False, excludes contextual attributes (e.g.
runtime timestamps) from serialization.
- :param sanitize: If True, performs some sanitization of before returning
- the JSON string. See :meth:`sanitize` for details.
- :param autogenerate_capital_annotations: If True, automatically convert
- any "pending" temporary properties from `Document` objects to
- `Annotation` objects. See :meth:`generate_capital_annotations` for
- details.
- :param factor_out_shared_properties: If True, lift properties shared
+ :param sanitize: If True, removes ``contains`` entries of types with
+ no annotations from the output, and validates the output against
+ the MMIF JSON schema.
+ :param autogenerate_capital_annotations: Deprecated and will be
+ removed in 2.0.0. Use :meth:`set_producer` instead. If True (the
+ default), write any "pending"
+ properties of `Document` objects as `Annotation` objects in the
+ output. See :meth:`generate_capital_annotations` for the rules.
+ :param factor_out_shared_properties: Deprecated and will be removed
+ in 2.0.0. Use :meth:`set_producer` instead. If True (the default),
+ lift properties shared
(same value) across all annotations of a type in a view into the
- view's ``contains`` metadata. See
- :meth:`factor_out_shared_properties` for details. Independent of
+ view's ``contains`` metadata in the output. See
+ :meth:`factor_out_shared_properties` for the rules. Independent of
``autogenerate_capital_annotations``, but runs after it (so
freshly-materialized pending properties are included) when both are
enabled.
:return: JSON string of the MMIF object.
"""
- if autogenerate_capital_annotations:
- self.generate_capital_annotations()
- if factor_out_shared_properties:
- self.factor_out_shared_properties()
- # sanitization should be done after `Annotation` annotations are generated
+ for name, value in (
+ ('autogenerate_capital_annotations', autogenerate_capital_annotations),
+ ('factor_out_shared_properties', factor_out_shared_properties)):
+ if value is not None:
+ warnings.warn(f'The {name} argument of serialize() is '
+ f'deprecated and will be removed in 2.0.0. '
+ f'Use set_producer() instead.',
+ DeprecationWarning, stacklevel=2)
+ output = self._serialize(include_context=include_context)
+ views_out = output['views']
+ if self._producer is not None:
+ self._producer._finalize_output(views_out, include_context)
+ else:
+ if autogenerate_capital_annotations is not False:
+ plan = self._plan_capital_annotations()
+ if plan:
+ warnings.warn('Writing document properties as Annotation '
+ 'objects in serialize() without a producer '
+ 'is deprecated and will be removed in '
+ '2.0.0. Use set_producer() instead.',
+ FutureWarning, stacklevel=2)
+ self._write_capital_annotations(plan, views_out, include_context)
+ if factor_out_shared_properties is not False:
+ self._factor_out_in_output(views_out)
+ # sanitization should be done after `Annotation` annotations are generated
+ if sanitize:
+ self._prune_contains_in_output(views_out)
+ serialized = json.dumps(output, indent=2 if pretty else None,
+ cls=MmifObjectEncoder)
if sanitize:
- self.sanitize()
- return super().serialize(pretty=pretty, include_context=include_context)
+ self.validate(serialized)
+ return serialized
+
+ def set_producer(self, signature: View) -> ProducerContext:
+ """
+ Turns on the write mode of this MMIF, for a producer that adds views
+ to it. Keep the serialization inside the ``with`` block::
+
+ with mmif_obj.set_producer(signature) as producer:
+ view = mmif_obj.new_view()
+ ...
+ output = mmif_obj.serialize()
+
+ Code that only reads, or re-saves without adding views, doesn't need
+ this. See :class:`mmif.serialize.producer.ProducerContext` for what
+ the mode does.
+
+ :param signature: a view whose metadata is signed by the producer,
+ copied to the views the mode writes
+ :return: the producer context; leaving its ``with`` block turns the
+ mode off
+ :raises RuntimeError: if a producer is already set
+ """
+ return ProducerContext(self, signature)
+
+ def _write_capital_annotations(self, plan: list[tuple[View, str, dict, bool]],
+ views_out: list[dict],
+ include_context: bool) -> None:
+ """
+ Writes capital ``Annotation`` records into the serialized views.
+
+ :param plan: the records, as returned by
+ :meth:`_plan_capital_annotations`
+ :param views_out: serialized views, changed in place
+ :param include_context: See :meth:`serialize` for details.
+ """
+ views_by_id = {view_out['id']: view_out for view_out in views_out}
+ last_nums = {}
+ for view, doc_id, props, in_place in plan:
+ view_out = views_by_id[view.id]
+ if in_place:
+ doc_out = next(ann_out for ann_out in view_out['annotations']
+ if ann_out['properties']['id'] == doc_id)
+ doc_out['properties'].update(props)
+ continue
+ record = Annotation()
+ record.at_type = AnnotationTypes.Annotation
+ prefix = record.at_type.get_prefix()
+ if view.id not in last_nums:
+ last_nums[view.id] = self._last_annotation_num(
+ view, view_out, prefix)
+ last_nums[view.id] += 1
+ record.id = f'{view.id}{self.id_delimiter}{prefix}_{last_nums[view.id]}'
+ for key, value in props.items():
+ record.add_property(key, value)
+ record.add_property('document', doc_id)
+ view_out['annotations'].append(
+ record._serialize(include_context=include_context))
+ view_out['metadata'].setdefault('contains', {}).setdefault(
+ str(record.at_type), {})
+
+ def _last_annotation_num(self, view: View, view_out: dict,
+ prefix: str) -> int:
+ """
+ Finds the largest number used in the IDs of a view's annotations with
+ the given prefix. Annotations added with an explicit ID are not
+ counted in ``View._id_counts``, so the serialized view is scanned too.
+
+ :param view: the view
+ :param view_out: the serialized view
+ :param prefix: an annotation ID prefix
+ :return: the largest number, 0 when there is none
+ """
+ last_num = view._id_counts.get(prefix, 0)
+ for ann_out in view_out['annotations']:
+ local_id = ann_out['properties']['id'].split(self.id_delimiter)[-1]
+ ann_prefix, _, num = local_id.rpartition('_')
+ if ann_prefix == prefix and num.isdigit():
+ last_num = max(last_num, int(num))
+ return last_num
+
+ def _factor_out_in_output(self, views_out: list[dict]) -> None:
+ """
+ Applies the rules of :meth:`factor_out_shared_properties` to the
+ serialized views.
+
+ :param views_out: serialized views, changed in place
+ """
+ for view_out in views_out:
+ props_by_type = defaultdict(list)
+ for ann_out in view_out['annotations']:
+ props_by_type[str(ann_out['@type'])].append(ann_out['properties'])
+ for at_type, props_list in props_by_type.items():
+ if len(props_list) < 2:
+ continue
+ shared = {key: value for key, value in props_list[0].items()
+ if key not in self._NON_FACTORABLE_PROPS
+ and all(key in props and props[key] == value
+ for props in props_list)}
+ if not shared:
+ continue
+ contain = view_out['metadata'].setdefault(
+ 'contains', {}).setdefault(at_type, {})
+ for key, value in shared.items():
+ contain[key] = value
+ for props in props_list:
+ del props[key]
+
+ @staticmethod
+ def _prune_contains_in_output(views_out: list[dict]) -> None:
+ """
+ Removes ``contains`` entries of types that have no annotations in a
+ serialized view. A view with an error or warnings drops an empty
+ ``contains``; any other view keeps it, as the schema requires.
+
+ :param views_out: serialized views, changed in place
+ """
+ for view_out in views_out:
+ metadata = view_out['metadata']
+ contains = metadata.get('contains', {})
+ existing_at_types = {str(ann_out['@type'])
+ for ann_out in view_out['annotations']}
+ for at_type in set(contains) - existing_at_types:
+ del contains[at_type]
+ if not contains and ('error' in metadata or 'warnings' in metadata):
+ metadata.pop('contains', None)
def _deserialize(self, input_dict: dict) -> None:
"""
Deserializes the MMIF JSON string into a Mmif object.
- After *regular* deserialization, this method will perform the following
- *special* handling of Annotation.properties that allows apps to access
- Annotation/Document properties that are not encoded in the objects
- themselves. This is to allow apps to access in a more intuitive way,
- without having too much hassle to iterate views and manually collect the properties.
-
- 1. This will read in existing *view*-scoped properties from *contains*
- metadata and attach them to the corresponding ``Annotation`` objects.
+ After *regular* deserialization, this method reads existing
+ ``Annotation`` typed annotations and copies the document-level
+ properties to the ``Document`` objects, in an ephemeral property dict,
+ so that apps don't need to iterate views to collect them.
- 1. This will read in existing ``Annotation`` typed annotations and
- attach the document-level properties to the ``Document`` objects,
- using an ephemeral property dict.
-
+ View-scoped properties in *contains* metadata are not copied. They
+ resolve through the parent view when an annotation is read.
"""
super()._deserialize(input_dict)
for view in self.views:
view._parent_mmif = self
for ann in view.get_annotations():
## for "capital" Annotation properties
- # first, distribute view-level defaults from `contains` metadata
- # to the Annotation objects as "ephemeral" properties
- ann._add_ephemeral_defaults(view.metadata.contains.get(ann.at_type, {}))
- # then, do the same to associated Document objects. Note that,
- # in a view, it is guaranteed that all Annotation objects are not duplicates
+ # copy the recorded properties onto the associated Document
+ # objects; view-level defaults resolve through the parent view
if ann.at_type == AnnotationTypes.Annotation:
doc_id = ann.get_property('document')
try:
- for prop_key, prop_value in ann.properties.items():
+ for prop_key, prop_value in ann._resolved_properties().items():
doc = cast(Document, self.__getitem__(doc_id))
if not isinstance(doc, Document):
raise KeyError
@@ -338,11 +501,35 @@ def _when_failed():
def generate_capital_annotations(self):
"""
Automatically convert any "pending" temporary properties from
- `Document` objects to `Annotation` objects . The generated `Annotation`
- objects are then added to the last `View` in the views lists.
+ `Document` objects to `Annotation` objects, and add them to the views
+ of this object. :meth:`serialize` applies the same rules to its output
+ without calling this method, and leaves the views unchanged.
See https://github.com/clamsproject/mmif-python/issues/226 for rationale
behind this behavior and discussion.
+
+ .. deprecated:: 1.7.0
+ Will be removed in 2.0.0. Use :meth:`set_producer` instead.
+ """
+ warnings.warn('generate_capital_annotations() is deprecated and will '
+ 'be removed in 2.0.0. Use set_producer() instead.',
+ DeprecationWarning, stacklevel=2)
+ for view, doc_id, props, in_place in self._plan_capital_annotations():
+ if in_place:
+ view[doc_id].properties.update(props)
+ else:
+ view.new_annotation(AnnotationTypes.Annotation,
+ **{**props, 'document': doc_id})
+
+ def _plan_capital_annotations(self) -> list[tuple[View, str, dict, bool]]:
+ """
+ Decides where the "pending" properties of `Document` objects are
+ recorded. Values that are already recorded for the document are left
+ out, and this method changes nothing.
+
+ :return: a list of (view to write to, document ID, properties to
+ record, whether the properties go into the document itself, which
+ is in that view, rather than into a new `Annotation`)
"""
# this view will be the default kitchen sink for all generated annotations.
# this can be an empty view, from an app that only adds document properties
@@ -350,64 +537,65 @@ def generate_capital_annotations(self):
if not v.has_error() and not v.has_warnings()), None)
# proceed only when there's at least one view
- if last_view:
-
- # this app name is used to check a view is generated by the "currently running" app.
- # knowing the currently running app is important so that properties of `Document` objects generated by the
- # current app can be properly recorded inside the `Document` objects (since they are "writable" to the
- # current app), instead of being recorded in a separate `Annotation` object.
- current_app = last_view.metadata.app
-
- # to avoid duplicate property recording, this will be populated with
- # existing Annotation objects from all existing views
- existing_anns = defaultdict(lambda: defaultdict(dict))
- # ideally, if we can "de-duplicate" props at `add_property()` time, that'd be more efficient,
- # but that is impossible without looking for the target `document` across other views and top documents list
-
- # new properties to record in the current serialization call
- anns_to_write = defaultdict(dict)
- for view in self.views:
- for ann in view.get_annotations(AnnotationTypes.Annotation):
- # view-level `contains` defaults are exposed on each
- # annotation as ephemeral properties; in MMIF made by old
- # SDKs, the id may be in the short form (no view prefix)
- existing_anns[ann.get_property('document')].update(
- ann.properties.items())
- for doc in view.get_documents():
- anns_to_write[doc.id].update(doc._props_pending.items())
- for doc in self.documents:
+ if last_view is None:
+ return []
+
+ # this app name is used to check a view is generated by the "currently running" app.
+ # knowing the currently running app is important so that properties of `Document` objects generated by the
+ # current app can be properly recorded inside the `Document` objects (since they are "writable" to the
+ # current app), instead of being recorded in a separate `Annotation` object.
+ current_app = last_view.metadata.app
+
+ # to avoid duplicate property recording, this will be populated with
+ # existing Annotation objects from all existing views
+ existing_anns: defaultdict[str, defaultdict[str, dict]] = defaultdict(
+ lambda: defaultdict(dict))
+ # ideally, if we can "de-duplicate" props at `add_property()` time, that'd be more efficient,
+ # but that is impossible without looking for the target `document` across other views and top documents list
+
+ # new properties to record in the current serialization call
+ anns_to_write: defaultdict[str, dict] = defaultdict(dict)
+ for view in self.views:
+ for ann in view.get_annotations(AnnotationTypes.Annotation):
+ # values recorded at view level (`contains`) count too;
+ # in MMIF made by old SDKs, the id may be in the short
+ # form (no view prefix)
+ existing_anns[ann.get_property('document')].update(
+ ann._resolved_properties())
+ for doc in view.get_documents():
anns_to_write[doc.id].update(doc._props_pending.items())
- # additional iteration of views, to find a proper view to add the
- # generated annotations. If none found, use the last view as the kitchen sink
- last_view_for_docs = defaultdict(lambda: last_view)
- doc_ids = set(anns_to_write.keys())
- for doc_id in doc_ids:
- if len(last_view.annotations) == 0:
- # meaning, this new app didn't generate any annotation except for these document properties
- # thus, we should add capital annotations to the last (empty) view
- last_view_for_docs[doc_id] = last_view
+ for doc in self.documents:
+ anns_to_write[doc.id].update(doc._props_pending.items())
+ # additional iteration of views, to find a proper view to add the
+ # generated annotations. If none found, use the last view as the kitchen sink
+ last_view_for_docs = defaultdict(lambda: last_view)
+ doc_ids = set(anns_to_write.keys())
+ for doc_id in doc_ids:
+ if len(last_view.annotations) == 0:
+ # meaning, this new app didn't generate any annotation except for these document properties
+ # thus, we should add capital annotations to the last (empty) view
+ last_view_for_docs[doc_id] = last_view
+ break
+ for view in reversed(self.views):
+ if any(view.get_annotations(document=doc_id)):
+ last_view_for_docs[doc_id] = view
break
- for view in reversed(self.views):
- if any(view.get_annotations(document=doc_id)):
- last_view_for_docs[doc_id] = view
- break
- for doc_id, found_props in anns_to_write.items():
- # ignore the "empty" id property from temporary dict
- # `id` is "required" attribute for `AnnotationProperty` class
- # thus will always be present in `props` dict as a key with emtpy value
- # also ignore duplicate k-v pairs
- props = {}
- for k, v in found_props.items():
- if k != 'id' and existing_anns[doc_id][k] != v:
- props[k] = v
- if props:
- view_to_write = last_view_for_docs[doc_id]
- if view_to_write.metadata.app == current_app and doc_id in view_to_write.annotations:
- view_to_write[doc_id].properties.update(props)
- else:
- props['document'] = doc_id
- view_to_write.new_annotation(AnnotationTypes.Annotation, **props)
-
+ plan = []
+ for doc_id, found_props in anns_to_write.items():
+ # ignore the "empty" id property from temporary dict
+ # `id` is "required" attribute for `AnnotationProperty` class
+ # thus will always be present in `props` dict as a key with emtpy value
+ # also ignore duplicate k-v pairs
+ props = {}
+ for k, v in found_props.items():
+ if k != 'id' and existing_anns[doc_id][k] != v:
+ props[k] = v
+ if props:
+ view_to_write = last_view_for_docs[doc_id]
+ in_place = (view_to_write.metadata.app == current_app
+ and doc_id in view_to_write.annotations)
+ plan.append((view_to_write, doc_id, props, in_place))
+ return plan
def factor_out_shared_properties(self):
"""
@@ -420,10 +608,17 @@ def factor_out_shared_properties(self):
**every** annotation of a given ``@type`` in the view, there are at
least two such annotations (so the value is genuinely shared), and the
key is not in the :attr:`_NON_FACTORABLE_PROPS` protection list.
- Factored values are mirrored into each annotation's ephemeral props at
- deserialization time, so in-memory access is unchanged, while the
- serialized output carries the value once, in ``contains``.
+ Factored values resolve through the view at read time, so
+ ``get_property`` is unchanged, while the view carries the value once,
+ in ``contains``. :meth:`serialize` applies the same rules to its
+ output without calling this method, and leaves the views unchanged.
+
+ .. deprecated:: 1.7.0
+ Will be removed in 2.0.0. Use :meth:`set_producer` instead.
"""
+ warnings.warn('factor_out_shared_properties() is deprecated and will '
+ 'be removed in 2.0.0. Use set_producer() instead.',
+ DeprecationWarning, stacklevel=2)
for view in self.views:
by_type = defaultdict(list)
for ann in view.get_annotations():
@@ -448,17 +643,25 @@ def factor_out_shared_properties(self):
contain[key] = value
for a in anns:
del a.properties[key]
- a._props_ephemeral[key] = value
def sanitize(self):
"""
- Sanitizes a Mmif object by running some safeguards.
- Concretely, it performs the following before returning the JSON string.
-
- #. validating output using built-in MMIF jsonschema
+ Sanitizes this Mmif object by running some safeguards.
+ Concretely, it performs the following.
+
#. remove non-existing annotation types from ``contains`` metadata
-
+ #. validating the serialized object using built-in MMIF jsonschema
+
+ ``serialize(sanitize=True)`` applies the same safeguards to its output
+ without calling this method, and leaves the views unchanged.
+
+ .. deprecated:: 1.7.0
+ Will be removed in 2.0.0. Use ``serialize(sanitize=True)``
+ instead.
"""
+ warnings.warn('sanitize() is deprecated and will be removed in '
+ '2.0.0. Use serialize(sanitize=True) instead.',
+ DeprecationWarning, stacklevel=2)
for view in self.views:
existing_at_types = set(annotation.at_type for annotation in view.annotations)
to_pop = set()
@@ -467,8 +670,7 @@ def sanitize(self):
to_pop.add(contains_at_type)
for key in to_pop:
view.metadata.contains.pop(key)
- serialized = self.serialize()
- self.validate(serialized)
+ self.validate(self.serialize())
def new_view_id(self) -> str:
"""
@@ -532,20 +734,20 @@ def add_document(self, document: Document, overwrite=False) -> None:
raise KeyError(f"{document.id} already exists in the views list. ")
self.documents.append(document, overwrite)
- def get_documents_in_view(self, vid: Optional[str] = None) -> List[Document]:
+ def get_documents_in_view(self, vid: str | None = None) -> list[Document]:
"""
Method to get all documents object queries by a view id.
:param vid: the source view ID to search for
:return: a list of documents matching the requested source view ID, or an empty list if the view not found
"""
- view = self.views.get(vid)
+ view = None if vid is None else self.views.get(vid)
if view is not None:
return view.get_documents()
else:
return []
- def get_documents_by_type(self, doc_type: DocumentTypesBase) -> List[Document]:
+ def get_documents_by_type(self, doc_type: DocumentTypesBase) -> list[Document]:
"""
Method to get all documents where the type matches a particular document type, which should be one of the CLAMS document types.
@@ -559,7 +761,7 @@ def get_documents_by_type(self, doc_type: DocumentTypesBase) -> List[Document]:
docs.extend([document for document in self.documents if document.is_type(doc_type)])
return docs
- def get_documents_by_app(self, app_id: str) -> List[Document]:
+ def get_documents_by_app(self, app_id: str) -> list[Document]:
"""
Method to get all documents object queries by its originated app name.
@@ -572,7 +774,7 @@ def get_documents_by_app(self, app_id: str) -> List[Document]:
docs.extend(view.get_documents())
return docs
- def get_documents_by_property(self, prop_key: str, prop_value: str) -> List[Document]:
+ def get_documents_by_property(self, prop_key: str, prop_value: str) -> list[Document]:
"""
Method to retrieve documents by an arbitrary key-value pair in the document properties objects.
@@ -588,7 +790,7 @@ def get_documents_by_property(self, prop_key: str, prop_value: str) -> List[Docu
docs.extend([document for document in self.documents if document[prop_key] == prop_value])
return docs
- def get_documents_locations(self, m_type: Union[DocumentTypesBase, str], path_only=False) -> List[Union[str, None]]:
+ def get_documents_locations(self, m_type: DocumentTypesBase | str, path_only=False) -> list[str | None]:
"""
This method returns the file paths of documents of given type.
Only top-level documents have locations, so we only check them.
@@ -603,7 +805,7 @@ def get_documents_locations(self, m_type: Union[DocumentTypesBase, str], path_on
else:
return [doc.location for doc in docs]
- def get_document_location(self, m_type: Union[DocumentTypesBase, str], path_only=False) -> Optional[str]:
+ def get_document_location(self, m_type: DocumentTypesBase | str, path_only=False) -> str | None:
"""
Method to get the location of *first* document of given type.
@@ -618,8 +820,7 @@ def get_document_location(self, m_type: Union[DocumentTypesBase, str], path_only
def get_document_by_id(self, doc_id: str) -> Document:
"""
.. deprecated:: 1.1.0
- Will be removed in 2.0.0.
- Use general ``__getitem__()`` method instead, e.g., ``mmif[doc_id]``.
+ Will be removed in 2.0.0. Use ``mmif[doc_id]`` instead.
Finds a Document object with the given ID.
@@ -627,10 +828,9 @@ def get_document_by_id(self, doc_id: str) -> Document:
:return: a reference to the corresponding document, if it exists
:raises KeyError: if there is no corresponding document
"""
- warnings.warn(
- "Mmif.get_document_by_id() is deprecated, use mmif[doc_id] instead.",
- DeprecationWarning
- )
+ warnings.warn('Mmif.get_document_by_id() is deprecated and will be '
+ 'removed in 2.0.0. Use mmif[doc_id] instead.',
+ DeprecationWarning)
doc_found = self.__getitem__(doc_id)
if not isinstance(doc_found, Document):
raise KeyError(f"Document with ID {doc_id} not found in the MMIF object.")
@@ -639,8 +839,7 @@ def get_document_by_id(self, doc_id: str) -> Document:
def get_view_by_id(self, view_id: str) -> View:
"""
.. deprecated:: 1.1.0
- Will be removed in 2.0.0.
- Use general ``__getitem__()`` method instead, e.g., ``mmif[view_id]``.
+ Will be removed in 2.0.0. Use ``mmif[view_id]`` instead.
Finds a View object with the given ID.
@@ -648,16 +847,15 @@ def get_view_by_id(self, view_id: str) -> View:
:return: a reference to the corresponding view, if it exists
:raises Exception: if there is no corresponding view
"""
- warnings.warn(
- "Mmif.get_view_by_id() is deprecated, use mmif[view_id] instead.",
- DeprecationWarning
- )
+ warnings.warn('get_view_by_id() is deprecated and will be removed in '
+ '2.0.0. Use mmif[view_id] instead.',
+ DeprecationWarning)
view_found = self.__getitem__(view_id)
if not isinstance(view_found, View):
raise KeyError(f"View with ID {view_id} not found in the MMIF object.")
return cast(View, view_found)
- def get_alignments(self, at_type1: Union[str, ThingTypesBase], at_type2: Union[str, ThingTypesBase]) -> Dict[str, List[Annotation]]:
+ def get_alignments(self, at_type1: str | ThingTypesBase, at_type2: str | ThingTypesBase) -> dict[str, list[Annotation]]:
"""
Finds views where alignments between two given annotation types occurred.
@@ -669,7 +867,7 @@ def get_alignments(self, at_type1: Union[str, ThingTypesBase], at_type2: Union[s
at_type1, at_type2 = [ThingTypesBase.from_str(x) if isinstance(x, str) else x for x in (at_type1, at_type2)]
assert at_type1 != at_type2, f"Alignment must be between two different types, given only one: {at_type1}"
for alignment_view in self.get_all_views_contain(AnnotationTypes.Alignment):
- alignments = []
+ alignments: list[Annotation] = []
contains_meta = alignment_view.metadata.contains[AnnotationTypes.Alignment]
if 'sourceType' in contains_meta and 'targetType' in contains_meta:
aligned_types = [ThingTypesBase.from_str(x)
@@ -678,19 +876,19 @@ def get_alignments(self, at_type1: Union[str, ThingTypesBase], at_type2: Union[s
alignments.extend(alignment_view.annotations)
else:
for alignment in alignment_view.get_annotations(AnnotationTypes.Alignment):
- aligned_types = set()
+ unique_types = set()
for ann_id in [alignment['target'], alignment['source']]:
ann_id = cast(str, ann_id)
aligned_type = cast(Annotation, self.__getitem__(ann_id)).at_type
- aligned_types.add(aligned_type)
- aligned_types = list(aligned_types) # because membership check for sets also checks hash() values
+ unique_types.add(aligned_type)
+ aligned_types = list(unique_types) # because membership check for sets also checks hash() values
if len(aligned_types) == 2 and at_type1 in aligned_types and at_type2 in aligned_types:
alignments.append(alignment)
if len(alignments) > 0:
v_and_a[alignment_view.id] = alignments
return v_and_a
- def get_views_for_document(self, doc_id: str) -> List[View]:
+ def get_views_for_document(self, doc_id: str) -> list[View]:
"""
Returns the list of all views that have annotations anchored on a particular document.
Note that when the document is inside a view (generated during the workflow's running),
@@ -706,7 +904,7 @@ def get_views_for_document(self, doc_id: str) -> List[View]:
pass
return views
- def get_all_views_with_error(self) -> List[View]:
+ def get_all_views_with_error(self) -> list[View]:
"""
Returns the list of all views in the MMIF that have errors.
@@ -716,7 +914,7 @@ def get_all_views_with_error(self) -> List[View]:
get_views_with_error = get_all_views_with_error
- def get_all_views_contain(self, *at_types: Union[ThingTypesBase, str]) -> List[View]:
+ def get_all_views_contain(self, *at_types: ThingTypesBase | str) -> list[View]:
"""
Returns the list of all views in the MMIF if given types
are present in that view's 'contains' metadata.
@@ -729,7 +927,7 @@ def get_all_views_contain(self, *at_types: Union[ThingTypesBase, str]) -> List[V
get_views_contain = get_all_views_contain
- def get_view_with_error(self) -> Optional[View]:
+ def get_view_with_error(self) -> View | None:
"""
Returns the last view appended that contains an error.
@@ -740,7 +938,7 @@ def get_view_with_error(self) -> Optional[View]:
return view
return None
- def get_last_error(self) -> Optional[str]:
+ def get_last_error(self) -> str | None:
"""
Returns the last error message found in the views.
@@ -749,7 +947,7 @@ def get_last_error(self) -> Optional[str]:
v = self.get_view_with_error()
return v.get_error() if v is not None else None
- def get_view_contains(self, at_types: Union[ThingTypesBase, str, List[Union[str, ThingTypesBase]]]) -> Optional[View]:
+ def get_view_contains(self, at_types: ThingTypesBase | str | list[str | ThingTypesBase]) -> View | None:
"""
Returns the last view appended that contains the given
types in its 'contains' metadata.
@@ -779,7 +977,7 @@ def _full_app_id(app_id: str) -> str:
return app_id
return f"{app_directory_baseurl}/{app_id.strip('/')}"
- def get_all_views_by_app(self, *app_ids: str) -> List[View]:
+ def get_all_views_by_app(self, *app_ids: str) -> list[View]:
"""
Returns the list of all views produced by any of the given apps.
@@ -800,7 +998,7 @@ def get_all_views_by_app(self, *app_ids: str) -> List[View]:
get_views_by_app = get_all_views_by_app
- def get_view_by_app(self, *app_ids: str) -> Optional[View]:
+ def get_view_by_app(self, *app_ids: str) -> View | None:
"""
Returns the last view appended that was produced by any of the given
apps. App IDs are matched as in :meth:`get_all_views_by_app`.
@@ -811,7 +1009,7 @@ def get_view_by_app(self, *app_ids: str) -> Optional[View]:
matches = self.get_all_views_by_app(*app_ids)
return matches[-1] if matches else None
- def get_all_views_by_apptag(self, *tags: str) -> List[View]:
+ def get_all_views_by_apptag(self, *tags: str) -> list[View]:
"""
Returns the list of all views whose ``appTags`` metadata contains
any of the given tags.
@@ -825,7 +1023,7 @@ def get_all_views_by_apptag(self, *tags: str) -> List[View]:
get_views_by_apptag = get_all_views_by_apptag
- def get_view_by_apptag(self, *tags: str) -> Optional[View]:
+ def get_view_by_apptag(self, *tags: str) -> View | None:
"""
Returns the last view appended whose ``appTags`` metadata contains
any of the given tags.
@@ -837,7 +1035,7 @@ def get_view_by_apptag(self, *tags: str) -> Optional[View]:
return matches[-1] if matches else None
def get_annotations_of_labels(
- self, at_type: Union[ThingTypesBase, str],
+ self, at_type: ThingTypesBase | str,
include_labels: Iterable[str] = (),
exclude_labels: Iterable[str] = ()) -> Iterator[Annotation]:
"""
@@ -873,7 +1071,7 @@ def get_annotations_of_labels(
continue
yield ann
- def _is_in_time_range(self, ann: Annotation, range_s: Union[int, float], range_e: Union[int, float]) -> bool:
+ def _is_in_time_range(self, ann: Annotation, range_s: int | float, range_e: int | float) -> bool:
"""
Checks if the annotation is anchored within the given time range. Any overlap is considered included.
@@ -886,8 +1084,8 @@ def _is_in_time_range(self, ann: Annotation, range_s: Union[int, float], range_e
ann_s, ann_e = self.get_start(ann), self.get_end(ann)
return (ann_s < range_s < ann_e) or (ann_s < range_e < ann_e) or (ann_s > range_s and ann_e < range_e)
- def get_annotations_between_time(self, start: Union[int, float], end: Union[int, float], time_unit: str = "ms",
- at_types: List[Union[ThingTypesBase, str]] = []) -> Iterator[Annotation]:
+ def get_annotations_between_time(self, start: int | float, end: int | float, time_unit: str = "ms",
+ at_types: list[ThingTypesBase | str] = []) -> Iterator[Annotation]:
"""
Finds annotations that are anchored between the given time points.
@@ -909,8 +1107,8 @@ def get_annotations_between_time(self, start: Union[int, float], end: Union[int,
for view in self.get_all_views_contain(AnnotationTypes.TimeFrame) + self.get_all_views_contain(AnnotationTypes.TimePoint):
time_unit_in_view = view.metadata.contains.get(AnnotationTypes.TimeFrame)["timeUnit"]
- start_time = convert(start, time_unit, time_unit_in_view, 1)
- end_time = convert(end, time_unit, time_unit_in_view, 1)
+ start_time = cast(int | float, convert(start, time_unit, time_unit_in_view, 1))
+ end_time = cast(int | float, convert(end, time_unit, time_unit_in_view, 1))
for ann in view.get_annotations():
if ann.at_type in (AnnotationTypes.TimeFrame, AnnotationTypes.TimePoint) and self._is_in_time_range(ann, start_time, end_time):
time_anchors_in_range.append(ann)
@@ -922,7 +1120,7 @@ def get_annotations_between_time(self, start: Union[int, float], end: Union[int,
if not uniq_types or aligned.at_type in uniq_types:
yield aligned
- def _get_linear_anchor_point(self, ann: Annotation, targets_sorted=False, start: bool = True) -> Union[int, float]:
+ def _get_linear_anchor_point(self, ann: Annotation, targets_sorted=False, start: bool = True) -> int | float:
# TODO (krim @ 2/5/24): Update the return type once timeunits are unified to `ms` as integers (https://github.com/clamsproject/mmif/issues/192)
"""
Retrieves the anchor point of the annotation. Currently, this method only supports linear anchors,
@@ -934,7 +1132,7 @@ def _get_linear_anchor_point(self, ann: Annotation, targets_sorted=False, start:
:return: the anchor point of the annotation. 1d for linear regions (time, text)
"""
props = ann.properties
- if 'timePoint' in props:
+ if 'timePoint' in ann:
return ann.get_property('timePoint')
elif 'targets' in props:
if 'start' in props or 'end' in props:
@@ -945,28 +1143,28 @@ def _get_linear_anchor_point(self, ann: Annotation, targets_sorted=False, start:
point = math.inf if start else -1
comp = min if start else max
for target_id in ann.get_property('targets'):
- point = comp(point, self._get_linear_anchor_point(self.__getitem__(target_id), start=start))
+ point = comp(point, self._get_linear_anchor_point(cast(Annotation, self.__getitem__(target_id)), start=start))
return point
target_id = ann.get_property('targets')[0 if start else -1]
- return self._get_linear_anchor_point(self.__getitem__(target_id), start=start)
+ return self._get_linear_anchor_point(cast(Annotation, self.__getitem__(target_id)), start=start)
elif (start and 'start' in props) or (not start and 'end' in props):
return ann.get_property('start' if start else 'end')
else:
raise ValueError(f"{ann.id} ({ann.at_type}) does not have a valid anchor point. Is it a valid 'Region' type?")
- def get_start(self, annotation: Annotation) -> Union[int, float]:
+ def get_start(self, annotation: Annotation) -> int | float:
"""
An alias to `get_anchor_point` method with `start=True`.
"""
return self._get_linear_anchor_point(annotation, start=True)
- def get_end(self, annotation: Annotation) -> Union[int, float]:
+ def get_end(self, annotation: Annotation) -> int | float:
"""
An alias to `get_anchor_point` method with `start=False`.
"""
return self._get_linear_anchor_point(annotation, start=False)
- def __getitem__(self, item: str) -> Union[Document, View, Annotation, MmifMetadata]:
+ def __getitem__(self, item: str) -> Document | View | Annotation | MmifMetadata:
"""
High-level string-based access to MMIF objects by their IDs.
@@ -1034,7 +1232,7 @@ def __getitem__(self, item: str) -> Union[Document, View, Annotation, MmifMetada
else:
# search for document first, then views
# raise KeyError if nothing is found
- ret = self.documents._items.get(item)
+ ret: Document | View | None = self.documents._items.get(item)
if ret is None:
ret = self.views._items.get(item)
if ret is None:
diff --git a/mmif/serialize/model.py b/mmif/serialize/model.py
index 827366f3..33ed3790 100644
--- a/mmif/serialize/model.py
+++ b/mmif/serialize/model.py
@@ -23,19 +23,19 @@
import json
import warnings
-from datetime import datetime
-from typing import Union, Any, Dict, Optional, TypeVar, Generic, Generator, Iterator, Type, Set, ClassVar, List
+from collections.abc import Generator, Iterator
+from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, Union
T = TypeVar('T')
S = TypeVar('S')
-PRMTV_TYPES: Type = Union[str, int, float, bool, None]
+PRMTV_TYPES: TypeAlias = Union[str, int, float, bool, None] # noqa: UP007
__all__ = [
- 'MmifObject',
- 'MmifObjectEncoder',
- 'DataList',
+ 'PRMTV_TYPES',
'DataDict',
- 'PRMTV_TYPES'
+ 'DataList',
+ 'MmifObject',
+ 'MmifObjectEncoder'
]
@@ -96,7 +96,7 @@ class MmifObject(object):
# these are the reserved names that cannot be used as attribute names, and
# they won't be serialized
- reserved_names: Set[str] = {
+ reserved_names: set[str] = {
'reserved_names',
'_unnamed_attributes',
'_attribute_classes',
@@ -104,12 +104,13 @@ class MmifObject(object):
'_exclude_from_diff',
'_contextual_attributes'
}
- _unnamed_attributes: Optional[dict]
- _exclude_from_diff: Set[str]
- _contextual_attributes: Set[str]
- _attribute_classes: Dict[str, Type] = {} # Mapping: str -> Type
+ _unnamed_attributes: dict | None
+ _required_attributes: list[str]
+ _exclude_from_diff: set[str]
+ _contextual_attributes: set[str]
+ _attribute_classes: dict[str, type] = {} # Mapping: str -> Type
- def __init__(self, mmif_obj: Optional[Union[bytes, str, dict]] = None, *_) -> None:
+ def __init__(self, mmif_obj: bytes | str | dict | None = None, *_) -> None:
if isinstance(mmif_obj, bytes):
mmif_obj = mmif_obj.decode('utf8')
if not hasattr(self, '_required_attributes'):
@@ -159,7 +160,7 @@ def set_additional_property(self, key: str, value: Any) -> None:
stacklevel=2)
self[attr] = value
return
- self._unnamed_attributes[key] = value # pytype: disable=unsupported-operands
+ self._unnamed_attributes[key] = value
def _named_attributes(self) -> Generator[str, None, None]:
"""
@@ -183,7 +184,7 @@ def serialize(self, pretty: bool = False, include_context: bool = True) -> str:
"""
return json.dumps(self._serialize(include_context=include_context), indent=2 if pretty else None, cls=MmifObjectEncoder)
- def _serialize(self, alt_container: Optional[Dict] = None, include_context: bool = True) -> dict:
+ def _serialize(self, alt_container: dict | None = None, include_context: bool = True) -> dict:
"""
Maps a MMIF object to a plain python dict object,
rewriting internal keys that start with '_' to
@@ -198,8 +199,9 @@ def _serialize(self, alt_container: Optional[Dict] = None, include_context: bool
"""
container = alt_container if alt_container is not None else self._unnamed_attributes
serializing_obj = {}
- try:
- for k, v in container.items(): # pytype: disable=attribute-error
+ # `container` is None when additional properties are disallowed
+ if container is not None:
+ for k, v in container.items():
if v is None:
continue
k = str(k)
@@ -212,9 +214,6 @@ def _serialize(self, alt_container: Optional[Dict] = None, include_context: bool
serializing_obj[k] = v._serialize(include_context=include_context)
else:
serializing_obj[k] = v
- except AttributeError as e:
- # means _unnamed_attributes is None, so nothing unnamed would be serialized
- pass
for k, v in self.__dict__.items():
if k in self.reserved_names:
continue
@@ -245,7 +244,7 @@ def is_empty(obj) -> bool:
return False
@staticmethod
- def _load_json(json_obj: Union[dict, str]) -> dict:
+ def _load_json(json_obj: dict | str) -> dict:
"""
Maps JSON-format MMIF strings and dicts into Python dicts
with identifier-compliant keys. To do this, it replaces "@"
@@ -259,7 +258,7 @@ def _load_json(json_obj: Union[dict, str]) -> dict:
:param json_str: the JSON data to load and process
:return: the mapped data as a dict
"""
- def from_atsign(d: Dict[str, Any]) -> dict:
+ def from_atsign(d: dict[str, Any]) -> dict:
for k in list(d.keys()):
if k.startswith('@'):
d[f'_{k[1:]}'] = d.pop(k)
@@ -280,7 +279,7 @@ def deep_from_atsign(d: dict) -> dict:
else:
raise TypeError(f"tried to load MMIF JSON in a format other than str or dict: {type(json_obj)}")
- def deserialize(self, mmif_json: Union[str, dict]) -> None:
+ def deserialize(self, mmif_json: str | dict) -> None:
"""
Takes a JSON-formatted string or a simple `dict` that's json-loaded from
such a string as an input and populates object's fields with the values
@@ -298,9 +297,9 @@ def _deserialize(self, input_dict: dict) -> None:
If a subclass needs a special treatment during the mapping, it needs to
override this method.
- This default method won't work for generic types (e.g. List[X], Dict[X, Y]).
+ This default method won't work for generic types (e.g. list[X], dict[X, Y]).
For now, lists are abstracted as DataList and dicts are abstracted as XXXMetadata classes.
- However, if an attribute uses a generic type (e.g. view_metadata.contains: Dict[str, Contain])
+ However, if an attribute uses a generic type (e.g. view_metadata.contains: dict[str, Contain])
that class should override _deserialize of its own.
:param input_dict: the prepared JSON data that defines the object
@@ -351,7 +350,7 @@ def __setitem__(self, key, value) -> None:
else:
self.set_additional_property(key, value)
- def __contains__(self, key: str) -> bool:
+ def __contains__(self, key: object) -> bool:
try:
self.__getitem__(key)
return True
@@ -450,15 +449,15 @@ class DataList(MmifObject, Generic[T]):
:param Union[str, list] mmif_obj: the data that the list contains
"""
- def __init__(self, mmif_obj: Optional[Union[bytes, str, list]] = None, *_):
+ def __init__(self, mmif_obj: bytes | str | list | None = None, *_):
self.reserved_names.add('_items')
- self._items: Dict[str, T] = dict()
+ self._items: dict[str, T] = dict()
self.disallow_additional_properties()
if mmif_obj is None:
mmif_obj = []
- super().__init__(mmif_obj)
+ super().__init__(mmif_obj) # type: ignore[arg-type]
- def _serialize(self, *args, **kwargs) -> list: # pytype: disable=signature-mismatch
+ def _serialize(self, *args, **kwargs) -> list: # type: ignore[override]
"""
Internal serialization method. Returns a list.
@@ -466,26 +465,26 @@ def _serialize(self, *args, **kwargs) -> list: # pytype: disable=signature-mism
"""
return list(super()._serialize(self._items, **kwargs).values())
- def deserialize(self, mmif_json: Union[str, list]) -> None: # pytype: disable=signature-mismatch
+ def deserialize(self, mmif_json: str | list) -> None: # type: ignore[override]
"""
Passes the input data into the internal deserializer.
"""
- super().deserialize(mmif_json)
+ super().deserialize(mmif_json) # type: ignore[arg-type]
@staticmethod
- def _load_json(json_list: Union[list, str]) -> list:
+ def _load_json(json_list: list | str) -> list: # type: ignore[override]
if type(json_list) is str:
json_list = json.loads(json_list)
return [MmifObject._load_json(obj) for obj in json_list]
- def _deserialize(self, input_list: list) -> None:
+ def _deserialize(self, input_list: list) -> None: # type: ignore[override]
raise NotImplementedError()
- def get(self, key: str, default=None) -> Optional[T]:
+ def get(self, key: str, default=None) -> T | None:
"""
.. deprecated:: 1.1.3
- Do not use in new code. Will be removed in 2.0.0.
- Use container-level access or positional indexing instead.
+ Will be removed in 2.0.0. Use container-level access or positional
+ indexing instead.
Deprecated method for retrieving list elements by string ID.
@@ -517,13 +516,11 @@ def get(self, key: str, default=None) -> Optional[T]:
--------
__getitem__ : List-style positional access with integers
"""
- warnings.warn(
- "The 'get' method on list-like collections is deprecated and "
- "will be removed in 2.0.0. Use container-level access "
- "(e.g., mmif['v1']) or positional indexing (e.g., views[0]).",
- DeprecationWarning,
- stacklevel=2
- )
+ warnings.warn("get() on list-like collections is deprecated and will "
+ "be removed in 2.0.0. Use container-level access (e.g.,"
+ " mmif['v1']) or positional indexing (e.g., views[0]) "
+ "instead.",
+ DeprecationWarning, stacklevel=2)
return self._items.get(key, default)
def _append_with_key(self, key: str, value: T, overwrite=False) -> None:
@@ -549,7 +546,7 @@ def _append_with_key(self, key: str, value: T, overwrite=False) -> None:
def append(self, value, overwrite):
raise NotImplementedError()
- def __getitem__(self, key: Union[int, slice]) -> Union[T, List[T]]:
+ def __getitem__(self, key: int | slice) -> T | list[T]:
"""
List-style positional access using integers or slices.
@@ -613,9 +610,9 @@ def empty(self):
class DataDict(MmifObject, Generic[T, S]):
- def __init__(self, mmif_obj: Optional[Union[bytes, str, dict]] = None, *_):
+ def __init__(self, mmif_obj: bytes | str | dict | None = None, *_):
self.reserved_names.add('_items')
- self._items: Dict[T, S] = dict()
+ self._items: dict[T, S] = dict()
self.disallow_additional_properties()
if mmif_obj is None:
mmif_obj = {}
@@ -624,7 +621,7 @@ def __init__(self, mmif_obj: Optional[Union[bytes, str, dict]] = None, *_):
def _serialize(self, *args, **kwargs) -> dict:
return super()._serialize(self._items, **kwargs)
- def get(self, key: T, default=None) -> Optional[S]:
+ def get(self, key: T, default=None) -> S | None:
"""
Dictionary-style safe access with optional default value.
diff --git a/mmif/serialize/producer.py b/mmif/serialize/producer.py
new file mode 100644
index 00000000..ac8e7a64
--- /dev/null
+++ b/mmif/serialize/producer.py
@@ -0,0 +1,236 @@
+"""
+The :mod:`producer` module contains the producer context, the write mode of a
+MMIF. See :meth:`mmif.serialize.mmif.Mmif.set_producer`.
+"""
+
+import copy
+import warnings
+from collections import defaultdict
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, cast
+
+from mmif.serialize.view import View
+from mmif.vocabulary import AnnotationTypes
+
+if TYPE_CHECKING:
+ from mmif.serialize.mmif import Mmif
+
+__all__ = ['EmptyOutputWarning', 'ProducerContext']
+
+
+class EmptyOutputWarning(UserWarning):
+ """
+ Issued when a run leaves no view: it produced no annotations, no error,
+ and no warnings.
+ """
+
+
+class ProducerContext:
+ """
+ The write mode of a MMIF, for one run of a producer. Holds what the run
+ needs to know: a snapshot taken when the producer is set, the signature,
+ and the warnings raised during the run.
+
+ While a producer is set, views that existed when it was set are serialized
+ as they are in memory, and :meth:`mmif.serialize.mmif.Mmif.serialize`
+ finalizes the views of the run in its output: capital ``Annotation``
+ records, factored properties, ``contains`` pruning, removal of empty
+ views, and one run ID. The ``Mmif`` object itself does not change, so
+ repeated serialization gives the same output.
+
+ Leaving the ``with`` block turns the mode off. An exception propagates
+ unchanged.
+
+ :param mmif: the MMIF the run works on
+ :param signature: a view whose metadata is signed by the producer, copied
+ to the views this context writes
+ :raises RuntimeError: if a producer is already set on the MMIF
+ """
+
+ def __init__(self, mmif: 'Mmif', signature: View) -> None:
+ if mmif._producer is not None:
+ raise RuntimeError('A producer is already set on this MMIF.')
+ self.mmif = mmif
+ self.signature = signature
+ self._warnings: list[Warning] = []
+ self._view_metadata: dict[str, Any] = {}
+ self._run_id: datetime | None = None
+ self._existing_view_ids = {view.id for view in mmif.views}
+ self._entry_pending = {doc.id: dict(doc._props_pending.items())
+ for doc in self._documents()}
+ mmif._producer = self
+
+ def __enter__(self) -> 'ProducerContext':
+ return self
+
+ def __exit__(self, exc_type, exc_value, exc_tb) -> None:
+ self.mmif._producer = None
+
+ def add_warnings(self,
+ *issued: Warning | warnings.WarningMessage) -> None:
+ """
+ Records warnings of this run, written to one signed warnings view.
+
+ :param issued: warnings, as raised or as caught by
+ ``warnings.catch_warnings(record=True)``
+ """
+ for warning in issued:
+ if isinstance(warning, warnings.WarningMessage):
+ # `warnings.warn()` turns a str message into a Warning
+ warning = cast(Warning, warning.message)
+ self._warnings.append(warning)
+
+ def set_view_metadata(self, key: str, value: Any) -> None:
+ """
+ Sets an additional metadata property on every view of this run,
+ including the views this context writes (e.g., ``appProfiling``).
+
+ :param key: the metadata key
+ :param value: the value
+ """
+ self._view_metadata[key] = value
+
+ def _finalize_output(self, views_out: list[dict],
+ include_context: bool) -> None:
+ """
+ Finalizes the views of this run in a serialized MMIF. Earlier views
+ are not touched.
+
+ :param views_out: serialized views, changed in place
+ :param include_context: See :meth:`mmif.serialize.mmif.Mmif.serialize`.
+ """
+ if self._run_id is None:
+ self._run_id = self._find_run_id()
+ plan = self._plan_capital_annotations(views_out, include_context)
+ self.mmif._write_capital_annotations(plan, views_out, include_context)
+ run_views_out = [view_out for view_out in views_out
+ if view_out['id'] not in self._existing_view_ids]
+ self.mmif._factor_out_in_output(run_views_out)
+ self.mmif._prune_contains_in_output(run_views_out)
+
+ empty_ids = {view_out['id'] for view_out in run_views_out
+ if self._is_empty(view_out)}
+ views_out[:] = [view_out for view_out in views_out
+ if view_out['id'] not in empty_ids]
+ run_views_out = [view_out for view_out in run_views_out
+ if view_out['id'] not in empty_ids]
+
+ run_warnings = list(self._warnings)
+ if not run_views_out:
+ run_warnings.append(EmptyOutputWarning(
+ f'App "{self.signature.metadata.app}" produced no annotations '
+ f'for this input; the input may lack expected content, or '
+ f'there may genuinely be nothing to annotate.'))
+ if run_warnings:
+ warnings_view = self._new_signed_view(views_out)
+ warnings_view.metadata.add_warnings(*run_warnings)
+ warnings_out = warnings_view._serialize(
+ include_context=include_context)
+ views_out.append(warnings_out)
+ run_views_out.append(warnings_out)
+
+ for view_out in run_views_out:
+ view_out['metadata'].update(self._view_metadata)
+ if include_context:
+ view_out['metadata']['timestamp'] = self._run_id
+
+ def _find_run_id(self) -> datetime:
+ """
+ Finds the run ID: the timestamp of the first view of this run, which
+ ``Mmif.new_view()`` assigned. A run that created no view gets the
+ current time.
+
+ :return: the run ID
+ """
+ return next((view.metadata.timestamp for view in self.mmif.views
+ if view.id not in self._existing_view_ids
+ and view.metadata.timestamp is not None),
+ datetime.now(timezone.utc))
+
+ def _plan_capital_annotations(
+ self, views_out: list[dict], include_context: bool,
+ ) -> list[tuple[View, str, dict, bool]]:
+ """
+ Decides where the pending properties of this run are recorded: those
+ that weren't pending on entry and that no view already records. A
+ document created in this run takes them itself, and the others get
+ records in the last view of this run. A signed view is added to the
+ output when the run created no view to hold the records.
+
+ :param views_out: serialized views, changed when a view is added
+ :param include_context: See :meth:`mmif.serialize.mmif.Mmif.serialize`.
+ :return: the plan, in the form that
+ :meth:`mmif.serialize.mmif.Mmif._write_capital_annotations` takes
+ """
+ recorded: defaultdict[str, dict] = defaultdict(dict)
+ for view in self.mmif.views:
+ for ann in view.get_annotations(AnnotationTypes.Annotation):
+ recorded[ann.get('document')].update(
+ ann._resolved_properties())
+
+ run_views = [view for view in self.mmif.views
+ if view.id not in self._existing_view_ids]
+ doc_views = {doc.id: view for view in run_views
+ for doc in view.get_documents()}
+ target = next((view for view in reversed(run_views)
+ if not view.has_error() and not view.has_warnings()),
+ None)
+
+ plan = []
+ for doc in self._documents():
+ on_entry = self._entry_pending.get(doc.id, {})
+ props = {key: value for key, value in doc._props_pending.items()
+ if key != 'id'
+ and not (key in on_entry and on_entry[key] == value)
+ and not (key in recorded[doc.id]
+ and recorded[doc.id][key] == value)}
+ if not props:
+ continue
+ if doc.id in doc_views:
+ plan.append((doc_views[doc.id], doc.id, props, True))
+ continue
+ if target is None:
+ target = self._new_signed_view(views_out)
+ views_out.append(
+ target._serialize(include_context=include_context))
+ plan.append((target, doc.id, props, False))
+ return plan
+
+ def _documents(self):
+ """
+ Iterates over all documents, in the top-level list and in the views.
+ """
+ yield from self.mmif.documents
+ for view in self.mmif.views:
+ yield from view.get_documents()
+
+ def _new_signed_view(self, views_out: list[dict]) -> View:
+ """
+ Creates a view for the output only, with an ID that is free in the
+ output and in the object, and with the metadata of the signature.
+
+ :param views_out: serialized views
+ :return: the new view, not added to the MMIF
+ """
+ used_ids = {view_out['id'] for view_out in views_out}
+ used_ids.update(view.id for view in self.mmif.views)
+ index = len(used_ids)
+ while f'{self.mmif.view_prefix}{index}' in used_ids:
+ index += 1
+ view = View()
+ view.id = f'{self.mmif.view_prefix}{index}'
+ view.metadata = copy.deepcopy(self.signature.metadata)
+ return view
+
+ @staticmethod
+ def _is_empty(view_out: dict) -> bool:
+ """
+ Checks that a serialized view has no annotations, no error, and no
+ warnings.
+
+ :param view_out: a serialized view
+ :return: True for an empty view
+ """
+ metadata = view_out['metadata']
+ return not (view_out['annotations'] or metadata.get('error')
+ or metadata.get('warnings'))
diff --git a/mmif/serialize/view.py b/mmif/serialize/view.py
index 2cb9fcda..30d776b8 100644
--- a/mmif/serialize/view.py
+++ b/mmif/serialize/view.py
@@ -14,14 +14,14 @@
import json
import warnings
from datetime import datetime
-from typing import Dict, Union, Optional, Generator, List, cast
+from collections.abc import Generator
+from typing import cast
-from mmif import DocumentTypes, AnnotationTypes, ThingTypesBase, ClamsTypesBase
+from mmif import AnnotationTypes, ClamsTypesBase, DocumentTypes, ThingTypesBase
from mmif.serialize.annotation import Annotation, Document, canonicalize_prop_name
-from mmif.serialize.model import PRMTV_TYPES, MmifObject, DataList, DataDict
-from mmif.vocabulary.base_types import AnnotationTypesBase
+from mmif.serialize.model import PRMTV_TYPES, DataDict, DataList, MmifObject
-__all__ = ['View', 'ViewMetadata', 'Contain']
+__all__ = ['Contain', 'View', 'ViewMetadata']
class View(MmifObject):
@@ -63,9 +63,9 @@ class View(MmifObject):
some_anns = view.annotations[1:5] # Slice of annotations
"""
- def __init__(self, view_obj: Optional[Union[bytes, str, dict]] = None, parent_mmif=None, *_) -> None:
+ def __init__(self, view_obj: bytes | str | dict | None = None, parent_mmif=None, *_) -> None:
# used to autogenerate annotation ids
- self._id_counts = {}
+ self._id_counts: dict[str, int] = {}
# used to access the parent MMIF object
self._parent_mmif = parent_mmif
self.reserved_names.update(("_parent_mmif", "_id_counts"))
@@ -81,6 +81,8 @@ def __init__(self, view_obj: Optional[Union[bytes, str, dict]] = None, parent_mm
}
self._required_attributes = ["id", "metadata", "annotations"]
super().__init__(view_obj)
+ for annotation in self.annotations:
+ annotation._parent_view = self
self._fix_old_short_ids()
self._rebuild_id_counts()
@@ -144,13 +146,14 @@ def _fix_old_short_ids(self):
propv[i] = f"{self.id}{self.id_delimiter}{item}"
self.annotations.append(annotation)
- def new_contain(self, at_type: Union[str, ThingTypesBase], **contains_metadata) -> Optional['Contain']:
+ def new_contain(self, at_type: str | ThingTypesBase, **contains_metadata) -> 'Contain':
"""
- Adds a new element to the ``contains`` metadata.
+ Adds a new element to the ``contains`` metadata. For a type that is
+ already there, the metadata is merged into its element.
:param at_type: the ``@type`` of the annotation type being added
:param contains_metadata: any metadata associated with the annotation type
- :return: the generated :class:`Contain` object
+ :return: the :class:`Contain` object of the type
"""
if MmifObject.is_empty(at_type):
raise ValueError("@type must not be empty.")
@@ -172,7 +175,7 @@ def _set_ann_id(self, annotation: Annotation, identifier):
self._id_counts[prefix] = new_num
annotation.id = self.id + self.id_delimiter + new_id
- def new_annotation(self, at_type: Union[str, ThingTypesBase], aid: Optional[str] = None,
+ def new_annotation(self, at_type: str | ThingTypesBase, aid: str | None = None,
overwrite=False, **properties) -> 'Annotation':
"""
Generates a new :class:`mmif.serialize.annotation.Annotation`
@@ -220,12 +223,12 @@ def add_annotation(self, annotation: 'Annotation', overwrite=False) -> 'Annotati
annotation.id = f"{self.id}{self.id_delimiter}{annotation.id}"
self.annotations.append(annotation, overwrite)
self.new_contain(annotation.at_type)
- annotation._add_ephemeral_defaults(self.metadata.contains.get(annotation.at_type, {}))
+ annotation._parent_view = self
if annotation.at_type == AnnotationTypes.Alignment:
self._parent_mmif._cache_alignment(annotation)
return annotation
- def new_textdocument(self, text: str, lang: str = "en", did: Optional[str] = None,
+ def new_textdocument(self, text: str, lang: str = "en", did: str | None = None,
overwrite=False, **properties) -> 'Document':
"""
Generates a new :class:`mmif.serialize.annotation.Document`
@@ -269,7 +272,7 @@ def add_document(self, document: Document, overwrite=False) -> Annotation:
"""
return self.add_annotation(document, overwrite)
- def get_annotations(self, at_type: Optional[Union[str, ThingTypesBase]] = None,
+ def get_annotations(self, at_type: str | ThingTypesBase | None = None,
**properties) -> Generator[Annotation, None, None]:
"""
Look for certain annotations in this view, specified by parameters
@@ -293,10 +296,8 @@ def prop_check(k, v, annotation, at_type_metadata):
def get_annotation_by_id(self, ann_id) -> Annotation:
"""
.. deprecated:: 1.1.0
- Will be removed in 2.0.0.
- Use general ``Mmif.__getitem__()`` method instead to retrieve
- any annotation across the MMIF, or View.__getitems__() to
- retrieve annotations within the view.
+ Will be removed in 2.0.0. Use ``view[ann_id]`` to look in this view,
+ or ``mmif[ann_id]`` to look across the MMIF, instead.
Thinly wraps the Mmif.__getitem__ method and returns an Annotation
object. Note that although this method is under View class, it can
@@ -306,25 +307,22 @@ def get_annotation_by_id(self, ann_id) -> Annotation:
:return: found :class:`mmif.serialize.annotation.Annotation` object.
:raises KeyError: if the annotation with the given ID is not found
"""
- warnings.warn(
- "View.get_annotation_by_id() is deprecated, use view[ann_id] instead.",
- DeprecationWarning
- )
+ warnings.warn('get_annotation_by_id() is deprecated and will be '
+ 'removed in 2.0.0. Use view[ann_id] instead.',
+ DeprecationWarning)
ann_found = self._parent_mmif.__getitem__(ann_id)
if not isinstance(ann_found, Annotation):
raise KeyError(f"Annotation with ID {ann_id} not found in the MMIF object.")
return cast(Annotation, ann_found)
- def get_documents(self) -> List[Document]:
+ def get_documents(self) -> list[Document]:
return [cast(Document, annotation) for annotation in self.annotations if annotation.is_document()]
def get_document_by_id(self, doc_id) -> Document:
"""
.. deprecated:: 1.1.0
- Will be removed in 2.0.0.
- Use general ``Mmif.__getitem__()`` method instead to retrieve
- any document across the MMIF, or View.__getitems__() to
- retrieve documents within the view.
+ Will be removed in 2.0.0. Use ``view[doc_id]`` to look in this view,
+ or ``mmif[doc_id]`` to look across the MMIF, instead.
Thinly wraps the Mmif.__getitem__ method and returns a Document
object. Note that although this method is under View class, it can
@@ -334,10 +332,9 @@ def get_document_by_id(self, doc_id) -> Document:
:return: found :class:`mmif.serialize.annotation.Document` object.
:raises KeyError: if the document with the given ID is not found
"""
- warnings.warn(
- "View.get_document_by_id() is deprecated, use view[doc_id] instead.",
- DeprecationWarning
- )
+ warnings.warn('View.get_document_by_id() is deprecated and will be '
+ 'removed in 2.0.0. Use view[doc_id] instead.',
+ DeprecationWarning)
doc_found = self.annotations._items.get(doc_id)
if not isinstance(doc_found, Document):
raise KeyError(f"Document \"{doc_id}\" not found in view {self.id}.")
@@ -401,7 +398,7 @@ def set_error(self, err_message: str, err_trace: str) -> None:
self.metadata.set_error(err_message, err_trace)
self.annotations.empty()
- def get_error(self) -> Optional[str]:
+ def get_error(self) -> str | None:
"""
Get the "text" representation of the error occurred during
processing. Text representation is supposed to be human-readable.
@@ -434,16 +431,16 @@ class ViewMetadata(MmifObject):
:param viewmetadata_obj: the JSON data that defines the metadata
"""
- def __init__(self, viewmetadata_obj: Optional[Union[bytes, str, dict]] = None, *_) -> None:
+ def __init__(self, viewmetadata_obj: bytes | str | dict | None = None, *_) -> None:
self.document: str = ''
- self.timestamp: Optional[datetime] = None
+ self.timestamp: datetime | None = None
self.app: str = ''
- self.appTags: List[str] = []
+ self.appTags: list[str] = []
self.contains: ContainsDict = ContainsDict()
- self.parameters: Dict[str, str] = {}
- self.appConfiguration: Dict[str, Union[PRMTV_TYPES, List[PRMTV_TYPES]]] = {}
- self.error: Union[dict, ErrorDict] = {}
- self.warnings: List[str] = []
+ self.parameters: dict[str, str] = {}
+ self.appConfiguration: dict[str, PRMTV_TYPES | list[PRMTV_TYPES]] = {}
+ self.error: dict | ErrorDict = {}
+ self.warnings: list[str] = []
self._required_attributes = ["app"]
self._contextual_attributes = {"timestamp"}
self._attribute_classes = {
@@ -482,32 +479,37 @@ def get_error_as_text(self) -> str:
else:
raise KeyError(f"No error found")
- def new_contain(self, at_type: Union[str, ThingTypesBase], **contains_metadata) -> Optional['Contain']:
+ def new_contain(self, at_type: str | ThingTypesBase, **contains_metadata) -> 'Contain':
"""
- Adds a new element to the ``contains`` dictionary.
+ Adds a new element to the ``contains`` dictionary. For a type that is
+ already there, the metadata is merged into its element.
:param at_type: the ``@type`` of the annotation type being added
:param contains_metadata: any metadata associated with the annotation type
- :return: the generated :class:`Contain` object
+ :return: the :class:`Contain` object of the type
"""
if isinstance(at_type, str):
at_type = ThingTypesBase.from_str(at_type)
+ new_contain = Contain._for_type(at_type, contains_metadata)
if at_type not in self.contains:
- new_contain = Contain._for_type(at_type, contains_metadata)
self.add_contain(new_contain, at_type)
return new_contain
+ existing = self.contains[at_type]
+ for key, value in new_contain.items():
+ existing[key] = value
+ return existing
- def add_contain(self, contain: 'Contain', at_type: Union[str, ThingTypesBase]) -> None:
+ def add_contain(self, contain: 'Contain', at_type: str | ThingTypesBase) -> None:
self.contains[at_type] = contain
- def add_app_configuration(self, config_key: str, config_value: Union[PRMTV_TYPES, List[PRMTV_TYPES]]) -> None:
+ def add_app_configuration(self, config_key: str, config_value: PRMTV_TYPES | list[PRMTV_TYPES]) -> None:
"""
Add a configuration key-value pair to the app_configuration dictionary.
"""
self.appConfiguration[config_key] = config_value
- def get_app_configuration(self, config_key: str) -> Union[PRMTV_TYPES, List[PRMTV_TYPES]]:
+ def get_app_configuration(self, config_key: str) -> PRMTV_TYPES | list[PRMTV_TYPES]:
"""
Get a configuration value from the app_configuration dictionary.
"""
@@ -556,7 +558,7 @@ class ErrorDict(MmifObject):
"""
Error object that stores information about error occurred during processing.
"""
- def __init__(self, error_obj: Optional[Union[bytes, str, dict]] = None, *_) -> None:
+ def __init__(self, error_obj: bytes | str | dict | None = None, *_) -> None:
self.message: str = ''
self.stackTrace: str = ''
self._contextual_attributes = {"stackTrace"}
@@ -582,14 +584,14 @@ def _for_type(cls, at_type: ThingTypesBase, contains_metadata: dict) -> 'Contain
return cls({canonicalize_prop_name(at_type, k): v for k, v in contains_metadata.items()})
-class AnnotationsList(DataList[Union[Annotation, Document]]):
+class AnnotationsList(DataList[Annotation | Document]):
"""
AnnotationsList object that implements :class:`mmif.serialize.model.DataList`
for :class:`mmif.serialize.annotation.Annotation`.
"""
- _items: Dict[str, Union[Annotation, Document]]
+ _items: dict[str, Annotation | Document]
- def _deserialize(self, input_list: list) -> None: # pytype: disable=signature-mismatch
+ def _deserialize(self, input_list: list) -> None: # type: ignore[override]
"""
Extends base ``_deserialize`` method to initialize ``items`` as a dict from
annotation IDs to :class:`mmif.serialize.annotation.Annotation` objects.
@@ -601,7 +603,7 @@ def _deserialize(self, input_list: list) -> None: # pytype: disable=signature-m
if ClamsTypesBase.attype_iri_isdocument(item['_type']) else Annotation(item)
for item in input_list}
- def append(self, value: Union[Annotation, Document], overwrite=False) -> None:
+ def append(self, value: Annotation | Document, overwrite=False) -> None:
"""
Appends an annotation to the list.
@@ -628,18 +630,18 @@ def _deserialize(self, input_dict: dict) -> None:
key = ThingTypesBase.from_str(key)
self._items[key] = Contain._for_type(key, value)
- def update(self, other: Union[dict, 'ContainsDict'], overwrite=False):
+ def update(self, other: 'dict | ContainsDict', overwrite=False):
for k, v in other.items():
if isinstance(k, str):
k = ThingTypesBase.from_str(k)
self._append_with_key(k, v, overwrite=overwrite)
- def get(self, key: Union[str, ThingTypesBase], default=None):
+ def get(self, key: str | ThingTypesBase, default=None):
if isinstance(key, str):
key = ThingTypesBase.from_str(key)
return self._items.get(key, default)
- def __contains__(self, item: Union[str, ThingTypesBase]):
+ def __contains__(self, item: str | ThingTypesBase):
if isinstance(item, str):
# in general, when querying with a string, do not use fuzzy equality
if 'vocab.lappsgrid.org' in item and item.split('/')[-1] in ThingTypesBase._old_lapps_type_shortnames:
diff --git a/mmif/utils/cli/__init__.py b/mmif/utils/cli/__init__.py
index f24248f2..124c2591 100644
--- a/mmif/utils/cli/__init__.py
+++ b/mmif/utils/cli/__init__.py
@@ -6,17 +6,22 @@
import io
import os
import sys
-from typing import Iterator, Optional, TextIO, Type, Union, cast, get_args, get_origin
+import types
+from collections.abc import Iterator
+from typing import TextIO, Union, cast, get_args, get_origin
from pydantic import BaseModel
+# ``Optional[X]`` / ``Union[X, Y]`` and ``X | Y`` have different origins
+_UNION_ORIGINS = (Union, types.UnionType)
+
@contextlib.contextmanager
def open_cli_io_arg(
- path_or_dash: Optional[str],
+ path_or_dash: str | None,
mode: str = "r",
- encoding: Optional[str] = None,
- errors: Optional[str] = None,
+ encoding: str | None = None,
+ errors: str | None = None,
default_stdin: bool = False,
) -> Iterator[TextIO]:
"""
@@ -72,7 +77,7 @@ def open_cli_io_arg(
should_use_stdio = path_or_dash == "-" or (path_or_dash is None and default_stdin)
- file_handle: Optional[TextIO] = None
+ file_handle: TextIO | None = None
should_close = False
try:
@@ -124,7 +129,7 @@ def open_cli_io_arg(
file_handle.close()
-def generate_model_summary(model: Type[BaseModel], indent: int = 0) -> str:
+def generate_model_summary(model: type[BaseModel], indent: int = 0) -> str:
lines = []
prefix = " " * indent
@@ -141,7 +146,7 @@ def format_type(t) -> str:
args = get_args(t)
# Handle Optional (Union[T, None])
- if origin is Union and type(None) in args:
+ if origin in _UNION_ORIGINS and type(None) in args:
non_none_args = [arg for arg in args if arg is not type(None)]
if len(non_none_args) == 1:
return f"{format_type(non_none_args[0])}, optional"
@@ -153,7 +158,7 @@ def format_type(t) -> str:
return "[]"
# Handle Dict
- if origin is dict:
+ if origin is dict or t is dict:
return "obj"
# Handle Pydantic Models (Custom Classes)
@@ -189,7 +194,7 @@ def format_type(t) -> str:
nested_model = None
# Handle Optional wrappers for nesting check
check_type = type_annotation
- if origin is Union and type(None) in args:
+ if origin in _UNION_ORIGINS and type(None) in args:
non_none_args = [arg for arg in args if arg is not type(None)]
if len(non_none_args) == 1:
check_type = non_none_args[0]
diff --git a/mmif/utils/cli/describe.py b/mmif/utils/cli/describe.py
index b8c79ced..ba6f5c15 100644
--- a/mmif/utils/cli/describe.py
+++ b/mmif/utils/cli/describe.py
@@ -3,7 +3,7 @@
import sys
import textwrap
from pathlib import Path
-from typing import Union, cast
+from typing import cast
from mmif.utils.cli import open_cli_io_arg, generate_model_summary
@@ -17,16 +17,27 @@
)
-def get_pipeline_specs(mmif_file: Union[str, Path]):
+def get_pipeline_specs(mmif_file: str | Path):
+ """
+ .. deprecated:: 1.2.1
+ Will be removed in 2.0.0. Use :func:`mmif.utils.workflow_helper.describe_single_mmif` instead.
+ """
import warnings
- warnings.warn("get_pipeline_specs is deprecated, use mmif.utils.workflow_helper.describe_single_mmif instead",
+ warnings.warn('get_pipeline_specs() is deprecated and will be removed in '
+ '2.0.0. Use describe_single_mmif() instead.',
DeprecationWarning)
return describe_single_mmif(mmif_file)
-def generate_pipeline_identifier(mmif_file: Union[str, Path]) -> str:
+def generate_pipeline_identifier(mmif_file: str | Path) -> str:
+ """
+ .. deprecated:: 1.2.1
+ Will be removed in 2.0.0. Use :func:`mmif.utils.workflow_helper.generate_workflow_identifier` instead.
+ """
import warnings
- warnings.warn("generate_pipeline_identifier is deprecated, use generate_workflow_identifier instead",
+ warnings.warn('generate_pipeline_identifier() is deprecated and will be '
+ 'removed in 2.0.0. Use generate_workflow_identifier() '
+ 'instead.',
DeprecationWarning)
return cast(str, generate_workflow_identifier(mmif_file))
@@ -97,6 +108,7 @@ def main(args):
"""
if hasattr(args, 'help_schema') and args.help_schema is not None:
schema_name = args.help_schema[0]
+ model_cls: type[SingleMmifDesc] | type[CollectionMmifDesc]
if schema_name == 'mmif-file':
model_cls = SingleMmifDesc
elif schema_name == 'mmif-dir':
diff --git a/mmif/utils/cli/source.py b/mmif/utils/cli/source.py
index 6c3b4b86..b6542329 100644
--- a/mmif/utils/cli/source.py
+++ b/mmif/utils/cli/source.py
@@ -4,7 +4,8 @@
import pathlib
import sys
import textwrap
-from typing import Union, Generator, List, Optional, Iterable
+from collections.abc import Generator, Iterable
+from typing import cast
from urllib.parse import urlparse
from mmif import Mmif, Document, DocumentTypes, __specver__
@@ -13,10 +14,10 @@
__all__ = ['WorkflowSource']
-DOC_JSON = Union[str, dict]
-DOC = Union[DOC_JSON, Document]
-METADATA_JSON = Union[str, dict]
-METADATA = Union[METADATA_JSON, MmifMetadata]
+DOC_JSON = str | dict
+DOC = DOC_JSON | Document
+METADATA_JSON = str | dict
+METADATA = METADATA_JSON | MmifMetadata
class WorkflowSource:
@@ -39,13 +40,15 @@ class WorkflowSource:
def __init__(
self,
- common_documents_json: Optional[List[DOC_JSON]] = None,
- common_metadata_json: Optional[METADATA_JSON] = None
+ common_documents_json: list[DOC_JSON] | None = None,
+ common_metadata_json: METADATA_JSON | None = None
) -> None:
if common_documents_json is None:
common_documents_json = []
if common_metadata_json is None:
common_metadata_json = dict()
+ if isinstance(common_metadata_json, str):
+ common_metadata_json = json.loads(common_metadata_json)
self.mmif_start: dict = {"documents": [json.loads(document)
if isinstance(document, str)
else document
@@ -53,11 +56,11 @@ def __init__(
"views": [],
"metadata": {
"mmif": f"http://mmif.clams.ai/{__specver__}",
- **common_metadata_json
+ **cast(dict, common_metadata_json)
}}
self.prime()
- def add_document(self, document: Union[str, dict, Document]) -> None:
+ def add_document(self, document: str | dict | Document) -> None:
"""
Adds a document to the working source MMIF.
@@ -103,8 +106,8 @@ def produce(self) -> Mmif:
def __call__(
self,
- documents: Optional[List[DOC]] = None,
- metadata: Optional[METADATA] = None
+ documents: list[DOC] | None = None,
+ metadata: METADATA | None = None
) -> Mmif:
"""
Callable API that produces a new MMIF object from
@@ -126,20 +129,20 @@ def __call__(
if isinstance(documents, str):
documents = json.loads(documents)
if isinstance(metadata, MmifMetadata):
- metadata = metadata.serialize() # pytype: disable=attribute-error # bug in pytype? (https://github.com/google/pytype/issues/533)
+ metadata = metadata.serialize()
if isinstance(metadata, str):
metadata = json.loads(metadata)
for document in documents:
self.add_document(document)
- for key, value in metadata.items():
+ for key, value in cast(dict, metadata).items():
self.change_metadata(key, value)
return self.produce()
def from_data(
self,
- doc_lists: Iterable[List[DOC]],
- metadata_objs: Optional[Iterable[Optional[METADATA]]] = None
+ doc_lists: Iterable[list[DOC]],
+ metadata_objs: Iterable[METADATA | None] | None = None
) -> Generator[Mmif, None, None]:
"""
Provided with an iterable of document lists and an
diff --git a/mmif/utils/sequence_helper.py b/mmif/utils/sequence_helper.py
index f4edcb19..7dd1e485 100644
--- a/mmif/utils/sequence_helper.py
+++ b/mmif/utils/sequence_helper.py
@@ -17,12 +17,16 @@
import importlib
import itertools
import warnings
-from typing import List, Tuple, Dict, Union, Iterable, Callable
+from collections.abc import Callable, Iterable
+from typing import TYPE_CHECKING
import mmif
from mmif import Annotation
from mmif.serialize.model import PRMTV_TYPES
+if TYPE_CHECKING:
+ import numpy
+
for seq_dep in ['numpy']:
try:
importlib.__import__(seq_dep)
@@ -34,7 +38,7 @@
NEG_LABEL = '-'
-def smooth_outlying_short_intervals(scores: List[float],
+def smooth_outlying_short_intervals(scores: list[float],
min_spseq_size: int,
min_snseq_size: int,
min_score: float = 0.5,
@@ -211,7 +215,7 @@ def trimmer(elems, min_width, target=False, keep_short_ends=False):
return _sequence_to_intervals(pass2)
-def _sequence_to_intervals(seq: Iterable[bool]) -> List[Tuple[int, int]]:
+def _sequence_to_intervals(seq: Iterable[bool]) -> list[tuple[int, int]]:
pos_ints = []
cur = 0
for positivity, members in itertools.groupby(seq):
@@ -222,7 +226,7 @@ def _sequence_to_intervals(seq: Iterable[bool]) -> List[Tuple[int, int]]:
return pos_ints
-def validate_labelset(annotations: Iterable[Annotation]) -> List[str]:
+def validate_labelset(annotations: Iterable[Annotation]) -> list[str]:
"""
Simple check for a list of annotations to see if they have the same label set.
@@ -248,7 +252,7 @@ def validate_labelset(annotations: Iterable[Annotation]) -> List[str]:
return list(src_labels)
-def build_label_remapper(src_labels: List[str], dst_labels: Dict[str, PRMTV_TYPES]) -> Dict[str, PRMTV_TYPES]:
+def build_label_remapper(src_labels: list[str], dst_labels: dict[str, PRMTV_TYPES]) -> dict[str, PRMTV_TYPES]:
"""
Build a label remapper dictionary from source and destination labels.
@@ -263,9 +267,9 @@ def build_label_remapper(src_labels: List[str], dst_labels: Dict[str, PRMTV_TYPE
return {**dst_labels, **dict((k, NEG_LABEL) for k in src_labels if k not in dst_labels)}
-def build_score_lists(classifications: List[Dict], label_remapper: Dict,
+def build_score_lists(classifications: list[dict], label_remapper: dict,
score_remap_op: Callable[..., float] = max) \
- -> Tuple[Dict[str, int], "numpy.ndarray"]: # pytype: disable=name-error
+ -> tuple[dict[str, int], "numpy.ndarray"]:
"""
Build lists of scores indexed by the label names.
@@ -278,7 +282,7 @@ def build_score_lists(classifications: List[Dict], label_remapper: Dict,
2. 2-d numpy array of scores, of which rows are indexed by label map dict (first return value)
"""
import numpy
- scores = {lbl: [] for lbl in label_remapper.values()}
+ scores: dict[str, list] = {lbl: [] for lbl in label_remapper.values()}
for c_idx, classification in enumerate(classifications):
for src_label, src_score in classification.items():
dst_label = label_remapper[src_label]
diff --git a/mmif/utils/summarizer/graph.py b/mmif/utils/summarizer/graph.py
index b5ea40a2..d75ba7ff 100644
--- a/mmif/utils/summarizer/graph.py
+++ b/mmif/utils/summarizer/graph.py
@@ -4,7 +4,6 @@
from pathlib import Path
import argparse
-from typing import Any
from mmif import Mmif
from mmif.utils.summarizer import config
@@ -31,13 +30,11 @@ class Graph(object):
"""
- def __init__(self, mmif: Any):
- # TODO: the type hint should really be "MMif | str", but pytype did not
- # like that.
- self.mmif = mmif if type(mmif) is Mmif else Mmif(mmif)
- self.documents = []
- self.nodes = {}
- self.alignments = []
+ def __init__(self, mmif: Mmif | str):
+ self.mmif = mmif if isinstance(mmif, Mmif) else Mmif(mmif)
+ self.documents: list = []
+ self.nodes: dict = {}
+ self.alignments: list = []
self._init_nodes()
self._init_edges()
# Third pass to add links between text elements, in particular from
@@ -115,7 +112,7 @@ def statistics(self) -> defaultdict:
"""
Collect counts for node types in each view.
"""
- stats = defaultdict(int)
+ stats: defaultdict[str, int] = defaultdict(int)
for node in self.nodes.values():
stats[f'{str(node.view_id):4} {node.at_type.shortname}'] += 1
return stats
@@ -191,7 +188,7 @@ class TokenIndex(object):
# targets to tokens.
def __init__(self, tokens):
- self.tokens = {}
+ self.tokens: dict = {}
self.token_count = len(tokens)
for t in tokens:
tup = ((t.properties['start'], t.properties['end']), t)
diff --git a/mmif/utils/summarizer/nodes.py b/mmif/utils/summarizer/nodes.py
index fa2e467c..e5a3142b 100644
--- a/mmif/utils/summarizer/nodes.py
+++ b/mmif/utils/summarizer/nodes.py
@@ -24,7 +24,8 @@ class NodeProperties(MutableMapping):
That round trip is not shape-preserving when a type has a single annotation
in the view, when the key is non-factorable (`start`, `source`, `target`, ...),
or when another annotation of the type overrides the default, so the SDK
- keeps defaults out of `.properties` and this class resolves them here."""
+ keeps defaults out of `.properties`, and `get_property` resolves them through
+ the parent view. This class does the same for its plain-dict view."""
def __init__(self, annotation, view):
self.annotation = annotation
@@ -342,7 +343,7 @@ def anchor(self) -> dict:
is always a time frame or a bounding box.
"""
# TODO: deal with the case where the primary document is not a video
- self.paths = self.paths_to_docs()
+ self.paths = self.paths_to_docs() # type: ignore[attr-defined]
bbtf = self.find_boundingbox_or_timeframe()
# for path in paths:
# print('... [')
@@ -369,7 +370,7 @@ def anchor2(self):
# where nodes have paths to anchors
# TODO: deal with the case where the primary document is not a video
if self._anchor is None:
- self._paths = self.paths_to_docs()
+ self._paths = self.paths_to_docs() # type: ignore[attr-defined]
bbtf = self.find_boundingbox_or_timeframe()
# for path in self._paths:
# print('... [')
diff --git a/mmif/utils/summarizer/summary.py b/mmif/utils/summarizer/summary.py
index 0db14e66..3d608ff8 100644
--- a/mmif/utils/summarizer/summary.py
+++ b/mmif/utils/summarizer/summary.py
@@ -13,6 +13,7 @@
import os
import pathlib
from collections import defaultdict
+from typing import Any
from mmif.serialize import Mmif
from mmif.utils.summarizer import config
@@ -109,7 +110,7 @@ def print_warnings(self):
def pp(self):
self.documents.pp()
self.views.pp()
- self.transcript.pp()
+ self.transcript.pp() # type: ignore[attr-defined]
self.timeframes.pp()
self.entities.pp()
print()
@@ -201,7 +202,7 @@ def __len__(self):
#@staticmethod
def get_view_summary(self, view):
- annotation_types = defaultdict(int)
+ annotation_types: defaultdict[str, int] = defaultdict(int)
for annotation in view.annotations:
annotation_types[annotation.at_type.shortname] += 1
basic_info = {
@@ -349,6 +350,9 @@ def __getitem__(self, i):
def __len__(self):
return len(self.nodes)
+ def __iter__(self):
+ return iter(self.nodes)
+
def add(self, node):
self.nodes.append(node)
@@ -445,7 +449,8 @@ def _collect_durations(self):
def _collect_other_morsels(self):
# First we want everything grouped by app and label
timeframes = self.timeframes.data
- grouped_timeframes = defaultdict(lambda: defaultdict(list))
+ grouped_timeframes: defaultdict[str, defaultdict[str, list]] = defaultdict(
+ lambda: defaultdict(list))
for app in timeframes:
for tf in timeframes[app]:
label = tf.get('label')
@@ -475,7 +480,7 @@ class Entities(Nodes):
def __init__(self, summary):
super().__init__(summary)
self.nodes_idx = {}
- self.bins = None
+ self.bins: Any = None
for ent in self.graph.get_nodes(config.NAMED_ENTITY):
self.add(ent)
self._create_node_index()
diff --git a/mmif/utils/summarizer/utils.py b/mmif/utils/summarizer/utils.py
index 897a3830..c62115ed 100644
--- a/mmif/utils/summarizer/utils.py
+++ b/mmif/utils/summarizer/utils.py
@@ -101,7 +101,7 @@ class AnnotationsIndex:
def __init__(self, view):
self.view = view
- self.idx = {}
+ self.idx: dict = {}
self.tokens = []
for annotation in view.annotations:
shortname = annotation.at_type.shortname
diff --git a/mmif/utils/text_document_helper.py b/mmif/utils/text_document_helper.py
index 87e8c478..5598b913 100644
--- a/mmif/utils/text_document_helper.py
+++ b/mmif/utils/text_document_helper.py
@@ -1,5 +1,3 @@
-import mmif
-from mmif import Annotation
from mmif.vocabulary import AnnotationTypes
diff --git a/mmif/utils/timeunit_helper.py b/mmif/utils/timeunit_helper.py
index c516b7c6..7f75be37 100644
--- a/mmif/utils/timeunit_helper.py
+++ b/mmif/utils/timeunit_helper.py
@@ -1,5 +1,5 @@
from datetime import datetime
-from typing import Union
+from typing import cast
UNIT_NORMALIZATION = {
@@ -28,7 +28,7 @@ def _isoformat_to_millisecond(isoformat: str) -> int:
return int(1000 * (t.hour * 3600 + t.minute * 60 + t.second + t.microsecond / 1000000))
-def _millisecond_to_isoformat(millisecond: int) -> str:
+def _millisecond_to_isoformat(millisecond: int | float) -> str:
t = datetime.utcfromtimestamp(millisecond / 1000)
return t.strftime('%H:%M:%S.%f')[:-3]
@@ -38,7 +38,7 @@ def _second_to_isoformat(second: float) -> str:
return t.strftime('%H:%M:%S.%f')[:-3] # python strftime will return "microsecond" with 6 digits
-def convert(t: Union[int, float, str], in_unit: str, out_unit: str, fps: float) -> Union[int, float, str]:
+def convert(t: int | float | str, in_unit: str, out_unit: str, fps: float) -> int | float | str:
"""
Converts time from one unit to another. Works with ``frames``, ``seconds``, ``milliseconds``.
@@ -62,6 +62,7 @@ def convert(t: Union[int, float, str], in_unit: str, out_unit: str, fps: float)
in_unit = 'millisecond'
else:
raise ValueError(f"Invalid time format: ISO format string expected, but got {t} of type {type(t)}")
+ t = cast(int | float, t)
# s>s, ms>ms, f>f
if in_unit == out_unit:
return t
@@ -70,11 +71,8 @@ def convert(t: Union[int, float, str], in_unit: str, out_unit: str, fps: float)
if in_unit == 'millisecond':
return round(t / 1000 * fps)
# s>f
- elif 'second' == in_unit:
- return round(t * fps)
- # i>f
else:
- return round(_isoformat_to_millisecond(t) / 1000 * fps)
+ return round(t * fps)
# s>(ms or i)
elif in_unit == 'second':
return round(t * 1000) if out_unit == 'millisecond' else _second_to_isoformat(t)
diff --git a/mmif/utils/video_document_helper.py b/mmif/utils/video_document_helper.py
index e3838bf1..9e03fdc8 100644
--- a/mmif/utils/video_document_helper.py
+++ b/mmif/utils/video_document_helper.py
@@ -1,15 +1,15 @@
import contextvars
import importlib
+import math
import shlex
import sys
+import warnings
+from collections.abc import Iterable, Iterator
+from collections.abc import Sequence
+from typing import cast
from contextlib import contextmanager
from enum import Enum
-
-import math
-import warnings
from io import StringIO
-from typing import Iterable # todo: replace with collections.abc.Iterable in Python 3.9
-from typing import Iterator, List, Optional, Union, Tuple
import mmif
from mmif import Annotation, Document, Mmif, View
@@ -162,7 +162,7 @@ def open_container(video_document: Document):
if video_document is None or video_document.at_type != DocumentTypes.VideoDocument:
raise ValueError(f'The document does not exist.')
- path = video_document.location_path(nonexist_ok=False)
+ path = cast(str, video_document.location_path(nonexist_ok=False))
container = None
with _ffmpeg_op('open', video_document, path):
try:
@@ -234,7 +234,7 @@ def get_framerate(video_document: Document) -> float:
container.close()
-def _decode_frames(container, stream, decode_errors: List[str]):
+def _decode_frames(container, stream, decode_errors: list[str]):
"""
Iterates decoded frames of a video stream, stopping at the first error.
@@ -267,9 +267,9 @@ def _decode_frames(container, stream, decode_errors: List[str]):
def _warn_missing_images(
video_document: Document,
path: str,
- missing_ms: List[int],
+ missing_ms: list[int],
n_missing_positions: int,
- decode_errors: List[str],
+ decode_errors: list[str],
) -> None:
"""
Issues a single warning for timepoints that have no image.
@@ -342,9 +342,9 @@ def extract_images_from_timepoints(
Image = _check_cv_dep('PIL.Image') if as_PIL else None
container = open_container(video_document)
- path = video_document.location_path()
+ path = cast(str, video_document.location_path())
result_map = {}
- decode_errors: List[str] = []
+ decode_errors: list[str] = []
try:
stream = container.streams.video[0]
time_base = float(stream.time_base)
@@ -406,7 +406,8 @@ def _emit(frame, t_ms):
# left out of `result_map` to become a `None` entry below.
while cur_ms is not None:
if (prev_frame is not None
- and cur_pts - prev_pts <= _tolerance_ticks(prev_frame)):
+ and cast(int, cur_pts) - cast(int, prev_pts)
+ <= _tolerance_ticks(prev_frame)):
_emit(prev_frame, cur_ms)
cur_ms, cur_pts = next(targets, (None, None))
finally:
@@ -421,7 +422,7 @@ def _emit(frame, t_ms):
return [result_map.get(t) for t in original_timepoints]
-def _tp_ids_to_timepoints_ms(mmif: Mmif, tp_ids: List[str]) -> List[int]:
+def _tp_ids_to_timepoints_ms(mmif: Mmif, tp_ids: list[str]) -> list[int]:
"""
Converts a list of timepoint annotation IDs to media-timeline timepoints in milliseconds.
@@ -434,37 +435,14 @@ def _tp_ids_to_timepoints_ms(mmif: Mmif, tp_ids: List[str]) -> List[int]:
# falls back to `frame / fps` ms math that ignores the container's PTS
# start offset. Fully resolving this requires retiring timeUnit='frame'
# (tracked in clams-vocabulary#15).
- return [int(round(convert_timepoint(mmif, mmif[tp_id], 'ms')))
+ return [int(round(cast(float, convert_timepoint(
+ mmif, cast(Annotation, mmif[tp_id]), 'ms'))))
for tp_id in tp_ids]
-def _resolve_video_document(mmif: Mmif, time_frame: Annotation):
- """
- Resolves the video document associated with a TimeFrame.
- Checks the TimeFrame's own ``document`` property first,
- then falls back to the ``document`` property of the first
- target timepoint.
-
- :param mmif: :py:class:`~mmif.serialize.mmif.Mmif` instance
- :param time_frame: :py:class:`~mmif.serialize.annotation.Annotation`
- instance of a TimeFrame
- :return: :py:class:`~mmif.serialize.annotation.Document`
- """
- if 'document' in time_frame.properties:
- return mmif[time_frame.get_property('document')]
- if 'targets' in time_frame.properties:
- targets = time_frame.get_property('targets')
- if targets:
- tp = mmif[targets[0]]
- return mmif[tp.get_property('document')]
- raise ValueError(
- f'Cannot resolve video document for TimeFrame '
- f'{time_frame.id}.')
-
-
def _timeframe_to_timepoint_range_ms(
mmif: Mmif, time_frame: Annotation
-) -> Tuple[int, int]:
+) -> tuple[int, int]:
"""
Converts a TimeFrame's start/end to media-timeline timepoints in ms.
@@ -476,12 +454,12 @@ def _timeframe_to_timepoint_range_ms(
:rtype: tuple
"""
start, end = convert_timeframe(mmif, time_frame, 'ms')
- return int(round(start)), int(round(end))
+ return int(round(cast(float, start))), int(round(cast(float, end)))
def _sample_all_timepoint_pairs_ms(
mmif: Mmif, time_frame: Annotation
-) -> List[Tuple[int, Optional[str]]]:
+) -> list[tuple[int, str | None]]:
"""
Samples all (timepoint_ms, source_id) pairs from a TimeFrame. Uses all
``targets`` if present (source is the TP annotation id), otherwise
@@ -495,12 +473,12 @@ def _sample_all_timepoint_pairs_ms(
the point came from interval sampling rather than a TP
:rtype: list
"""
- if 'targets' in time_frame.properties:
+ if 'targets' in time_frame:
target_ids = time_frame.get_property('targets')
ms_list = _tp_ids_to_timepoints_ms(mmif, target_ids)
return list(zip(ms_list, target_ids))
start_ms, end_ms = _timeframe_to_timepoint_range_ms(mmif, time_frame)
- video_document = _resolve_video_document(mmif, time_frame)
+ video_document = cast(Document, mmif[time_frame.get_property('document')])
fps = get_framerate(video_document)
step_ms = 1000.0 / fps
return [(t, None) for t in sample_timepoints(start_ms, end_ms, step_ms)]
@@ -508,7 +486,7 @@ def _sample_all_timepoint_pairs_ms(
def _sample_representatives_timepoint_pairs_ms(
mmif: Mmif, time_frame: Annotation
-) -> List[Tuple[int, str]]:
+) -> list[tuple[int, str]]:
"""
Samples (timepoint_ms, source_id) pairs from a TimeFrame's
representatives. Returns an empty list if ``representatives`` is not
@@ -520,7 +498,7 @@ def _sample_representatives_timepoint_pairs_ms(
:return: list of (ms, rep_id) pairs (empty if no representatives)
:rtype: list
"""
- if 'representatives' in time_frame.properties:
+ if 'representatives' in time_frame:
reps = time_frame.get_property('representatives')
if reps:
ms_list = _tp_ids_to_timepoints_ms(mmif, reps)
@@ -530,7 +508,7 @@ def _sample_representatives_timepoint_pairs_ms(
def _sample_single_timepoint_pair_ms(
mmif: Mmif, time_frame: Annotation
-) -> List[Tuple[int, Optional[str]]]:
+) -> list[tuple[int, str | None]]:
"""
Samples a single (timepoint_ms, source_id) pair from a TimeFrame.
Uses the middle representative if ``representatives`` is present
@@ -544,7 +522,7 @@ def _sample_single_timepoint_pair_ms(
``None`` when the point came from the interval midpoint fallback
:rtype: list
"""
- if 'representatives' in time_frame.properties:
+ if 'representatives' in time_frame:
reps = time_frame.get_property('representatives')
if reps:
mid = reps[len(reps) // 2]
@@ -561,7 +539,7 @@ def extract_images_by_count_with_sources(
max_timepoints: int = sys.maxsize,
fraction: float = 1.0,
as_PIL: bool = False
- ) -> Tuple[List, List[str]]:
+ ) -> tuple[list, list[str]]:
"""
Extracts images at a count-controlled subset of the timepoints listed
in the ``targets`` property of an annotation, alongside the IDs of the
@@ -586,7 +564,7 @@ def extract_images_by_count_with_sources(
:py:func:`extract_images_from_timepoints`)
:rtype: tuple
"""
- if 'targets' not in annotation.properties:
+ if 'targets' not in annotation:
raise ValueError(
f'Annotation {annotation.id} does not have a "targets" property.')
@@ -608,7 +586,7 @@ def extract_images_by_count_with_sources(
selected_target_ids = [targets[i] for i in indices]
timepoints_ms = _tp_ids_to_timepoints_ms(mmif, selected_target_ids)
- video_doc = _resolve_video_document(mmif, annotation)
+ video_doc = cast(Document, mmif[annotation.get_property('document')])
images = extract_images_from_timepoints(
video_doc, timepoints_ms, as_PIL=as_PIL)
return images, selected_target_ids
@@ -621,7 +599,7 @@ def extract_images_by_count(
max_timepoints: int = sys.maxsize,
fraction: float = 1.0,
as_PIL: bool = False
- ) -> List:
+ ) -> list:
"""
Extracts images at a count-controlled subset of the timepoints listed
in the ``targets`` property of an annotation. See
@@ -653,9 +631,9 @@ def extract_images_by_count(
def extract_images_by_mode_with_sources(
mmif: Mmif,
time_frame: Annotation,
- mode: Union[SamplingMode, None] = None,
+ mode: SamplingMode | None = None,
as_PIL: bool = False,
-) -> Tuple[List, List[Union[str, int]]]:
+) -> tuple[list, list[str | int]]:
"""
Extracts images from a TimeFrame using a :py:class:`SamplingMode`,
alongside the per-image source: a TP annotation id (``str``) when the
@@ -681,6 +659,7 @@ def extract_images_by_mode_with_sources(
"""
if mode is None:
mode = _sampling_mode.get()
+ pairs: Sequence[tuple[int, str | None]]
if mode == SamplingMode.ALL:
pairs = _sample_all_timepoint_pairs_ms(mmif, time_frame)
elif mode == SamplingMode.REPRESENTATIVES:
@@ -690,10 +669,10 @@ def extract_images_by_mode_with_sources(
if not pairs:
return [], []
timepoints_ms = [ms for ms, _ in pairs]
- sources: List[Union[str, int]] = [
+ sources: list[str | int] = [
tp_id if tp_id is not None else ms for ms, tp_id in pairs
]
- video_doc = _resolve_video_document(mmif, time_frame)
+ video_doc = cast(Document, mmif[time_frame.get_property('document')])
images = extract_images_from_timepoints(
video_doc, timepoints_ms, as_PIL=as_PIL)
return images, sources
@@ -702,8 +681,8 @@ def extract_images_by_mode_with_sources(
def _to_timepoints(
output_view: View,
video_document: Document,
- sources: List[Union[str, int]],
-) -> List[str]:
+ sources: list[str | int],
+) -> list[str]:
"""
Turns per-image sources, as returned by
:py:func:`extract_images_by_mode_with_sources`, into ``TimePoint``
@@ -721,7 +700,7 @@ def _to_timepoints(
:param sources: per-image sources
:return: ``TimePoint`` ids, parallel to ``sources``
"""
- ids: List[str] = []
+ ids: list[str] = []
for source in sources:
if isinstance(source, str):
ids.append(source)
@@ -738,9 +717,9 @@ def iter_timeframe_images(
output_view: View,
include_labels: Iterable[str] = (),
exclude_labels: Iterable[str] = (),
- mode: Union[SamplingMode, None] = None,
+ mode: SamplingMode | None = None,
as_PIL: bool = False,
-) -> Iterator[Tuple[Annotation, List, List[str]]]:
+) -> Iterator[tuple[Annotation, list, list[str]]]:
"""
Samples images from every ``TimeFrame`` in the MMIF, filtered by label,
and yields them with the ``TimePoint`` each image was taken from.
@@ -782,7 +761,7 @@ def iter_timeframe_images(
if image is not None]
if not readable:
continue
- video_document = _resolve_video_document(mmif, time_frame)
+ video_document = cast(Document, mmif[time_frame.get_property('document')])
timepoint_ids = _to_timepoints(
output_view, video_document, [source for _, source in readable])
yield time_frame, [image for image, _ in readable], timepoint_ids
@@ -791,9 +770,9 @@ def iter_timeframe_images(
def extract_images_by_mode(
mmif: Mmif,
time_frame: Annotation,
- mode: Union[SamplingMode, None] = None,
+ mode: SamplingMode | None = None,
as_PIL: bool = False,
-) -> List:
+) -> list:
"""
Extracts images from a TimeFrame using a :py:class:`SamplingMode`.
See :py:func:`extract_images_by_mode_with_sources` for the variant
@@ -821,8 +800,8 @@ def extract_images_by_mode(
def sample_timepoints(
start_ms: int,
end_ms: int,
- step_ms: Union[int, float],
-) -> List[int]:
+ step_ms: int | float,
+) -> list[int]:
"""
Samples timepoints (in ms) from a half-open time interval
``[start_ms, end_ms)`` with a fixed step.
@@ -839,7 +818,7 @@ def sample_timepoints(
if step_ms <= 0:
raise ValueError(
f'step_ms must be positive, got {step_ms}')
- timepoints: List[int] = []
+ timepoints: list[int] = []
i = 0
while True:
t = start_ms + i * step_ms
@@ -850,7 +829,7 @@ def sample_timepoints(
return timepoints
-def convert_timepoint(mmif: Mmif, timepoint: Annotation, out_unit: str) -> Union[int, float, str]:
+def convert_timepoint(mmif: Mmif, timepoint: Annotation, out_unit: str) -> int | float | str:
"""
Converts a time point included in an annotation to a different time unit.
The input annotation must have ``timePoint`` property.
@@ -861,11 +840,11 @@ def convert_timepoint(mmif: Mmif, timepoint: Annotation, out_unit: str) -> Union
:return: frame number (integer) or second/millisecond (float) of input timepoint
"""
in_unit = timepoint.get_property('timeUnit')
- vd = mmif[timepoint.get_property('document')]
+ vd = cast(Document, mmif[timepoint.get_property('document')])
return convert(timepoint.get_property('timePoint'), in_unit, out_unit, get_framerate(vd))
-def convert_timeframe(mmif: Mmif, time_frame: Annotation, out_unit: str) -> Tuple[Union[int, float, str], Union[int, float, str]]:
+def convert_timeframe(mmif: Mmif, time_frame: Annotation, out_unit: str) -> tuple[int | float | str, int | float | str]:
"""
Converts start and end points in a ``TimeFrame`` annotation a different time unit.
@@ -875,26 +854,26 @@ def convert_timeframe(mmif: Mmif, time_frame: Annotation, out_unit: str) -> Tupl
:return: tuple of frame numbers, seconds/milliseconds, or ISO notation of TimeFrame's start and end
"""
in_unit = time_frame.get_property('timeUnit')
- vd = mmif[time_frame.get_property('document')]
+ vd = cast(Document, mmif[time_frame.get_property('document')])
fps = get_framerate(vd)
return convert(time_frame.get_property('start'), in_unit, out_unit, fps), convert(time_frame.get_property('end'), in_unit, out_unit, fps)
def capture(video_document: Document):
"""
- .. deprecated::
- Use :py:func:`open_container` instead. See issue #379.
+ .. deprecated:: 1.4.0
+ Will be removed in 2.0.0. Use :func:`open_container` instead. See issue
+ #379.
Captures a video file using OpenCV and adds fps, frame count, and duration as properties to the document.
:param video_document: :py:class:`~mmif.serialize.annotation.Document` instance that holds a video document (``"@type": ".../VideoDocument/..."``)
:return: `OpenCV VideoCapture `_ object
"""
- warnings.warn(
- f'capture() is deprecated; use open_container() instead. '
- f'{_PTS_BUG_NOTICE}',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('capture() is deprecated and will be removed in 2.0.0. Use '
+ 'open_container() instead. '
+ f'{_PTS_BUG_NOTICE}',
+ DeprecationWarning, stacklevel=2)
cv2 = _check_cv_dep('cv2')
if video_document is None or video_document.at_type != DocumentTypes.VideoDocument:
raise ValueError(f'The document does not exist.')
@@ -911,8 +890,9 @@ def capture(video_document: Document):
def extract_frames_as_images(video_document: Document, framenums: Iterable[int], as_PIL: bool = False, record_ffmpeg_errors: bool = False):
"""
- .. deprecated::
- Use :py:func:`extract_images_from_timepoints` instead. See issue #379.
+ .. deprecated:: 1.4.0
+ Will be removed in 2.0.0. Use :func:`extract_images_from_timepoints`
+ instead. See issue #379.
Extracts frames from a video document as a list of :py:class:`numpy.ndarray`.
Use with :py:func:`sample_frames` function to get the list of frame numbers first.
@@ -923,11 +903,11 @@ def extract_frames_as_images(video_document: Document, framenums: Iterable[int],
:param record_ffmpeg_errors: if True, records and warns about FFmpeg stderr output during extraction
:return: frames as a list of :py:class:`~numpy.ndarray` or :py:class:`~PIL.Image.Image`
"""
- warnings.warn(
- f'extract_frames_as_images() is deprecated; use '
- f'extract_images_from_timepoints() instead. {_PTS_BUG_NOTICE}',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('extract_frames_as_images() is deprecated and will be '
+ 'removed in 2.0.0. Use extract_images_from_timepoints() '
+ 'instead. '
+ f'{_PTS_BUG_NOTICE}',
+ DeprecationWarning, stacklevel=2)
cv2 = _check_cv_dep('cv2')
# deduplicate and sort frame numbers for extraction, then map back to original order
original_framenums = list(framenums)
@@ -959,14 +939,22 @@ def extract_frames_as_images(video_document: Document, framenums: Iterable[int],
if cur_f == next_target_f:
if not ret:
sec = convert(cur_f, 'f', 's', video_document.get_property(FPS_DOCPROP_KEY))
- warnings.warn(f'Frame #{cur_f} ({sec}s) could not be read from the video {video_document.id} @ {video_document.location} .')
+ warnings.warn('extract_frames_as_images() is deprecated '
+ 'and will be removed in 2.0.0. Use '
+ 'extract_images_from_timepoints() instead. '
+ f'{_PTS_BUG_NOTICE}',
+ DeprecationWarning)
else:
unique_frames[cur_f] = Image.fromarray(frame[:, :, ::-1]) if as_PIL else frame
next_target_f = next(framenumi, None)
cur_f += 1
ffmpeg_err_str = ffmpeg_errs.getvalue()
if ffmpeg_err_str and record_ffmpeg_errors:
- warnings.warn(f'FFmpeg output during extracting frames: {ffmpeg_err_str}')
+ warnings.warn('extract_frames_as_images() is deprecated and will be '
+ 'removed in 2.0.0. Use extract_images_from_timepoints()'
+ ' instead. '
+ f'{_PTS_BUG_NOTICE}',
+ DeprecationWarning)
video.release()
# return frames in original input order, duplicating where needed
return [unique_frames[f] for f in original_framenums if f in unique_frames]
@@ -982,24 +970,26 @@ def _get_mid_framenum(mmif: Mmif, time_frame: Annotation) -> int:
:return: middle frame number as an integer
"""
timeunit = time_frame.get_property('timeUnit')
- video_document = mmif[time_frame.get_property('document')]
+ video_document = cast(Document, mmif[time_frame.get_property('document')])
fps = get_framerate(video_document)
- return int(convert(time_frame.get_property('start') + time_frame.get_property('end'), timeunit, 'frame', fps) // 2)
+ return int(cast(int, convert(time_frame.get_property('start') + time_frame.get_property('end'), timeunit, 'frame', fps)) // 2)
def get_mid_framenum(mmif: Mmif, time_frame: Annotation) -> int:
"""
- .. deprecated::
- Use :py:func:`extract_frames_by_mode` instead.
+ .. deprecated:: 1.3.0
+ Will be removed in 2.0.0. Use :func:`extract_images_by_mode` instead.
"""
- warnings.warn('This function is deprecated. Use ``extract_frames_by_mode()`` instead.', DeprecationWarning, stacklevel=2)
+ warnings.warn('get_mid_framenum() is deprecated and will be removed in '
+ '2.0.0. Use extract_images_by_mode() instead.',
+ DeprecationWarning, stacklevel=2)
return _get_mid_framenum(mmif, time_frame)
def extract_mid_frame(mmif: Mmif, time_frame: Annotation, as_PIL: bool = False):
"""
- .. deprecated::
- Use :py:func:`extract_frames_by_mode` instead.
+ .. deprecated:: 1.3.0
+ Will be removed in 2.0.0. Use :func:`extract_images_by_mode` instead.
Extracts the middle frame of a time interval annotation as a numpy ndarray.
@@ -1008,16 +998,18 @@ def extract_mid_frame(mmif: Mmif, time_frame: Annotation, as_PIL: bool = False):
:param as_PIL: return :py:class:`~PIL.Image.Image` instead of :py:class:`~numpy.ndarray`
:return: frame as a :py:class:`numpy.ndarray` or :py:class:`PIL.Image.Image`
"""
- warnings.warn('This function is deprecated. Use ``extract_frames_by_mode()`` instead.', DeprecationWarning, stacklevel=2)
- vd = mmif[time_frame.get_property('document')]
+ warnings.warn('extract_mid_frame() is deprecated and will be removed in '
+ '2.0.0. Use extract_images_by_mode() instead.',
+ DeprecationWarning, stacklevel=2)
+ vd = cast(Document, mmif[time_frame.get_property('document')])
fn = get_mid_framenum(mmif, time_frame)
return extract_frames_as_images(vd, [fn], as_PIL=as_PIL)[0]
-def get_representative_framenums(mmif: Mmif, time_frame: Annotation) -> List[int]:
+def get_representative_framenums(mmif: Mmif, time_frame: Annotation) -> list[int]:
"""
- .. deprecated::
- Use :py:func:`extract_frames_by_mode` instead.
+ .. deprecated:: 1.3.0
+ Will be removed in 2.0.0. Use :func:`extract_images_by_mode` instead.
Calculates the representative frame numbers from an annotation. To pick the representative frames, it first looks
up the ``representatives`` property of the ``TimeFrame`` annotation. If it is not found, it will calculate the
@@ -1027,17 +1019,19 @@ def get_representative_framenums(mmif: Mmif, time_frame: Annotation) -> List[int
:param time_frame: :py:class:`~mmif.serialize.annotation.Annotation` instance that holds a time interval annotation containing a `representatives` property (``"@type": ".../TimeFrame/..."``)
:return: representative frame number as an integer
"""
- warnings.warn('This function is deprecated. Use ``extract_frames_by_mode()`` instead.', DeprecationWarning, stacklevel=2)
- if 'representatives' not in time_frame.properties:
+ warnings.warn('get_representative_framenums() is deprecated and will be '
+ 'removed in 2.0.0. Use extract_images_by_mode() instead.',
+ DeprecationWarning, stacklevel=2)
+ if 'representatives' not in time_frame:
return [_get_mid_framenum(mmif, time_frame)]
timeunit = time_frame.get_property('timeUnit')
- video_document = mmif[time_frame.get_property('document')]
+ video_document = cast(Document, mmif[time_frame.get_property('document')])
fps = get_framerate(video_document)
representatives = time_frame.get_property('representatives')
ref_frams = []
for rep_id in representatives:
try:
- rep_anno = mmif[rep_id]
+ rep_anno = cast(Annotation, mmif[rep_id])
except KeyError as ke:
raise ValueError(f'Representative timepoint {rep_id} not found in any view. ({ke})')
ref_frams.append(int(convert(rep_anno.get_property('timePoint'), timeunit, 'frame', fps)))
@@ -1046,13 +1040,15 @@ def get_representative_framenums(mmif: Mmif, time_frame: Annotation) -> List[int
def get_representative_framenum(mmif: Mmif, time_frame: Annotation) -> int:
"""
- .. deprecated::
- Use :py:func:`extract_frames_by_mode` instead.
+ .. deprecated:: 1.3.0
+ Will be removed in 2.0.0. Use :func:`extract_images_by_mode` instead.
A thin wrapper around :py:func:`get_representative_framenums` to return a single representative frame number. Always
return the first frame number found.
"""
- warnings.warn('This function is deprecated. Use ``extract_frames_by_mode()`` instead.', DeprecationWarning, stacklevel=2)
+ warnings.warn('get_representative_framenum() is deprecated and will be '
+ 'removed in 2.0.0. Use extract_images_by_mode() instead.',
+ DeprecationWarning, stacklevel=2)
try:
return get_representative_framenums(mmif, time_frame)[0]
except IndexError:
@@ -1061,8 +1057,8 @@ def get_representative_framenum(mmif: Mmif, time_frame: Annotation) -> int:
def extract_representative_frame(mmif: Mmif, time_frame: Annotation, as_PIL: bool = False, first_only: bool = True):
"""
- .. deprecated::
- Use :py:func:`extract_frames_by_mode` instead.
+ .. deprecated:: 1.3.0
+ Will be removed in 2.0.0. Use :func:`extract_images_by_mode` instead.
Extracts the representative frame of an annotation as a numpy ndarray or PIL Image.
@@ -1072,16 +1068,19 @@ def extract_representative_frame(mmif: Mmif, time_frame: Annotation, as_PIL: boo
:param first_only: return the first representative frame only
:return: frame as a :py:class:`numpy.ndarray` or :py:class:`PIL.Image.Image`
"""
- warnings.warn('This function is deprecated. Use ``extract_frames_by_mode()`` instead.', DeprecationWarning, stacklevel=2)
- video_document = mmif[time_frame.get_property('document')]
+ warnings.warn('extract_representative_frame() is deprecated and will be '
+ 'removed in 2.0.0. Use extract_images_by_mode() instead.',
+ DeprecationWarning, stacklevel=2)
+ video_document = cast(Document, mmif[time_frame.get_property('document')])
rep_frame_num = [get_representative_framenum(mmif, time_frame)] if first_only else get_representative_framenums(mmif, time_frame)
return extract_frames_as_images(video_document, rep_frame_num, as_PIL=as_PIL)[0]
-def sample_frames(start_frame: int, end_frame: int, sample_rate: float = 1) -> List[int]:
+def sample_frames(start_frame: int, end_frame: int, sample_rate: float = 1) -> list[int]:
"""
- .. deprecated::
- Use :py:func:`sample_timepoints` instead. See issue #379.
+ .. deprecated:: 1.4.0
+ Will be removed in 2.0.0. Use :func:`sample_timepoints` instead. See
+ issue #379.
Helper function to sample frames from a time interval.
Can also be used as a "cutoff" function when used with ``start_frame==0`` and ``sample_rate==1``.
@@ -1091,15 +1090,14 @@ def sample_frames(start_frame: int, end_frame: int, sample_rate: float = 1) -> L
:param sample_rate: sampling rate (or step) to configure how often to take a frame, default is 1, meaning all consecutive frames are sampled
:return: list of frame numbers to extract
"""
- warnings.warn(
- f'sample_frames() is deprecated; use sample_timepoints() instead. '
- f'{_PTS_BUG_NOTICE}',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('sample_frames() is deprecated and will be removed in '
+ '2.0.0. Use sample_timepoints() instead. '
+ f'{_PTS_BUG_NOTICE}',
+ DeprecationWarning, stacklevel=2)
if sample_rate < 1:
raise ValueError(f"Sample rate must be greater than 1, but got {sample_rate}")
- frame_nums: List[int] = []
- cur_f = start_frame
+ frame_nums: list[int] = []
+ cur_f: float = start_frame
while cur_f < end_frame:
ceiling = math.ceil(cur_f)
if ceiling < end_frame:
@@ -1111,8 +1109,8 @@ def sample_frames(start_frame: int, end_frame: int, sample_rate: float = 1) -> L
def get_annotation_property(mmif, annotation, prop_name):
"""
.. deprecated:: 1.0.8
- Will be removed in 2.0.0.
- Use :py:meth:`mmif.serialize.annotation.Annotation.get_property` method instead.
+ Will be removed in 2.0.0. Use
+ :meth:`mmif.serialize.annotation.Annotation.get_property` instead.
Get a property value from an annotation. If the property is not found in the annotation, it will look up the metadata of the annotation's parent view and return the value from there.
@@ -1121,104 +1119,104 @@ def get_annotation_property(mmif, annotation, prop_name):
:param prop_name: name of the property to retrieve
:return: the property value
"""
- warnings.warn(f'{__name__}() is deprecated. '
- f'Directly ask the annotation for a property by calling annotation.get_property() instead.',
+ warnings.warn('get_annotation_property() is deprecated and will be '
+ 'removed in 2.0.0. Use Annotation.get_property() instead.',
DeprecationWarning)
return annotation.get_property(prop_name)
def framenum_to_second(video_doc: Document, frame: int):
"""
- .. deprecated::
- Use :py:func:`~mmif.utils.timeunit_helper.convert` with ``ms``/``s``
- directly. See issue #379.
+ .. deprecated:: 1.4.0
+ Will be removed in 2.0.0. Use :func:`mmif.utils.timeunit_helper.convert`
+ with ``ms``/``s`` instead. See issue #379.
"""
- warnings.warn(
- f'framenum_to_second() is deprecated. {_PTS_BUG_NOTICE}',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('framenum_to_second() is deprecated and will be removed in '
+ '2.0.0. Use timeunit_helper.convert() instead. '
+ f'{_PTS_BUG_NOTICE}',
+ DeprecationWarning, stacklevel=2)
fps = get_framerate(video_doc)
return convert(frame, 'f', 's', fps)
def framenum_to_millisecond(video_doc: Document, frame: int):
"""
- .. deprecated::
- Use :py:func:`~mmif.utils.timeunit_helper.convert` with ``ms``/``s``
- directly. See issue #379.
+ .. deprecated:: 1.4.0
+ Will be removed in 2.0.0. Use :func:`mmif.utils.timeunit_helper.convert`
+ with ``ms``/``s`` instead. See issue #379.
"""
- warnings.warn(
- f'framenum_to_millisecond() is deprecated. {_PTS_BUG_NOTICE}',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('framenum_to_millisecond() is deprecated and will be '
+ 'removed in 2.0.0. Use timeunit_helper.convert() instead. '
+ f'{_PTS_BUG_NOTICE}',
+ DeprecationWarning, stacklevel=2)
fps = get_framerate(video_doc)
return convert(frame, 'f', 'ms', fps)
def second_to_framenum(video_doc: Document, second) -> int:
"""
- .. deprecated::
- Use :py:func:`extract_images_from_timepoints` or stay in the time
- domain. See issue #379.
+ .. deprecated:: 1.4.0
+ Will be removed in 2.0.0. Use :func:`extract_images_from_timepoints`
+ instead, or stay in the time domain. See issue #379.
"""
- warnings.warn(
- f'second_to_framenum() is deprecated. {_PTS_BUG_NOTICE}',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('second_to_framenum() is deprecated and will be removed in '
+ '2.0.0. Use extract_images_from_timepoints() instead. '
+ f'{_PTS_BUG_NOTICE}',
+ DeprecationWarning, stacklevel=2)
fps = get_framerate(video_doc)
return int(convert(second, 's', 'f', fps))
def millisecond_to_framenum(video_doc: Document, millisecond: float) -> int:
"""
- .. deprecated::
- Use :py:func:`extract_images_from_timepoints` or stay in the time
- domain. See issue #379.
+ .. deprecated:: 1.4.0
+ Will be removed in 2.0.0. Use :func:`extract_images_from_timepoints`
+ instead, or stay in the time domain. See issue #379.
"""
- warnings.warn(
- f'millisecond_to_framenum() is deprecated. {_PTS_BUG_NOTICE}',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('millisecond_to_framenum() is deprecated and will be '
+ 'removed in 2.0.0. Use extract_images_from_timepoints() '
+ 'instead. '
+ f'{_PTS_BUG_NOTICE}',
+ DeprecationWarning, stacklevel=2)
fps = get_framerate(video_doc)
return int(convert(millisecond, 'ms', 'f', fps))
def extract_timepoints_as_images(*args, **kwargs):
"""
- .. deprecated::
- Renamed to :py:func:`extract_images_from_timepoints`.
+ .. deprecated:: 1.5.0
+ Will be removed in 2.0.0. Use :func:`extract_images_from_timepoints`
+ instead.
"""
- warnings.warn(
- 'extract_timepoints_as_images() is deprecated; '
- 'use extract_images_from_timepoints() instead.',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('extract_timepoints_as_images() is deprecated and will be '
+ 'removed in 2.0.0. Use extract_images_from_timepoints() '
+ 'instead.',
+ DeprecationWarning, stacklevel=2)
return extract_images_from_timepoints(*args, **kwargs)
def extract_target_frames(*args, **kwargs):
"""
- .. deprecated::
- Renamed to :py:func:`extract_images_by_count_with_sources`.
- For a bare-images variant, use :py:func:`extract_images_by_count`.
+ .. deprecated:: 1.5.0
+ Will be removed in 2.0.0. Use
+ :func:`extract_images_by_count_with_sources` instead. For a bare-images
+ variant, use :func:`extract_images_by_count`.
"""
- warnings.warn(
- 'extract_target_frames() is deprecated; '
- 'use extract_images_by_count_with_sources() instead.',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('extract_target_frames() is deprecated and will be removed '
+ 'in 2.0.0. Use extract_images_by_count_with_sources() '
+ 'instead.',
+ DeprecationWarning, stacklevel=2)
return extract_images_by_count_with_sources(*args, **kwargs)
def extract_frames_by_mode(*args, **kwargs):
"""
- .. deprecated::
- Renamed to :py:func:`extract_images_by_mode`. For per-image source
- IDs, use :py:func:`extract_images_by_mode_with_sources`.
+ .. deprecated:: 1.5.0
+ Will be removed in 2.0.0. Use :func:`extract_images_by_mode` instead.
+ For per-image source IDs, use
+ :func:`extract_images_by_mode_with_sources`.
"""
- warnings.warn(
- 'extract_frames_by_mode() is deprecated; '
- 'use extract_images_by_mode() instead.',
- DeprecationWarning, stacklevel=2,
- )
+ warnings.warn('extract_frames_by_mode() is deprecated and will be removed'
+ ' in 2.0.0. Use extract_images_by_mode() instead.',
+ DeprecationWarning, stacklevel=2)
return extract_images_by_mode(*args, **kwargs)
diff --git a/mmif/utils/workflow_helper.py b/mmif/utils/workflow_helper.py
index 2b54a10b..b6dee070 100644
--- a/mmif/utils/workflow_helper.py
+++ b/mmif/utils/workflow_helper.py
@@ -3,14 +3,14 @@
import itertools
from collections import Counter
from pathlib import Path
-from typing import Any, Dict, List, Literal, Optional, Tuple, Union, overload
+from typing import Any, Literal, overload
from pydantic import BaseModel, ConfigDict, Field
from mmif.serialize.mmif import Mmif, ViewsList
-def group_views_by_app(views: ViewsList) -> List[List[Any]]:
+def group_views_by_app(views: ViewsList) -> list[list[Any]]:
"""
Groups views into app executions based on app and timestamp.
@@ -50,7 +50,7 @@ def group_views_by_app(views: ViewsList) -> List[List[Any]]:
def _split_appname_appversion(
long_app_id: str
-) -> Tuple[Optional[str], Optional[str]]:
+) -> tuple[str | None, str | None]:
"""
Split app name and version from a long app identifier.
@@ -90,7 +90,7 @@ def generate_param_hash(params: dict) -> str:
return hashlib.md5(param_string.encode('utf-8')).hexdigest()
-def _read_mmif_from_path(mmif_input: Union[str, Path, Mmif]) -> Mmif:
+def _read_mmif_from_path(mmif_input: str | Path | Mmif) -> Mmif:
"""
Helper function to get a Mmif object from various input types.
@@ -111,20 +111,20 @@ def _read_mmif_from_path(mmif_input: Union[str, Path, Mmif]) -> Mmif:
@overload
-def generate_workflow_identifier(mmif_input: Union[str, Path, Mmif],
+def generate_workflow_identifier(mmif_input: str | Path | Mmif,
return_param_dicts: Literal[True]
- ) -> Tuple[str, List[dict]]: ...
+ ) -> tuple[str, list[dict]]: ...
@overload
-def generate_workflow_identifier(mmif_input: Union[str, Path, Mmif],
+def generate_workflow_identifier(mmif_input: str | Path | Mmif,
return_param_dicts: Literal[False] = False
) -> str: ...
-def generate_workflow_identifier(mmif_input: Union[str, Path, Mmif],
+def generate_workflow_identifier(mmif_input: str | Path | Mmif,
return_param_dicts: bool = False
- ) -> Union[str, Tuple[str, List[dict]]]:
+ ) -> str | tuple[str, list[dict]]:
"""
Generate a workflow identifier string from a MMIF file or object.
@@ -197,10 +197,10 @@ class SingleMmifStats(BaseModel):
model_config = ConfigDict(populate_by_name=True)
app_count: int = Field(..., alias="appCount", description="Total number of app executions identified.")
- error_views: List[str] = Field(default_factory=list, alias="errorViews", description="List of view IDs that contain errors.")
- warning_views: List[str] = Field(default_factory=list, alias="warningViews", description="List of view IDs that contain warnings.")
- empty_views: List[str] = Field(default_factory=list, alias="emptyViews", description="List of view IDs that contain no annotations, error, or warnings.")
- annotation_count_by_type: Dict[str, int] = Field(default_factory=dict, alias="annotationCountByType", description="Total annotation counts across the file.")
+ error_views: list[str] = Field(default_factory=list, alias="errorViews", description="List of view IDs that contain errors.")
+ warning_views: list[str] = Field(default_factory=list, alias="warningViews", description="List of view IDs that contain warnings.")
+ empty_views: list[str] = Field(default_factory=list, alias="emptyViews", description="List of view IDs that contain no annotations, error, or warnings.")
+ annotation_count_by_type: dict[str, int] = Field(default_factory=dict, alias="annotationCountByType", description="Total annotation counts across the file.")
class AppProfiling(BaseModel):
"""
@@ -208,7 +208,7 @@ class AppProfiling(BaseModel):
"""
model_config = ConfigDict(populate_by_name=True)
- running_time_ms: Optional[int] = Field(default=None, alias="runningTimeMS", description="Execution time in milliseconds.")
+ running_time_ms: int | None = Field(default=None, alias="runningTimeMS", description="Execution time in milliseconds.")
class AppExecution(BaseModel):
"""
@@ -217,10 +217,10 @@ class AppExecution(BaseModel):
model_config = ConfigDict(populate_by_name=True)
app: str = Field(..., description="The URI of the app.")
- view_ids: List[str] = Field(..., alias="viewIds", description="List of view IDs generated by this execution.")
- app_configuration: Dict = Field(default_factory=dict, alias="appConfiguration", description="Configuration parameters used for this execution.")
+ view_ids: list[str] = Field(..., alias="viewIds", description="List of view IDs generated by this execution.")
+ app_configuration: dict = Field(default_factory=dict, alias="appConfiguration", description="Configuration parameters used for this execution.")
app_profiling: AppProfiling = Field(default_factory=lambda: AppProfiling(), alias="appProfiling", description="Profiling data for this execution.")
- annotation_count_by_type: Dict[str, int] = Field(default_factory=dict, alias="annotationCountByType", description="Counts of annotations produced, grouped by type.")
+ annotation_count_by_type: dict[str, int] = Field(default_factory=dict, alias="annotationCountByType", description="Counts of annotations produced, grouped by type.")
class SingleMmifDesc(BaseModel):
@@ -231,7 +231,7 @@ class SingleMmifDesc(BaseModel):
workflow_id: str = Field(..., alias="workflowId", description="Unique identifier for the workflow structure.")
stats: SingleMmifStats = Field(..., description="Statistics about the views and annotations.")
- apps: List[AppExecution] = Field(..., description="Sequence of app executions in the workflow.")
+ apps: list[AppExecution] = Field(..., description="Sequence of app executions in the workflow.")
def _get_profile_data(view) -> AppProfiling:
@@ -262,7 +262,7 @@ def _get_profile_data(view) -> AppProfiling:
return AppProfiling(runningTimeMS=milliseconds)
-def describe_single_mmif(mmif_input: Union[str, Path, Mmif]) -> dict:
+def describe_single_mmif(mmif_input: str | Path | Mmif) -> dict:
"""
Reads a MMIF file or object and extracts the workflow specification from it.
@@ -319,7 +319,7 @@ def describe_single_mmif(mmif_input: Union[str, Path, Mmif]) -> dict:
for group in view_groups:
first_view = group[0]
- execution_ann_counter = Counter()
+ execution_ann_counter: Counter[str] = Counter()
for view in group:
execution_ann_counter.update(Counter(str(ann.at_type) for ann in view.annotations))
@@ -358,7 +358,7 @@ def describe_single_mmif(mmif_input: Union[str, Path, Mmif]) -> dict:
))
# aggregate total annotation counts
- total_annotations_by_type = Counter()
+ total_annotations_by_type: Counter[str] = Counter()
for execution in grouped_apps:
# Only aggregate from actual apps, not the special unassigned entry
if execution.app != "http://apps.clams.ai/non-existing-app/v1":
@@ -394,10 +394,10 @@ class AppProfilingStats(BaseModel):
"""
model_config = ConfigDict(populate_by_name=True)
- avg_running_time_ms: Optional[float] = Field(default=None, alias="avgRunningTimeMS", description="Average execution time in milliseconds.")
- min_running_time_ms: Optional[float] = Field(default=None, alias="minRunningTimeMS", description="Minimum execution time in milliseconds.")
- max_running_time_ms: Optional[float] = Field(default=None, alias="maxRunningTimeMS", description="Maximum execution time in milliseconds.")
- stdev_running_time_ms: Optional[float] = Field(default=None, alias="stdevRunningTimeMS", description="Standard deviation of execution time.")
+ avg_running_time_ms: float | None = Field(default=None, alias="avgRunningTimeMS", description="Average execution time in milliseconds.")
+ min_running_time_ms: float | None = Field(default=None, alias="minRunningTimeMS", description="Minimum execution time in milliseconds.")
+ max_running_time_ms: float | None = Field(default=None, alias="maxRunningTimeMS", description="Maximum execution time in milliseconds.")
+ stdev_running_time_ms: float | None = Field(default=None, alias="stdevRunningTimeMS", description="Standard deviation of execution time.")
@@ -409,7 +409,7 @@ class WorkflowAppExecution(BaseModel):
model_config = ConfigDict(populate_by_name=True)
app: str = Field(..., description="The URI of the app.")
- app_configuration: Dict = Field(default_factory=dict, alias="appConfiguration", description="Representative configuration (usually from the first occurrence).")
+ app_configuration: dict = Field(default_factory=dict, alias="appConfiguration", description="Representative configuration (usually from the first occurrence).")
app_profiling: AppProfilingStats = Field(default_factory=lambda: AppProfilingStats(), alias="appProfiling", description="Aggregated profiling statistics.")
@@ -420,9 +420,9 @@ class WorkflowCollectionEntry(BaseModel):
model_config = ConfigDict(populate_by_name=True)
workflow_id: str = Field(..., alias="workflowId", description="Unique identifier for the workflow.")
- mmifs: List[str] = Field(..., description="List of filenames belonging to this workflow.")
+ mmifs: list[str] = Field(..., description="List of filenames belonging to this workflow.")
mmif_count: int = Field(..., alias="mmifCount", description="Number of MMIF files matching this workflow.")
- apps: List[WorkflowAppExecution] = Field(..., description="Sequence of apps in this workflow with aggregated stats.")
+ apps: list[WorkflowAppExecution] = Field(..., description="Sequence of apps in this workflow with aggregated stats.")
class MmifCountByStatus(BaseModel):
"""
@@ -444,11 +444,11 @@ class CollectionMmifDesc(BaseModel):
model_config = ConfigDict(populate_by_name=True)
mmif_count_by_status: MmifCountByStatus = Field(..., alias="mmifCountByStatus", description="Counts of MMIF files by status.")
- workflows: List[WorkflowCollectionEntry] = Field(..., description="List of unique workflows identified in the collection.")
- annotation_count_by_type: Dict[str, int] = Field(default_factory=dict, alias="annotationCountByType", description="Total annotation counts across the entire collection.")
+ workflows: list[WorkflowCollectionEntry] = Field(..., description="List of unique workflows identified in the collection.")
+ annotation_count_by_type: dict[str, int] = Field(default_factory=dict, alias="annotationCountByType", description="Total annotation counts across the entire collection.")
-def describe_mmif_collection(mmif_dir: Union[str, Path]) -> dict:
+def describe_mmif_collection(mmif_dir: str | Path) -> dict:
"""
Reads all MMIF files in a directory and extracts a summarized workflow specification.
@@ -476,10 +476,10 @@ def describe_mmif_collection(mmif_dir: Union[str, Path]) -> dict:
invalid=0
)
- aggregated_counts = Counter()
+ aggregated_counts: Counter[str] = Counter()
# Structure: {workflow_id: {'mmifs': [...], 'apps': {app_uri: {'appConfiguration': ..., 'execution_times': [...]}}}}
- workflows_data: Dict[str, Dict] = {}
+ workflows_data: dict[str, dict] = {}
for mmif_file in mmif_files:
try:
diff --git a/mmif/ver/__init__.py b/mmif/ver/__init__.py
index bddd3a5c..c4ab731e 100644
--- a/mmif/ver/__init__.py
+++ b/mmif/ver/__init__.py
@@ -9,7 +9,7 @@
_urls = {
k: v for k, v in (
u.split(", ", 1)
- for u in metadata("mmif-python").get_all("Project-URL")
+ for u in metadata("mmif-python").get_all("Project-URL", [])
)
}
__specver__ = _urls["mmif-spec"].rstrip("/").rsplit("/", 1)[-1]
diff --git a/mmif_docloc_http/__init__.py b/mmif_docloc_http/__init__.py
index 1d954474..7697c594 100644
--- a/mmif_docloc_http/__init__.py
+++ b/mmif_docloc_http/__init__.py
@@ -9,7 +9,7 @@
import urllib.request
import urllib.error
-_cache = {}
+_cache: dict = {}
def resolve(docloc):
diff --git a/pyproject.toml b/pyproject.toml
index 97611807..5691624b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -45,7 +45,7 @@ docs = ["sphinx>=7.0,<