diff --git a/.circleci/config.yml b/.circleci/config.yml index 9cfcbc9..139830d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -156,9 +156,6 @@ workflows: matrix: parameters: version: - - "3.8" - - "3.9" - - "3.10" - "3.11" - "3.12" - "3.13" @@ -202,9 +199,6 @@ workflows: matrix: parameters: version: - - "3.8" - - "3.9" - - "3.10" - "3.11" - "3.12" - "3.13" diff --git a/HISTORY.rst b/HISTORY.rst index cefd768..a2bb445 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,17 @@ History ======= +unreleased +---------- + +Fixed +^^^^^ +- SOFA attributes and string variables are not always written as netCDF `char` data as required by the SOFA standard. For backwards compatibility and user friendliness, those data can be read even if they are saved as `string` data (PR #149) + +Deprecated +^^^^^^^^^^ +- Python versions below 3.11 and netCDF4 versions below 1.7.5 (PR #149) + 1.3.0 (2026-08-27) ------------------ diff --git a/README.md b/README.md index 7fa6c1b..e6ba462 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Use pip to install sofar pip install sofar -(Requires Python >= 3.8) +(Requires Python >= 3.11) If the installation fails, please check out the [help section](https://pyfar-gallery.readthedocs.io/en/latest/help). diff --git a/pyproject.toml b/pyproject.toml index be1a9f6..57f608e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ version = "1.3.0" description = "Maybe the most complete python package for SOFA files so far." readme = "README.md" license = {file = "LICENSE"} -requires-python = ">=3.8" +requires-python = ">=3.11" authors = [ { name = "The pyfar developers", email = "info@pyfar.org" }, ] @@ -18,16 +18,13 @@ classifiers = [ "License :: OSI Approved :: MIT License", "Natural Language :: English", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", ] dependencies = [ - 'netCDF4!=1.7.4', - 'numpy>=1.14.0,<2.5.0', + 'netCDF4>=1.7.5', + 'numpy>=1.14.0', 'beautifulsoup4', 'requests', 'packaging', diff --git a/sofar/io.py b/sofar/io.py index 558622a..7925ba6 100644 --- a/sofar/io.py +++ b/sofar/io.py @@ -2,7 +2,7 @@ import contextlib import os import numpy as np -from netCDF4 import Dataset, chartostring, stringtochar +from netCDF4 import Dataset import warnings import pathlib from packaging.version import parse @@ -126,6 +126,10 @@ def _read_netcdf(filename, verify, verbose, mode): # open new NETCDF4 file for reading with Dataset(filename, "r", format="NETCDF4") as file: + # return character variables as raw bytes for decoding in + # `_format_value_from_netcdf` + file.set_auto_chartostring(False) + if mode == "sofa": # get convention name and version convention = file.SOFAConventions @@ -145,7 +149,7 @@ def _read_netcdf(filename, verify, verbose, mode): # load global attributes for attr in file.ncattrs(): - value = getattr(file, attr) + value = _format_value_from_netcdf(getattr(file, attr), attr) all_attr.append(f"GLOBAL_{attr}") if not hasattr(sofa, f"GLOBAL_{attr}"): @@ -166,8 +170,15 @@ def _read_netcdf(filename, verify, verbose, mode): setattr(sofa, var.replace(".", "_"), value) else: dimensions = "".join(list(file[var].dimensions)) - # SOFA only uses dtypes 'double' and 'S1' but netCDF has more - dtype = "string" if file[var].datatype == "S1" else "double" + # Translate data type from netCDF to SOFA. + # SOFA uses dtypes 'double' (also double in netCDF) and + # 'string'. To be user friendly, sofar reads netCDF data of + # type 'char' ("S1") and 'string' (str). + # (the datatype of 'string' is a VLType but its dtype is str) + if file[var].dtype in ["S1", str]: + dtype = "string" + else: + dtype = "double" sofa._add_custom_api_entry(var.replace(".", "_"), value, None, dimensions, dtype) custom.append(var.replace(".", "_")) @@ -176,7 +187,8 @@ def _read_netcdf(filename, verify, verbose, mode): # load variable attributes for attr in [a for a in file[var].ncattrs() if a not in skip]: - value = getattr(file[var], attr) + value = _format_value_from_netcdf( + getattr(file[var], attr), attr) all_attr.append(var.replace(".", "_") + "_" + attr) if not hasattr(sofa, var.replace(".", "_") + "_" + attr): @@ -301,7 +313,9 @@ def _write_sofa(filename: str, sofa: sf.Sofa, compression=4, verify=True): # write global attributes keys = [key for key in all_keys if key.startswith("GLOBAL_")] for key in keys: - setattr(file, key[7:], str(getattr(sofa, key))) + value, _ = _format_value_for_netcdf( + getattr(sofa, key), key, "attribute", None, None) + setattr(file, key[7:], value) # write data for key in all_keys: @@ -327,16 +341,14 @@ def _write_sofa(filename: str, sofa: sf.Sofa, compression=4, verify=True): tmp_var = file.createVariable( key.replace("Data_", "Data."), dtype, shape, zlib=use_zlib, complevel=compression) - if dtype == "f8": - tmp_var[:] = value - else: - tmp_var[:] = stringtochar(value, encoding='utf-8') + tmp_var[:] = value # write variable attributes sub_keys = [k for k in all_keys if k.startswith(f"{key}_")] for sub_key in sub_keys: - setattr(tmp_var, sub_key[len(key)+1:], - str(getattr(sofa, sub_key))) + value, _ = _format_value_for_netcdf( + getattr(sofa, sub_key), sub_key, "attribute", None, None) + setattr(tmp_var, sub_key[len(key)+1:], value) def _format_value_for_netcdf(value, key, dtype, dimensions, S): @@ -350,7 +362,7 @@ def _format_value_for_netcdf(value, key, dtype, dimensions, S): key : str The name of the current attribute. Needed for verbose errors. dtype : str - The the data type of value + The the data type of value. Must be "attribute", "double", or "string". dimensions : str The intended dimensions from ``sofa.dimensions`` S : int @@ -358,8 +370,11 @@ def _format_value_for_netcdf(value, key, dtype, dimensions, S): Returns ------- - value : str, numpy array - The formatted value. + value : bytes, numpy array + The formatted value. Attributes are returned as UTF-8 encoded bytes and + strings as UTF-8 encoded character arrays of dtype ``S1`` with the + characters along the last dimension. This makes sure that attributes + and string variables are written as the netCDF type ``char``. netcdf_dtype : str The data type as a string for writing to a NETCDF4 file ('attribute', 'f8', or 'S1'). @@ -370,14 +385,33 @@ def _format_value_for_netcdf(value, key, dtype, dimensions, S): # parse data if dtype == "attribute": - value = str(value) + # Format attributes as UTF-8 encoded byte strings, which netCDF4 writes + # as 1D `char` arrays (see AES69-2022, 4.5, note 3). Returning a str + # would cause netCDF4 to write a `string` if it contains non-ASCII + # characters. + value = str(value).encode("utf-8") netcdf_dtype = "attribute" elif dtype == "double": value = _atleast_nd(value, len(dimensions)) netcdf_dtype = "f8" elif dtype == "string": - value = np.array(value, dtype=f"S{str(S)}") - value = _atleast_nd(value, len(dimensions)) + # Format strings as character arrays of numpy dtype `S1` (one byte per + # element), which netCDF4 writes as netCDF type `char` (as required by + # AES69-2022, Section 4.4). + # First, encode the strings as UTF-8 bytes. This allows to write + # non-ASCII characters. Then pad them with null bytes to the length S, + # which is the maximum length of the UTF-8 encoded strings in bytes + # (see Sofa._get_size_and_shape_of_string_var). + value = np.asarray(value) + if value.dtype.kind != "S": + value = np.char.encode(value.astype("U"), "utf-8") + value = value.astype(f"S{str(S)}") + value = np.ascontiguousarray(_atleast_nd(value, len(dimensions))) + # Second, split the strings of dtype `S{S}` into characters of dtype + # `S1` along a new last dimension, which is the dimension S of the + # netCDF variable. For example, an array of shape (M, 1) and dtype + # `S10` becomes an array of shape (M, 1, 10) and dtype `S1`. + value = value.view("S1").reshape(value.shape + (value.itemsize, )) netcdf_dtype = 'S1' else: raise ValueError(f"Unknown type {dtype} for {key}") @@ -391,8 +425,11 @@ def _format_value_from_netcdf(value, key): Parameters ---------- - value : np.array of dtype float or S - The value to be formatted + value : str, np.array of dtype float, int, S, U, or O + The value to be formatted. Values that are not numpy arrays, e.g., + attributes returned as str by netCDF4, are returned unchanged. + Character arrays of dtype ``S1`` (netCDF type ``char``) are decoded + as UTF-8 strings along the last dimension. key : str The variable name of the current value. Needed for verbose errors. @@ -402,21 +439,37 @@ def _format_value_from_netcdf(value, key): The formatted value. """ + # No need to format SOFA attributes: attributes can be of netCDF4 types + # `char` and `string` and are both already returned as the required str + # object. They should be of type `char`, but sofar also reads `str` to be + # user friendly. + if not isinstance(value, np.ndarray): + return value + if "float" in str(value.dtype) or "int" in str(value.dtype): if np.ma.is_masked(value): warnings.warn(f"Entry {key} contains missing data", stacklevel=3) else: # Convert to numpy array or scalar value = np.asarray(value) - elif str(value.dtype)[1] in ["S", "U"]: - # string arrays are stored in masked arrays with empty strings '' being - # masked. Convert to regular arrays with unmasked empty strings - if str(value.dtype)[1] == "S": - value = chartostring(value, encoding="ascii") - value = np.atleast_1d(value).astype("U") + elif value.dtype.kind == "S": + # character arrays (netCDF type 'char') are stored in masked arrays + # with empty characters being masked. Convert to regular arrays + value = np.ascontiguousarray(np.ma.filled(value, b"")) + # join characters (bytes) along the last dimension + if value.dtype.itemsize == 1 and value.ndim > 0: + if value.shape[-1] == 0: + value = np.zeros(value.shape[:-1], dtype="S1") + else: + value = value.view(f"S{value.shape[-1]}")[..., 0] + value = np.atleast_1d(np.char.decode(value, "utf-8")) + elif value.dtype.kind in ["U", "O"]: + # string arrays (netCDF type 'string') + value = np.atleast_1d(np.asarray(value).astype("U")) else: - raise TypeError( - f"{key}: value.dtype is {value.dtype} but must be float, S or, U") + raise TypeError(( + f"{key}: value.dtype is {value.dtype} but must be float, int, S, " + "U, or O")) # convert arrays to scalars if they do not store data that is usually used # as scalar metadata, e.g., the SamplingRate diff --git a/sofar/sofa.py b/sofar/sofa.py index ee13fe0..4bbee31 100644 --- a/sofar/sofa.py +++ b/sofar/sofa.py @@ -1666,19 +1666,26 @@ def _get_size_and_shape_of_string_var(value, key): String variables can be strings, list of strings, or numpy arrays of strings. This functions returns the length of the longest string S inside the string variable and the shape of the string variable as - required by the SOFA definition. Note that the shape is the shape of - the array that holds the strings. NETCDF stores all string variables in - arrays. + required by the SOFA definition. S is the number of bytes of the UTF-8 + encoded string, because strings are written as UTF-8 encoded character + arrays. Note that the shape is the shape of the array that holds the + strings. NETCDF stores all string variables in arrays. """ + def n_bytes(string): + """Return the UTF-8 byte length of a string.""" + if isinstance(string, bytes): + return len(string) + return len(str(string).encode("utf-8")) + if isinstance(value, str): - S = len(value) + S = n_bytes(value) shape = (1, 1) elif isinstance(value, list): - S = len(max(value, key=len)) + S = max(np.vectorize(n_bytes)(np.array(value)).flatten()) shape = np.array(value).shape elif isinstance(value, np.ndarray): - S = max(np.vectorize(len)(value)) + S = max(np.vectorize(n_bytes)(value).flatten()) shape = value.shape else: raise TypeError((f"{key} must be a string, numpy string array, " diff --git a/sofar/sofastream.py b/sofar/sofastream.py index 3579a78..ee93dc3 100644 --- a/sofar/sofastream.py +++ b/sofar/sofastream.py @@ -182,7 +182,7 @@ def _read_open_file_for_verification(self): # load global attributes for attr in self._file.ncattrs(): - value = getattr(self._file, attr) + value = _format_value_from_netcdf(getattr(self._file, attr), attr) key = f"GLOBAL_{attr}" all_attr.append(key) @@ -198,7 +198,13 @@ def _read_open_file_for_verification(self): key = var.replace(".", "_") all_attr.append(key) - if netcdf_variable.datatype == "S1": + # SOFA string variables should be saved as `char` arrays in a SOFA + # file (see AES69-2022, Section 4.4), which translates to a numpy + # array of dtype "S1". To be user friendly, sofar also reads + # str, which translates to the netCDF type 'string' (the datatype + # of 'string' is a VLType but its dtype is str) in case other + # applications are less strinct than sofar. + if netcdf_variable.dtype in ["S1", str]: value = _format_value_from_netcdf(netcdf_variable[:], var) dtype = "string" else: @@ -217,7 +223,8 @@ def _read_open_file_for_verification(self): # load variable attributes for attr in [a for a in netcdf_variable.ncattrs() if a not in skip]: - value = getattr(netcdf_variable, attr) + value = _format_value_from_netcdf( + getattr(netcdf_variable, attr), attr) attr_key = key + "_" + attr all_attr.append(attr_key) diff --git a/sofar/utils.py b/sofar/utils.py index 136bcd2..9c92284 100644 --- a/sofar/utils.py +++ b/sofar/utils.py @@ -221,7 +221,7 @@ def equals(sofa_a, sofa_b, verbose=True, exclude=None): elif type_a == "string" and type_b == "string": try: assert np.all( - np.squeeze(a).astype("S") == np.squeeze(b).astype("S")) + np.squeeze(a).astype("U") == np.squeeze(b).astype("U")) except AssertionError: is_identical = _equals_raise_warning( "not identical: different values for {key}", verbose) diff --git a/tests/test_io.py b/tests/test_io.py index 6186b4d..c42ef74 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -6,11 +6,13 @@ from sofar.io import (_format_value_for_netcdf, _format_value_from_netcdf) import os +import ctypes import pathlib from tempfile import TemporaryDirectory import pytest import numpy as np import numpy.testing as npt +import netCDF4 from netCDF4 import Dataset from packaging.version import parse @@ -212,46 +214,181 @@ def test_roundtrip_multidimensional_string_variable(): assert identical +def _netcdf_attribute_type(filename, attribute, variable=None): + """ + Get the netCDF type of an attribute from the netCDF C library that netCDF4 + is linked against. The netCDF4 Python API does not expose the type of + attributes. + + Returns 2 for netCDF type 'char' (NC_CHAR) and 12 for + netCDF type 'string' (NC_STRING). + """ + + # load the C functions through the netCDF4 extension module, which links + # the netCDF C library. Skip if this is not possible, e.g., on Windows + try: + lib = ctypes.CDLL(netCDF4._netCDF4.__file__) + nc_open, nc_close = lib.nc_open, lib.nc_close + nc_inq_varid, nc_inq_atttype = lib.nc_inq_varid, lib.nc_inq_atttype + except (OSError, AttributeError): + pytest.skip("netCDF C library can not be accessed via ctypes") + + # open file in read mode (0 = NC_NOWRITE). The C functions return 0 on + # success and write their results to the passed pointers + ncid, varid, xtype = ctypes.c_int(), ctypes.c_int(-1), ctypes.c_int() + assert nc_open(str(filename).encode(), 0, ctypes.byref(ncid)) == 0 + + # query the attribute type and make sure the file is closed afterwards + try: + # get the variable id (varid -1 = NC_GLOBAL for global attributes) + if variable is not None: + assert nc_inq_varid( + ncid, variable.encode(), ctypes.byref(varid)) == 0 + assert nc_inq_atttype( + ncid, varid, attribute.encode(), ctypes.byref(xtype)) == 0 + finally: + nc_close(ncid) + + return xtype.value + + +@pytest.mark.parametrize("text", ["Only ASCII text", "Möre than ASCII text"]) +def test_write_netcdf_char(text): + """ + Test that attributes and string variables are written as netCDF type + 'char' and UTF-8 encoded. + """ + + temp_dir = TemporaryDirectory() + file = os.path.join(temp_dir.name, "HeadphoneIR.sofa") + + sofa = sf.Sofa("SimpleHeadphoneIR") + # add dummy matrix that contains 2 measurements + sofa.Data_IR = np.zeros((2, 2, 10)) + + # add text to global attribute, variable attribute, and string variable + sofa.GLOBAL_Comment = text + sofa.add_attribute("SourcePosition_Comment", text) + sofa.SourceManufacturer = [[text], ["b"]] + + # remove other string variables for simplicity + delattr(sofa, "SourceModel") + delattr(sofa, "ReceiverDescriptions") + delattr(sofa, "EmitterDescriptions") + delattr(sofa, "MeasurementDate") + sf.write_sofa(file, sofa) + + # Test that attributes are of netCDF type 'char' (NC_CHAR = 2) + # This must checked with netCDF and cannot be checked with sofar. + assert _netcdf_attribute_type(file, "Comment") == 2 + assert _netcdf_attribute_type(file, "Comment", "SourcePosition") == 2 + + # Test SOFA string variable + with Dataset(file, "r", format="NETCDF4") as nc: + # attributes contain the text + assert nc.Comment == text + assert nc["SourcePosition"].Comment == text + + # string variable is of netCDF type 'char' with S as last dimension + assert nc["SourceManufacturer"].datatype == "S1" + assert nc["SourceManufacturer"].dimensions[-1] == "S" + + # S is the number of bytes of the longest UTF-8 encoded string + n_bytes = len(text.encode("utf-8")) + assert nc.dimensions["S"].size == n_bytes + + # characters are the UTF-8 encoded bytes padded with null characters + nc.set_auto_chartostring(False) + characters = np.ma.filled(nc["SourceManufacturer"][:], b"") + assert characters.tobytes() == \ + text.encode("utf-8") + b"b".ljust(n_bytes, b"\x00") + + +def test_read_netcdf_string(): + """ + Test reading attributes and string variables of netCDF type 'string' that + are not written by sofar but can be contained in SOFA files written by + other applications. Sofar reads those as well to be user friendly, despite + they violate the AES69-2022 standard. + """ + + temp_dir = TemporaryDirectory() + file = os.path.join(temp_dir.name, "HeadphoneIR.sofa") + text = "Möre than ASCII" + + sofa = sf.Sofa("SimpleHeadphoneIR") + sf.write_sofa(file, sofa) + + # add attributes and string variable of netCDF type 'string' + with Dataset(file, "a", format="NETCDF4") as nc: + nc.setncattr_string("Comment", text) + nc["SourcePosition"].setncattr_string("Comment", text) + variable = nc.createVariable("CustomStrings", str, ("M", )) + variable[0] = text + + assert _netcdf_attribute_type(file, "Comment") == 12 + + sofa_r = sf.read_sofa(file, verify=False, verbose=False) + assert sofa_r.GLOBAL_Comment == text + assert sofa_r.SourcePosition_Comment == text + assert sofa_r.CustomStrings == text + assert sofa_r._convention["CustomStrings"]["type"] == "string" + + def test_format_value_for_netcdf(): # string and None dimensions (a.k.a NETCDF attribute) value, dtype = _format_value_for_netcdf( "string", "test_attr", "attribute", None, 12) - assert value == "string" + assert value == b"string" assert dtype == "attribute" # int that should be converted to a string value, dtype = _format_value_for_netcdf( 1, "test_attr", "attribute", None, 12) - assert value == "1" + assert value == b"1" assert dtype == "attribute" # float that should be converted to a string value, dtype = _format_value_for_netcdf( 0.2, "test_attr", "attribute", None, 12) - assert value == "0.2" + assert value == b"0.2" + assert dtype == "attribute" + + # non-ASCII string that should be encoded as UTF-8 + value, dtype = _format_value_for_netcdf( + "Möre than ASCII", "test_attr", "attribute", None, 12) + assert value == "Möre than ASCII".encode("utf-8") assert dtype == "attribute" # string and IS dimensions value, dtype = _format_value_for_netcdf( "string", "TestVar", "string", "IS", 12) - assert value == np.array("string", dtype="S12") + assert value.view("S12")[..., 0] == np.array("string", dtype="S12") assert dtype == "S1" - assert value.ndim == 2 + assert value.shape == (1, 1, 12) # single entry array and none Dimensions value, dtype = _format_value_for_netcdf( ["string"], "TestVar", "string", "IS", 12) - assert value == np.array(["string"], dtype="S12") + assert value.view("S12")[..., 0] == np.array(["string"], dtype="S12") assert dtype == "S1" - assert value.ndim == 2 + assert value.shape == (1, 1, 12) # array of strings value, dtype = _format_value_for_netcdf( [["a"], ["bc"]], "TestVar", "string", "MS", 12) - assert all(value == np.array([["a"], ["bc"]], "S12")) + assert all(value.view("S12")[..., 0] == np.array([["a"], ["bc"]], "S12")) assert dtype == "S1" - assert value.ndim == 2 + assert value.shape == (2, 1, 12) + + # non-ASCII string that should be encoded as UTF-8 (15 characters but 16 + # bytes because 'ö' is encoded with 2 bytes) + value, dtype = _format_value_for_netcdf( + "Möre than ASCII", "TestVar", "string", "IS", 16) + assert value.view("S16")[..., 0] == "Möre than ASCII".encode("utf-8") + assert dtype == "S1" + assert value.shape == (1, 1, 16) # test with list value, dtype = _format_value_for_netcdf( @@ -285,6 +422,26 @@ def test_format_value_from_netcdf(): value = _format_value_from_netcdf(value_in, "Some_Attribute") assert all(value == np.array(["str1", "str2"], dtype="U")) + # non-ASCII string (emulate NetCDF binary format of UTF-8 string) + value_in = np.array(list("Möre than ASCII".encode("utf-8")), dtype="u1") + value = _format_value_from_netcdf(value_in.view("S1"), "Some_Attribute") + assert value == "Möre than ASCII" + + # array of strings with empty characters (emulate masked NetCDF data) + value_in = np.ma.masked_equal( + np.array([["a", "b"], ["c", ""]], dtype="S1"), b"") + value = _format_value_from_netcdf(value_in, "Some_Attribute") + assert all(value == np.array(["ab", "c"], dtype="U")) + + # array of strings (emulate NetCDF type string) + value_in = np.array(["Möre than ASCII", "str2"], dtype="O") + value = _format_value_from_netcdf(value_in, "Some_Attribute") + assert all(value == np.array(["Möre than ASCII", "str2"], dtype="U")) + + # attributes are returned unchanged + value = _format_value_from_netcdf("Möre than ASCII", "Some_Attribute") + assert value == "Möre than ASCII" + # single string (emulate NetCDF ascii encoding) value_in = np.array(["str"], dtype="U") value = _format_value_from_netcdf(value_in, "Some_Attribute")