Feature/deserialization optimization

serde_arrow/Cargo.toml

@@ -37,6 +37,7 @@ bytemuck = { version = "1", default-features = false }
 chrono = { version = "0.4", features = ["std"], default-features = false }
 half = { version = "2", features = ["bytemuck"], default-features = false }
 serde = { version = "1.0", features = ["derive", "std"], default-features = false }
+simdutf8 = { version = "0.1", default-features = false }
 
 # arrow-version:insert: arrow-array-{version} = {{ package = "arrow-array", version = "{version}", optional = true, default-features = false }}
 arrow-array-60 = { package = "arrow-array", version = "60", optional = true, default-features = false }

serde_arrow/src/internal/deserialization/array_deserializer.rs

@@ -6,7 +6,7 @@ use serde::{
 };
 
 use crate::internal::{
-    error::{fail, Context, Error, Result},
+    error::{fail, try_, Context, ContextSupport, Error, Result},
     schema::Strategy,
 };
 
@@ -276,6 +276,22 @@ impl<'de> RandomAccessDeserializer<'de> for ArrayDeserializer<'de> {
         dispatch!(self, Self(this) => this.is_some(idx))
     }
 
+    fn deserialize_option<V: Visitor<'de>>(&self, visitor: V, idx: usize) -> Result<V::Value> {
+        match self {
+            Self::Utf8(this) => this.deserialize_option(visitor, idx),
+            Self::LargeUtf8(this) => this.deserialize_option(visitor, idx),
+            Self::Utf8View(this) => this.deserialize_option(visitor, idx),
+            _ => try_(|| {
+                if self.is_some(idx)? {
+                    visitor.visit_some(self.at(idx))
+                } else {
+                    visitor.visit_none()
+                }
+            })
+            .ctx(self),
+        }
+    }
+
     fn deserialize_any_some<V: Visitor<'de>>(&self, visitor: V, idx: usize) -> Result<V::Value> {
         dispatch!(self, Self(this) => this.deserialize_any_some(visitor, idx))
     }

serde_arrow/src/internal/deserialization/dictionary_deserializer.rs

@@ -8,7 +8,7 @@ use crate::internal::{
 
 use super::{
     enums_as_string_impl::EnumAccess, integer_deserializer::Integer,
-    random_access_deserializer::RandomAccessDeserializer,
+    random_access_deserializer::RandomAccessDeserializer, string_deserializer::get_utf8,
 };
 
 pub struct DictionaryDeserializer<'a, K: Integer, V: Offset> {
@@ -28,7 +28,7 @@ impl<'a, K: Integer, V: Offset> DictionaryDeserializer<'a, K, V> {
 
     pub fn get_str(&self, idx: usize) -> Result<&str> {
         let key: usize = self.keys.get_required(idx)?.into_i64()?.try_into()?;
-        let value: &str = self.values.get_required(key)?;
+        let value: &str = get_utf8(&self.values, key)?;
         Ok(value)
     }
 }

serde_arrow/src/internal/deserialization/list_deserializer.rs

@@ -131,4 +131,8 @@ impl<'de> SeqAccess<'de> for ListItemDeserializer<'_, 'de> {
         self.start += 1;
         Ok(Some(item))
     }
+
+    fn size_hint(&self) -> Option<usize> {
+        Some(self.end.saturating_sub(self.start).min(1024))
+    }
 }

serde_arrow/src/internal/deserialization/string_deserializer.rs

@@ -36,6 +36,13 @@ impl<V> StringDeserializer<V> {
     }
 }
 
+pub(super) fn get_utf8<'a, V>(view: &V, idx: usize) -> Result<&'a str>
+where
+    V: ViewAccess<'a, str> + 'a,
+{
+    ViewAccess::<str>::get_required(view, idx)
+}
+
 impl<V: StringDeserializerDataType> Context for StringDeserializer<V> {
     fn annotate(&self, annotations: &mut std::collections::BTreeMap<String, String>) {
         set_default(annotations, "field", &self.path);
@@ -51,6 +58,24 @@ where
         self.view.is_some(idx)
     }
 
+    fn deserialize_option<V: serde::de::Visitor<'a>>(
+        &self,
+        visitor: V,
+        idx: usize,
+    ) -> Result<V::Value> {
+        try_(|| match ViewAccess::<str>::get(&self.view, idx)? {
+            Some(value) => visitor.visit_some(
+                ValidatedStringDeserializer {
+                    original: self,
+                    value,
+                }
+                .at(idx),
+            ),
+            None => visitor.visit_none(),
+        })
+        .ctx(self)
+    }
+
     fn deserialize_any_some<V: serde::de::Visitor<'a>>(
         &self,
         visitor: V,
@@ -64,31 +89,31 @@ where
         visitor: V,
         idx: usize,
     ) -> Result<V::Value> {
-        try_(|| visitor.visit_borrowed_str(self.view.get_required(idx)?)).ctx(self)
+        try_(|| visitor.visit_borrowed_str(get_utf8(&self.view, idx)?)).ctx(self)
     }
 
     fn deserialize_string<V: serde::de::Visitor<'a>>(
         &self,
         visitor: V,
         idx: usize,
     ) -> Result<V::Value> {
-        try_(|| visitor.visit_string(self.view.get_required(idx)?.to_owned())).ctx(self)
+        try_(|| visitor.visit_string(get_utf8(&self.view, idx)?.to_owned())).ctx(self)
     }
 
     fn deserialize_bytes<V: serde::de::Visitor<'a>>(
         &self,
         visitor: V,
         idx: usize,
     ) -> Result<V::Value> {
-        try_(|| visitor.visit_bytes(self.view.get_required(idx)?.as_bytes())).ctx(self)
+        try_(|| visitor.visit_bytes(get_utf8(&self.view, idx)?.as_bytes())).ctx(self)
     }
 
     fn deserialize_byte_buf<V: serde::de::Visitor<'a>>(
         &self,
         visitor: V,
         idx: usize,
     ) -> Result<V::Value> {
-        try_(|| visitor.visit_byte_buf(self.view.get_required(idx)?.to_owned().into_bytes()))
+        try_(|| visitor.visit_byte_buf(get_utf8(&self.view, idx)?.to_owned().into_bytes()))
             .ctx(self)
     }
 
@@ -100,9 +125,137 @@ where
         idx: usize,
     ) -> Result<V::Value> {
         try_(|| {
-            let variant = self.view.get_required(idx)?;
+            let variant = get_utf8(&self.view, idx)?;
             visitor.visit_enum(EnumAccess(variant))
         })
         .ctx(self)
     }
 }
+
+struct ValidatedStringDeserializer<'this, 'a, VV> {
+    original: &'this StringDeserializer<VV>,
+    value: &'a str,
+}
+
+impl<VV: StringDeserializerDataType> Context for ValidatedStringDeserializer<'_, '_, VV> {
+    fn annotate(&self, annotations: &mut std::collections::BTreeMap<String, String>) {
+        self.original.annotate(annotations);
+    }
+}
+
+impl<'a, VV> RandomAccessDeserializer<'a> for ValidatedStringDeserializer<'_, 'a, VV>
+where
+    VV: StringDeserializerDataType,
+{
+    fn is_some(&self, _idx: usize) -> Result<bool> {
+        Ok(true)
+    }
+
+    fn deserialize_any_some<V: serde::de::Visitor<'a>>(
+        &self,
+        visitor: V,
+        idx: usize,
+    ) -> Result<V::Value> {
+        self.deserialize_str(visitor, idx)
+    }
+
+    fn deserialize_str<V: serde::de::Visitor<'a>>(
+        &self,
+        visitor: V,
+        _idx: usize,
+    ) -> Result<V::Value> {
+        try_(|| visitor.visit_borrowed_str(self.value)).ctx(self)
+    }
+
+    fn deserialize_string<V: serde::de::Visitor<'a>>(
+        &self,
+        visitor: V,
+        _idx: usize,
+    ) -> Result<V::Value> {
+        try_(|| visitor.visit_string(self.value.to_owned())).ctx(self)
+    }
+
+    fn deserialize_bytes<V: serde::de::Visitor<'a>>(
+        &self,
+        visitor: V,
+        _idx: usize,
+    ) -> Result<V::Value> {
+        try_(|| visitor.visit_bytes(self.value.as_bytes())).ctx(self)
+    }
+
+    fn deserialize_byte_buf<V: serde::de::Visitor<'a>>(
+        &self,
+        visitor: V,
+        _idx: usize,
+    ) -> Result<V::Value> {
+        try_(|| visitor.visit_byte_buf(self.value.to_owned().into_bytes())).ctx(self)
+    }
+
+    fn deserialize_enum<V: serde::de::Visitor<'a>>(
+        &self,
+        _name: &'static str,
+        _variants: &'static [&'static str],
+        visitor: V,
+        _idx: usize,
+    ) -> Result<V::Value> {
+        try_(|| visitor.visit_enum(EnumAccess(self.value))).ctx(self)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use marrow::{
+        datatypes::{DataType, Field},
+        view::{BitsWithOffset, BytesView, View},
+    };
+    use serde::Deserialize;
+
+    #[derive(Debug, Deserialize, PartialEq)]
+    struct Row<'a> {
+        #[serde(borrow)]
+        value: Option<&'a str>,
+    }
+
+    #[test]
+    fn nullable_marrow_string_borrows_valid_values_and_reports_invalid_utf8() {
+        let fields = [Field {
+            name: "value".into(),
+            data_type: DataType::LargeUtf8,
+            nullable: true,
+            metadata: Default::default(),
+        }];
+        let offsets = [0_i64, 5, 5];
+        let valid = [0b0000_0001];
+        let views = [View::LargeUtf8(BytesView {
+            offsets: &offsets,
+            data: "café".as_bytes(),
+            validity: Some(BitsWithOffset {
+                offset: 0,
+                data: &valid,
+            }),
+        })];
+        let decoded: Vec<Row<'_>> = crate::from_marrow(&fields, &views).unwrap();
+        assert_eq!(
+            decoded,
+            vec![
+                Row {
+                    value: Some("café")
+                },
+                Row { value: None }
+            ]
+        );
+
+        let invalid_offsets = [0_i64, 2];
+        let invalid = [b'a', 0xff];
+        let views = [View::LargeUtf8(BytesView {
+            offsets: &invalid_offsets,
+            data: &invalid,
+            validity: Some(BitsWithOffset {
+                offset: 0,
+                data: &valid,
+            }),
+        })];
+        let error = crate::from_marrow::<Vec<Row<'_>>>(&fields, &views).unwrap_err();
+        assert!(error.to_string().contains("invalid UTF-8 data"));
+    }
+}