diff --git a/crates/build/re_types_builder/src/codegen/rust/api.rs b/crates/build/re_types_builder/src/codegen/rust/api.rs index beefef0528f9..3c10c6260b76 100644 --- a/crates/build/re_types_builder/src/codegen/rust/api.rs +++ b/crates/build/re_types_builder/src/codegen/rust/api.rs @@ -958,13 +958,13 @@ fn quote_trait_impls_for_datatype_or_component( let quoted_serializer = if let Some(forwarded_type) = forwarded_type.as_ref() { quote! { - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - #forwarded_type::to_arrow2_opt(data.into_iter().map(|datum| { + #forwarded_type::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), @@ -978,9 +978,9 @@ fn quote_trait_impls_for_datatype_or_component( quote! { // NOTE: Don't inline this, this gets _huge_. - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a { @@ -989,10 +989,14 @@ fn quote_trait_impls_for_datatype_or_component( #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; use ::re_types_core::{Loggable as _, ResultExt as _}; + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } + Ok(#quoted_serializer) } } diff --git a/crates/build/re_types_builder/src/codegen/rust/arrow.rs b/crates/build/re_types_builder/src/codegen/rust/arrow.rs index 3d5e5ed978cc..a20fe53a6f84 100644 --- a/crates/build/re_types_builder/src/codegen/rust/arrow.rs +++ b/crates/build/re_types_builder/src/codegen/rust/arrow.rs @@ -164,3 +164,22 @@ pub fn is_backed_by_arrow_buffer(typ: &DataType) -> bool { | DataType::Float64 ) } + +pub fn quoted_arrow_primitive_type(datatype: &DataType) -> TokenStream { + match datatype { + DataType::Null => quote!(arrow::datatypes::NullType), + DataType::Boolean => quote!(arrow::datatypes::BooleanType), + DataType::Int8 => quote!(arrow::datatypes::Int8Type), + DataType::Int16 => quote!(arrow::datatypes::Int16Type), + DataType::Int32 => quote!(arrow::datatypes::Int32Type), + DataType::Int64 => quote!(arrow::datatypes::Int64Type), + DataType::UInt8 => quote!(arrow::datatypes::UInt8Type), + DataType::UInt16 => quote!(arrow::datatypes::UInt16Type), + DataType::UInt32 => quote!(arrow::datatypes::UInt32Type), + DataType::UInt64 => quote!(arrow::datatypes::UInt64Type), + DataType::Float16 => quote!(arrow::datatypes::Float16Type), + DataType::Float32 => quote!(arrow::datatypes::Float32Type), + DataType::Float64 => quote!(arrow::datatypes::Float64Type), + _ => unimplemented!("Not a primitive type: {datatype:#?}"), + } +} diff --git a/crates/build/re_types_builder/src/codegen/rust/serializer.rs b/crates/build/re_types_builder/src/codegen/rust/serializer.rs index 31d28f71b95f..0e40c842f79e 100644 --- a/crates/build/re_types_builder/src/codegen/rust/serializer.rs +++ b/crates/build/re_types_builder/src/codegen/rust/serializer.rs @@ -1,11 +1,14 @@ use arrow2::datatypes::DataType; -use proc_macro2::TokenStream; +use proc_macro2::{Literal, TokenStream}; use quote::{format_ident, quote}; -use crate::{ArrowRegistry, Object, Objects}; +use crate::{codegen::rust::arrow::ArrowDataTypeTokenizer, ArrowRegistry, Object, Objects}; use super::{ - arrow::{is_backed_by_arrow_buffer, quote_fqname_as_type_path}, + arrow::{ + is_backed_by_arrow_buffer, quote_fqname_as_type_path, quoted_arrow_primitive_type, + ArrowFieldTokenizer, + }, util::{is_tuple_struct_from_obj, quote_comment}, }; @@ -35,10 +38,10 @@ pub fn quote_arrow_serializer( } }; - let quoted_bitmap = |var| { + let quoted_validity = |var| { quote! { - let #var: Option = { - // NOTE: Don't compute a bitmap if there isn't at least one null element. + let #var: Option = { + // NOTE: Don't compute a validity if there isn't at least one null element. let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) } @@ -48,10 +51,10 @@ pub fn quote_arrow_serializer( if is_enum { let quoted_data_src = data_src.clone(); let quoted_data_dst = format_ident!("data0"); - let bitmap_dst = format_ident!("{quoted_data_dst}_bitmap"); + let validity_dst = format_ident!("{quoted_data_dst}_validity"); // The choice of true or false for `elements_are_nullable` here is a bit confusing. - // This code-gen path forms the basis of `to_arrow2_opt`, which implies that we + // This code-gen path forms the basis of `to_arrow_opt`, which implies that we // support nullable elements. Additionally, this MAY be used as a recursive code // path when using an enum within a struct, and that struct within the field may // be null, as such the elements are always handled as nullable. @@ -62,13 +65,13 @@ pub fn quote_arrow_serializer( objects, datatype.to_logical_type(), "ed_datatype, - &bitmap_dst, + &validity_dst, elements_are_nullable, "ed_data_dst, InnerRepr::NativeIterable, ); - let quoted_bitmap = quoted_bitmap(bitmap_dst); + let quoted_validity = quoted_validity(validity_dst); quote! {{ let (somes, #quoted_data_dst): (Vec<_>, Vec<_>) = #quoted_data_src @@ -84,7 +87,7 @@ pub fn quote_arrow_serializer( .unzip(); - #quoted_bitmap; + #quoted_validity; #quoted_serializer }} @@ -102,7 +105,7 @@ pub fn quote_arrow_serializer( obj_field.name.as_str() } ); - let bitmap_dst = format_ident!("{quoted_data_dst}_bitmap"); + let validity_dst = format_ident!("{quoted_data_dst}_validity"); let quoted_member_accessor = if is_tuple_struct { quote!(0) @@ -117,13 +120,13 @@ pub fn quote_arrow_serializer( objects, datatype, "ed_datatype, - &bitmap_dst, + &validity_dst, elements_are_nullable, "ed_data_dst, InnerRepr::NativeIterable, ); - let quoted_bitmap = quoted_bitmap(bitmap_dst); + let quoted_validity = quoted_validity(validity_dst); let quoted_flatten = quoted_flatten(obj_field.is_nullable); @@ -144,17 +147,24 @@ pub fn quote_arrow_serializer( .unzip(); - #quoted_bitmap; + #quoted_validity; #quoted_serializer }} } else { // NOTE: This can only be struct or union/enum at this point. match datatype.to_logical_type() { - DataType::Struct(_) => { + DataType::Struct(fields) => { + let quoted_fields = fields.iter().map(|field| { + let field_name = &field.name; + let data_type = ArrowDataTypeTokenizer(&field.data_type, false); + let is_nullable = &field.is_nullable; + quote! { Field::new(#field_name, #data_type, #is_nullable) } + }); + let quoted_field_serializers = obj.fields.iter().map(|obj_field| { let data_dst = format_ident!("{}", obj_field.name); - let bitmap_dst = format_ident!("{data_dst}_bitmap"); + let validity_dst = format_ident!("{data_dst}_validity"); let inner_datatype = &arrow_registry.get(&obj_field.fqname); let quoted_inner_datatype = @@ -165,7 +175,7 @@ pub fn quote_arrow_serializer( objects, inner_datatype, "ed_inner_datatype, - &bitmap_dst, + &validity_dst, elements_are_nullable, &data_dst, InnerRepr::NativeIterable, @@ -173,7 +183,7 @@ pub fn quote_arrow_serializer( let quoted_flatten = quoted_flatten(obj_field.is_nullable); - let quoted_bitmap = quoted_bitmap(bitmap_dst); + let quoted_validity = quoted_validity(validity_dst); quote! {{ let (somes, #data_dst): (Vec<_>, Vec<_>) = #data_src @@ -191,15 +201,17 @@ pub fn quote_arrow_serializer( .unzip(); - #quoted_bitmap; + #quoted_validity; #quoted_serializer }} }); - let quoted_bitmap = quoted_bitmap(format_ident!("bitmap")); + let quoted_declare_validity = quoted_validity(format_ident!("validity")); quote! {{ + let fields = Fields::from(vec![#(#quoted_fields,)*]); + let (somes, #data_src): (Vec<_>, Vec<_>) = #data_src .into_iter() .map(|datum| { @@ -208,20 +220,25 @@ pub fn quote_arrow_serializer( }) .unzip(); - #quoted_bitmap; + #quoted_declare_validity; - StructArray::new( - #quoted_datatype.into(), + as_array_ref(StructArray::new( + fields, vec![#(#quoted_field_serializers,)*], - bitmap, - ).boxed() + validity, + )) }} } - DataType::Union(_, _, arrow2::datatypes::UnionMode::Sparse) => { + DataType::Union(fields, _, arrow2::datatypes::UnionMode::Sparse) => { // We use sparse unions for enums, which means only 8 bits is required for each field, // and nulls are encoded with a special 0-index `_null_markers` variant. - + let quoted_fields = fields.iter().map(|field| { + let field_name = &field.name; + let data_type = ArrowDataTypeTokenizer(&field.data_type, false); + let is_nullable = &field.is_nullable; + quote! { Field::new(#field_name, #data_type, #is_nullable) } + }); let quoted_data_collect = quote! { let #data_src: Vec<_> = #data_src .into_iter() @@ -232,7 +249,7 @@ pub fn quote_arrow_serializer( .collect(); }; - let quoted_types = quote! { + let quoted_type_ids = quote! { #data_src .iter() .map(|a| match a.as_deref() { @@ -251,28 +268,37 @@ pub fn quote_arrow_serializer( #quoted_data_collect + let fields = vec![#(#quoted_fields,)*]; + let type_ids: Vec = #quoted_type_ids; let num_variants = #num_variants; - let types = #quoted_types; - - let fields: Vec<_> = std::iter::repeat( - NullArray::new( - arrow2::datatypes::DataType::Null, + let children: Vec<_> = std::iter::repeat( + as_array_ref(NullArray::new( #data_src.len(), - ).boxed() + )) ).take(1 + num_variants) // +1 for the virtual `nulls` arm .collect(); - UnionArray::new( - #quoted_datatype.into(), - types, - fields, + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + + as_array_ref(UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), None, - ).boxed() + children, + ).unwrap().) // TODO: can this fail? }} } - DataType::Union(_, _, arrow2::datatypes::UnionMode::Dense) => { + DataType::Union(fields, _, arrow2::datatypes::UnionMode::Dense) => { + let quoted_field_type_ids = (0..fields.len()).map(Literal::usize_unsuffixed); + let quoted_fields = fields.iter().map(|field| { + let field_name = &field.name; + let data_type = ArrowDataTypeTokenizer(&field.data_type, false); + let is_nullable = &field.is_nullable; + quote! { Field::new(#field_name, #data_type, #is_nullable) } + }); let quoted_data_collect = quote! { let #data_src: Vec<_> = #data_src .into_iter() @@ -289,13 +315,12 @@ pub fn quote_arrow_serializer( // Short circuit for empty variants since they're trivial to solve at this level: if obj_field.typ == crate::Type::Unit { return quote! { - NullArray::new( - arrow2::datatypes::DataType::Null, + as_array_ref(NullArray::new( #data_src .iter() .filter(|datum| matches!(datum.as_deref(), Some(Self::#quoted_obj_field_name))) .count(), - ).boxed() + )) }; } @@ -303,7 +328,7 @@ pub fn quote_arrow_serializer( // We handle nullability with a special null variant that is always present. let elements_are_nullable = false; - let bitmap_dst = format_ident!("{}_bitmap", data_dst); + let validity_dst = format_ident!("{}_validity", data_dst); let inner_datatype = &arrow_registry.get(&obj_field.fqname); let quoted_inner_datatype = @@ -313,7 +338,7 @@ pub fn quote_arrow_serializer( objects, inner_datatype, "ed_inner_datatype, - &bitmap_dst, + &validity_dst, elements_are_nullable, &data_dst, InnerRepr::NativeIterable, @@ -328,18 +353,19 @@ pub fn quote_arrow_serializer( }) .collect(); - let #bitmap_dst: Option = None; + let #validity_dst: Option = None; #quoted_serializer }} }); - let quoted_fields = quote! { + let quoted_children_serializers = quoted_field_serializers.clone(); + + let quoted_children = quote! { vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, + as_array_ref(NullArray::new( #data_src.iter().filter(|v| v.is_none()).count(), - ).boxed(), - #(#quoted_field_serializers,)* + )), + #(#quoted_children_serializers,)* ] }; @@ -351,7 +377,7 @@ pub fn quote_arrow_serializer( } }; - let quoted_types = { + let quoted_type_ids = { let quoted_branches = obj.fields.iter().enumerate().map(|(i, obj_field)| { let i = 1 + i as i8; // NOTE: +1 to account for `nulls` virtual arm let quoted_obj_field_name = @@ -421,16 +447,21 @@ pub fn quote_arrow_serializer( #quoted_data_collect - let types = #quoted_types; - let fields = #quoted_fields; - let offsets = Some(#quoted_offsets); - - UnionArray::new( - #quoted_datatype.into(), - types, - fields, - offsets, - ).boxed() + let field_type_ids = [#(#quoted_field_type_ids,)*]; + let fields = vec![#(#quoted_fields,)*]; + let type_ids: Vec = #quoted_type_ids; + let offsets = #quoted_offsets; + let children = #quoted_children; + + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + + as_array_ref(UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + ).unwrap()) // TODO: can this fail? }} } @@ -456,14 +487,14 @@ enum InnerRepr { /// * unions: nullability is encoded as a separate variant /// * lists inside of fields that are lists: we don't support intra-list nullability /// -/// TODO(#2993): However, we still emit a validity bitmap for lists inside lists -/// to `validity_bitmap_ident`since Python and Rust do so. +/// TODO(#2993): However, we still emit a validity validity for lists inside lists +/// to `validity_validity_ident`since Python and Rust do so. #[allow(clippy::too_many_arguments)] fn quote_arrow_field_serializer( objects: &Objects, datatype: &DataType, quoted_datatype: &dyn quote::ToTokens, - bitmap_src: &proc_macro2::Ident, + validity_src: &proc_macro2::Ident, elements_are_nullable: bool, data_src: &proc_macro2::Ident, inner_repr: InnerRepr, @@ -485,8 +516,8 @@ fn quote_arrow_field_serializer( }; return quote! {{ - _ = #bitmap_src; - #fqname_use::to_arrow2_opt(#data_src #option_wrapper)? + _ = #validity_src; + #fqname_use::to_arrow_opt(#data_src #option_wrapper)? }}; } } @@ -506,7 +537,7 @@ fn quote_arrow_field_serializer( | DataType::Float16 | DataType::Float32 | DataType::Float64 => { - // NOTE: We need values for all slots, regardless of what the bitmap says, + // NOTE: We need values for all slots, regardless of what the validity says, // hence `unwrap_or_default` (unless elements_are_nullable is false) let quoted_transparent_mapping = if inner_is_arrow_transparent { let inner_obj = inner_obj.as_ref().unwrap(); @@ -546,29 +577,28 @@ fn quote_arrow_field_serializer( if datatype.to_logical_type() == &DataType::Boolean { quote! { - BooleanArray::new( - #quoted_datatype.into(), - #data_src.into_iter() #quoted_transparent_mapping .collect(), - #bitmap_src, - ).boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from(#data_src.into_iter() #quoted_transparent_mapping .collect::>()), + #validity_src, + )) } } else { + let arrow_primitive_type = quoted_arrow_primitive_type(datatype.to_logical_type()); + match inner_repr { // A primitive that's an inner element of a list will already have been mapped // to a buffer type. InnerRepr::ArrowBuffer => quote! { - PrimitiveArray::new( - #quoted_datatype.into(), + as_array_ref(PrimitiveArray::<#arrow_primitive_type>::new( #data_src, - #bitmap_src, - ).boxed() + #validity_src, + )) }, InnerRepr::NativeIterable => quote! { - PrimitiveArray::new( - #quoted_datatype.into(), - #data_src.into_iter() #quoted_transparent_mapping .collect(), - #bitmap_src, - ).boxed() + as_array_ref(PrimitiveArray::<#arrow_primitive_type>::new( + ScalarBuffer::from(#data_src.into_iter() #quoted_transparent_mapping .collect::>()), + #validity_src, + )) }, } } @@ -579,7 +609,7 @@ fn quote_arrow_field_serializer( } DataType::Utf8 => { - // NOTE: We need values for all slots, regardless of what the bitmap says, + // NOTE: We need values for all slots, regardless of what the validity says, // hence `unwrap_or_default`. let (quoted_transparent_mapping, quoted_transparent_length) = if inner_is_arrow_transparent { @@ -624,24 +654,22 @@ fn quote_arrow_field_serializer( let inner_data_and_offsets = if elements_are_nullable { quote! { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( #data_src.iter().map(|opt| opt.as_ref() #quoted_transparent_length .unwrap_or_default()) - )? - .into(); + ); // NOTE: Flattening to remove the guaranteed layer of nullability: we don't care // about it while building the backing buffer since it's all offsets driven. - let inner_data: arrow2::buffer::Buffer = + let inner_data: arrow::buffer::Buffer = #data_src.into_iter().flatten() #quoted_transparent_mapping.collect(); } } else { quote! { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( #data_src.iter() #quoted_transparent_length - )? - .into(); + ); - let inner_data: arrow2::buffer::Buffer = + let inner_data: arrow::buffer::Buffer = #data_src.into_iter() #quoted_transparent_mapping.collect(); } }; @@ -654,12 +682,12 @@ fn quote_arrow_field_serializer( // It would be nice to use quote_comment here and put this safety notice in the generated code, // but that seems to push us over some complexity limit causing rustfmt to fail. #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { Utf8Array::::new_unchecked(#quoted_datatype.into(), offsets, inner_data, #bitmap_src) }.boxed() + as_array_ref(unsafe { StringArray::new_unchecked(offsets, inner_data, #validity_src) }) }} } - DataType::List(inner) | DataType::FixedSizeList(inner, _) => { - let inner_datatype = inner.data_type(); + DataType::List(inner_field) | DataType::FixedSizeList(inner_field, _) => { + let inner_datatype = inner_field.data_type(); let quoted_inner_datatype = super::arrow::ArrowDataTypeTokenizer(inner_datatype, false); // Note: We only use the ArrowBuffer optimization for `Lists` but not `FixedSizeList`. @@ -668,7 +696,7 @@ fn quote_arrow_field_serializer( // // TODO(jleibs): If we need to support large FixedSizeList types where the `ArrowBuffer` // optimization would be significant, we can introduce a new attribute to force this. - let inner_repr = if is_backed_by_arrow_buffer(inner.data_type()) + let inner_repr = if is_backed_by_arrow_buffer(inner_field.data_type()) && matches!(datatype, DataType::List(_)) { InnerRepr::ArrowBuffer @@ -677,14 +705,14 @@ fn quote_arrow_field_serializer( }; let quoted_inner_data = format_ident!("{data_src}_inner_data"); - let inner_bitmap_ident = format_ident!("{data_src}_inner_bitmap"); + let inner_validity_ident = format_ident!("{data_src}_inner_validity"); let inner_elements_are_nullable = false; // We don't support intra-list nullability. let quoted_inner = quote_arrow_field_serializer( objects, inner_datatype, "ed_inner_datatype, - &inner_bitmap_ident, + &inner_validity_ident, inner_elements_are_nullable, "ed_inner_data, inner_repr, @@ -798,31 +826,40 @@ fn quote_arrow_field_serializer( }; quote! { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( #data_src.iter(). #map_to_length - )?.into(); + ); } } else { quote! {} }; - let quoted_create = if let DataType::List(_) = datatype { - quote! { - ListArray::try_new( - #quoted_datatype.into(), - offsets, - #quoted_inner, - #bitmap_src, - )?.boxed() + let field = ArrowFieldTokenizer(inner_field); + let quoted_field = quote!(std::sync::Arc::new(#field)); + + let quoted_create = match datatype.to_logical_type() { + DataType::List(_) => { + quote! { + as_array_ref(ListArray::try_new( + #quoted_field, + offsets, + #quoted_inner, + #validity_src, + )?) + } } - } else { - quote! { - FixedSizeListArray::new( - #quoted_datatype.into(), - #quoted_inner, - #bitmap_src, - ).boxed() + DataType::FixedSizeList(_, size) => { + let quoted_size = Literal::usize_unsuffixed(*size); + quote! { + as_array_ref(FixedSizeListArray::new( + #quoted_field, + #quoted_size, + #quoted_inner, + #validity_src, + )) + } } + _ => unreachable!(), }; // TODO(#2993): The inner @@ -834,15 +871,15 @@ fn quote_arrow_field_serializer( // if we make the C++ and Python codegen match the rust behavior or // make our comparison tests more lenient. // - // This workaround does not apply if we don't have any validity bitmap on the outer type. + // This workaround does not apply if we don't have any validity validity on the outer type. // (as it is always the case with unions where the nullability is encoded as a separate variant) - let quoted_inner_bitmap = if let (true, DataType::FixedSizeList(_, count)) = + let quoted_inner_validity = if let (true, DataType::FixedSizeList(_, count)) = (elements_are_nullable, datatype.to_logical_type()) { quote! { - let #inner_bitmap_ident: Option = - #bitmap_src.as_ref().map(|bitmap| { - bitmap + let #inner_validity_ident: Option = + #validity_src.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(#count)) .flatten() @@ -853,20 +890,18 @@ fn quote_arrow_field_serializer( } else { // TODO(cmc): We don't support intra-list nullability in our IDL at the moment. quote! { - let #inner_bitmap_ident: Option = None; + let #inner_validity_ident: Option = None; } }; match inner_repr { InnerRepr::ArrowBuffer => { quote! {{ - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - #quoted_declare_offsets - let #quoted_inner_data: Buffer<_> = #quoted_transparent_mapping; + let #quoted_inner_data: ScalarBuffer<_> = #quoted_transparent_mapping; - #quoted_inner_bitmap + #quoted_inner_validity #quoted_create }} @@ -874,15 +909,13 @@ fn quote_arrow_field_serializer( InnerRepr::NativeIterable => { quote! {{ - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - #quoted_declare_offsets let #quoted_inner_data: Vec<_> = #quoted_transparent_mapping .collect(); - #quoted_inner_bitmap + #quoted_inner_validity #quoted_create }} @@ -903,8 +936,8 @@ fn quote_arrow_field_serializer( }; quote! {{ - _ = #bitmap_src; - #fqname_use::to_arrow2_opt(#data_src #option_wrapper)? + _ = #validity_src; + #fqname_use::to_arrow_opt(#data_src #option_wrapper)? }} } diff --git a/crates/store/re_types/src/blueprint/components/active_tab.rs b/crates/store/re_types/src/blueprint/components/active_tab.rs index d372ddddb791..144cde6a2651 100644 --- a/crates/store/re_types/src/blueprint/components/active_tab.rs +++ b/crates/store/re_types/src/blueprint/components/active_tab.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for ActiveTab { crate::datatypes::EntityPath::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::EntityPath::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/apply_latest_at.rs b/crates/store/re_types/src/blueprint/components/apply_latest_at.rs index 440068bdec6d..8afa9418e09f 100644 --- a/crates/store/re_types/src/blueprint/components/apply_latest_at.rs +++ b/crates/store/re_types/src/blueprint/components/apply_latest_at.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for ApplyLatestAt { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/background_kind.rs b/crates/store/re_types/src/blueprint/components/background_kind.rs index c993bcbd6f35..9e2a86150542 100644 --- a/crates/store/re_types/src/blueprint/components/background_kind.rs +++ b/crates/store/re_types/src/blueprint/components/background_kind.rs @@ -90,17 +90,21 @@ impl ::re_types_core::Loggable for BackgroundKind { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -110,16 +114,19 @@ impl ::re_types_core::Loggable for BackgroundKind { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/column_share.rs b/crates/store/re_types/src/blueprint/components/column_share.rs index e23a7d39cc3e..130b7eab8eea 100644 --- a/crates/store/re_types/src/blueprint/components/column_share.rs +++ b/crates/store/re_types/src/blueprint/components/column_share.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for ColumnShare { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/component_column_selector.rs b/crates/store/re_types/src/blueprint/components/component_column_selector.rs index 39675fda0703..a7e9de8c3668 100644 --- a/crates/store/re_types/src/blueprint/components/component_column_selector.rs +++ b/crates/store/re_types/src/blueprint/components/component_column_selector.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { crate::blueprint::datatypes::ComponentColumnSelector::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::ComponentColumnSelector::to_arrow2_opt(data.into_iter().map( + crate::blueprint::datatypes::ComponentColumnSelector::to_arrow_opt(data.into_iter().map( |datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), diff --git a/crates/store/re_types/src/blueprint/components/corner2d.rs b/crates/store/re_types/src/blueprint/components/corner2d.rs index 62d9e96351c6..4089023d062b 100644 --- a/crates/store/re_types/src/blueprint/components/corner2d.rs +++ b/crates/store/re_types/src/blueprint/components/corner2d.rs @@ -92,17 +92,21 @@ impl ::re_types_core::Loggable for Corner2D { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -112,16 +116,19 @@ impl ::re_types_core::Loggable for Corner2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/filter_by_range.rs b/crates/store/re_types/src/blueprint/components/filter_by_range.rs index 4538c20fd6ac..f48e3f65fcc5 100644 --- a/crates/store/re_types/src/blueprint/components/filter_by_range.rs +++ b/crates/store/re_types/src/blueprint/components/filter_by_range.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for FilterByRange { crate::blueprint::datatypes::FilterByRange::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::FilterByRange::to_arrow2_opt(data.into_iter().map(|datum| { + crate::blueprint::datatypes::FilterByRange::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/filter_is_not_null.rs b/crates/store/re_types/src/blueprint/components/filter_is_not_null.rs index fa8a3c20851a..47da6800b958 100644 --- a/crates/store/re_types/src/blueprint/components/filter_is_not_null.rs +++ b/crates/store/re_types/src/blueprint/components/filter_is_not_null.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for FilterIsNotNull { crate::blueprint::datatypes::FilterIsNotNull::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::FilterIsNotNull::to_arrow2_opt(data.into_iter().map(|datum| { + crate::blueprint::datatypes::FilterIsNotNull::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/included_content.rs b/crates/store/re_types/src/blueprint/components/included_content.rs index fd5ca2170ae2..590900ece0ff 100644 --- a/crates/store/re_types/src/blueprint/components/included_content.rs +++ b/crates/store/re_types/src/blueprint/components/included_content.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for IncludedContent { crate::datatypes::EntityPath::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::EntityPath::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/interactive.rs b/crates/store/re_types/src/blueprint/components/interactive.rs index 4fbec8d71dc6..a99f7505c0af 100644 --- a/crates/store/re_types/src/blueprint/components/interactive.rs +++ b/crates/store/re_types/src/blueprint/components/interactive.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for Interactive { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/lock_range_during_zoom.rs b/crates/store/re_types/src/blueprint/components/lock_range_during_zoom.rs index 13280b5e1e2a..a0d60c1e2444 100644 --- a/crates/store/re_types/src/blueprint/components/lock_range_during_zoom.rs +++ b/crates/store/re_types/src/blueprint/components/lock_range_during_zoom.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for LockRangeDuringZoom { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/map_provider.rs b/crates/store/re_types/src/blueprint/components/map_provider.rs index 0de6496dcf70..9364b25b4178 100644 --- a/crates/store/re_types/src/blueprint/components/map_provider.rs +++ b/crates/store/re_types/src/blueprint/components/map_provider.rs @@ -92,17 +92,21 @@ impl ::re_types_core::Loggable for MapProvider { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -112,16 +116,19 @@ impl ::re_types_core::Loggable for MapProvider { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/panel_state.rs b/crates/store/re_types/src/blueprint/components/panel_state.rs index 2b9decb690b8..415b1e506d1a 100644 --- a/crates/store/re_types/src/blueprint/components/panel_state.rs +++ b/crates/store/re_types/src/blueprint/components/panel_state.rs @@ -82,17 +82,21 @@ impl ::re_types_core::Loggable for PanelState { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -102,16 +106,19 @@ impl ::re_types_core::Loggable for PanelState { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/query_expression.rs b/crates/store/re_types/src/blueprint/components/query_expression.rs index 41549031ce0a..ec60f3a0a29d 100644 --- a/crates/store/re_types/src/blueprint/components/query_expression.rs +++ b/crates/store/re_types/src/blueprint/components/query_expression.rs @@ -81,13 +81,13 @@ impl ::re_types_core::Loggable for QueryExpression { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/row_share.rs b/crates/store/re_types/src/blueprint/components/row_share.rs index cc75f807a264..2b70f7fe32bf 100644 --- a/crates/store/re_types/src/blueprint/components/row_share.rs +++ b/crates/store/re_types/src/blueprint/components/row_share.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for RowShare { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/selected_columns.rs b/crates/store/re_types/src/blueprint/components/selected_columns.rs index 1290721b396f..83b30d1acc1b 100644 --- a/crates/store/re_types/src/blueprint/components/selected_columns.rs +++ b/crates/store/re_types/src/blueprint/components/selected_columns.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for SelectedColumns { crate::blueprint::datatypes::SelectedColumns::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::SelectedColumns::to_arrow2_opt(data.into_iter().map(|datum| { + crate::blueprint::datatypes::SelectedColumns::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/space_view_class.rs b/crates/store/re_types/src/blueprint/components/space_view_class.rs index 7e17ee0bd137..6e0c0cf363c2 100644 --- a/crates/store/re_types/src/blueprint/components/space_view_class.rs +++ b/crates/store/re_types/src/blueprint/components/space_view_class.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for SpaceViewClass { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/space_view_origin.rs b/crates/store/re_types/src/blueprint/components/space_view_origin.rs index b3e01469cad5..82e0ecf6e155 100644 --- a/crates/store/re_types/src/blueprint/components/space_view_origin.rs +++ b/crates/store/re_types/src/blueprint/components/space_view_origin.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for SpaceViewOrigin { crate::datatypes::EntityPath::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::EntityPath::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/tensor_dimension_index_slider.rs b/crates/store/re_types/src/blueprint/components/tensor_dimension_index_slider.rs index db8295ed7625..350db3ae3dff 100644 --- a/crates/store/re_types/src/blueprint/components/tensor_dimension_index_slider.rs +++ b/crates/store/re_types/src/blueprint/components/tensor_dimension_index_slider.rs @@ -76,20 +76,20 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSlider { crate::blueprint::datatypes::TensorDimensionIndexSlider::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::TensorDimensionIndexSlider::to_arrow2_opt( - data.into_iter().map(|datum| { + crate::blueprint::datatypes::TensorDimensionIndexSlider::to_arrow_opt(data.into_iter().map( + |datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), }) - }), - ) + }, + )) } fn from_arrow2_opt( diff --git a/crates/store/re_types/src/blueprint/components/timeline_name.rs b/crates/store/re_types/src/blueprint/components/timeline_name.rs index b8efd276ac94..fd7ebbe0398f 100644 --- a/crates/store/re_types/src/blueprint/components/timeline_name.rs +++ b/crates/store/re_types/src/blueprint/components/timeline_name.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for TimelineName { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/view_fit.rs b/crates/store/re_types/src/blueprint/components/view_fit.rs index c0a6f9e10f19..d4a9e58bbc31 100644 --- a/crates/store/re_types/src/blueprint/components/view_fit.rs +++ b/crates/store/re_types/src/blueprint/components/view_fit.rs @@ -88,17 +88,21 @@ impl ::re_types_core::Loggable for ViewFit { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -108,16 +112,19 @@ impl ::re_types_core::Loggable for ViewFit { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/components/viewer_recommendation_hash.rs b/crates/store/re_types/src/blueprint/components/viewer_recommendation_hash.rs index ea8a3ec28da2..c52a0f6137ca 100644 --- a/crates/store/re_types/src/blueprint/components/viewer_recommendation_hash.rs +++ b/crates/store/re_types/src/blueprint/components/viewer_recommendation_hash.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for ViewerRecommendationHash { crate::datatypes::UInt64::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::UInt64::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::UInt64::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/visible.rs b/crates/store/re_types/src/blueprint/components/visible.rs index 0abcf7cc4dda..372c85ca7cfe 100644 --- a/crates/store/re_types/src/blueprint/components/visible.rs +++ b/crates/store/re_types/src/blueprint/components/visible.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Visible { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/visible_time_range.rs b/crates/store/re_types/src/blueprint/components/visible_time_range.rs index 462bfae52d23..2875854faf72 100644 --- a/crates/store/re_types/src/blueprint/components/visible_time_range.rs +++ b/crates/store/re_types/src/blueprint/components/visible_time_range.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for VisibleTimeRange { crate::datatypes::VisibleTimeRange::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::VisibleTimeRange::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::VisibleTimeRange::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/visual_bounds2d.rs b/crates/store/re_types/src/blueprint/components/visual_bounds2d.rs index 2db4a32793fa..e5355ecb9d5c 100644 --- a/crates/store/re_types/src/blueprint/components/visual_bounds2d.rs +++ b/crates/store/re_types/src/blueprint/components/visual_bounds2d.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for VisualBounds2D { crate::datatypes::Range2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Range2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Range2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/components/zoom_level.rs b/crates/store/re_types/src/blueprint/components/zoom_level.rs index 4d1b883f30f4..1708693a90b5 100644 --- a/crates/store/re_types/src/blueprint/components/zoom_level.rs +++ b/crates/store/re_types/src/blueprint/components/zoom_level.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for ZoomLevel { crate::datatypes::Float64::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float64::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float64::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs index 580c81ac7be7..2836ce494983 100644 --- a/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs +++ b/crates/store/re_types/src/blueprint/datatypes/component_column_selector.rs @@ -61,18 +61,26 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("entity_path", DataType::Utf8, false), + Field::new("component", DataType::Utf8, false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -80,12 +88,12 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, entity_path): (Vec<_>, Vec<_>) = data @@ -95,33 +103,29 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { (datum.is_some(), datum) }) .unzip(); - let entity_path_bitmap: Option = { + let entity_path_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( entity_path.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = entity_path + ); + let inner_data: arrow::buffer::Buffer = entity_path .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - entity_path_bitmap, + entity_path_validity, ) - } - .boxed() + }) } }, { @@ -132,39 +136,31 @@ impl ::re_types_core::Loggable for ComponentColumnSelector { (datum.is_some(), datum) }) .unzip(); - let component_bitmap: Option = { + let component_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( component.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = component + ); + let inner_data: arrow::buffer::Buffer = component .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - component_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, component_validity) + }) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/datatypes/filter_by_range.rs b/crates/store/re_types/src/blueprint/datatypes/filter_by_range.rs index 56548ef92a32..fe7becb59f65 100644 --- a/crates/store/re_types/src/blueprint/datatypes/filter_by_range.rs +++ b/crates/store/re_types/src/blueprint/datatypes/filter_by_range.rs @@ -57,18 +57,26 @@ impl ::re_types_core::Loggable for FilterByRange { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("start", DataType::Int64, false), + Field::new("end", DataType::Int64, false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -76,12 +84,12 @@ impl ::re_types_core::Loggable for FilterByRange { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, start): (Vec<_>, Vec<_>) = data @@ -91,19 +99,19 @@ impl ::re_types_core::Loggable for FilterByRange { (datum.is_some(), datum) }) .unzip(); - let start_bitmap: Option = { + let start_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Int64.into(), - start - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - start_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + start + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + start_validity, + )) }, { let (somes, end): (Vec<_>, Vec<_>) = data @@ -113,23 +121,22 @@ impl ::re_types_core::Loggable for FilterByRange { (datum.is_some(), datum) }) .unzip(); - let end_bitmap: Option = { + let end_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Int64.into(), - end.into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - end_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + end.into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + end_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/datatypes/filter_is_not_null.rs b/crates/store/re_types/src/blueprint/datatypes/filter_is_not_null.rs index 1f1d20ebf0d1..3c83814ea311 100644 --- a/crates/store/re_types/src/blueprint/datatypes/filter_is_not_null.rs +++ b/crates/store/re_types/src/blueprint/datatypes/filter_is_not_null.rs @@ -58,18 +58,41 @@ impl ::re_types_core::Loggable for FilterIsNotNull { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("active", DataType::Boolean, false), + Field::new( + "column", + DataType::Struct(Fields::from(vec![ + Field::new( + "entity_path", + ::arrow_datatype(), + false, + ), + Field::new( + "component", + ::arrow_datatype(), + false, + ), + ])), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -77,12 +100,12 @@ impl ::re_types_core::Loggable for FilterIsNotNull { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, active): (Vec<_>, Vec<_>) = data @@ -92,19 +115,19 @@ impl ::re_types_core::Loggable for FilterIsNotNull { (datum.is_some(), datum) }) .unzip(); - let active_bitmap: Option = { + let active_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - BooleanArray::new( - DataType::Boolean.into(), - active - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - active_bitmap, - ) - .boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from( + active + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + active_validity, + )) }, { let (somes, column): (Vec<_>, Vec<_>) = data @@ -114,21 +137,20 @@ impl ::re_types_core::Loggable for FilterIsNotNull { (datum.is_some(), datum) }) .unzip(); - let column_bitmap: Option = { + let column_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = column_bitmap; - crate::blueprint::datatypes::ComponentColumnSelector::to_arrow2_opt( + _ = column_validity; + crate::blueprint::datatypes::ComponentColumnSelector::to_arrow_opt( column, )? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs index e2282cbe6a72..cff91ae89b29 100644 --- a/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs +++ b/crates/store/re_types/src/blueprint/datatypes/selected_columns.rs @@ -70,18 +70,42 @@ impl ::re_types_core::Loggable for SelectedColumns { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "time_columns", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + Field::new( + "component_columns", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -89,63 +113,98 @@ impl ::re_types_core::Loggable for SelectedColumns { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), - vec![ - { let (somes, time_columns) : (Vec < _ >, Vec < _ >) = data - .iter().map(| datum | { let datum = datum.as_ref().map(| datum | - { datum.time_columns.clone() }); (datum.is_some(), datum) }) - .unzip(); let time_columns_bitmap : Option < - arrow2::bitmap::Bitmap > = { let any_nones = somes.iter().any(| - some | ! * some); any_nones.then(|| somes.into()) }; { use - arrow2:: { buffer::Buffer, offset::OffsetsBuffer }; let offsets = - arrow2::offset::Offsets:: < i32 > ::try_from_lengths(time_columns - .iter().map(| opt | opt.as_ref().map_or(0, | datum | datum - .len()))) ? .into(); let time_columns_inner_data : Vec < _ > = - time_columns.into_iter().flatten().flatten().collect(); let - time_columns_inner_bitmap : Option < arrow2::bitmap::Bitmap > = - None; - ListArray::try_new(DataType::List(std::sync::Arc::new(Field::new("item", - < crate ::datatypes::Utf8 > ::arrow_datatype(), false))).into(), - offsets, { let offsets = arrow2::offset::Offsets:: < i32 > - ::try_from_lengths(time_columns_inner_data.iter().map(| datum | { - datum.0.len() })) ? .into(); let inner_data : - arrow2::buffer::Buffer < u8 > = time_columns_inner_data - .into_iter().flat_map(| datum | { datum.0.0 }).collect(); - - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] unsafe - { Utf8Array:: < i32 > ::new_unchecked(DataType::Utf8.into(), - offsets, inner_data, time_columns_inner_bitmap) } .boxed() }, - time_columns_bitmap,) ? .boxed() } }, { let (somes, - component_columns) : (Vec < _ >, Vec < _ >) = data.iter().map(| - datum | { let datum = datum.as_ref().map(| datum | { datum - .component_columns.clone() }); (datum.is_some(), datum) }) - .unzip(); let component_columns_bitmap : Option < - arrow2::bitmap::Bitmap > = { let any_nones = somes.iter().any(| - some | ! * some); any_nones.then(|| somes.into()) }; { use - arrow2:: { buffer::Buffer, offset::OffsetsBuffer }; let offsets = - arrow2::offset::Offsets:: < i32 > - ::try_from_lengths(component_columns.iter().map(| opt | opt - .as_ref().map_or(0, | datum | datum.len()))) ? .into(); let - component_columns_inner_data : Vec < _ > = component_columns - .into_iter().flatten().flatten().collect(); let - component_columns_inner_bitmap : Option < arrow2::bitmap::Bitmap - > = None; - ListArray::try_new(DataType::List(std::sync::Arc::new(Field::new("item", + as_array_ref(StructArray::new( + fields, + vec![ + { + let (somes, time_columns): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = datum.as_ref().map(|datum| datum.time_columns.clone()); + (datum.is_some(), datum) + }) + .unzip(); + let time_columns_validity: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( + time_columns + .iter() + .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), + ); + let time_columns_inner_data: Vec<_> = + time_columns.into_iter().flatten().flatten().collect(); + let time_columns_inner_validity: Option = + None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), + offsets, + { + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( + time_columns_inner_data.iter().map(|datum| datum.0.len()), + ); + let inner_data: arrow::buffer::Buffer = time_columns_inner_data + .into_iter() + .flat_map(|datum| datum.0 .0) + .collect(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] + as_array_ref(unsafe { + StringArray::new_unchecked( + offsets, + inner_data, + time_columns_inner_validity, + ) + }) + }, + time_columns_validity, + )?) + } + }, + { + let (somes, component_columns): (Vec<_>, Vec<_>) = data + .iter() + .map(|datum| { + let datum = + datum.as_ref().map(|datum| datum.component_columns.clone()); + (datum.is_some(), datum) + }) + .unzip(); + let component_columns_validity: Option = { + let any_nones = somes.iter().any(|some| !*some); + any_nones.then(|| somes.into()) + }; + { + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( + component_columns + .iter() + .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), + ); + let component_columns_inner_data: Vec<_> = + component_columns.into_iter().flatten().flatten().collect(); + let component_columns_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new(std::sync::Arc::new(Field::new("item", < crate ::blueprint::datatypes::ComponentColumnSelector > - ::arrow_datatype(), false))).into(), offsets, { _ = - component_columns_inner_bitmap; crate - ::blueprint::datatypes::ComponentColumnSelector::to_arrow2_opt(component_columns_inner_data - .into_iter().map(Some)) ? }, component_columns_bitmap,) ? - .boxed() } }, - ], - bitmap, - ) - .boxed() + ::arrow_datatype(), false)), offsets, { _ = + component_columns_inner_validity; crate + ::blueprint::datatypes::ComponentColumnSelector::to_arrow_opt(component_columns_inner_data + .into_iter().map(Some)) ? }, component_columns_validity,) ?) + } + }, + ], + validity, + )) }) } diff --git a/crates/store/re_types/src/blueprint/datatypes/tensor_dimension_index_slider.rs b/crates/store/re_types/src/blueprint/datatypes/tensor_dimension_index_slider.rs index 4c3c4c1e0d0b..38a51e2466e1 100644 --- a/crates/store/re_types/src/blueprint/datatypes/tensor_dimension_index_slider.rs +++ b/crates/store/re_types/src/blueprint/datatypes/tensor_dimension_index_slider.rs @@ -65,18 +65,23 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSlider { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![Field::new("dimension", DataType::UInt32, false)]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -84,12 +89,12 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSlider { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![{ let (somes, dimension): (Vec<_>, Vec<_>) = data .iter() @@ -98,23 +103,22 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSlider { (datum.is_some(), datum) }) .unzip(); - let dimension_bitmap: Option = { + let dimension_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - dimension - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - dimension_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + dimension + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + dimension_validity, + )) }], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/components/aggregation_policy.rs b/crates/store/re_types/src/components/aggregation_policy.rs index c76c4317ec92..b4909f752805 100644 --- a/crates/store/re_types/src/components/aggregation_policy.rs +++ b/crates/store/re_types/src/components/aggregation_policy.rs @@ -114,17 +114,21 @@ impl ::re_types_core::Loggable for AggregationPolicy { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -134,16 +138,19 @@ impl ::re_types_core::Loggable for AggregationPolicy { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/albedo_factor.rs b/crates/store/re_types/src/components/albedo_factor.rs index d5ebcc94b5e7..a11f0a6d9ee6 100644 --- a/crates/store/re_types/src/components/albedo_factor.rs +++ b/crates/store/re_types/src/components/albedo_factor.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for AlbedoFactor { crate::datatypes::Rgba32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Rgba32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Rgba32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/annotation_context.rs b/crates/store/re_types/src/components/annotation_context.rs index 04a17d807543..0d433c1ab4ac 100644 --- a/crates/store/re_types/src/components/annotation_context.rs +++ b/crates/store/re_types/src/components/annotation_context.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for AnnotationContext { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,32 +89,33 @@ impl ::re_types_core::Loggable for AnnotationContext { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - _ = data0_inner_bitmap; - crate::datatypes::ClassDescriptionMapElem::to_arrow2_opt( + _ = data0_inner_validity; + crate::datatypes::ClassDescriptionMapElem::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/components/axis_length.rs b/crates/store/re_types/src/components/axis_length.rs index 52c80dc16e84..0d81cb23a7d1 100644 --- a/crates/store/re_types/src/components/axis_length.rs +++ b/crates/store/re_types/src/components/axis_length.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for AxisLength { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/blob.rs b/crates/store/re_types/src/components/blob.rs index 561a6ac5529b..78dd2967aa50 100644 --- a/crates/store/re_types/src/components/blob.rs +++ b/crates/store/re_types/src/components/blob.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for Blob { crate::datatypes::Blob::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Blob::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Blob::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/class_id.rs b/crates/store/re_types/src/components/class_id.rs index 43d038b2be57..7df2edaa9226 100644 --- a/crates/store/re_types/src/components/class_id.rs +++ b/crates/store/re_types/src/components/class_id.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for ClassId { crate::datatypes::ClassId::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::ClassId::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::ClassId::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/color.rs b/crates/store/re_types/src/components/color.rs index de259fe82e2d..731c1ba07f04 100644 --- a/crates/store/re_types/src/components/color.rs +++ b/crates/store/re_types/src/components/color.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for Color { crate::datatypes::Rgba32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Rgba32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Rgba32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/colormap.rs b/crates/store/re_types/src/components/colormap.rs index 3f01fa78fd88..b75a623b092f 100644 --- a/crates/store/re_types/src/components/colormap.rs +++ b/crates/store/re_types/src/components/colormap.rs @@ -151,17 +151,21 @@ impl ::re_types_core::Loggable for Colormap { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -171,16 +175,19 @@ impl ::re_types_core::Loggable for Colormap { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/depth_meter.rs b/crates/store/re_types/src/components/depth_meter.rs index a9ffd0b01ead..6f9c37f3a046 100644 --- a/crates/store/re_types/src/components/depth_meter.rs +++ b/crates/store/re_types/src/components/depth_meter.rs @@ -79,13 +79,13 @@ impl ::re_types_core::Loggable for DepthMeter { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/disconnected_space.rs b/crates/store/re_types/src/components/disconnected_space.rs index 2d3f6068a756..ed96d1dd7406 100644 --- a/crates/store/re_types/src/components/disconnected_space.rs +++ b/crates/store/re_types/src/components/disconnected_space.rs @@ -83,13 +83,13 @@ impl ::re_types_core::Loggable for DisconnectedSpace { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/draw_order.rs b/crates/store/re_types/src/components/draw_order.rs index c7ab0a195754..a3ae26ecbc25 100644 --- a/crates/store/re_types/src/components/draw_order.rs +++ b/crates/store/re_types/src/components/draw_order.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for DrawOrder { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/entity_path.rs b/crates/store/re_types/src/components/entity_path.rs index d2894dd7813c..b9b6c23c4ada 100644 --- a/crates/store/re_types/src/components/entity_path.rs +++ b/crates/store/re_types/src/components/entity_path.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for EntityPath { crate::datatypes::EntityPath::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::EntityPath::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::EntityPath::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/fill_mode.rs b/crates/store/re_types/src/components/fill_mode.rs index 8931e51b40c2..5fc470d14fc1 100644 --- a/crates/store/re_types/src/components/fill_mode.rs +++ b/crates/store/re_types/src/components/fill_mode.rs @@ -100,17 +100,21 @@ impl ::re_types_core::Loggable for FillMode { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -120,16 +124,19 @@ impl ::re_types_core::Loggable for FillMode { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/fill_ratio.rs b/crates/store/re_types/src/components/fill_ratio.rs index d8863dc39964..53f23b6af66c 100644 --- a/crates/store/re_types/src/components/fill_ratio.rs +++ b/crates/store/re_types/src/components/fill_ratio.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for FillRatio { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/gamma_correction.rs b/crates/store/re_types/src/components/gamma_correction.rs index 75c2cd3bedda..456903d5b6e4 100644 --- a/crates/store/re_types/src/components/gamma_correction.rs +++ b/crates/store/re_types/src/components/gamma_correction.rs @@ -78,13 +78,13 @@ impl ::re_types_core::Loggable for GammaCorrection { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/geo_line_string.rs b/crates/store/re_types/src/components/geo_line_string.rs index ca171c2adcdb..b97e09657bda 100644 --- a/crates/store/re_types/src/components/geo_line_string.rs +++ b/crates/store/re_types/src/components/geo_line_string.rs @@ -55,17 +55,21 @@ impl ::re_types_core::Loggable for GeoLineString { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,50 +79,47 @@ impl ::re_types_core::Loggable for GeoLineString { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data_inner_data: Vec<_> = data0_inner_data .into_iter() .map(|datum| datum.0) .flatten() .collect(); - let data0_inner_data_inner_bitmap: Option = None; - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::Float64, false)), - 2, - ) - .into(), - PrimitiveArray::new( - DataType::Float64.into(), - data0_inner_data_inner_data.into_iter().collect(), - data0_inner_data_inner_bitmap, - ) - .boxed(), - data0_inner_bitmap, - ) - .boxed() + let data0_inner_data_inner_validity: Option = + None; + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0_inner_data_inner_data.into_iter().collect::>(), + ), + data0_inner_data_inner_validity, + )), + data0_inner_validity, + )) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/components/half_size2d.rs b/crates/store/re_types/src/components/half_size2d.rs index 183c4a2cc54c..d6bf57e23794 100644 --- a/crates/store/re_types/src/components/half_size2d.rs +++ b/crates/store/re_types/src/components/half_size2d.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for HalfSize2D { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/half_size3d.rs b/crates/store/re_types/src/components/half_size3d.rs index fb4c6917e9de..d4ba44268531 100644 --- a/crates/store/re_types/src/components/half_size3d.rs +++ b/crates/store/re_types/src/components/half_size3d.rs @@ -77,13 +77,13 @@ impl ::re_types_core::Loggable for HalfSize3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/image_buffer.rs b/crates/store/re_types/src/components/image_buffer.rs index e9a643de855d..79bafe36e5e3 100644 --- a/crates/store/re_types/src/components/image_buffer.rs +++ b/crates/store/re_types/src/components/image_buffer.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for ImageBuffer { crate::datatypes::Blob::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Blob::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Blob::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/image_format.rs b/crates/store/re_types/src/components/image_format.rs index d816d249afe1..74afc7ba4df4 100644 --- a/crates/store/re_types/src/components/image_format.rs +++ b/crates/store/re_types/src/components/image_format.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for ImageFormat { crate::datatypes::ImageFormat::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::ImageFormat::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::ImageFormat::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/image_plane_distance.rs b/crates/store/re_types/src/components/image_plane_distance.rs index 5e297e9b198b..45935f9f1c45 100644 --- a/crates/store/re_types/src/components/image_plane_distance.rs +++ b/crates/store/re_types/src/components/image_plane_distance.rs @@ -73,13 +73,13 @@ impl ::re_types_core::Loggable for ImagePlaneDistance { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/keypoint_id.rs b/crates/store/re_types/src/components/keypoint_id.rs index bdb8aae1063b..ecc81d62c883 100644 --- a/crates/store/re_types/src/components/keypoint_id.rs +++ b/crates/store/re_types/src/components/keypoint_id.rs @@ -89,13 +89,13 @@ impl ::re_types_core::Loggable for KeypointId { crate::datatypes::KeypointId::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::KeypointId::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::KeypointId::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/lat_lon.rs b/crates/store/re_types/src/components/lat_lon.rs index 3350cfbaa9fc..a6e64499ab02 100644 --- a/crates/store/re_types/src/components/lat_lon.rs +++ b/crates/store/re_types/src/components/lat_lon.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for LatLon { crate::datatypes::DVec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::DVec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::DVec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/length.rs b/crates/store/re_types/src/components/length.rs index a576a8827fd9..26127dce80ae 100644 --- a/crates/store/re_types/src/components/length.rs +++ b/crates/store/re_types/src/components/length.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for Length { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/line_strip2d.rs b/crates/store/re_types/src/components/line_strip2d.rs index e4cc494c81e2..2c748243220f 100644 --- a/crates/store/re_types/src/components/line_strip2d.rs +++ b/crates/store/re_types/src/components/line_strip2d.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for LineStrip2D { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,50 +89,47 @@ impl ::re_types_core::Loggable for LineStrip2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data_inner_data: Vec<_> = data0_inner_data .into_iter() .map(|datum| datum.0) .flatten() .collect(); - let data0_inner_data_inner_bitmap: Option = None; - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::Float32, false)), - 2, - ) - .into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data_inner_data.into_iter().collect(), - data0_inner_data_inner_bitmap, - ) - .boxed(), - data0_inner_bitmap, - ) - .boxed() + let data0_inner_data_inner_validity: Option = + None; + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0_inner_data_inner_data.into_iter().collect::>(), + ), + data0_inner_data_inner_validity, + )), + data0_inner_validity, + )) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/components/line_strip3d.rs b/crates/store/re_types/src/components/line_strip3d.rs index 241ab65e8130..3fb8ddba6699 100644 --- a/crates/store/re_types/src/components/line_strip3d.rs +++ b/crates/store/re_types/src/components/line_strip3d.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for LineStrip3D { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,50 +89,47 @@ impl ::re_types_core::Loggable for LineStrip3D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data_inner_data: Vec<_> = data0_inner_data .into_iter() .map(|datum| datum.0) .flatten() .collect(); - let data0_inner_data_inner_bitmap: Option = None; - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::Float32, false)), - 3, - ) - .into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data_inner_data.into_iter().collect(), - data0_inner_data_inner_bitmap, - ) - .boxed(), - data0_inner_bitmap, - ) - .boxed() + let data0_inner_data_inner_validity: Option = + None; + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0_inner_data_inner_data.into_iter().collect::>(), + ), + data0_inner_data_inner_validity, + )), + data0_inner_validity, + )) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/components/magnification_filter.rs b/crates/store/re_types/src/components/magnification_filter.rs index b1bd2c038cc7..996805d808cd 100644 --- a/crates/store/re_types/src/components/magnification_filter.rs +++ b/crates/store/re_types/src/components/magnification_filter.rs @@ -86,17 +86,21 @@ impl ::re_types_core::Loggable for MagnificationFilter { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -106,16 +110,19 @@ impl ::re_types_core::Loggable for MagnificationFilter { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/marker_shape.rs b/crates/store/re_types/src/components/marker_shape.rs index 388ee5ad5e94..2661501806da 100644 --- a/crates/store/re_types/src/components/marker_shape.rs +++ b/crates/store/re_types/src/components/marker_shape.rs @@ -128,17 +128,21 @@ impl ::re_types_core::Loggable for MarkerShape { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -148,16 +152,19 @@ impl ::re_types_core::Loggable for MarkerShape { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/marker_size.rs b/crates/store/re_types/src/components/marker_size.rs index e1927f83ddee..206022bf21f5 100644 --- a/crates/store/re_types/src/components/marker_size.rs +++ b/crates/store/re_types/src/components/marker_size.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for MarkerSize { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/media_type.rs b/crates/store/re_types/src/components/media_type.rs index cf29f6909bd4..2eacec30e60e 100644 --- a/crates/store/re_types/src/components/media_type.rs +++ b/crates/store/re_types/src/components/media_type.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for MediaType { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/name.rs b/crates/store/re_types/src/components/name.rs index 94e94354b0cf..06f9fc3bea2c 100644 --- a/crates/store/re_types/src/components/name.rs +++ b/crates/store/re_types/src/components/name.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Name { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/opacity.rs b/crates/store/re_types/src/components/opacity.rs index 146e6cf15654..a8c7048b63b9 100644 --- a/crates/store/re_types/src/components/opacity.rs +++ b/crates/store/re_types/src/components/opacity.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for Opacity { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pinhole_projection.rs b/crates/store/re_types/src/components/pinhole_projection.rs index 355e268fc373..847249910dde 100644 --- a/crates/store/re_types/src/components/pinhole_projection.rs +++ b/crates/store/re_types/src/components/pinhole_projection.rs @@ -81,13 +81,13 @@ impl ::re_types_core::Loggable for PinholeProjection { crate::datatypes::Mat3x3::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Mat3x3::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Mat3x3::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_rotation_axis_angle.rs b/crates/store/re_types/src/components/pose_rotation_axis_angle.rs index d41ea1b0a22c..c36f231806c9 100644 --- a/crates/store/re_types/src/components/pose_rotation_axis_angle.rs +++ b/crates/store/re_types/src/components/pose_rotation_axis_angle.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for PoseRotationAxisAngle { crate::datatypes::RotationAxisAngle::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::RotationAxisAngle::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::RotationAxisAngle::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_rotation_quat.rs b/crates/store/re_types/src/components/pose_rotation_quat.rs index 05541e505641..fa2abc7d118d 100644 --- a/crates/store/re_types/src/components/pose_rotation_quat.rs +++ b/crates/store/re_types/src/components/pose_rotation_quat.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for PoseRotationQuat { crate::datatypes::Quaternion::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Quaternion::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Quaternion::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_scale3d.rs b/crates/store/re_types/src/components/pose_scale3d.rs index b4378b6c23b4..ec58d7e8c816 100644 --- a/crates/store/re_types/src/components/pose_scale3d.rs +++ b/crates/store/re_types/src/components/pose_scale3d.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for PoseScale3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_transform_mat3x3.rs b/crates/store/re_types/src/components/pose_transform_mat3x3.rs index 4ddcd78a1b71..83d0da756fe7 100644 --- a/crates/store/re_types/src/components/pose_transform_mat3x3.rs +++ b/crates/store/re_types/src/components/pose_transform_mat3x3.rs @@ -84,13 +84,13 @@ impl ::re_types_core::Loggable for PoseTransformMat3x3 { crate::datatypes::Mat3x3::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Mat3x3::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Mat3x3::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/pose_translation3d.rs b/crates/store/re_types/src/components/pose_translation3d.rs index a4d4e6424676..2fba1c14cfd3 100644 --- a/crates/store/re_types/src/components/pose_translation3d.rs +++ b/crates/store/re_types/src/components/pose_translation3d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for PoseTranslation3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/position2d.rs b/crates/store/re_types/src/components/position2d.rs index 145668f91d1f..a4bd51d415cb 100644 --- a/crates/store/re_types/src/components/position2d.rs +++ b/crates/store/re_types/src/components/position2d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Position2D { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/position3d.rs b/crates/store/re_types/src/components/position3d.rs index dc03a9b250f6..959582b1e83a 100644 --- a/crates/store/re_types/src/components/position3d.rs +++ b/crates/store/re_types/src/components/position3d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Position3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/radius.rs b/crates/store/re_types/src/components/radius.rs index 15bd75ee62c6..a835ec36faed 100644 --- a/crates/store/re_types/src/components/radius.rs +++ b/crates/store/re_types/src/components/radius.rs @@ -79,13 +79,13 @@ impl ::re_types_core::Loggable for Radius { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/range1d.rs b/crates/store/re_types/src/components/range1d.rs index 90404d0457cf..8126bd996ea4 100644 --- a/crates/store/re_types/src/components/range1d.rs +++ b/crates/store/re_types/src/components/range1d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Range1D { crate::datatypes::Range1D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Range1D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Range1D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/resolution.rs b/crates/store/re_types/src/components/resolution.rs index a5395a62b4e7..391bf37f9a0c 100644 --- a/crates/store/re_types/src/components/resolution.rs +++ b/crates/store/re_types/src/components/resolution.rs @@ -73,13 +73,13 @@ impl ::re_types_core::Loggable for Resolution { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/rotation_axis_angle.rs b/crates/store/re_types/src/components/rotation_axis_angle.rs index be681335c48b..494c8c726f0a 100644 --- a/crates/store/re_types/src/components/rotation_axis_angle.rs +++ b/crates/store/re_types/src/components/rotation_axis_angle.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for RotationAxisAngle { crate::datatypes::RotationAxisAngle::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::RotationAxisAngle::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::RotationAxisAngle::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/rotation_quat.rs b/crates/store/re_types/src/components/rotation_quat.rs index ff8ded9db655..796fb844cb59 100644 --- a/crates/store/re_types/src/components/rotation_quat.rs +++ b/crates/store/re_types/src/components/rotation_quat.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for RotationQuat { crate::datatypes::Quaternion::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Quaternion::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Quaternion::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/scalar.rs b/crates/store/re_types/src/components/scalar.rs index 9521649b7603..1058e369e2d6 100644 --- a/crates/store/re_types/src/components/scalar.rs +++ b/crates/store/re_types/src/components/scalar.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for Scalar { crate::datatypes::Float64::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float64::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float64::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/scale3d.rs b/crates/store/re_types/src/components/scale3d.rs index 33beb6bd70c9..3ff9af13e107 100644 --- a/crates/store/re_types/src/components/scale3d.rs +++ b/crates/store/re_types/src/components/scale3d.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for Scale3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/show_labels.rs b/crates/store/re_types/src/components/show_labels.rs index 9c72df4a18bb..b20c59d4b517 100644 --- a/crates/store/re_types/src/components/show_labels.rs +++ b/crates/store/re_types/src/components/show_labels.rs @@ -78,13 +78,13 @@ impl ::re_types_core::Loggable for ShowLabels { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/stroke_width.rs b/crates/store/re_types/src/components/stroke_width.rs index ae14c912af90..5a6f522cc994 100644 --- a/crates/store/re_types/src/components/stroke_width.rs +++ b/crates/store/re_types/src/components/stroke_width.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for StrokeWidth { crate::datatypes::Float32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Float32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Float32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/tensor_data.rs b/crates/store/re_types/src/components/tensor_data.rs index 5bc6b96067ea..2faca36bb69e 100644 --- a/crates/store/re_types/src/components/tensor_data.rs +++ b/crates/store/re_types/src/components/tensor_data.rs @@ -79,13 +79,13 @@ impl ::re_types_core::Loggable for TensorData { crate::datatypes::TensorData::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::TensorData::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::TensorData::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/tensor_dimension_index_selection.rs b/crates/store/re_types/src/components/tensor_dimension_index_selection.rs index 405b26b96193..01ede1b23e6c 100644 --- a/crates/store/re_types/src/components/tensor_dimension_index_selection.rs +++ b/crates/store/re_types/src/components/tensor_dimension_index_selection.rs @@ -76,13 +76,13 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { crate::datatypes::TensorDimensionIndexSelection::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::TensorDimensionIndexSelection::to_arrow2_opt(data.into_iter().map( + crate::datatypes::TensorDimensionIndexSelection::to_arrow_opt(data.into_iter().map( |datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), diff --git a/crates/store/re_types/src/components/tensor_height_dimension.rs b/crates/store/re_types/src/components/tensor_height_dimension.rs index 26f053539042..119c5933267f 100644 --- a/crates/store/re_types/src/components/tensor_height_dimension.rs +++ b/crates/store/re_types/src/components/tensor_height_dimension.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for TensorHeightDimension { crate::datatypes::TensorDimensionSelection::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::TensorDimensionSelection::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::TensorDimensionSelection::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/tensor_width_dimension.rs b/crates/store/re_types/src/components/tensor_width_dimension.rs index 6877c4b1474d..fc7da7124b9b 100644 --- a/crates/store/re_types/src/components/tensor_width_dimension.rs +++ b/crates/store/re_types/src/components/tensor_width_dimension.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for TensorWidthDimension { crate::datatypes::TensorDimensionSelection::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::TensorDimensionSelection::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::TensorDimensionSelection::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/texcoord2d.rs b/crates/store/re_types/src/components/texcoord2d.rs index c9c5a724232f..ff9ea66ed5fe 100644 --- a/crates/store/re_types/src/components/texcoord2d.rs +++ b/crates/store/re_types/src/components/texcoord2d.rs @@ -87,13 +87,13 @@ impl ::re_types_core::Loggable for Texcoord2D { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/text.rs b/crates/store/re_types/src/components/text.rs index 5d5afa486884..14829ca69e11 100644 --- a/crates/store/re_types/src/components/text.rs +++ b/crates/store/re_types/src/components/text.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Text { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/text_log_level.rs b/crates/store/re_types/src/components/text_log_level.rs index a64239622976..8cb71910bb95 100644 --- a/crates/store/re_types/src/components/text_log_level.rs +++ b/crates/store/re_types/src/components/text_log_level.rs @@ -80,13 +80,13 @@ impl ::re_types_core::Loggable for TextLogLevel { crate::datatypes::Utf8::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Utf8::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Utf8::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/transform_mat3x3.rs b/crates/store/re_types/src/components/transform_mat3x3.rs index 8b9f349d1f2b..c60364a8cb21 100644 --- a/crates/store/re_types/src/components/transform_mat3x3.rs +++ b/crates/store/re_types/src/components/transform_mat3x3.rs @@ -84,13 +84,13 @@ impl ::re_types_core::Loggable for TransformMat3x3 { crate::datatypes::Mat3x3::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Mat3x3::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Mat3x3::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/transform_relation.rs b/crates/store/re_types/src/components/transform_relation.rs index 0acb7a41088b..1af141515d9c 100644 --- a/crates/store/re_types/src/components/transform_relation.rs +++ b/crates/store/re_types/src/components/transform_relation.rs @@ -89,17 +89,21 @@ impl ::re_types_core::Loggable for TransformRelation { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -109,16 +113,19 @@ impl ::re_types_core::Loggable for TransformRelation { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/components/translation3d.rs b/crates/store/re_types/src/components/translation3d.rs index f3b0ea611f9c..8c7bbe48e03a 100644 --- a/crates/store/re_types/src/components/translation3d.rs +++ b/crates/store/re_types/src/components/translation3d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Translation3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/triangle_indices.rs b/crates/store/re_types/src/components/triangle_indices.rs index 7253c0499b19..f22283f0c1a7 100644 --- a/crates/store/re_types/src/components/triangle_indices.rs +++ b/crates/store/re_types/src/components/triangle_indices.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for TriangleIndices { crate::datatypes::UVec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::UVec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::UVec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/value_range.rs b/crates/store/re_types/src/components/value_range.rs index 4fdaa772d331..d4dc347e9e86 100644 --- a/crates/store/re_types/src/components/value_range.rs +++ b/crates/store/re_types/src/components/value_range.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for ValueRange { crate::datatypes::Range1D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Range1D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Range1D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/vector2d.rs b/crates/store/re_types/src/components/vector2d.rs index 8500bd3aabc8..6769f40a04ac 100644 --- a/crates/store/re_types/src/components/vector2d.rs +++ b/crates/store/re_types/src/components/vector2d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Vector2D { crate::datatypes::Vec2D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec2D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec2D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/vector3d.rs b/crates/store/re_types/src/components/vector3d.rs index 795985b9fb80..42ca53ba7c37 100644 --- a/crates/store/re_types/src/components/vector3d.rs +++ b/crates/store/re_types/src/components/vector3d.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for Vector3D { crate::datatypes::Vec3D::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Vec3D::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Vec3D::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/video_timestamp.rs b/crates/store/re_types/src/components/video_timestamp.rs index b9a75cb1023e..96b6d1cc4d2a 100644 --- a/crates/store/re_types/src/components/video_timestamp.rs +++ b/crates/store/re_types/src/components/video_timestamp.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for VideoTimestamp { crate::datatypes::VideoTimestamp::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::VideoTimestamp::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::VideoTimestamp::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/components/view_coordinates.rs b/crates/store/re_types/src/components/view_coordinates.rs index ae377dfa1fa1..4160466dc856 100644 --- a/crates/store/re_types/src/components/view_coordinates.rs +++ b/crates/store/re_types/src/components/view_coordinates.rs @@ -92,13 +92,13 @@ impl ::re_types_core::Loggable for ViewCoordinates { crate::datatypes::ViewCoordinates::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::ViewCoordinates::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::ViewCoordinates::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/datatypes/angle.rs b/crates/store/re_types/src/datatypes/angle.rs index e00e35c0a074..2a7c9a15751d 100644 --- a/crates/store/re_types/src/datatypes/angle.rs +++ b/crates/store/re_types/src/datatypes/angle.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Angle { DataType::Float32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, radians): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,16 +86,19 @@ impl ::re_types_core::Loggable for Angle { (datum.is_some(), datum) }) .unzip(); - let radians_bitmap: Option = { + let radians_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - radians.into_iter().map(|v| v.unwrap_or_default()).collect(), - radians_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + radians + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + radians_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/annotation_info.rs b/crates/store/re_types/src/datatypes/annotation_info.rs index 3fbe50c49993..109ad16a58eb 100644 --- a/crates/store/re_types/src/datatypes/annotation_info.rs +++ b/crates/store/re_types/src/datatypes/annotation_info.rs @@ -62,18 +62,27 @@ impl ::re_types_core::Loggable for AnnotationInfo { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("id", DataType::UInt16, false), + Field::new("label", DataType::Utf8, true), + Field::new("color", DataType::UInt32, true), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -81,12 +90,12 @@ impl ::re_types_core::Loggable for AnnotationInfo { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, id): (Vec<_>, Vec<_>) = data @@ -96,16 +105,18 @@ impl ::re_types_core::Loggable for AnnotationInfo { (datum.is_some(), datum) }) .unzip(); - let id_bitmap: Option = { + let id_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt16.into(), - id.into_iter().map(|v| v.unwrap_or_default()).collect(), - id_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + id.into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + id_validity, + )) }, { let (somes, label): (Vec<_>, Vec<_>) = data @@ -116,33 +127,25 @@ impl ::re_types_core::Loggable for AnnotationInfo { (datum.is_some(), datum) }) .unzip(); - let label_bitmap: Option = { + let label_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( label.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = label + ); + let inner_data: arrow::buffer::Buffer = label .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - label_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, label_validity) + }) } }, { @@ -154,24 +157,23 @@ impl ::re_types_core::Loggable for AnnotationInfo { (datum.is_some(), datum) }) .unzip(); - let color_bitmap: Option = { + let color_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - color - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - color_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + color + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + color_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/blob.rs b/crates/store/re_types/src/datatypes/blob.rs index 6501ddfe5194..2795984b0601 100644 --- a/crates/store/re_types/src/datatypes/blob.rs +++ b/crates/store/re_types/src/datatypes/blob.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for Blob { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,38 +89,33 @@ impl ::re_types_core::Loggable for Blob { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.num_instances())), - )? - .into(); - let data0_inner_data: Buffer<_> = data0 + ); + let data0_inner_data: ScalarBuffer<_> = data0 .iter() .flatten() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), offsets, - PrimitiveArray::new( - DataType::UInt8.into(), + as_array_ref(PrimitiveArray::::new( data0_inner_data, - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - )? - .boxed() + data0_inner_validity, + )), + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/datatypes/channel_datatype.rs b/crates/store/re_types/src/datatypes/channel_datatype.rs index 204d22bf0c2b..4690450a3b61 100644 --- a/crates/store/re_types/src/datatypes/channel_datatype.rs +++ b/crates/store/re_types/src/datatypes/channel_datatype.rs @@ -136,17 +136,21 @@ impl ::re_types_core::Loggable for ChannelDatatype { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -156,16 +160,19 @@ impl ::re_types_core::Loggable for ChannelDatatype { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/class_description.rs b/crates/store/re_types/src/datatypes/class_description.rs index dfbdf480eccd..a30a7aa9d651 100644 --- a/crates/store/re_types/src/datatypes/class_description.rs +++ b/crates/store/re_types/src/datatypes/class_description.rs @@ -94,18 +94,51 @@ impl ::re_types_core::Loggable for ClassDescription { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "info", + DataType::Struct(Fields::from(vec![ + Field::new("id", DataType::UInt16, false), + Field::new("label", ::arrow_datatype(), true), + Field::new("color", ::arrow_datatype(), true), + ])), + false, + ), + Field::new( + "keypoint_annotations", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + Field::new( + "keypoint_connections", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -113,12 +146,12 @@ impl ::re_types_core::Loggable for ClassDescription { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, info): (Vec<_>, Vec<_>) = data @@ -128,13 +161,13 @@ impl ::re_types_core::Loggable for ClassDescription { (datum.is_some(), datum) }) .unzip(); - let info_bitmap: Option = { + let info_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = info_bitmap; - crate::datatypes::AnnotationInfo::to_arrow2_opt(info)? + _ = info_validity; + crate::datatypes::AnnotationInfo::to_arrow_opt(info)? } }, { @@ -147,42 +180,39 @@ impl ::re_types_core::Loggable for ClassDescription { (datum.is_some(), datum) }) .unzip(); - let keypoint_annotations_bitmap: Option = { + let keypoint_annotations_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( keypoint_annotations .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let keypoint_annotations_inner_data: Vec<_> = keypoint_annotations .into_iter() .flatten() .flatten() .collect(); - let keypoint_annotations_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let keypoint_annotations_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), false, - ))) - .into(), + )), offsets, { - _ = keypoint_annotations_inner_bitmap; - crate::datatypes::AnnotationInfo::to_arrow2_opt( + _ = keypoint_annotations_inner_validity; + crate::datatypes::AnnotationInfo::to_arrow_opt( keypoint_annotations_inner_data.into_iter().map(Some), )? }, - keypoint_annotations_bitmap, - )? - .boxed() + keypoint_annotations_validity, + )?) } }, { @@ -195,48 +225,44 @@ impl ::re_types_core::Loggable for ClassDescription { (datum.is_some(), datum) }) .unzip(); - let keypoint_connections_bitmap: Option = { + let keypoint_connections_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( keypoint_connections .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let keypoint_connections_inner_data: Vec<_> = keypoint_connections .into_iter() .flatten() .flatten() .collect(); - let keypoint_connections_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let keypoint_connections_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), false, - ))) - .into(), + )), offsets, { - _ = keypoint_connections_inner_bitmap; - crate::datatypes::KeypointPair::to_arrow2_opt( + _ = keypoint_connections_inner_validity; + crate::datatypes::KeypointPair::to_arrow_opt( keypoint_connections_inner_data.into_iter().map(Some), )? }, - keypoint_connections_bitmap, - )? - .boxed() + keypoint_connections_validity, + )?) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/class_description_map_elem.rs b/crates/store/re_types/src/datatypes/class_description_map_elem.rs index 6edbc87efc3d..41029d9db135 100644 --- a/crates/store/re_types/src/datatypes/class_description_map_elem.rs +++ b/crates/store/re_types/src/datatypes/class_description_map_elem.rs @@ -63,18 +63,54 @@ impl ::re_types_core::Loggable for ClassDescriptionMapElem { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("class_id", DataType::UInt16, false), + Field::new( + "class_description", + DataType::Struct(Fields::from(vec![ + Field::new( + "info", + ::arrow_datatype(), + false, + ), + Field::new( + "keypoint_annotations", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + Field::new( + "keypoint_connections", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + ])), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -82,12 +118,12 @@ impl ::re_types_core::Loggable for ClassDescriptionMapElem { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, class_id): (Vec<_>, Vec<_>) = data @@ -97,19 +133,19 @@ impl ::re_types_core::Loggable for ClassDescriptionMapElem { (datum.is_some(), datum) }) .unzip(); - let class_id_bitmap: Option = { + let class_id_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt16.into(), - class_id - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - class_id_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + class_id + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + class_id_validity, + )) }, { let (somes, class_description): (Vec<_>, Vec<_>) = data @@ -120,19 +156,18 @@ impl ::re_types_core::Loggable for ClassDescriptionMapElem { (datum.is_some(), datum) }) .unzip(); - let class_description_bitmap: Option = { + let class_description_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = class_description_bitmap; - crate::datatypes::ClassDescription::to_arrow2_opt(class_description)? + _ = class_description_validity; + crate::datatypes::ClassDescription::to_arrow_opt(class_description)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/class_id.rs b/crates/store/re_types/src/datatypes/class_id.rs index cfaec84e2b23..bdd44fd8ed72 100644 --- a/crates/store/re_types/src/datatypes/class_id.rs +++ b/crates/store/re_types/src/datatypes/class_id.rs @@ -74,17 +74,21 @@ impl ::re_types_core::Loggable for ClassId { DataType::UInt16 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -94,16 +98,19 @@ impl ::re_types_core::Loggable for ClassId { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/color_model.rs b/crates/store/re_types/src/datatypes/color_model.rs index 17595e694f15..14856538bb9f 100644 --- a/crates/store/re_types/src/datatypes/color_model.rs +++ b/crates/store/re_types/src/datatypes/color_model.rs @@ -98,17 +98,21 @@ impl ::re_types_core::Loggable for ColorModel { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -118,16 +122,19 @@ impl ::re_types_core::Loggable for ColorModel { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/dvec2d.rs b/crates/store/re_types/src/datatypes/dvec2d.rs index 1001799afe77..025d82e970ad 100644 --- a/crates/store/re_types/src/datatypes/dvec2d.rs +++ b/crates/store/re_types/src/datatypes/dvec2d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for DVec2D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for DVec2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for DVec2D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float64.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/image_format.rs b/crates/store/re_types/src/datatypes/image_format.rs index f28ef43b48b7..503c1b09f148 100644 --- a/crates/store/re_types/src/datatypes/image_format.rs +++ b/crates/store/re_types/src/datatypes/image_format.rs @@ -91,18 +91,29 @@ impl ::re_types_core::Loggable for ImageFormat { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("width", DataType::UInt32, false), + Field::new("height", DataType::UInt32, false), + Field::new("pixel_format", DataType::UInt8, true), + Field::new("color_model", DataType::UInt8, true), + Field::new("channel_datatype", DataType::UInt8, true), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -110,12 +121,12 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, width): (Vec<_>, Vec<_>) = data @@ -125,16 +136,19 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let width_bitmap: Option = { + let width_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - width.into_iter().map(|v| v.unwrap_or_default()).collect(), - width_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + width + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + width_validity, + )) }, { let (somes, height): (Vec<_>, Vec<_>) = data @@ -144,16 +158,19 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let height_bitmap: Option = { + let height_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - height.into_iter().map(|v| v.unwrap_or_default()).collect(), - height_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + height + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + height_validity, + )) }, { let (somes, pixel_format): (Vec<_>, Vec<_>) = data @@ -166,13 +183,13 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let pixel_format_bitmap: Option = { + let pixel_format_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = pixel_format_bitmap; - crate::datatypes::PixelFormat::to_arrow2_opt(pixel_format)? + _ = pixel_format_validity; + crate::datatypes::PixelFormat::to_arrow_opt(pixel_format)? } }, { @@ -186,13 +203,13 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let color_model_bitmap: Option = { + let color_model_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = color_model_bitmap; - crate::datatypes::ColorModel::to_arrow2_opt(color_model)? + _ = color_model_validity; + crate::datatypes::ColorModel::to_arrow_opt(color_model)? } }, { @@ -206,19 +223,18 @@ impl ::re_types_core::Loggable for ImageFormat { (datum.is_some(), datum) }) .unzip(); - let channel_datatype_bitmap: Option = { + let channel_datatype_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = channel_datatype_bitmap; - crate::datatypes::ChannelDatatype::to_arrow2_opt(channel_datatype)? + _ = channel_datatype_validity; + crate::datatypes::ChannelDatatype::to_arrow_opt(channel_datatype)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/keypoint_id.rs b/crates/store/re_types/src/datatypes/keypoint_id.rs index 363f740ceb49..f27a53ae1da6 100644 --- a/crates/store/re_types/src/datatypes/keypoint_id.rs +++ b/crates/store/re_types/src/datatypes/keypoint_id.rs @@ -76,17 +76,21 @@ impl ::re_types_core::Loggable for KeypointId { DataType::UInt16 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -96,16 +100,19 @@ impl ::re_types_core::Loggable for KeypointId { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/keypoint_pair.rs b/crates/store/re_types/src/datatypes/keypoint_pair.rs index 0eb42c747454..a769a97b4c30 100644 --- a/crates/store/re_types/src/datatypes/keypoint_pair.rs +++ b/crates/store/re_types/src/datatypes/keypoint_pair.rs @@ -61,18 +61,26 @@ impl ::re_types_core::Loggable for KeypointPair { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("keypoint0", DataType::UInt16, false), + Field::new("keypoint1", DataType::UInt16, false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -80,12 +88,12 @@ impl ::re_types_core::Loggable for KeypointPair { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, keypoint0): (Vec<_>, Vec<_>) = data @@ -95,19 +103,19 @@ impl ::re_types_core::Loggable for KeypointPair { (datum.is_some(), datum) }) .unzip(); - let keypoint0_bitmap: Option = { + let keypoint0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt16.into(), - keypoint0 - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - keypoint0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + keypoint0 + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + keypoint0_validity, + )) }, { let (somes, keypoint1): (Vec<_>, Vec<_>) = data @@ -117,24 +125,23 @@ impl ::re_types_core::Loggable for KeypointPair { (datum.is_some(), datum) }) .unzip(); - let keypoint1_bitmap: Option = { + let keypoint1_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt16.into(), - keypoint1 - .into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - keypoint1_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + keypoint1 + .into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + keypoint1_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/mat3x3.rs b/crates/store/re_types/src/datatypes/mat3x3.rs index 303f14de9774..8625a5ff906c 100644 --- a/crates/store/re_types/src/datatypes/mat3x3.rs +++ b/crates/store/re_types/src/datatypes/mat3x3.rs @@ -74,17 +74,21 @@ impl ::re_types_core::Loggable for Mat3x3 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -94,12 +98,11 @@ impl ::re_types_core::Loggable for Mat3x3 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -109,26 +112,24 @@ impl ::re_types_core::Loggable for Mat3x3 { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(9usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 9, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/mat4x4.rs b/crates/store/re_types/src/datatypes/mat4x4.rs index 4a9b9081fae3..a52bfd58ccd6 100644 --- a/crates/store/re_types/src/datatypes/mat4x4.rs +++ b/crates/store/re_types/src/datatypes/mat4x4.rs @@ -74,17 +74,21 @@ impl ::re_types_core::Loggable for Mat4x4 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -94,12 +98,11 @@ impl ::re_types_core::Loggable for Mat4x4 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -109,26 +112,24 @@ impl ::re_types_core::Loggable for Mat4x4 { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(16usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 16, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/pixel_format.rs b/crates/store/re_types/src/datatypes/pixel_format.rs index 13c80f4c84f3..d45adae69c2e 100644 --- a/crates/store/re_types/src/datatypes/pixel_format.rs +++ b/crates/store/re_types/src/datatypes/pixel_format.rs @@ -218,17 +218,21 @@ impl ::re_types_core::Loggable for PixelFormat { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -238,16 +242,19 @@ impl ::re_types_core::Loggable for PixelFormat { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/quaternion.rs b/crates/store/re_types/src/datatypes/quaternion.rs index b45e589dc61c..6b7fa846115a 100644 --- a/crates/store/re_types/src/datatypes/quaternion.rs +++ b/crates/store/re_types/src/datatypes/quaternion.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for Quaternion { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,12 +89,11 @@ impl ::re_types_core::Loggable for Quaternion { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -100,26 +103,24 @@ impl ::re_types_core::Loggable for Quaternion { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(4usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 4, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/range1d.rs b/crates/store/re_types/src/datatypes/range1d.rs index 24fc4122531c..dab8fb2e2bfd 100644 --- a/crates/store/re_types/src/datatypes/range1d.rs +++ b/crates/store/re_types/src/datatypes/range1d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Range1D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for Range1D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for Range1D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float64.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/range2d.rs b/crates/store/re_types/src/datatypes/range2d.rs index d38ca45ee68f..ea744ba6ee08 100644 --- a/crates/store/re_types/src/datatypes/range2d.rs +++ b/crates/store/re_types/src/datatypes/range2d.rs @@ -62,18 +62,40 @@ impl ::re_types_core::Loggable for Range2D { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "x_range", + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + ), + false, + ), + Field::new( + "y_range", + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + ), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -81,12 +103,12 @@ impl ::re_types_core::Loggable for Range2D { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, x_range): (Vec<_>, Vec<_>) = data @@ -96,45 +118,36 @@ impl ::re_types_core::Loggable for Range2D { (datum.is_some(), datum) }) .unzip(); - let x_range_bitmap: Option = { + let x_range_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let x_range_inner_data: Vec<_> = x_range .into_iter() .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) .flatten() .collect(); - let x_range_inner_bitmap: Option = - x_range_bitmap.as_ref().map(|bitmap| { - bitmap + let x_range_inner_validity: Option = + x_range_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new( - "item", - DataType::Float64, - false, - )), - 2, - ) - .into(), - PrimitiveArray::new( - DataType::Float64.into(), - x_range_inner_data.into_iter().collect(), - x_range_inner_bitmap, - ) - .boxed(), - x_range_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + x_range_inner_data.into_iter().collect::>(), + ), + x_range_inner_validity, + )), + x_range_validity, + )) } }, { @@ -145,51 +158,41 @@ impl ::re_types_core::Loggable for Range2D { (datum.is_some(), datum) }) .unzip(); - let y_range_bitmap: Option = { + let y_range_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let y_range_inner_data: Vec<_> = y_range .into_iter() .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) .flatten() .collect(); - let y_range_inner_bitmap: Option = - y_range_bitmap.as_ref().map(|bitmap| { - bitmap + let y_range_inner_validity: Option = + y_range_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new( - "item", - DataType::Float64, - false, - )), - 2, - ) - .into(), - PrimitiveArray::new( - DataType::Float64.into(), - y_range_inner_data.into_iter().collect(), - y_range_inner_bitmap, - ) - .boxed(), - y_range_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + y_range_inner_data.into_iter().collect::>(), + ), + y_range_inner_validity, + )), + y_range_validity, + )) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/rgba32.rs b/crates/store/re_types/src/datatypes/rgba32.rs index 3d6862855ddb..e476ba850bb3 100644 --- a/crates/store/re_types/src/datatypes/rgba32.rs +++ b/crates/store/re_types/src/datatypes/rgba32.rs @@ -64,17 +64,21 @@ impl ::re_types_core::Loggable for Rgba32 { DataType::UInt32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -84,16 +88,19 @@ impl ::re_types_core::Loggable for Rgba32 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/rotation_axis_angle.rs b/crates/store/re_types/src/datatypes/rotation_axis_angle.rs index 9b74207cd837..f9dfa2d96fc7 100644 --- a/crates/store/re_types/src/datatypes/rotation_axis_angle.rs +++ b/crates/store/re_types/src/datatypes/rotation_axis_angle.rs @@ -57,18 +57,33 @@ impl ::re_types_core::Loggable for RotationAxisAngle { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "axis", + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + ), + false, + ), + Field::new("angle", DataType::Float32, false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -76,12 +91,12 @@ impl ::re_types_core::Loggable for RotationAxisAngle { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, axis): (Vec<_>, Vec<_>) = data @@ -91,45 +106,36 @@ impl ::re_types_core::Loggable for RotationAxisAngle { (datum.is_some(), datum) }) .unzip(); - let axis_bitmap: Option = { + let axis_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let axis_inner_data: Vec<_> = axis .into_iter() .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) .flatten() .collect(); - let axis_inner_bitmap: Option = - axis_bitmap.as_ref().map(|bitmap| { - bitmap + let axis_inner_validity: Option = + axis_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(3usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new( - "item", - DataType::Float32, - false, - )), - 3, - ) - .into(), - PrimitiveArray::new( - DataType::Float32.into(), - axis_inner_data.into_iter().collect(), - axis_inner_bitmap, - ) - .boxed(), - axis_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + axis_inner_data.into_iter().collect::>(), + ), + axis_inner_validity, + )), + axis_validity, + )) } }, { @@ -140,24 +146,25 @@ impl ::re_types_core::Loggable for RotationAxisAngle { (datum.is_some(), datum) }) .unzip(); - let angle_bitmap: Option = { + let angle_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float32.into(), - angle - .into_iter() - .map(|datum| datum.map(|datum| datum.radians).unwrap_or_default()) - .collect(), - angle_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + angle + .into_iter() + .map(|datum| { + datum.map(|datum| datum.radians).unwrap_or_default() + }) + .collect::>(), + ), + angle_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_buffer.rs b/crates/store/re_types/src/datatypes/tensor_buffer.rs index 7e5055659005..d14bf3fc92e0 100644 --- a/crates/store/re_types/src/datatypes/tensor_buffer.rs +++ b/crates/store/re_types/src/datatypes/tensor_buffer.rs @@ -209,17 +209,21 @@ impl ::re_types_core::Loggable for TensorBuffer { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ // Dense Arrow union let data: Vec<_> = data @@ -229,7 +233,110 @@ impl ::re_types_core::Loggable for TensorBuffer { datum }) .collect(); - let types = data + let field_type_ids = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]; + let fields = vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new( + "U8", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt8, + false, + ))), + false, + ), + Field::new( + "U16", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt16, + false, + ))), + false, + ), + Field::new( + "U32", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt32, + false, + ))), + false, + ), + Field::new( + "U64", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt64, + false, + ))), + false, + ), + Field::new( + "I8", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int8, + false, + ))), + false, + ), + Field::new( + "I16", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int16, + false, + ))), + false, + ), + Field::new( + "I32", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + false, + ))), + false, + ), + Field::new( + "I64", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int64, + false, + ))), + false, + ), + Field::new( + "F16", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float16, + false, + ))), + false, + ), + Field::new( + "F32", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float32, + false, + ))), + false, + ), + Field::new( + "F64", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float64, + false, + ))), + false, + ), + ]; + let type_ids: Vec = data .iter() .map(|a| match a.as_deref() { None => 0, @@ -246,12 +353,86 @@ impl ::re_types_core::Loggable for TensorBuffer { Some(Self::F64(_)) => 11i8, }) .collect(); - let fields = vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter().filter(|v| v.is_none()).count(), - ) - .boxed(), + let offsets = { + let mut u8_offset = 0; + let mut u16_offset = 0; + let mut u32_offset = 0; + let mut u64_offset = 0; + let mut i8_offset = 0; + let mut i16_offset = 0; + let mut i32_offset = 0; + let mut i64_offset = 0; + let mut f16_offset = 0; + let mut f32_offset = 0; + let mut f64_offset = 0; + let mut nulls_offset = 0; + data.iter() + .map(|v| match v.as_deref() { + None => { + let offset = nulls_offset; + nulls_offset += 1; + offset + } + Some(Self::U8(_)) => { + let offset = u8_offset; + u8_offset += 1; + offset + } + Some(Self::U16(_)) => { + let offset = u16_offset; + u16_offset += 1; + offset + } + Some(Self::U32(_)) => { + let offset = u32_offset; + u32_offset += 1; + offset + } + Some(Self::U64(_)) => { + let offset = u64_offset; + u64_offset += 1; + offset + } + Some(Self::I8(_)) => { + let offset = i8_offset; + i8_offset += 1; + offset + } + Some(Self::I16(_)) => { + let offset = i16_offset; + i16_offset += 1; + offset + } + Some(Self::I32(_)) => { + let offset = i32_offset; + i32_offset += 1; + offset + } + Some(Self::I64(_)) => { + let offset = i64_offset; + i64_offset += 1; + offset + } + Some(Self::F16(_)) => { + let offset = f16_offset; + f16_offset += 1; + offset + } + Some(Self::F32(_)) => { + let offset = f32_offset; + f32_offset += 1; + offset + } + Some(Self::F64(_)) => { + let offset = f64_offset; + f64_offset += 1; + offset + } + }) + .collect() + }; + let children = vec![ + as_array_ref(NullArray::new(data.iter().filter(|v| v.is_none()).count())), { let u8: Vec<_> = data .iter() @@ -260,37 +441,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let u8_bitmap: Option = None; + let u8_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( u8.iter().map(|datum| datum.num_instances()), - )? - .into(); - let u8_inner_data: Buffer<_> = u8 + ); + let u8_inner_data: ScalarBuffer<_> = u8 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let u8_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::UInt8, - false, - ))) - .into(), + let u8_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), offsets, - PrimitiveArray::new( - DataType::UInt8.into(), + as_array_ref(PrimitiveArray::::new( u8_inner_data, - u8_inner_bitmap, - ) - .boxed(), - u8_bitmap, - )? - .boxed() + u8_inner_validity, + )), + u8_validity, + )?) } }, { @@ -301,37 +472,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let u16_bitmap: Option = None; + let u16_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( u16.iter().map(|datum| datum.num_instances()), - )? - .into(); - let u16_inner_data: Buffer<_> = u16 + ); + let u16_inner_data: ScalarBuffer<_> = u16 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let u16_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::UInt16, - false, - ))) - .into(), + let u16_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt16, false)), offsets, - PrimitiveArray::new( - DataType::UInt16.into(), + as_array_ref(PrimitiveArray::::new( u16_inner_data, - u16_inner_bitmap, - ) - .boxed(), - u16_bitmap, - )? - .boxed() + u16_inner_validity, + )), + u16_validity, + )?) } }, { @@ -342,37 +503,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let u32_bitmap: Option = None; + let u32_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( u32.iter().map(|datum| datum.num_instances()), - )? - .into(); - let u32_inner_data: Buffer<_> = u32 + ); + let u32_inner_data: ScalarBuffer<_> = u32 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let u32_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::UInt32, - false, - ))) - .into(), + let u32_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt32, false)), offsets, - PrimitiveArray::new( - DataType::UInt32.into(), + as_array_ref(PrimitiveArray::::new( u32_inner_data, - u32_inner_bitmap, - ) - .boxed(), - u32_bitmap, - )? - .boxed() + u32_inner_validity, + )), + u32_validity, + )?) } }, { @@ -383,37 +534,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let u64_bitmap: Option = None; + let u64_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( u64.iter().map(|datum| datum.num_instances()), - )? - .into(); - let u64_inner_data: Buffer<_> = u64 + ); + let u64_inner_data: ScalarBuffer<_> = u64 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let u64_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::UInt64, - false, - ))) - .into(), + let u64_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::UInt64, false)), offsets, - PrimitiveArray::new( - DataType::UInt64.into(), + as_array_ref(PrimitiveArray::::new( u64_inner_data, - u64_inner_bitmap, - ) - .boxed(), - u64_bitmap, - )? - .boxed() + u64_inner_validity, + )), + u64_validity, + )?) } }, { @@ -424,37 +565,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let i8_bitmap: Option = None; + let i8_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( i8.iter().map(|datum| datum.num_instances()), - )? - .into(); - let i8_inner_data: Buffer<_> = i8 + ); + let i8_inner_data: ScalarBuffer<_> = i8 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let i8_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Int8, - false, - ))) - .into(), + let i8_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Int8, false)), offsets, - PrimitiveArray::new( - DataType::Int8.into(), + as_array_ref(PrimitiveArray::::new( i8_inner_data, - i8_inner_bitmap, - ) - .boxed(), - i8_bitmap, - )? - .boxed() + i8_inner_validity, + )), + i8_validity, + )?) } }, { @@ -465,37 +596,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let i16_bitmap: Option = None; + let i16_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( i16.iter().map(|datum| datum.num_instances()), - )? - .into(); - let i16_inner_data: Buffer<_> = i16 + ); + let i16_inner_data: ScalarBuffer<_> = i16 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let i16_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Int16, - false, - ))) - .into(), + let i16_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Int16, false)), offsets, - PrimitiveArray::new( - DataType::Int16.into(), + as_array_ref(PrimitiveArray::::new( i16_inner_data, - i16_inner_bitmap, - ) - .boxed(), - i16_bitmap, - )? - .boxed() + i16_inner_validity, + )), + i16_validity, + )?) } }, { @@ -506,37 +627,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let i32_bitmap: Option = None; + let i32_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( i32.iter().map(|datum| datum.num_instances()), - )? - .into(); - let i32_inner_data: Buffer<_> = i32 + ); + let i32_inner_data: ScalarBuffer<_> = i32 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let i32_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Int32, - false, - ))) - .into(), + let i32_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Int32, false)), offsets, - PrimitiveArray::new( - DataType::Int32.into(), + as_array_ref(PrimitiveArray::::new( i32_inner_data, - i32_inner_bitmap, - ) - .boxed(), - i32_bitmap, - )? - .boxed() + i32_inner_validity, + )), + i32_validity, + )?) } }, { @@ -547,37 +658,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let i64_bitmap: Option = None; + let i64_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( i64.iter().map(|datum| datum.num_instances()), - )? - .into(); - let i64_inner_data: Buffer<_> = i64 + ); + let i64_inner_data: ScalarBuffer<_> = i64 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let i64_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Int64, - false, - ))) - .into(), + let i64_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Int64, false)), offsets, - PrimitiveArray::new( - DataType::Int64.into(), + as_array_ref(PrimitiveArray::::new( i64_inner_data, - i64_inner_bitmap, - ) - .boxed(), - i64_bitmap, - )? - .boxed() + i64_inner_validity, + )), + i64_validity, + )?) } }, { @@ -588,37 +689,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let f16_bitmap: Option = None; + let f16_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( f16.iter().map(|datum| datum.num_instances()), - )? - .into(); - let f16_inner_data: Buffer<_> = f16 + ); + let f16_inner_data: ScalarBuffer<_> = f16 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let f16_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float16, - false, - ))) - .into(), + let f16_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float16, false)), offsets, - PrimitiveArray::new( - DataType::Float16.into(), + as_array_ref(PrimitiveArray::::new( f16_inner_data, - f16_inner_bitmap, - ) - .boxed(), - f16_bitmap, - )? - .boxed() + f16_inner_validity, + )), + f16_validity, + )?) } }, { @@ -629,37 +720,27 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let f32_bitmap: Option = None; + let f32_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( f32.iter().map(|datum| datum.num_instances()), - )? - .into(); - let f32_inner_data: Buffer<_> = f32 + ); + let f32_inner_data: ScalarBuffer<_> = f32 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let f32_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float32, - false, - ))) - .into(), + let f32_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), offsets, - PrimitiveArray::new( - DataType::Float32.into(), + as_array_ref(PrimitiveArray::::new( f32_inner_data, - f32_inner_bitmap, - ) - .boxed(), - f32_bitmap, - )? - .boxed() + f32_inner_validity, + )), + f32_validity, + )?) } }, { @@ -670,119 +751,41 @@ impl ::re_types_core::Loggable for TensorBuffer { _ => None, }) .collect(); - let f64_bitmap: Option = None; + let f64_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( f64.iter().map(|datum| datum.num_instances()), - )? - .into(); - let f64_inner_data: Buffer<_> = f64 + ); + let f64_inner_data: ScalarBuffer<_> = f64 .iter() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let f64_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float64, - false, - ))) - .into(), + let f64_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float64, false)), offsets, - PrimitiveArray::new( - DataType::Float64.into(), + as_array_ref(PrimitiveArray::::new( f64_inner_data, - f64_inner_bitmap, - ) - .boxed(), - f64_bitmap, - )? - .boxed() + f64_inner_validity, + )), + f64_validity, + )?) } }, ]; - let offsets = Some({ - let mut u8_offset = 0; - let mut u16_offset = 0; - let mut u32_offset = 0; - let mut u64_offset = 0; - let mut i8_offset = 0; - let mut i16_offset = 0; - let mut i32_offset = 0; - let mut i64_offset = 0; - let mut f16_offset = 0; - let mut f32_offset = 0; - let mut f64_offset = 0; - let mut nulls_offset = 0; - data.iter() - .map(|v| match v.as_deref() { - None => { - let offset = nulls_offset; - nulls_offset += 1; - offset - } - Some(Self::U8(_)) => { - let offset = u8_offset; - u8_offset += 1; - offset - } - Some(Self::U16(_)) => { - let offset = u16_offset; - u16_offset += 1; - offset - } - Some(Self::U32(_)) => { - let offset = u32_offset; - u32_offset += 1; - offset - } - Some(Self::U64(_)) => { - let offset = u64_offset; - u64_offset += 1; - offset - } - Some(Self::I8(_)) => { - let offset = i8_offset; - i8_offset += 1; - offset - } - Some(Self::I16(_)) => { - let offset = i16_offset; - i16_offset += 1; - offset - } - Some(Self::I32(_)) => { - let offset = i32_offset; - i32_offset += 1; - offset - } - Some(Self::I64(_)) => { - let offset = i64_offset; - i64_offset += 1; - offset - } - Some(Self::F16(_)) => { - let offset = f16_offset; - f16_offset += 1; - offset - } - Some(Self::F32(_)) => { - let offset = f32_offset; - f32_offset += 1; - offset - } - Some(Self::F64(_)) => { - let offset = f64_offset; - f64_offset += 1; - offset - } - }) - .collect() - }); - UnionArray::new(Self::arrow_datatype().into(), types, fields, offsets).boxed() + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + as_array_ref( + UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + ) + .unwrap(), + ) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_data.rs b/crates/store/re_types/src/datatypes/tensor_data.rs index 644a0b8c69ee..4f7b33005d4c 100644 --- a/crates/store/re_types/src/datatypes/tensor_data.rs +++ b/crates/store/re_types/src/datatypes/tensor_data.rs @@ -73,18 +73,145 @@ impl ::re_types_core::Loggable for TensorData { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "shape", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + Field::new( + "buffer", + DataType::Union( + UnionFields::new( + vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], + vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new( + "U8", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt8, + false, + ))), + false, + ), + Field::new( + "U16", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt16, + false, + ))), + false, + ), + Field::new( + "U32", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt32, + false, + ))), + false, + ), + Field::new( + "U64", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::UInt64, + false, + ))), + false, + ), + Field::new( + "I8", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int8, + false, + ))), + false, + ), + Field::new( + "I16", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int16, + false, + ))), + false, + ), + Field::new( + "I32", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int32, + false, + ))), + false, + ), + Field::new( + "I64", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Int64, + false, + ))), + false, + ), + Field::new( + "F16", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float16, + false, + ))), + false, + ), + Field::new( + "F32", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float32, + false, + ))), + false, + ), + Field::new( + "F64", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float64, + false, + ))), + false, + ), + ], + ), + UnionMode::Dense, + ), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -92,12 +219,12 @@ impl ::re_types_core::Loggable for TensorData { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, shape): (Vec<_>, Vec<_>) = data @@ -107,38 +234,34 @@ impl ::re_types_core::Loggable for TensorData { (datum.is_some(), datum) }) .unzip(); - let shape_bitmap: Option = { + let shape_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( shape .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let shape_inner_data: Vec<_> = shape.into_iter().flatten().flatten().collect(); - let shape_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let shape_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), false, - ))) - .into(), + )), offsets, { - _ = shape_inner_bitmap; - crate::datatypes::TensorDimension::to_arrow2_opt( + _ = shape_inner_validity; + crate::datatypes::TensorDimension::to_arrow_opt( shape_inner_data.into_iter().map(Some), )? }, - shape_bitmap, - )? - .boxed() + shape_validity, + )?) } }, { @@ -149,19 +272,18 @@ impl ::re_types_core::Loggable for TensorData { (datum.is_some(), datum) }) .unzip(); - let buffer_bitmap: Option = { + let buffer_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = buffer_bitmap; - crate::datatypes::TensorBuffer::to_arrow2_opt(buffer)? + _ = buffer_validity; + crate::datatypes::TensorBuffer::to_arrow_opt(buffer)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_dimension.rs b/crates/store/re_types/src/datatypes/tensor_dimension.rs index f0c30b88529a..11989ab39a29 100644 --- a/crates/store/re_types/src/datatypes/tensor_dimension.rs +++ b/crates/store/re_types/src/datatypes/tensor_dimension.rs @@ -53,18 +53,26 @@ impl ::re_types_core::Loggable for TensorDimension { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("size", DataType::UInt64, false), + Field::new("name", DataType::Utf8, true), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -72,12 +80,12 @@ impl ::re_types_core::Loggable for TensorDimension { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, size): (Vec<_>, Vec<_>) = data @@ -87,16 +95,18 @@ impl ::re_types_core::Loggable for TensorDimension { (datum.is_some(), datum) }) .unzip(); - let size_bitmap: Option = { + let size_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt64.into(), - size.into_iter().map(|v| v.unwrap_or_default()).collect(), - size_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + size.into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + size_validity, + )) }, { let (somes, name): (Vec<_>, Vec<_>) = data @@ -107,34 +117,26 @@ impl ::re_types_core::Loggable for TensorDimension { (datum.is_some(), datum) }) .unzip(); - let name_bitmap: Option = { + let name_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { let offsets = - arrow2::offset::Offsets::::try_from_lengths(name.iter().map( + arrow::buffer::OffsetBuffer::::from_lengths(name.iter().map( |opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default(), - ))? - .into(); - let inner_data: arrow2::buffer::Buffer = + )); + let inner_data: arrow::buffer::Buffer = name.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - name_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, name_validity) + }) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_dimension_index_selection.rs b/crates/store/re_types/src/datatypes/tensor_dimension_index_selection.rs index b2d55eabfca3..6d09b01079be 100644 --- a/crates/store/re_types/src/datatypes/tensor_dimension_index_selection.rs +++ b/crates/store/re_types/src/datatypes/tensor_dimension_index_selection.rs @@ -55,18 +55,26 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("dimension", DataType::UInt32, false), + Field::new("index", DataType::UInt64, false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -74,12 +82,12 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, dimension): (Vec<_>, Vec<_>) = data @@ -89,19 +97,19 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { (datum.is_some(), datum) }) .unzip(); - let dimension_bitmap: Option = { + let dimension_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - dimension - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - dimension_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + dimension + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + dimension_validity, + )) }, { let (somes, index): (Vec<_>, Vec<_>) = data @@ -111,21 +119,23 @@ impl ::re_types_core::Loggable for TensorDimensionIndexSelection { (datum.is_some(), datum) }) .unzip(); - let index_bitmap: Option = { + let index_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt64.into(), - index.into_iter().map(|v| v.unwrap_or_default()).collect(), - index_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + index + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + index_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/tensor_dimension_selection.rs b/crates/store/re_types/src/datatypes/tensor_dimension_selection.rs index 74fe871edc8c..77c8bde57d77 100644 --- a/crates/store/re_types/src/datatypes/tensor_dimension_selection.rs +++ b/crates/store/re_types/src/datatypes/tensor_dimension_selection.rs @@ -53,18 +53,26 @@ impl ::re_types_core::Loggable for TensorDimensionSelection { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("dimension", DataType::UInt32, false), + Field::new("invert", DataType::Boolean, false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -72,12 +80,12 @@ impl ::re_types_core::Loggable for TensorDimensionSelection { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, dimension): (Vec<_>, Vec<_>) = data @@ -87,19 +95,19 @@ impl ::re_types_core::Loggable for TensorDimensionSelection { (datum.is_some(), datum) }) .unzip(); - let dimension_bitmap: Option = { + let dimension_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - dimension - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - dimension_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + dimension + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + dimension_validity, + )) }, { let (somes, invert): (Vec<_>, Vec<_>) = data @@ -109,21 +117,23 @@ impl ::re_types_core::Loggable for TensorDimensionSelection { (datum.is_some(), datum) }) .unzip(); - let invert_bitmap: Option = { + let invert_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - BooleanArray::new( - DataType::Boolean.into(), - invert.into_iter().map(|v| v.unwrap_or_default()).collect(), - invert_bitmap, - ) - .boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from( + invert + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + invert_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/uuid.rs b/crates/store/re_types/src/datatypes/uuid.rs index 61f6aef39274..7b6b892bec06 100644 --- a/crates/store/re_types/src/datatypes/uuid.rs +++ b/crates/store/re_types/src/datatypes/uuid.rs @@ -65,17 +65,21 @@ impl ::re_types_core::Loggable for Uuid { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, bytes): (Vec<_>, Vec<_>) = data .into_iter() @@ -85,12 +89,11 @@ impl ::re_types_core::Loggable for Uuid { (datum.is_some(), datum) }) .unzip(); - let bytes_bitmap: Option = { + let bytes_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let bytes_inner_data: Vec<_> = bytes .into_iter() .flat_map(|v| match v { @@ -100,26 +103,24 @@ impl ::re_types_core::Loggable for Uuid { ), }) .collect(); - let bytes_inner_bitmap: Option = - bytes_bitmap.as_ref().map(|bitmap| { - bitmap + let bytes_inner_validity: Option = + bytes_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(16usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt8.into(), - bytes_inner_data.into_iter().collect(), - bytes_inner_bitmap, - ) - .boxed(), - bytes_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), + 16, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(bytes_inner_data.into_iter().collect::>()), + bytes_inner_validity, + )), + bytes_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/uvec2d.rs b/crates/store/re_types/src/datatypes/uvec2d.rs index 8a3220a543e0..0e48f8ff1cf5 100644 --- a/crates/store/re_types/src/datatypes/uvec2d.rs +++ b/crates/store/re_types/src/datatypes/uvec2d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for UVec2D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for UVec2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for UVec2D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt32, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/uvec3d.rs b/crates/store/re_types/src/datatypes/uvec3d.rs index 3d22d04e5070..bad023d2daa0 100644 --- a/crates/store/re_types/src/datatypes/uvec3d.rs +++ b/crates/store/re_types/src/datatypes/uvec3d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for UVec3D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for UVec3D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for UVec3D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(3usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/uvec4d.rs b/crates/store/re_types/src/datatypes/uvec4d.rs index da202b7ae2f4..997b406ebd0a 100644 --- a/crates/store/re_types/src/datatypes/uvec4d.rs +++ b/crates/store/re_types/src/datatypes/uvec4d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for UVec4D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for UVec4D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for UVec4D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(4usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt32, false)), + 4, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/vec2d.rs b/crates/store/re_types/src/datatypes/vec2d.rs index 85ad4879b70e..991cef08a77c 100644 --- a/crates/store/re_types/src/datatypes/vec2d.rs +++ b/crates/store/re_types/src/datatypes/vec2d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Vec2D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for Vec2D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for Vec2D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(2usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 2, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/vec3d.rs b/crates/store/re_types/src/datatypes/vec3d.rs index ff1dcc07fe86..66fb58c9223b 100644 --- a/crates/store/re_types/src/datatypes/vec3d.rs +++ b/crates/store/re_types/src/datatypes/vec3d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Vec3D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for Vec3D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for Vec3D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(3usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/vec4d.rs b/crates/store/re_types/src/datatypes/vec4d.rs index 6f6fb956fb89..bb97cee3eb36 100644 --- a/crates/store/re_types/src/datatypes/vec4d.rs +++ b/crates/store/re_types/src/datatypes/vec4d.rs @@ -62,17 +62,21 @@ impl ::re_types_core::Loggable for Vec4D { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -82,12 +86,11 @@ impl ::re_types_core::Loggable for Vec4D { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -97,26 +100,24 @@ impl ::re_types_core::Loggable for Vec4D { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(4usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::Float32.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 4, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/datatypes/video_timestamp.rs b/crates/store/re_types/src/datatypes/video_timestamp.rs index 14bf4065b8af..926881c7be7a 100644 --- a/crates/store/re_types/src/datatypes/video_timestamp.rs +++ b/crates/store/re_types/src/datatypes/video_timestamp.rs @@ -64,17 +64,21 @@ impl ::re_types_core::Loggable for VideoTimestamp { DataType::Int64 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -84,16 +88,19 @@ impl ::re_types_core::Loggable for VideoTimestamp { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/datatypes/view_coordinates.rs b/crates/store/re_types/src/datatypes/view_coordinates.rs index f7de8aeb8d8a..139be29735cb 100644 --- a/crates/store/re_types/src/datatypes/view_coordinates.rs +++ b/crates/store/re_types/src/datatypes/view_coordinates.rs @@ -82,17 +82,21 @@ impl ::re_types_core::Loggable for ViewCoordinates { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -102,12 +106,11 @@ impl ::re_types_core::Loggable for ViewCoordinates { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let data0_inner_data: Vec<_> = data0 .into_iter() .flat_map(|v| match v { @@ -117,26 +120,24 @@ impl ::re_types_core::Loggable for ViewCoordinates { ), }) .collect(); - let data0_inner_bitmap: Option = - data0_bitmap.as_ref().map(|bitmap| { - bitmap + let data0_inner_validity: Option = + data0_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(3usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - Self::arrow_datatype().into(), - PrimitiveArray::new( - DataType::UInt8.into(), - data0_inner_data.into_iter().collect(), - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(data0_inner_data.into_iter().collect::>()), + data0_inner_validity, + )), + data0_validity, + )) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer1.rs b/crates/store/re_types/src/testing/components/affix_fuzzer1.rs index 23fc88eb7ef5..929831427396 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer1.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer1.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { crate::testing::datatypes::AffixFuzzer1::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs index ca252cc37b6d..b73afe90515b 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer10.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer10.rs @@ -73,17 +73,21 @@ impl ::re_types_core::Loggable for AffixFuzzer10 { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -93,30 +97,23 @@ impl ::re_types_core::Loggable for AffixFuzzer10 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer11.rs b/crates/store/re_types/src/testing/components/affix_fuzzer11.rs index 24a487f42cf3..b3ee7f6b9e42 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer11.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer11.rs @@ -77,17 +77,21 @@ impl ::re_types_core::Loggable for AffixFuzzer11 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -97,38 +101,33 @@ impl ::re_types_core::Loggable for AffixFuzzer11 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.num_instances())), - )? - .into(); - let data0_inner_data: Buffer<_> = data0 + ); + let data0_inner_data: ScalarBuffer<_> = data0 .iter() .flatten() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), offsets, - PrimitiveArray::new( - DataType::Float32.into(), + as_array_ref(PrimitiveArray::::new( data0_inner_data, - data0_inner_bitmap, - ) - .boxed(), - data0_bitmap, - )? - .boxed() + data0_inner_validity, + )), + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs index baf938edb179..956c0138a3c5 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer12.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer12.rs @@ -77,17 +77,21 @@ impl ::re_types_core::Loggable for AffixFuzzer12 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -97,45 +101,35 @@ impl ::re_types_core::Loggable for AffixFuzzer12 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0_inner_data.iter().map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0_inner_data.into_iter().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - data0_inner_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_inner_validity) + }) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs index c2e4aea264f2..6f1383be46fd 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer13.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer13.rs @@ -77,17 +77,21 @@ impl ::re_types_core::Loggable for AffixFuzzer13 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -97,45 +101,35 @@ impl ::re_types_core::Loggable for AffixFuzzer13 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0_inner_data.iter().map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0_inner_data.into_iter().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - data0_inner_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_inner_validity) + }) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer14.rs b/crates/store/re_types/src/testing/components/affix_fuzzer14.rs index 28c697c5aace..54ad6a98da8a 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer14.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer14.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer14 { crate::testing::datatypes::AffixFuzzer3::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer15.rs b/crates/store/re_types/src/testing/components/affix_fuzzer15.rs index 97940644f3c7..bf4d7ada4791 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer15.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer15.rs @@ -99,17 +99,21 @@ impl ::re_types_core::Loggable for AffixFuzzer15 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -119,13 +123,13 @@ impl ::re_types_core::Loggable for AffixFuzzer15 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer16.rs b/crates/store/re_types/src/testing/components/affix_fuzzer16.rs index 9a6715a98a42..3473217995c4 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer16.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer16.rs @@ -55,17 +55,21 @@ impl ::re_types_core::Loggable for AffixFuzzer16 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,32 +79,33 @@ impl ::re_types_core::Loggable for AffixFuzzer16 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - _ = data0_inner_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt( + _ = data0_inner_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer17.rs b/crates/store/re_types/src/testing/components/affix_fuzzer17.rs index 848fefbf891f..3d40c8e0c760 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer17.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer17.rs @@ -55,17 +55,21 @@ impl ::re_types_core::Loggable for AffixFuzzer17 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,32 +79,33 @@ impl ::re_types_core::Loggable for AffixFuzzer17 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - _ = data0_inner_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt( + _ = data0_inner_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer18.rs b/crates/store/re_types/src/testing/components/affix_fuzzer18.rs index 69fdddd48a7e..208ba34fa8e9 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer18.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer18.rs @@ -55,17 +55,21 @@ impl ::re_types_core::Loggable for AffixFuzzer18 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,32 +79,33 @@ impl ::re_types_core::Loggable for AffixFuzzer18 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - _ = data0_inner_bitmap; - crate::testing::datatypes::AffixFuzzer4::to_arrow2_opt( + _ = data0_inner_validity; + crate::testing::datatypes::AffixFuzzer4::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer19.rs b/crates/store/re_types/src/testing/components/affix_fuzzer19.rs index d2388aa54a9a..5e14eb383e73 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer19.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer19.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer19 { crate::testing::datatypes::AffixFuzzer5::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer5::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer5::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer2.rs b/crates/store/re_types/src/testing/components/affix_fuzzer2.rs index 6198ffce321b..f8d0d3f36646 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer2.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer2.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer2 { crate::testing::datatypes::AffixFuzzer1::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer20.rs b/crates/store/re_types/src/testing/components/affix_fuzzer20.rs index 0b051434900f..e856fac201c0 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer20.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer20.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { crate::testing::datatypes::AffixFuzzer20::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer20::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer20::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer21.rs b/crates/store/re_types/src/testing/components/affix_fuzzer21.rs index b16f44aba56e..c0e0e19df83b 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer21.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer21.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { crate::testing::datatypes::AffixFuzzer21::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer21::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer21::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer22.rs b/crates/store/re_types/src/testing/components/affix_fuzzer22.rs index 6f3e7bed2cec..5e7ca35ae2ef 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer22.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer22.rs @@ -79,17 +79,21 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -99,13 +103,13 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer22::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer22::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer23.rs b/crates/store/re_types/src/testing/components/affix_fuzzer23.rs index 5a3629e8eefa..c35246cbc236 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer23.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer23.rs @@ -83,17 +83,21 @@ impl ::re_types_core::Loggable for AffixFuzzer23 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -103,13 +107,13 @@ impl ::re_types_core::Loggable for AffixFuzzer23 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::MultiEnum::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::MultiEnum::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer3.rs b/crates/store/re_types/src/testing/components/affix_fuzzer3.rs index a6161782c1bd..ba87966d50a9 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer3.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer3.rs @@ -70,13 +70,13 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { crate::testing::datatypes::AffixFuzzer1::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data.into_iter().map(|datum| { + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer4.rs b/crates/store/re_types/src/testing/components/affix_fuzzer4.rs index 31631ff304e3..4f161e22f277 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer4.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer4.rs @@ -110,17 +110,21 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -130,13 +134,13 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer5.rs b/crates/store/re_types/src/testing/components/affix_fuzzer5.rs index 2a1e60c30ec2..aedec46b86cc 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer5.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer5.rs @@ -110,17 +110,21 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -130,13 +134,13 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer6.rs b/crates/store/re_types/src/testing/components/affix_fuzzer6.rs index 7201c7e465b4..f682a2f627ea 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer6.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer6.rs @@ -110,17 +110,21 @@ impl ::re_types_core::Loggable for AffixFuzzer6 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -130,13 +134,13 @@ impl ::re_types_core::Loggable for AffixFuzzer6 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = data0_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt(data0)? + _ = data0_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt(data0)? } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer7.rs b/crates/store/re_types/src/testing/components/affix_fuzzer7.rs index a28471b432a4..7bfb69f49e2c 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer7.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer7.rs @@ -55,17 +55,21 @@ impl ::re_types_core::Loggable for AffixFuzzer7 { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -75,32 +79,33 @@ impl ::re_types_core::Loggable for AffixFuzzer7 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + )), offsets, { - _ = data0_inner_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt( + _ = data0_inner_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt( data0_inner_data.into_iter().map(Some), )? }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer8.rs b/crates/store/re_types/src/testing/components/affix_fuzzer8.rs index 6584bb83cb90..7ce515217964 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer8.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer8.rs @@ -73,17 +73,21 @@ impl ::re_types_core::Loggable for AffixFuzzer8 { DataType::Float32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -93,16 +97,19 @@ impl ::re_types_core::Loggable for AffixFuzzer8 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs index d7a58f1781cb..fe012f10e97d 100644 --- a/crates/store/re_types/src/testing/components/affix_fuzzer9.rs +++ b/crates/store/re_types/src/testing/components/affix_fuzzer9.rs @@ -73,17 +73,21 @@ impl ::re_types_core::Loggable for AffixFuzzer9 { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -93,30 +97,23 @@ impl ::re_types_core::Loggable for AffixFuzzer9 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs index 40f874c2c97a..afbf99b85899 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer1.rs @@ -107,18 +107,65 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("single_float_optional", DataType::Float32, true), + Field::new("single_string_required", DataType::Utf8, false), + Field::new("single_string_optional", DataType::Utf8, true), + Field::new( + "many_floats_optional", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float32, + false, + ))), + true, + ), + Field::new( + "many_strings_required", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Utf8, + false, + ))), + false, + ), + Field::new( + "many_strings_optional", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Utf8, + false, + ))), + true, + ), + Field::new("flattened_scalar", DataType::Float32, false), + Field::new( + "almost_flattened_scalar", + DataType::Struct(Fields::from(vec![Field::new( + "value", + DataType::Float32, + false, + )])), + false, + ), + Field::new("from_parent", DataType::Boolean, true), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -126,12 +173,12 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, single_float_optional): (Vec<_>, Vec<_>) = data @@ -144,19 +191,19 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let single_float_optional_bitmap: Option = { + let single_float_optional_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float32.into(), - single_float_optional - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - single_float_optional_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + single_float_optional + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + single_float_optional_validity, + )) }, { let (somes, single_string_required): (Vec<_>, Vec<_>) = data @@ -168,32 +215,30 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let single_string_required_bitmap: Option = { + let single_string_required_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( single_string_required.iter().map(|opt| { opt.as_ref().map(|datum| datum.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = single_string_required + ); + let inner_data: arrow::buffer::Buffer = single_string_required .into_iter() .flatten() .flat_map(|s| s.0) .collect(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - single_string_required_bitmap, + single_string_required_validity, ) - } - .boxed() + }) } }, { @@ -207,32 +252,30 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let single_string_optional_bitmap: Option = { + let single_string_optional_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( single_string_optional.iter().map(|opt| { opt.as_ref().map(|datum| datum.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = single_string_optional + ); + let inner_data: arrow::buffer::Buffer = single_string_optional .into_iter() .flatten() .flat_map(|s| s.0) .collect(); + #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - single_string_optional_bitmap, + single_string_optional_validity, ) - } - .boxed() + }) } }, { @@ -246,44 +289,36 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let many_floats_optional_bitmap: Option = { + let many_floats_optional_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_floats_optional.iter().map(|opt| { opt.as_ref().map_or(0, |datum| datum.num_instances()) }), - )? - .into(); - let many_floats_optional_inner_data: Buffer<_> = many_floats_optional - .iter() - .flatten() - .map(|b| b.as_slice()) - .collect::>() - .concat() - .into(); - let many_floats_optional_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float32, - false, - ))) - .into(), + ); + let many_floats_optional_inner_data: ScalarBuffer<_> = + many_floats_optional + .iter() + .flatten() + .map(|b| b.as_slice()) + .collect::>() + .concat() + .into(); + let many_floats_optional_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), offsets, - PrimitiveArray::new( - DataType::Float32.into(), + as_array_ref(PrimitiveArray::::new( many_floats_optional_inner_data, - many_floats_optional_inner_bitmap, - ) - .boxed(), - many_floats_optional_bitmap, - )? - .boxed() + many_floats_optional_inner_validity, + )), + many_floats_optional_validity, + )?) } }, { @@ -296,59 +331,49 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let many_strings_required_bitmap: Option = { + let many_strings_required_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_strings_required .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let many_strings_required_inner_data: Vec<_> = many_strings_required .into_iter() .flatten() .flatten() .collect(); - let many_strings_required_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Utf8, - false, - ))) - .into(), + let many_strings_required_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_strings_required_inner_data .iter() .map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = many_strings_required_inner_data .into_iter() .flat_map(|s| s.0) .collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - many_strings_required_inner_bitmap, + many_strings_required_inner_validity, ) - } - .boxed() + }) }, - many_strings_required_bitmap, - )? - .boxed() + many_strings_required_validity, + )?) } }, { @@ -362,59 +387,49 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let many_strings_optional_bitmap: Option = { + let many_strings_optional_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_strings_optional .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let many_strings_optional_inner_data: Vec<_> = many_strings_optional .into_iter() .flatten() .flatten() .collect(); - let many_strings_optional_inner_bitmap: Option = - None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Utf8, - false, - ))) - .into(), + let many_strings_optional_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_strings_optional_inner_data .iter() .map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = many_strings_optional_inner_data .into_iter() .flat_map(|s| s.0) .collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), + as_array_ref(unsafe { + StringArray::new_unchecked( offsets, inner_data, - many_strings_optional_inner_bitmap, + many_strings_optional_inner_validity, ) - } - .boxed() + }) }, - many_strings_optional_bitmap, - )? - .boxed() + many_strings_optional_validity, + )?) } }, { @@ -426,19 +441,19 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let flattened_scalar_bitmap: Option = { + let flattened_scalar_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float32.into(), - flattened_scalar - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - flattened_scalar_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + flattened_scalar + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + flattened_scalar_validity, + )) }, { let (somes, almost_flattened_scalar): (Vec<_>, Vec<_>) = data @@ -450,13 +465,13 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let almost_flattened_scalar_bitmap: Option = { + let almost_flattened_scalar_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = almost_flattened_scalar_bitmap; - crate::testing::datatypes::FlattenedScalar::to_arrow2_opt( + _ = almost_flattened_scalar_validity; + crate::testing::datatypes::FlattenedScalar::to_arrow_opt( almost_flattened_scalar, )? } @@ -472,24 +487,23 @@ impl ::re_types_core::Loggable for AffixFuzzer1 { (datum.is_some(), datum) }) .unzip(); - let from_parent_bitmap: Option = { + let from_parent_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - BooleanArray::new( - DataType::Boolean.into(), - from_parent - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - from_parent_bitmap, - ) - .boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from( + from_parent + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + from_parent_validity, + )) }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer2.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer2.rs index 070ae5cf38ae..e387b0667362 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer2.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer2.rs @@ -57,17 +57,21 @@ impl ::re_types_core::Loggable for AffixFuzzer2 { DataType::Float32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -77,16 +81,19 @@ impl ::re_types_core::Loggable for AffixFuzzer2 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs index 663d5160ef6f..df20b73b8657 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer20.rs @@ -58,18 +58,26 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("p", DataType::UInt32, false), + Field::new("s", DataType::Utf8, false), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -77,12 +85,12 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, p): (Vec<_>, Vec<_>) = data @@ -92,18 +100,18 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { (datum.is_some(), datum) }) .unzip(); - let p_bitmap: Option = { + let p_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::UInt32.into(), - p.into_iter() - .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) - .collect(), - p_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + p.into_iter() + .map(|datum| datum.map(|datum| datum.0).unwrap_or_default()) + .collect::>(), + ), + p_validity, + )) }, { let (somes, s): (Vec<_>, Vec<_>) = data @@ -113,39 +121,30 @@ impl ::re_types_core::Loggable for AffixFuzzer20 { (datum.is_some(), datum) }) .unzip(); - let s_bitmap: Option = { + let s_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( s.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = s + ); + let inner_data: arrow::buffer::Buffer = s .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - s_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, s_validity) + }) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs index 3aef143f90d8..d6af2059c7eb 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer21.rs @@ -57,18 +57,34 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("single_half", DataType::Float16, false), + Field::new( + "many_halves", + DataType::List(std::sync::Arc::new(Field::new( + "item", + DataType::Float16, + false, + ))), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -76,12 +92,12 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, single_half): (Vec<_>, Vec<_>) = data @@ -91,19 +107,19 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { (datum.is_some(), datum) }) .unzip(); - let single_half_bitmap: Option = { + let single_half_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float16.into(), - single_half - .into_iter() - .map(|v| v.unwrap_or_default()) - .collect(), - single_half_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + single_half + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + single_half_validity, + )) }, { let (somes, many_halves): (Vec<_>, Vec<_>) = data @@ -113,49 +129,39 @@ impl ::re_types_core::Loggable for AffixFuzzer21 { (datum.is_some(), datum) }) .unzip(); - let many_halves_bitmap: Option = { + let many_halves_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_halves.iter().map(|opt| { opt.as_ref().map_or(0, |datum| datum.num_instances()) }), - )? - .into(); - let many_halves_inner_data: Buffer<_> = many_halves + ); + let many_halves_inner_data: ScalarBuffer<_> = many_halves .iter() .flatten() .map(|b| b.as_slice()) .collect::>() .concat() .into(); - let many_halves_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( - "item", - DataType::Float16, - false, - ))) - .into(), + let many_halves_inner_validity: Option = + None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Float16, false)), offsets, - PrimitiveArray::new( - DataType::Float16.into(), + as_array_ref(PrimitiveArray::::new( many_halves_inner_data, - many_halves_inner_bitmap, - ) - .boxed(), - many_halves_bitmap, - )? - .boxed() + many_halves_inner_validity, + )), + many_halves_validity, + )?) } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer22.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer22.rs index 9fc33457b8b9..90ee87532484 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer22.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer22.rs @@ -66,18 +66,30 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![Field::new( + "fixed_sized_native", + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), + 4, + ), + false, + )]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -85,12 +97,12 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![{ let (somes, fixed_sized_native): (Vec<_>, Vec<_>) = data .iter() @@ -100,12 +112,11 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { (datum.is_some(), datum) }) .unzip(); - let fixed_sized_native_bitmap: Option = { + let fixed_sized_native_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let fixed_sized_native_inner_data: Vec<_> = fixed_sized_native .into_iter() .flat_map(|v| match v { @@ -115,35 +126,32 @@ impl ::re_types_core::Loggable for AffixFuzzer22 { ), }) .collect(); - let fixed_sized_native_inner_bitmap: Option = - fixed_sized_native_bitmap.as_ref().map(|bitmap| { - bitmap + let fixed_sized_native_inner_validity: Option = + fixed_sized_native_validity.as_ref().map(|validity| { + validity .iter() .map(|b| std::iter::repeat(b).take(4usize)) .flatten() .collect::>() .into() }); - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), - 4, - ) - .into(), - PrimitiveArray::new( - DataType::UInt8.into(), - fixed_sized_native_inner_data.into_iter().collect(), - fixed_sized_native_inner_bitmap, - ) - .boxed(), - fixed_sized_native_bitmap, - ) - .boxed() + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::UInt8, false)), + 4, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + fixed_sized_native_inner_data + .into_iter() + .collect::>(), + ), + fixed_sized_native_inner_validity, + )), + fixed_sized_native_validity, + )) } }], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs index f32c9e182238..04d6963c7882 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer3.rs @@ -83,17 +83,21 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ // Dense Arrow union let data: Vec<_> = data @@ -103,7 +107,30 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { datum }) .collect(); - let types = data + let field_type_ids = [0, 1, 2, 3, 4]; + let fields = vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new("degrees", DataType::Float32, false), + Field::new( + "craziness", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + Field::new( + "fixed_size_shenanigans", + DataType::FixedSizeList( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + ), + false, + ), + Field::new("empty_variant", DataType::Null, true), + ]; + let type_ids: Vec = data .iter() .map(|a| match a.as_deref() { None => 0, @@ -113,12 +140,44 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { Some(Self::EmptyVariant) => 4i8, }) .collect(); - let fields = vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter().filter(|v| v.is_none()).count(), - ) - .boxed(), + let offsets = { + let mut degrees_offset = 0; + let mut craziness_offset = 0; + let mut fixed_size_shenanigans_offset = 0; + let mut empty_variant_offset = 0; + let mut nulls_offset = 0; + data.iter() + .map(|v| match v.as_deref() { + None => { + let offset = nulls_offset; + nulls_offset += 1; + offset + } + Some(Self::Degrees(_)) => { + let offset = degrees_offset; + degrees_offset += 1; + offset + } + Some(Self::Craziness(_)) => { + let offset = craziness_offset; + craziness_offset += 1; + offset + } + Some(Self::FixedSizeShenanigans(_)) => { + let offset = fixed_size_shenanigans_offset; + fixed_size_shenanigans_offset += 1; + offset + } + Some(Self::EmptyVariant) => { + let offset = empty_variant_offset; + empty_variant_offset += 1; + offset + } + }) + .collect() + }; + let children = vec![ + as_array_ref(NullArray::new(data.iter().filter(|v| v.is_none()).count())), { let degrees: Vec<_> = data .iter() @@ -127,13 +186,11 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { _ => None, }) .collect(); - let degrees_bitmap: Option = None; - PrimitiveArray::new( - DataType::Float32.into(), - degrees.into_iter().collect(), - degrees_bitmap, - ) - .boxed() + let degrees_validity: Option = None; + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from(degrees.into_iter().collect::>()), + degrees_validity, + )) }, { let craziness: Vec<_> = data @@ -143,33 +200,29 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { _ => None, }) .collect(); - let craziness_bitmap: Option = None; + let craziness_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( craziness.iter().map(|datum| datum.len()), - )? - .into(); + ); let craziness_inner_data: Vec<_> = craziness.into_iter().flatten().collect(); - let craziness_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let craziness_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), false, - ))) - .into(), + )), offsets, { - _ = craziness_inner_bitmap; - crate::testing::datatypes::AffixFuzzer1::to_arrow2_opt( + _ = craziness_inner_validity; + crate::testing::datatypes::AffixFuzzer1::to_arrow_opt( craziness_inner_data.into_iter().map(Some), )? }, - craziness_bitmap, - )? - .boxed() + craziness_validity, + )?) } }, { @@ -180,75 +233,45 @@ impl ::re_types_core::Loggable for AffixFuzzer3 { _ => None, }) .collect(); - let fixed_size_shenanigans_bitmap: Option = None; + let fixed_size_shenanigans_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; let fixed_size_shenanigans_inner_data: Vec<_> = fixed_size_shenanigans.into_iter().flatten().collect(); - let fixed_size_shenanigans_inner_bitmap: Option = - None; - FixedSizeListArray::new( - DataType::FixedSizeList( - std::sync::Arc::new(Field::new("item", DataType::Float32, false)), - 3, - ) - .into(), - PrimitiveArray::new( - DataType::Float32.into(), - fixed_size_shenanigans_inner_data.into_iter().collect(), - fixed_size_shenanigans_inner_bitmap, - ) - .boxed(), - fixed_size_shenanigans_bitmap, - ) - .boxed() + let fixed_size_shenanigans_inner_validity: Option< + arrow::buffer::NullBuffer, + > = None; + as_array_ref(FixedSizeListArray::new( + std::sync::Arc::new(Field::new("item", DataType::Float32, false)), + 3, + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + fixed_size_shenanigans_inner_data + .into_iter() + .collect::>(), + ), + fixed_size_shenanigans_inner_validity, + )), + fixed_size_shenanigans_validity, + )) } }, - NullArray::new( - arrow2::datatypes::DataType::Null, + as_array_ref(NullArray::new( data.iter() .filter(|datum| matches!(datum.as_deref(), Some(Self::EmptyVariant))) .count(), - ) - .boxed(), + )), ]; - let offsets = Some({ - let mut degrees_offset = 0; - let mut craziness_offset = 0; - let mut fixed_size_shenanigans_offset = 0; - let mut empty_variant_offset = 0; - let mut nulls_offset = 0; - data.iter() - .map(|v| match v.as_deref() { - None => { - let offset = nulls_offset; - nulls_offset += 1; - offset - } - Some(Self::Degrees(_)) => { - let offset = degrees_offset; - degrees_offset += 1; - offset - } - Some(Self::Craziness(_)) => { - let offset = craziness_offset; - craziness_offset += 1; - offset - } - Some(Self::FixedSizeShenanigans(_)) => { - let offset = fixed_size_shenanigans_offset; - fixed_size_shenanigans_offset += 1; - offset - } - Some(Self::EmptyVariant) => { - let offset = empty_variant_offset; - empty_variant_offset += 1; - offset - } - }) - .collect() - }); - UnionArray::new(Self::arrow_datatype().into(), types, fields, offsets).boxed() + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + as_array_ref( + UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + ) + .unwrap(), + ) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs index cb52d1e578ea..8cbbc58ff707 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer4.rs @@ -73,17 +73,21 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ // Dense Arrow union let data: Vec<_> = data @@ -93,7 +97,39 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { datum }) .collect(); - let types = data + let field_type_ids = [0, 1, 2]; + let fields = + vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new( + "single_required", + DataType::Union( + UnionFields::new( + vec![0, 1, 2, 3, 4], + vec![Field::new("_null_markers", DataType::Null, true), + Field::new("degrees", DataType::Float32, false), Field::new("craziness", + DataType::List(std::sync::Arc::new(Field::new("item", < crate + ::testing::datatypes::AffixFuzzer1 > ::arrow_datatype(), false))), + false), Field::new("fixed_size_shenanigans", + DataType::FixedSizeList(std::sync::Arc::new(Field::new("item", + DataType::Float32, false)), 3), false), Field::new("empty_variant", + DataType::Null, true),], + ), + UnionMode::Dense, + ), + false, + ), + Field::new( + "many_required", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + ]; + let type_ids: Vec = data .iter() .map(|a| match a.as_deref() { None => 0, @@ -101,12 +137,32 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { Some(Self::ManyRequired(_)) => 2i8, }) .collect(); - let fields = vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter().filter(|v| v.is_none()).count(), - ) - .boxed(), + let offsets = { + let mut single_required_offset = 0; + let mut many_required_offset = 0; + let mut nulls_offset = 0; + data.iter() + .map(|v| match v.as_deref() { + None => { + let offset = nulls_offset; + nulls_offset += 1; + offset + } + Some(Self::SingleRequired(_)) => { + let offset = single_required_offset; + single_required_offset += 1; + offset + } + Some(Self::ManyRequired(_)) => { + let offset = many_required_offset; + many_required_offset += 1; + offset + } + }) + .collect() + }; + let children = vec![ + as_array_ref(NullArray::new(data.iter().filter(|v| v.is_none()).count())), { let single_required: Vec<_> = data .iter() @@ -115,10 +171,10 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { _ => None, }) .collect(); - let single_required_bitmap: Option = None; + let single_required_validity: Option = None; { - _ = single_required_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt( + _ = single_required_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt( single_required.into_iter().map(Some), )? } @@ -131,61 +187,43 @@ impl ::re_types_core::Loggable for AffixFuzzer4 { _ => None, }) .collect(); - let many_required_bitmap: Option = None; + let many_required_validity: Option = None; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( many_required.iter().map(|datum| datum.len()), - )? - .into(); + ); let many_required_inner_data: Vec<_> = many_required.into_iter().flatten().collect(); - let many_required_inner_bitmap: Option = None; - ListArray::try_new( - DataType::List(std::sync::Arc::new(Field::new( + let many_required_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new( "item", ::arrow_datatype(), false, - ))) - .into(), + )), offsets, { - _ = many_required_inner_bitmap; - crate::testing::datatypes::AffixFuzzer3::to_arrow2_opt( + _ = many_required_inner_validity; + crate::testing::datatypes::AffixFuzzer3::to_arrow_opt( many_required_inner_data.into_iter().map(Some), )? }, - many_required_bitmap, - )? - .boxed() + many_required_validity, + )?) } }, ]; - let offsets = Some({ - let mut single_required_offset = 0; - let mut many_required_offset = 0; - let mut nulls_offset = 0; - data.iter() - .map(|v| match v.as_deref() { - None => { - let offset = nulls_offset; - nulls_offset += 1; - offset - } - Some(Self::SingleRequired(_)) => { - let offset = single_required_offset; - single_required_offset += 1; - offset - } - Some(Self::ManyRequired(_)) => { - let offset = many_required_offset; - many_required_offset += 1; - offset - } - }) - .collect() - }); - UnionArray::new(Self::arrow_datatype().into(), types, fields, offsets).boxed() + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + as_array_ref( + UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + ) + .unwrap(), + ) }) } diff --git a/crates/store/re_types/src/testing/datatypes/affix_fuzzer5.rs b/crates/store/re_types/src/testing/datatypes/affix_fuzzer5.rs index 468ab9becfc6..a867eb0c3a02 100644 --- a/crates/store/re_types/src/testing/datatypes/affix_fuzzer5.rs +++ b/crates/store/re_types/src/testing/datatypes/affix_fuzzer5.rs @@ -80,18 +80,49 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![Field::new( + "single_optional_union", + DataType::Union( + UnionFields::new( + vec![0, 1, 2], + vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new( + "single_required", + ::arrow_datatype(), + false, + ), + Field::new( + "many_required", + DataType::List(std::sync::Arc::new(Field::new( + "item", + ::arrow_datatype(), + false, + ))), + false, + ), + ], + ), + UnionMode::Dense, + ), + true, + )]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -99,12 +130,12 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![{ let (somes, single_optional_union): (Vec<_>, Vec<_>) = data .iter() @@ -116,20 +147,19 @@ impl ::re_types_core::Loggable for AffixFuzzer5 { (datum.is_some(), datum) }) .unzip(); - let single_optional_union_bitmap: Option = { + let single_optional_union_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = single_optional_union_bitmap; - crate::testing::datatypes::AffixFuzzer4::to_arrow2_opt( + _ = single_optional_union_validity; + crate::testing::datatypes::AffixFuzzer4::to_arrow_opt( single_optional_union, )? } }], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/enum_test.rs b/crates/store/re_types/src/testing/datatypes/enum_test.rs index 18bd4f253eca..ddd7269c4ad2 100644 --- a/crates/store/re_types/src/testing/datatypes/enum_test.rs +++ b/crates/store/re_types/src/testing/datatypes/enum_test.rs @@ -104,17 +104,21 @@ impl ::re_types_core::Loggable for EnumTest { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -124,16 +128,19 @@ impl ::re_types_core::Loggable for EnumTest { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/flattened_scalar.rs b/crates/store/re_types/src/testing/datatypes/flattened_scalar.rs index 5d703e045abd..8e344801cb3d 100644 --- a/crates/store/re_types/src/testing/datatypes/flattened_scalar.rs +++ b/crates/store/re_types/src/testing/datatypes/flattened_scalar.rs @@ -63,18 +63,23 @@ impl ::re_types_core::Loggable for FlattenedScalar { )])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![Field::new("value", DataType::Float32, false)]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -82,12 +87,12 @@ impl ::re_types_core::Loggable for FlattenedScalar { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![{ let (somes, value): (Vec<_>, Vec<_>) = data .iter() @@ -96,20 +101,22 @@ impl ::re_types_core::Loggable for FlattenedScalar { (datum.is_some(), datum) }) .unzip(); - let value_bitmap: Option = { + let value_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - DataType::Float32.into(), - value.into_iter().map(|v| v.unwrap_or_default()).collect(), - value_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + value + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + value_validity, + )) }], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/multi_enum.rs b/crates/store/re_types/src/testing/datatypes/multi_enum.rs index ee199b47177f..5013e84fea9b 100644 --- a/crates/store/re_types/src/testing/datatypes/multi_enum.rs +++ b/crates/store/re_types/src/testing/datatypes/multi_enum.rs @@ -61,18 +61,26 @@ impl ::re_types_core::Loggable for MultiEnum { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("value1", DataType::UInt8, false), + Field::new("value2", DataType::UInt8, true), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -80,12 +88,12 @@ impl ::re_types_core::Loggable for MultiEnum { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, value1): (Vec<_>, Vec<_>) = data @@ -95,13 +103,13 @@ impl ::re_types_core::Loggable for MultiEnum { (datum.is_some(), datum) }) .unzip(); - let value1_bitmap: Option = { + let value1_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = value1_bitmap; - crate::testing::datatypes::EnumTest::to_arrow2_opt(value1)? + _ = value1_validity; + crate::testing::datatypes::EnumTest::to_arrow_opt(value1)? } }, { @@ -113,19 +121,18 @@ impl ::re_types_core::Loggable for MultiEnum { (datum.is_some(), datum) }) .unzip(); - let value2_bitmap: Option = { + let value2_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = value2_bitmap; - crate::testing::datatypes::ValuedEnum::to_arrow2_opt(value2)? + _ = value2_validity; + crate::testing::datatypes::ValuedEnum::to_arrow_opt(value2)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/primitive_component.rs b/crates/store/re_types/src/testing/datatypes/primitive_component.rs index ce330f882d93..e3bbf8e7b750 100644 --- a/crates/store/re_types/src/testing/datatypes/primitive_component.rs +++ b/crates/store/re_types/src/testing/datatypes/primitive_component.rs @@ -58,17 +58,21 @@ impl ::re_types_core::Loggable for PrimitiveComponent { DataType::UInt32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl ::re_types_core::Loggable for PrimitiveComponent { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types/src/testing/datatypes/string_component.rs b/crates/store/re_types/src/testing/datatypes/string_component.rs index 49d4e3e9cb15..b001198a7d2e 100644 --- a/crates/store/re_types/src/testing/datatypes/string_component.rs +++ b/crates/store/re_types/src/testing/datatypes/string_component.rs @@ -58,17 +58,21 @@ impl ::re_types_core::Loggable for StringComponent { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,30 +82,23 @@ impl ::re_types_core::Loggable for StringComponent { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types/src/testing/datatypes/valued_enum.rs b/crates/store/re_types/src/testing/datatypes/valued_enum.rs index 33cd899e675a..29fb11118411 100644 --- a/crates/store/re_types/src/testing/datatypes/valued_enum.rs +++ b/crates/store/re_types/src/testing/datatypes/valued_enum.rs @@ -86,17 +86,21 @@ impl ::re_types_core::Loggable for ValuedEnum { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -106,16 +110,19 @@ impl ::re_types_core::Loggable for ValuedEnum { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_blueprint/src/blueprint/components/auto_layout.rs b/crates/store/re_types_blueprint/src/blueprint/components/auto_layout.rs index c976534556d0..e76dc9edd155 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/auto_layout.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/auto_layout.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for AutoLayout { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/auto_space_views.rs b/crates/store/re_types_blueprint/src/blueprint/components/auto_space_views.rs index 797a9f14876d..b23a9816b680 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/auto_space_views.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/auto_space_views.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for AutoSpaceViews { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/container_kind.rs b/crates/store/re_types_blueprint/src/blueprint/components/container_kind.rs index af6b6fd6d5b3..ae35a53d4c6c 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/container_kind.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/container_kind.rs @@ -87,17 +87,21 @@ impl ::re_types_core::Loggable for ContainerKind { DataType::UInt8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -107,16 +111,19 @@ impl ::re_types_core::Loggable for ContainerKind { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_blueprint/src/blueprint/components/grid_columns.rs b/crates/store/re_types_blueprint/src/blueprint/components/grid_columns.rs index 468eb395f924..6d5c2cd36447 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/grid_columns.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/grid_columns.rs @@ -74,13 +74,13 @@ impl ::re_types_core::Loggable for GridColumns { crate::datatypes::UInt32::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::UInt32::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::UInt32::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/included_space_view.rs b/crates/store/re_types_blueprint/src/blueprint/components/included_space_view.rs index 4e2538c66fb9..020fbd69bbdb 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/included_space_view.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/included_space_view.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for IncludedSpaceView { crate::datatypes::Uuid::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Uuid::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Uuid::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/root_container.rs b/crates/store/re_types_blueprint/src/blueprint/components/root_container.rs index da8acada3789..0c754d3403ce 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/root_container.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/root_container.rs @@ -75,13 +75,13 @@ impl ::re_types_core::Loggable for RootContainer { crate::datatypes::Uuid::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Uuid::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Uuid::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/space_view_maximized.rs b/crates/store/re_types_blueprint/src/blueprint/components/space_view_maximized.rs index 062f4e5d3bc0..482494005730 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/space_view_maximized.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/space_view_maximized.rs @@ -72,13 +72,13 @@ impl ::re_types_core::Loggable for SpaceViewMaximized { crate::datatypes::Uuid::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Uuid::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Uuid::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/components/visualizer_overrides.rs b/crates/store/re_types_blueprint/src/blueprint/components/visualizer_overrides.rs index 82bc233e67e1..a5c7020c1cd1 100644 --- a/crates/store/re_types_blueprint/src/blueprint/components/visualizer_overrides.rs +++ b/crates/store/re_types_blueprint/src/blueprint/components/visualizer_overrides.rs @@ -107,13 +107,13 @@ impl ::re_types_core::Loggable for VisualizerOverrides { crate::blueprint::datatypes::Utf8List::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::blueprint::datatypes::Utf8List::to_arrow2_opt(data.into_iter().map(|datum| { + crate::blueprint::datatypes::Utf8List::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs index 7b079ff28f06..0dd4a91fef63 100644 --- a/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs +++ b/crates/store/re_types_blueprint/src/blueprint/datatypes/utf8list.rs @@ -63,17 +63,21 @@ impl ::re_types_core::Loggable for Utf8List { ))) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use ::re_types_core::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -83,45 +87,35 @@ impl ::re_types_core::Loggable for Utf8List { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - use arrow2::{buffer::Buffer, offset::OffsetsBuffer}; - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map_or(0, |datum| datum.len())), - )? - .into(); + ); let data0_inner_data: Vec<_> = data0.into_iter().flatten().flatten().collect(); - let data0_inner_bitmap: Option = None; - ListArray::try_new( - Self::arrow_datatype().into(), + let data0_inner_validity: Option = None; + as_array_ref(ListArray::try_new( + std::sync::Arc::new(Field::new("item", DataType::Utf8, false)), offsets, { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0_inner_data.iter().map(|datum| datum.len()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0_inner_data.into_iter().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - data0_inner_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_inner_validity) + }) }, - data0_bitmap, - )? - .boxed() + data0_validity, + )?) } }) } diff --git a/crates/store/re_types_core/src/components/clear_is_recursive.rs b/crates/store/re_types_core/src/components/clear_is_recursive.rs index f45db2e65b45..04d68f0ab05c 100644 --- a/crates/store/re_types_core/src/components/clear_is_recursive.rs +++ b/crates/store/re_types_core/src/components/clear_is_recursive.rs @@ -74,13 +74,13 @@ impl crate::Loggable for ClearIsRecursive { crate::datatypes::Bool::arrow_datatype() } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { - crate::datatypes::Bool::to_arrow2_opt(data.into_iter().map(|datum| { + crate::datatypes::Bool::to_arrow_opt(data.into_iter().map(|datum| { datum.map(|datum| match datum.into() { ::std::borrow::Cow::Borrowed(datum) => ::std::borrow::Cow::Borrowed(&datum.0), ::std::borrow::Cow::Owned(datum) => ::std::borrow::Cow::Owned(datum.0), diff --git a/crates/store/re_types_core/src/datatypes/bool.rs b/crates/store/re_types_core/src/datatypes/bool.rs index afcd6aa88995..4ff9c81191f2 100644 --- a/crates/store/re_types_core/src/datatypes/bool.rs +++ b/crates/store/re_types_core/src/datatypes/bool.rs @@ -59,17 +59,21 @@ impl crate::Loggable for Bool { DataType::Boolean } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,16 +83,19 @@ impl crate::Loggable for Bool { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - BooleanArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(BooleanArray::new( + BooleanBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/entity_path.rs b/crates/store/re_types_core/src/datatypes/entity_path.rs index 57714f3eeb01..6c276a78e811 100644 --- a/crates/store/re_types_core/src/datatypes/entity_path.rs +++ b/crates/store/re_types_core/src/datatypes/entity_path.rs @@ -59,17 +59,21 @@ impl crate::Loggable for EntityPath { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,30 +83,23 @@ impl crate::Loggable for EntityPath { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types_core/src/datatypes/float32.rs b/crates/store/re_types_core/src/datatypes/float32.rs index af808a56775c..afc596ff7bf9 100644 --- a/crates/store/re_types_core/src/datatypes/float32.rs +++ b/crates/store/re_types_core/src/datatypes/float32.rs @@ -59,17 +59,21 @@ impl crate::Loggable for Float32 { DataType::Float32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,16 +83,19 @@ impl crate::Loggable for Float32 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/float64.rs b/crates/store/re_types_core/src/datatypes/float64.rs index b7395356b011..c037c5ae9351 100644 --- a/crates/store/re_types_core/src/datatypes/float64.rs +++ b/crates/store/re_types_core/src/datatypes/float64.rs @@ -59,17 +59,21 @@ impl crate::Loggable for Float64 { DataType::Float64 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,16 +83,19 @@ impl crate::Loggable for Float64 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/time_int.rs b/crates/store/re_types_core/src/datatypes/time_int.rs index d84e9b5aee4a..fb4ccb27584e 100644 --- a/crates/store/re_types_core/src/datatypes/time_int.rs +++ b/crates/store/re_types_core/src/datatypes/time_int.rs @@ -58,17 +58,21 @@ impl crate::Loggable for TimeInt { DataType::Int64 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl crate::Loggable for TimeInt { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/time_range.rs b/crates/store/re_types_core/src/datatypes/time_range.rs index 7089e285a08d..71f5e96d1b61 100644 --- a/crates/store/re_types_core/src/datatypes/time_range.rs +++ b/crates/store/re_types_core/src/datatypes/time_range.rs @@ -62,18 +62,72 @@ impl crate::Loggable for TimeRange { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new( + "start", + DataType::Union( + UnionFields::new( + vec![0, 1, 2, 3], + vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new( + "CursorRelative", + ::arrow_datatype(), + false, + ), + Field::new( + "Absolute", + ::arrow_datatype(), + false, + ), + Field::new("Infinite", DataType::Null, true), + ], + ), + UnionMode::Dense, + ), + false, + ), + Field::new( + "end", + DataType::Union( + UnionFields::new( + vec![0, 1, 2, 3], + vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new( + "CursorRelative", + ::arrow_datatype(), + false, + ), + Field::new( + "Absolute", + ::arrow_datatype(), + false, + ), + Field::new("Infinite", DataType::Null, true), + ], + ), + UnionMode::Dense, + ), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -81,12 +135,12 @@ impl crate::Loggable for TimeRange { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, start): (Vec<_>, Vec<_>) = data @@ -96,13 +150,13 @@ impl crate::Loggable for TimeRange { (datum.is_some(), datum) }) .unzip(); - let start_bitmap: Option = { + let start_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = start_bitmap; - crate::datatypes::TimeRangeBoundary::to_arrow2_opt(start)? + _ = start_validity; + crate::datatypes::TimeRangeBoundary::to_arrow_opt(start)? } }, { @@ -113,19 +167,18 @@ impl crate::Loggable for TimeRange { (datum.is_some(), datum) }) .unzip(); - let end_bitmap: Option = { + let end_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = end_bitmap; - crate::datatypes::TimeRangeBoundary::to_arrow2_opt(end)? + _ = end_validity; + crate::datatypes::TimeRangeBoundary::to_arrow_opt(end)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs index aaa3095c2f7b..4404c191ca3e 100644 --- a/crates/store/re_types_core/src/datatypes/time_range_boundary.rs +++ b/crates/store/re_types_core/src/datatypes/time_range_boundary.rs @@ -77,17 +77,21 @@ impl crate::Loggable for TimeRangeBoundary { ) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ // Dense Arrow union let data: Vec<_> = data @@ -97,7 +101,14 @@ impl crate::Loggable for TimeRangeBoundary { datum }) .collect(); - let types = data + let field_type_ids = [0, 1, 2, 3]; + let fields = vec![ + Field::new("_null_markers", DataType::Null, true), + Field::new("CursorRelative", DataType::Int64, false), + Field::new("Absolute", DataType::Int64, false), + Field::new("Infinite", DataType::Null, true), + ]; + let type_ids: Vec = data .iter() .map(|a| match a.as_deref() { None => 0, @@ -106,53 +117,7 @@ impl crate::Loggable for TimeRangeBoundary { Some(Self::Infinite) => 3i8, }) .collect(); - let fields = vec![ - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter().filter(|v| v.is_none()).count(), - ) - .boxed(), - { - let cursor_relative: Vec<_> = data - .iter() - .filter_map(|datum| match datum.as_deref() { - Some(Self::CursorRelative(v)) => Some(v.clone()), - _ => None, - }) - .collect(); - let cursor_relative_bitmap: Option = None; - PrimitiveArray::new( - DataType::Int64.into(), - cursor_relative.into_iter().map(|datum| datum.0).collect(), - cursor_relative_bitmap, - ) - .boxed() - }, - { - let absolute: Vec<_> = data - .iter() - .filter_map(|datum| match datum.as_deref() { - Some(Self::Absolute(v)) => Some(v.clone()), - _ => None, - }) - .collect(); - let absolute_bitmap: Option = None; - PrimitiveArray::new( - DataType::Int64.into(), - absolute.into_iter().map(|datum| datum.0).collect(), - absolute_bitmap, - ) - .boxed() - }, - NullArray::new( - arrow2::datatypes::DataType::Null, - data.iter() - .filter(|datum| matches!(datum.as_deref(), Some(Self::Infinite))) - .count(), - ) - .boxed(), - ]; - let offsets = Some({ + let offsets = { let mut cursor_relative_offset = 0; let mut absolute_offset = 0; let mut infinite_offset = 0; @@ -181,8 +146,64 @@ impl crate::Loggable for TimeRangeBoundary { } }) .collect() - }); - UnionArray::new(Self::arrow_datatype().into(), types, fields, offsets).boxed() + }; + let children = vec![ + as_array_ref(NullArray::new(data.iter().filter(|v| v.is_none()).count())), + { + let cursor_relative: Vec<_> = data + .iter() + .filter_map(|datum| match datum.as_deref() { + Some(Self::CursorRelative(v)) => Some(v.clone()), + _ => None, + }) + .collect(); + let cursor_relative_validity: Option = None; + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + cursor_relative + .into_iter() + .map(|datum| datum.0) + .collect::>(), + ), + cursor_relative_validity, + )) + }, + { + let absolute: Vec<_> = data + .iter() + .filter_map(|datum| match datum.as_deref() { + Some(Self::Absolute(v)) => Some(v.clone()), + _ => None, + }) + .collect(); + let absolute_validity: Option = None; + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + absolute + .into_iter() + .map(|datum| datum.0) + .collect::>(), + ), + absolute_validity, + )) + }, + as_array_ref(NullArray::new( + data.iter() + .filter(|datum| matches!(datum.as_deref(), Some(Self::Infinite))) + .count(), + )), + ]; + debug_assert_eq!(field_type_ids.len(), fields.len()); + debug_assert_eq!(fields.len(), children.len()); + as_array_ref( + UnionArray::try_new( + UnionFields::new(field_type_ids, fields), + ScalarBuffer::from(type_ids), + Some(offsets), + children, + ) + .unwrap(), + ) }) } diff --git a/crates/store/re_types_core/src/datatypes/uint16.rs b/crates/store/re_types_core/src/datatypes/uint16.rs index d72659834ab0..0657d43cbf82 100644 --- a/crates/store/re_types_core/src/datatypes/uint16.rs +++ b/crates/store/re_types_core/src/datatypes/uint16.rs @@ -58,17 +58,21 @@ impl crate::Loggable for UInt16 { DataType::UInt16 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl crate::Loggable for UInt16 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/uint32.rs b/crates/store/re_types_core/src/datatypes/uint32.rs index 9f64b394645c..c26fc9b74a3a 100644 --- a/crates/store/re_types_core/src/datatypes/uint32.rs +++ b/crates/store/re_types_core/src/datatypes/uint32.rs @@ -58,17 +58,21 @@ impl crate::Loggable for UInt32 { DataType::UInt32 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl crate::Loggable for UInt32 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/uint64.rs b/crates/store/re_types_core/src/datatypes/uint64.rs index e3bd7be8887b..9ba27af65628 100644 --- a/crates/store/re_types_core/src/datatypes/uint64.rs +++ b/crates/store/re_types_core/src/datatypes/uint64.rs @@ -58,17 +58,21 @@ impl crate::Loggable for UInt64 { DataType::UInt64 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -78,16 +82,19 @@ impl crate::Loggable for UInt64 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - PrimitiveArray::new( - Self::arrow_datatype().into(), - data0.into_iter().map(|v| v.unwrap_or_default()).collect(), - data0_bitmap, - ) - .boxed() + as_array_ref(PrimitiveArray::::new( + ScalarBuffer::from( + data0 + .into_iter() + .map(|v| v.unwrap_or_default()) + .collect::>(), + ), + data0_validity, + )) }) } diff --git a/crates/store/re_types_core/src/datatypes/utf8.rs b/crates/store/re_types_core/src/datatypes/utf8.rs index d835960f3054..1cf49c0065d0 100644 --- a/crates/store/re_types_core/src/datatypes/utf8.rs +++ b/crates/store/re_types_core/src/datatypes/utf8.rs @@ -59,17 +59,21 @@ impl crate::Loggable for Utf8 { DataType::Utf8 } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ let (somes, data0): (Vec<_>, Vec<_>) = data .into_iter() @@ -79,30 +83,23 @@ impl crate::Loggable for Utf8 { (datum.is_some(), datum) }) .unzip(); - let data0_bitmap: Option = { + let data0_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( data0 .iter() .map(|opt| opt.as_ref().map(|datum| datum.len()).unwrap_or_default()), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = + ); + let inner_data: arrow::buffer::Buffer = data0.into_iter().flatten().flat_map(|s| s.0).collect(); #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - Self::arrow_datatype().into(), - offsets, - inner_data, - data0_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, data0_validity) + }) } }) } diff --git a/crates/store/re_types_core/src/datatypes/visible_time_range.rs b/crates/store/re_types_core/src/datatypes/visible_time_range.rs index c2fcf4ff6902..08e73f814803 100644 --- a/crates/store/re_types_core/src/datatypes/visible_time_range.rs +++ b/crates/store/re_types_core/src/datatypes/visible_time_range.rs @@ -61,18 +61,41 @@ impl crate::Loggable for VisibleTimeRange { ])) } - fn to_arrow2_opt<'a>( + fn to_arrow_opt<'a>( data: impl IntoIterator>>>, - ) -> SerializationResult> + ) -> SerializationResult where Self: Clone + 'a, { #![allow(clippy::wildcard_imports)] #![allow(clippy::manual_is_variant_and)] use crate::{Loggable as _, ResultExt as _}; - use arrow::datatypes::*; - use arrow2::array::*; + use arrow::{array::*, buffer::*, datatypes::*}; + + #[allow(unused)] + fn as_array_ref(t: T) -> ArrayRef { + std::sync::Arc::new(t) as ArrayRef + } Ok({ + let fields = Fields::from(vec![ + Field::new("timeline", DataType::Utf8, false), + Field::new( + "range", + DataType::Struct(Fields::from(vec![ + Field::new( + "start", + ::arrow_datatype(), + false, + ), + Field::new( + "end", + ::arrow_datatype(), + false, + ), + ])), + false, + ), + ]); let (somes, data): (Vec<_>, Vec<_>) = data .into_iter() .map(|datum| { @@ -80,12 +103,12 @@ impl crate::Loggable for VisibleTimeRange { (datum.is_some(), datum) }) .unzip(); - let bitmap: Option = { + let validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; - StructArray::new( - Self::arrow_datatype().into(), + as_array_ref(StructArray::new( + fields, vec![ { let (somes, timeline): (Vec<_>, Vec<_>) = data @@ -95,33 +118,25 @@ impl crate::Loggable for VisibleTimeRange { (datum.is_some(), datum) }) .unzip(); - let timeline_bitmap: Option = { + let timeline_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - let offsets = arrow2::offset::Offsets::::try_from_lengths( + let offsets = arrow::buffer::OffsetBuffer::::from_lengths( timeline.iter().map(|opt| { opt.as_ref().map(|datum| datum.0.len()).unwrap_or_default() }), - )? - .into(); - let inner_data: arrow2::buffer::Buffer = timeline + ); + let inner_data: arrow::buffer::Buffer = timeline .into_iter() .flatten() .flat_map(|datum| datum.0 .0) .collect(); - #[allow(unsafe_code, clippy::undocumented_unsafe_blocks)] - unsafe { - Utf8Array::::new_unchecked( - DataType::Utf8.into(), - offsets, - inner_data, - timeline_bitmap, - ) - } - .boxed() + as_array_ref(unsafe { + StringArray::new_unchecked(offsets, inner_data, timeline_validity) + }) } }, { @@ -132,19 +147,18 @@ impl crate::Loggable for VisibleTimeRange { (datum.is_some(), datum) }) .unzip(); - let range_bitmap: Option = { + let range_validity: Option = { let any_nones = somes.iter().any(|some| !*some); any_nones.then(|| somes.into()) }; { - _ = range_bitmap; - crate::datatypes::TimeRange::to_arrow2_opt(range)? + _ = range_validity; + crate::datatypes::TimeRange::to_arrow_opt(range)? } }, ], - bitmap, - ) - .boxed() + validity, + )) }) } diff --git a/crates/store/re_types_core/src/loggable.rs b/crates/store/re_types_core/src/loggable.rs index c4f143969d9d..15c561d921c9 100644 --- a/crates/store/re_types_core/src/loggable.rs +++ b/crates/store/re_types_core/src/loggable.rs @@ -51,7 +51,10 @@ pub trait Loggable: 'static + Send + Sync + Clone + Sized + SizeBytes { data: impl IntoIterator>>>, ) -> SerializationResult> where - Self: 'a; + Self: 'a, + { + Self::to_arrow_opt(data).map(|array| array.into()) + } // --- Optional serialization methods ---