Skip to content

Commit cb3babc

Browse files
authored
Improve performance reading ByteViewArray from parquet by removing an implicit copy (#6031)
* update byte view array to not implicit copy * Add small comments
1 parent 3ce8e84 commit cb3babc

File tree

2 files changed

+8
-4
lines changed

2 files changed

+8
-4
lines changed

parquet/src/arrow/array_reader/byte_view_array.rs

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ struct ByteViewArrayReader {
7171
}
7272

7373
impl ByteViewArrayReader {
74-
#[allow(unused)]
7574
fn new(
7675
pages: Box<dyn PageIterator>,
7776
data_type: ArrowType,
@@ -316,7 +315,10 @@ impl ByteViewArrayDecoderPlain {
316315
}
317316

318317
pub fn read(&mut self, output: &mut ViewBuffer, len: usize) -> Result<usize> {
319-
let block_id = output.append_block(self.buf.clone().into());
318+
// Here we convert `bytes::Bytes` into `arrow_buffer::Bytes`, which is zero copy
319+
// Then we convert `arrow_buffer::Bytes` into `arrow_buffer:Buffer`, which is also zero copy
320+
let buf = arrow_buffer::Buffer::from_bytes(self.buf.clone().into());
321+
let block_id = output.append_block(buf);
320322

321323
let to_read = len.min(self.max_remaining_values);
322324

@@ -546,7 +548,10 @@ impl ByteViewArrayDecoderDeltaLength {
546548

547549
let src_lengths = &self.lengths[self.length_offset..self.length_offset + to_read];
548550

549-
let block_id = output.append_block(self.data.clone().into());
551+
// Here we convert `bytes::Bytes` into `arrow_buffer::Bytes`, which is zero copy
552+
// Then we convert `arrow_buffer::Bytes` into `arrow_buffer:Buffer`, which is also zero copy
553+
let bytes = arrow_buffer::Buffer::from_bytes(self.data.clone().into());
554+
let block_id = output.append_block(bytes);
550555

551556
let mut current_offset = self.data_offset;
552557
let initial_offset = current_offset;

parquet/src/arrow/buffer/view_buffer.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@ impl ViewBuffer {
6868
}
6969

7070
/// Converts this into an [`ArrayRef`] with the provided `data_type` and `null_buffer`
71-
#[allow(unused)]
7271
pub fn into_array(self, null_buffer: Option<Buffer>, data_type: &ArrowType) -> ArrayRef {
7372
let len = self.views.len();
7473
let views = Buffer::from_vec(self.views);

0 commit comments

Comments
 (0)