Datafusion 46 (#2417)

I mostly just like to see changes in Github.
We can merge this once Datafusion 46 released, assuming it includes the
following PRs:
- apache/datafusion#14754
- apache/datafusion#14671
- also hoping for apache/datafusion#14798

Seems like everything we wanted to get in made it, this branch now
compiles AND passes all checks, so we're only waiting for the actual
release in apache/datafusion#14123

---------

Co-authored-by: Alexander Droste <[email protected]>

Author: AdamGS

Cargo.lock

@@ -53,32 +53,32 @@ version = "0.25.2"
 anyhow = "1.0.95"
 arbitrary = "1.3.2"
 arrayref = "0.3.7"
-arrow = "54.2"
-arrow-arith = "54.2"
-arrow-array = "54.2"
-arrow-buffer = "54.2"
-arrow-cast = "54.2"
-arrow-ord = "54.2"
-arrow-schema = "54.2"
-arrow-select = "54.2"
-arrow-string = "54.2"
+arrow = "54.2.1"
+arrow-arith = "54.2.1"
+arrow-array = "54.2.1"
+arrow-buffer = "54.2.1"
+arrow-cast = "54.2.1"
+arrow-ord = "54.2.1"
+arrow-schema = "54.2.1"
+arrow-select = "54.2.1"
+arrow-string = "54.2.1"
 async-once-cell = "0.5.4"
 async-trait = "0.1.86"
 backtrace = "0.3.74"
 bit-vec = "0.8.0"
-bytes = "1.9"
+bytes = "1.10"
 bzip2 = "0.5.0"
 cfg-if = "1"
 chrono = "0.4.38"
 clap = "4"
 compio = { version = "0.13", features = ["io-uring"], default-features = false }
 crossterm = "0.28"
-datafusion = { version = "45", default-features = false }
-datafusion-common = { version = "45", default-features = false }
-datafusion-execution = "45"
-datafusion-expr = "45"
-datafusion-physical-expr = "45"
-datafusion-physical-plan = "45"
+datafusion = { version = "46", default-features = false }
+datafusion-common = { version = "46" }
+datafusion-execution = { version = "46" }
+datafusion-expr = { version = "46" }
+datafusion-physical-expr = { version = "46" }
+datafusion-physical-plan = { version = "46" }
 divan = { package = "codspeed-divan-compat", version = "2.8.0" }
 dyn-hash = "0.2.0"
 enum-iterator = "2.0.0"
@@ -113,7 +113,7 @@ opentelemetry = "0.28.0"
 opentelemetry-otlp = "0.28.0"
 opentelemetry_sdk = "0.28.0"
 parking_lot = "0.12.3"
-parquet = "54.2"
+parquet = "54.2.1"
 paste = "1.0.15"
 pin-project = "1.1.5"
 pin-project-lite = "0.2.15"
@@ -153,13 +153,14 @@ tar = "0.4"
 tempfile = "3"
 thiserror = "2.0.3"
 tikv-jemallocator = "0.6"
-tokio = "1.36"
+tokio = "1.43"
+tokio-stream = "0.1.17"
 tracing = { version = "0.1.41" }
 tracing-chrome = "0.7.2"
 tracing-futures = "0.2.5"
 tracing-subscriber = "0.3.19"
 url = "2.5.4"
-uuid = { version = "1.14", features = ["js"] }
+uuid = { version = "1.15", features = ["js"] }
 wasm-bindgen-futures = "0.4.39"
 witchcraft-metrics = "1.0.1"
 

Cargo.toml

@@ -32,6 +32,7 @@ object_store = { workspace = true }
 pin-project = { workspace = true }
 rand = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "fs"] }
+tokio-stream = { workspace = true }
 tracing = { workspace = true, optional = true }
 tracing-futures = { workspace = true, features = [
     "futures-03",

vortex-datafusion/Cargo.toml

@@ -1,6 +1,5 @@
 //! Connectors to enable DataFusion to read Vortex data.
 #![deny(missing_docs)]
-#![allow(clippy::nonminimal_bool)]
 #![allow(clippy::cast_possible_truncation)]
 
 use std::fmt::Debug;

vortex-datafusion/src/lib.rs

@@ -185,7 +185,7 @@ mod test {
     use arrow_schema::{DataType, Field, Schema};
     use datafusion::functions_aggregate::count::count_distinct;
     use datafusion::prelude::SessionContext;
-    use datafusion_common::{Column, TableReference};
+    use datafusion_common::{Column, Spans, TableReference};
     use datafusion_expr::{BinaryExpr, Expr, Operator, and, col, lit};
     use vortex_array::arrays::{PrimitiveArray, StructArray, VarBinViewArray};
     use vortex_array::{Array, ArrayRef};
@@ -249,6 +249,7 @@ mod test {
                         table: "orders".into(),
                     }),
                     name: "o_orderstatus".to_string(),
+                    spans: Spans::new(),
                 }
                 .into(),
             ),

vortex-datafusion/src/memory/provider.rs

@@ -35,6 +35,7 @@ pub struct ConfigProjection {
     pub arrow_schema: SchemaRef,
     pub constraints: Constraints,
     pub statistics: Statistics,
+    #[allow(dead_code)]
     pub orderings: Vec<LexOrdering>,
     pub projection_expr: Arc<dyn VortexExpr>,
 }