From b972e266a3c662f9e843a754847c905ef2707b0f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Oct 2022 11:38:06 -0400 Subject: [PATCH 1/3] Update to arrow 24.0.0 --- datafusion-cli/Cargo.toml | 2 +- datafusion-examples/Cargo.toml | 2 +- datafusion/common/Cargo.toml | 4 ++-- datafusion/core/Cargo.toml | 6 +++--- datafusion/core/fuzz-utils/Cargo.toml | 2 +- datafusion/expr/Cargo.toml | 2 +- datafusion/jit/Cargo.toml | 2 +- datafusion/optimizer/Cargo.toml | 2 +- datafusion/physical-expr/Cargo.toml | 2 +- datafusion/proto/Cargo.toml | 2 +- datafusion/row/Cargo.toml | 2 +- datafusion/sql/Cargo.toml | 2 +- 12 files changed, 15 insertions(+), 15 deletions(-) diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index e74b81a36baf..84c4bab48c64 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -29,7 +29,7 @@ rust-version = "1.62" readme = "README.md" [dependencies] -arrow = "23.0.0" +arrow = "24.0.0" clap = { version = "3", features = ["derive", "cargo"] } datafusion = { path = "../datafusion/core", version = "12.0.0" } dirs = "4.0.0" diff --git a/datafusion-examples/Cargo.toml b/datafusion-examples/Cargo.toml index c1feb1542fbd..12c267832fb1 100644 --- a/datafusion-examples/Cargo.toml +++ b/datafusion-examples/Cargo.toml @@ -34,7 +34,7 @@ path = "examples/avro_sql.rs" required-features = ["datafusion/avro"] [dev-dependencies] -arrow-flight = "23.0.0" +arrow-flight = "24.0.0" async-trait = "0.1.41" datafusion = { path = "../datafusion/core" } futures = "0.3" diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index 48bc61f5368f..994c39befbc7 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -40,10 +40,10 @@ pyarrow = ["pyo3", "arrow/pyarrow"] [dependencies] apache-avro = { version = "0.14", default-features = false, features = ["snappy"], optional = true } -arrow = { version = "23.0.0", default-features = false } +arrow = { version = "24.0.0", default-features = false } cranelift-module = { version = "0.88.0", optional = true } object_store = { version = "0.5.0", default-features = false, optional = true } ordered-float = "3.0" -parquet = { version = "23.0.0", default-features = false, optional = true } +parquet = { version = "24.0.0", default-features = false, optional = true } pyo3 = { version = "0.17.1", optional = true } sqlparser = "0.24" diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 1e338c5d106c..8c275e4dadeb 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -56,7 +56,7 @@ unicode_expressions = ["datafusion-physical-expr/regex_expressions", "datafusion [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } apache-avro = { version = "0.14", optional = true } -arrow = { version = "23.0.0", features = ["prettyprint"] } +arrow = { version = "24.0.0", features = ["prettyprint"] } async-trait = "0.1.41" bytes = "1.1" chrono = { version = "0.4", default-features = false } @@ -78,7 +78,7 @@ num_cpus = "1.13.0" object_store = "0.5.0" ordered-float = "3.0" parking_lot = "0.12" -parquet = { version = "23.0.0", features = ["arrow", "async"] } +parquet = { version = "24.0.0", features = ["arrow", "async"] } paste = "^1.0" pin-project-lite = "^0.2.7" pyo3 = { version = "0.17.1", optional = true } @@ -93,7 +93,7 @@ url = "2.2" uuid = { version = "1.0", features = ["v4"] } [dev-dependencies] -arrow = { version = "23.0.0", features = ["prettyprint", "dyn_cmp_dict"] } +arrow = { version = "24.0.0", features = ["prettyprint", "dyn_cmp_dict"] } async-trait = "0.1.53" criterion = "0.4" csv = "1.1.6" diff --git a/datafusion/core/fuzz-utils/Cargo.toml b/datafusion/core/fuzz-utils/Cargo.toml index 07f09b6362cc..3f1f2ed765a7 100644 --- a/datafusion/core/fuzz-utils/Cargo.toml +++ b/datafusion/core/fuzz-utils/Cargo.toml @@ -23,6 +23,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -arrow = { version = "23.0.0", features = ["prettyprint"] } +arrow = { version = "24.0.0", features = ["prettyprint"] } env_logger = "0.9.0" rand = "0.8" diff --git a/datafusion/expr/Cargo.toml b/datafusion/expr/Cargo.toml index e0b8e2d52532..a3111517e361 100644 --- a/datafusion/expr/Cargo.toml +++ b/datafusion/expr/Cargo.toml @@ -36,6 +36,6 @@ path = "src/lib.rs" [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } -arrow = { version = "23.0.0", default-features = false } +arrow = { version = "24.0.0", default-features = false } datafusion-common = { path = "../common", version = "12.0.0" } sqlparser = "0.24" diff --git a/datafusion/jit/Cargo.toml b/datafusion/jit/Cargo.toml index 3ac81631a88d..6166e91747e9 100644 --- a/datafusion/jit/Cargo.toml +++ b/datafusion/jit/Cargo.toml @@ -36,7 +36,7 @@ path = "src/lib.rs" jit = [] [dependencies] -arrow = { version = "23.0.0", default-features = false } +arrow = { version = "24.0.0", default-features = false } cranelift = "0.88.0" cranelift-jit = "0.88.0" cranelift-module = "0.88.0" diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml index 409521088bf6..862f51024266 100644 --- a/datafusion/optimizer/Cargo.toml +++ b/datafusion/optimizer/Cargo.toml @@ -37,7 +37,7 @@ default = ["unicode_expressions"] unicode_expressions = [] [dependencies] -arrow = { version = "23.0.0", features = ["prettyprint"] } +arrow = { version = "24.0.0", features = ["prettyprint"] } async-trait = "0.1.41" chrono = { version = "0.4", default-features = false } datafusion-common = { path = "../common", version = "12.0.0" } diff --git a/datafusion/physical-expr/Cargo.toml b/datafusion/physical-expr/Cargo.toml index ba618acb64bf..a2473fb75333 100644 --- a/datafusion/physical-expr/Cargo.toml +++ b/datafusion/physical-expr/Cargo.toml @@ -40,7 +40,7 @@ unicode_expressions = ["unicode-segmentation"] [dependencies] ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } -arrow = { version = "23.0.0", features = ["prettyprint"] } +arrow = { version = "24.0.0", features = ["prettyprint"] } blake2 = { version = "^0.10.2", optional = true } blake3 = { version = "1.0", optional = true } chrono = { version = "0.4", default-features = false } diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index adab4b356977..122b980cfa7f 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -40,7 +40,7 @@ default = [] json = ["pbjson", "pbjson-build", "serde", "serde_json"] [dependencies] -arrow = "23.0.0" +arrow = "24.0.0" datafusion = { path = "../core", version = "12.0.0" } datafusion-common = { path = "../common", version = "12.0.0" } datafusion-expr = { path = "../expr", version = "12.0.0" } diff --git a/datafusion/row/Cargo.toml b/datafusion/row/Cargo.toml index d1c7fdf0cf44..306ab7b3cbab 100644 --- a/datafusion/row/Cargo.toml +++ b/datafusion/row/Cargo.toml @@ -37,7 +37,7 @@ path = "src/lib.rs" jit = ["datafusion-jit"] [dependencies] -arrow = "23.0.0" +arrow = "24.0.0" datafusion-common = { path = "../common", version = "12.0.0" } datafusion-jit = { path = "../jit", version = "12.0.0", optional = true } paste = "^1.0" diff --git a/datafusion/sql/Cargo.toml b/datafusion/sql/Cargo.toml index cf057a562aba..55aabd48cbce 100644 --- a/datafusion/sql/Cargo.toml +++ b/datafusion/sql/Cargo.toml @@ -37,7 +37,7 @@ default = ["unicode_expressions"] unicode_expressions = [] [dependencies] -arrow = { version = "23.0.0", default-features = false } +arrow = { version = "24.0.0", default-features = false } datafusion-common = { path = "../common", version = "12.0.0" } datafusion-expr = { path = "../expr", version = "12.0.0" } sqlparser = "0.24" From 687ff9e485a6e18c484d14246246d125085a3fcd Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Oct 2022 11:54:44 -0400 Subject: [PATCH 2/3] Update pyo3 interface --- datafusion/common/src/pyarrow.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datafusion/common/src/pyarrow.rs b/datafusion/common/src/pyarrow.rs index bb3d9a6d7bac..6d9cab44cc9d 100644 --- a/datafusion/common/src/pyarrow.rs +++ b/datafusion/common/src/pyarrow.rs @@ -43,8 +43,8 @@ impl PyArrowConvert for ScalarValue { let array = factory.call1((args, typ))?; // convert the pyarrow array to rust array using C data interface - let array = array.extract::()?; - let scalar = ScalarValue::try_from_array(&array.into(), 0)?; + let array = arrow::array::make_array(ArrayData::from_pyarrow(array)?); + let scalar = ScalarValue::try_from_array(&array, 0)?; Ok(scalar) } @@ -52,7 +52,7 @@ impl PyArrowConvert for ScalarValue { fn to_pyarrow(&self, py: Python) -> PyResult { let array = self.to_array(); // convert to pyarrow array using C data interface - let pyarray = array.data_ref().clone().into_py(py); + let pyarray = array.data().to_pyarrow(py)?; let pyscalar = pyarray.call_method1(py, "__getitem__", (0,))?; Ok(pyscalar) From 6c058625a93322e072248ce8f40782f60e798379 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Oct 2022 17:02:20 -0400 Subject: [PATCH 3/3] Update datafusion-cli lockfile --- datafusion-cli/Cargo.lock | 104 +++++++++++++++++++++++++------------- 1 file changed, 69 insertions(+), 35 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index bdb0fdbaee37..bc4e084f10d9 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -79,12 +79,15 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fedc767fbaa36ea50f086215f54f1a007d22046fc4754b0448c657bcbe9f8413" +checksum = "d68391300d5237f6725f0f869ae7cb65d45fcf8a6d18f6ceecd328fb803bef93" dependencies = [ "ahash 0.8.0", + "arrow-array", "arrow-buffer", + "arrow-data", + "arrow-schema", "bitflags", "chrono", "comfy-table", @@ -99,20 +102,52 @@ dependencies = [ "num", "regex", "regex-syntax", - "serde", "serde_json", ] +[[package]] +name = "arrow-array" +version = "24.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0bb00c5862b5eea683812083c495bef01a9a5149da46ad2f4c0e4aa8800f64d" +dependencies = [ + "ahash 0.8.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown", + "num", +] + [[package]] name = "arrow-buffer" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d290050c6e12a81a24ad08525cef2203c4156a6350f75508d49885d677e88ea9" +checksum = "3e594d0fe0026a8bc2459bdc5ac9623e5fb666724a715e0acbc96ba30c5d4cc7" dependencies = [ "half", +] + +[[package]] +name = "arrow-data" +version = "24.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8500df05060d86fdc53e9b5cb32e51bfeaacc040fdeced3eb99ac0d59200ff45" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", "num", ] +[[package]] +name = "arrow-schema" +version = "24.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d1fef01f25e1452c86fa6887f078de8e0aaeeb828370feab205944cfc30e27" + [[package]] name = "async-trait" version = "0.1.57" @@ -438,7 +473,7 @@ dependencies = [ "log", "num_cpus", "object_store", - "ordered-float 3.1.0", + "ordered-float 3.2.0", "parking_lot", "parquet", "paste", @@ -475,7 +510,7 @@ version = "12.0.0" dependencies = [ "arrow", "object_store", - "ordered-float 3.1.0", + "ordered-float 3.2.0", "parquet", "sqlparser", ] @@ -519,7 +554,7 @@ dependencies = [ "hashbrown", "lazy_static", "md-5", - "ordered-float 3.1.0", + "ordered-float 3.2.0", "paste", "rand", "regex", @@ -974,9 +1009,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.49" +version = "0.1.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bbaead50122b06e9a973ac20bc7445074d99ad9a0a0654934876908a9cec82c" +checksum = "fd911b35d940d2bd0bea0f9100068e5b97b51a1cbe13d13382f132e0365257a0" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1143,9 +1178,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.133" +version = "0.2.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f80d65747a3e43d1596c7c5492d95d5edddaabd45a7fcdb02b95f644164966" +checksum = "329c933548736bc49fd575ee68c89e8be4d260064184389a5b77517cddd99ffb" [[package]] name = "libm" @@ -1430,9 +1465,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98ffdb14730ed2ef599c65810c15b000896e21e8776b512de0db0c3d7335cc2a" +checksum = "129d36517b53c461acc6e1580aeb919c8ae6708a4b1eae61c4463a615d4f0411" dependencies = [ "num-traits", ] @@ -1468,9 +1503,9 @@ dependencies = [ [[package]] name = "parquet" -version = "23.0.0" +version = "24.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc38abcd826e52e80a45abe46707745e6d12d31031a9b11c2eab112def7efe71" +checksum = "74fd590f0672998df84503d1bcbebc69732583d03cc3495c7dd8d3e5a1d8437f" dependencies = [ "ahash 0.8.0", "arrow", @@ -1554,9 +1589,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.43" +version = "1.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2ca2c61bc9f3d74d2886294ab7b9853abd9c1ad903a3ac7815c58989bb7bab" +checksum = "94e2ef8dbfc347b10c094890f778ee2e36ca9bb4262e86dc99cd217e35f3470b" dependencies = [ "unicode-ident", ] @@ -1729,9 +1764,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.35.10" +version = "0.35.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af895b90e5c071badc3136fc10ff0bcfc98747eadbaf43ed8f214e07ba8f8477" +checksum = "fbb2fda4666def1433b1b05431ab402e42a1084285477222b72d6c564c417cef" dependencies = [ "bitflags", "errno", @@ -1893,9 +1928,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd0db749597d91ff862fd1d55ea87f7855a744a8425a64695b6fca237d1dad1" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "snafu" @@ -1943,9 +1978,9 @@ checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] name = "sqlparser" -version = "0.23.0" +version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beb13adabbdda01b63d595f38c8bfd19a361e697fd94ce0098a634077bc5b25" +checksum = "dac9c312566fdfc45a38ecf1924013c82af2a7d5315e46f67b1cc987f12be260" dependencies = [ "log", ] @@ -1995,9 +2030,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.100" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52205623b1b0f064a4e71182c3b18ae902267282930c6d5462c91b859668426e" +checksum = "e90cde112c4b9690b8cbe810cba9ddd8bc1d7472e2cae317b69e9438c1cba7d2" dependencies = [ "proc-macro2", "quote", @@ -2035,18 +2070,18 @@ checksum = "949517c0cf1bf4ee812e2e07e08ab448e3ae0d23472aee8a06c985f0c8815b16" [[package]] name = "thiserror" -version = "1.0.35" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c53f98874615aea268107765aa1ed8f6116782501d18e53d08b471733bea6c85" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.35" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b463991b4eab2d801e724172285ec4195c650e8ec79b149e6c2a8e6dd3f783" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" dependencies = [ "proc-macro2", "quote", @@ -2090,9 +2125,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.21.1" +version = "1.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0020c875007ad96677dcc890298f4b942882c5d4eb7cc8f439fc3bf813dc9c95" +checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" dependencies = [ "autocfg", "bytes", @@ -2100,7 +2135,6 @@ dependencies = [ "memchr", "mio", "num_cpus", - "once_cell", "parking_lot", "pin-project-lite", "socket2", @@ -2391,9 +2425,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.22.4" +version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1c760f0d366a6c24a02ed7816e23e691f5d92291f94d15e836006fd11b04daf" +checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" dependencies = [ "webpki", ]