From 5eb35f94ff0ab6c6302f5b9b81b60e7dfbccd04f Mon Sep 17 00:00:00 2001 From: nuno-faria Date: Sat, 7 Nov 2026 16:17:42 +0000 Subject: [PATCH 1/3] Upgrade DataFusion to version 56 --- Cargo.lock | 695 +++++++++++------- Cargo.toml | 43 +- crates/core/src/context.rs | 4 +- crates/core/src/dataframe.rs | 23 +- crates/core/src/dataset.rs | 4 +- crates/core/src/dataset_exec.rs | 2 +- crates/core/src/expr.rs | 14 +- crates/core/src/expr/asof_join.rs | 175 +++++ .../core/src/expr/create_external_catalog.rs | 110 +++ crates/core/src/expr/drop_catalog.rs | 93 +++ crates/core/src/expr/join.rs | 2 +- crates/core/src/expr/materialized_cte.rs | 167 +++++ crates/core/src/expr/show.rs | 114 +++ crates/core/src/sql/logical.rs | 23 +- crates/core/src/table.rs | 5 +- docs/source/user-guide/upgrade-guides.md | 22 + python/datafusion/expr.py | 16 +- python/datafusion/functions/__init__.py | 4 +- python/tests/test_context.py | 6 +- python/tests/test_dataframe.py | 4 +- python/tests/test_expr.py | 80 +- python/tests/test_functions.py | 2 +- rust-toolchain.toml | 2 +- 23 files changed, 1307 insertions(+), 303 deletions(-) create mode 100644 crates/core/src/expr/asof_join.rs create mode 100644 crates/core/src/expr/create_external_catalog.rs create mode 100644 crates/core/src/expr/drop_catalog.rs create mode 100644 crates/core/src/expr/materialized_cte.rs create mode 100644 crates/core/src/expr/show.rs diff --git a/Cargo.lock b/Cargo.lock index ec665e285..e42c8b193 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -33,15 +33,15 @@ dependencies = [ [[package]] name = "alloc-no-stdlib" -version = "2.0.4" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" [[package]] name = "alloc-stdlib" -version = "0.2.4" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" dependencies = [ "alloc-no-stdlib", ] @@ -93,9 +93,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" [[package]] name = "arrow" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c14b3d39f306bc28fd639d59f06e17a0f377d0021e1b7e9054e4d6fedc98774" +checksum = "d965156f2220e94dcf676e04322b7b0b41c97e85cb3e1c365b8e56d0f70be999" dependencies = [ "arrow-arith", "arrow-array", @@ -115,9 +115,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2961626677665b2195eb59242af4c7befe7b8737ca2050295389362380104e" +checksum = "ca16fdd1daeb1eccefb42df52ec65448ef33eec72488598675d46dfa9ce58923" dependencies = [ "arrow-array", "arrow-buffer", @@ -129,9 +129,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f6adeffdf587d7a31db5d2266189624b526730cd3627f9ff9fedae97ad584" +checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4" dependencies = [ "ahash", "arrow-buffer", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9145b685d586482102e4fe3d40349fe00086c0b219d51cae9daaead4698849ea" +checksum = "cfd40a71a38e4366b28742cc46912b9289d89e0c858895c8276ac0fc83dacdc6" dependencies = [ "arrow-array", "arrow-buffer", @@ -162,20 +162,20 @@ dependencies = [ "flate2", "indexmap", "liblzma", - "rand 0.9.5", + "rand 0.10.3", "serde", "serde_json", "snap", "strum_macros", "uuid", - "zstd 0.13.3", + "zstd", ] [[package]] name = "arrow-buffer" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "097d193003ce7995d5d087089069ec2a6e0187faf5a6f8c9f38af2645d987182" +checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade" dependencies = [ "bytes", "half", @@ -185,9 +185,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "635c9c635668ad26adf76cce8fb276c4be7cf06e63bd516de7da514f9680ee53" +checksum = "baa83a0752814a1922d5ee75032524ca9bbfb7f60ee35a67ed4ab8b275260aaf" dependencies = [ "arrow-array", "arrow-buffer", @@ -205,11 +205,22 @@ dependencies = [ "ryu", ] +[[package]] +name = "arrow-cmp" +version = "60.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", +] + [[package]] name = "arrow-csv" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2ebf8d631e79b02c16cf5ae860561272c26024ec88fce389a56aaddd558e86" +checksum = "e4572af05fadde76a063139805c089d0915b7eb64ed5bef14456c6e458a80b28" dependencies = [ "arrow-array", "arrow-cast", @@ -222,9 +233,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba2f832eaeca24b8f26143dba750e42ee4ab51cf7d65e701ca9607cfda9f358" +checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -235,9 +246,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcc41681ea80f521df14c36725b74d4c60702c47f0793af2be469c04527e2599" +checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba" dependencies = [ "arrow-array", "arrow-buffer", @@ -246,14 +257,14 @@ dependencies = [ "arrow-select", "flatbuffers", "lz4_flex", - "zstd 0.13.3", + "zstd", ] [[package]] name = "arrow-json" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f57d7a81969f24ccf80809587b76c09897e6f829d2d65a5976bfb3218851f1" +checksum = "d843cca4b5361d7667564083b151a30d816d2b7e7ce2baa53948e2188f087f06" dependencies = [ "arrow-array", "arrow-buffer", @@ -276,12 +287,13 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c900759f3bd8354fd4196bc4403eee846894dc2adf66b4225472006a0bf18c5" +checksum = "4c58f2ae6dd1480fc413f7320a547e39dcc499c23d4f36ca19572b685811e6e4" dependencies = [ "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "arrow-select", @@ -289,9 +301,9 @@ dependencies = [ [[package]] name = "arrow-pyarrow" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801aed7e607dbcc60ed6359025383b27a490327d4282ba87a9a3b78e25959c9a" +checksum = "3b6dfd82e81f080a076cb8ae5d53ea55b18db98edd4abb3960a6ca9d5d3c8184" dependencies = [ "arrow-array", "arrow-data", @@ -301,9 +313,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c6425032e28266e3fc4ff680805e57e670d6ea92473043f3e65b7ed6ac79f2" +checksum = "68f9db8345b0638416d3b56d117d262d13070fdef788c6d75b53a180215c477e" dependencies = [ "arrow-array", "arrow-buffer", @@ -314,9 +326,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10fab8d4563491417ba801fab29d205104d20d4bdf37bda6cd1cf425cff598cd" +checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3" dependencies = [ "bitflags", "serde_core", @@ -325,13 +337,14 @@ dependencies = [ [[package]] name = "arrow-select" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc58569193c2525915f3cc6310edba3792f1200f65d6e9ed330aa33e691493b8" +checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30" dependencies = [ "ahash", "arrow-array", "arrow-buffer", + "arrow-cmp", "arrow-data", "arrow-schema", "num-traits", @@ -339,9 +352,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e0813f3c35c1cfea65e14c20a953440f7783c088b7ad2d0db162ccdeefcec14" +checksum = "ab1be9306b9a413b5b004e2a5a19caac728870d76e8891b351f040d554c93862" dependencies = [ "arrow-array", "arrow-buffer", @@ -396,9 +409,9 @@ dependencies = [ [[package]] name = "atoi" -version = "2.0.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +checksum = "d12c37257553c19e9aee6cba77814fdc9e07844abfc8b3a2c377cd920ee5a993" dependencies = [ "num-traits", ] @@ -415,6 +428,29 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "aws-lc-rs" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.47.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + [[package]] name = "base64" version = "0.22.1" @@ -448,11 +484,11 @@ checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "blake2" -version = "0.10.6" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "5b5d4d889834ee8ecfc0f8426ad30faf7cdcb10f741a8e6d7224d95325479f6f" dependencies = [ - "digest 0.10.7", + "digest 0.11.3", ] [[package]] @@ -468,15 +504,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.12.1" @@ -488,9 +515,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.4" +version = "9.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" +checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -499,9 +526,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "5.0.3" +version = "6.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" +checksum = "97d9071e5b45944c5a9c06baf13a7a65ffa2c88c204015967852c90869692c05" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -594,11 +621,27 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "combine" +version = "4.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "comfy-table" -version = "7.2.2" +version = "8.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +checksum = "b96a46ad395a5e00676eddaca4a45ed78b29c08be5a878d35e4b071c625637dd" dependencies = [ "unicode-segmentation", "unicode-width", @@ -615,8 +658,8 @@ dependencies = [ "flate2", "liblzma", "memchr", - "zstd 0.14.0", - "zstd-safe 8.0.0", + "zstd", + "zstd-safe", ] [[package]] @@ -697,6 +740,18 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85d9be5297a59f1b7651fd2711a1f4461929f53b182b394df0df15b3a387ef51" +dependencies = [ + "crc", + "digest 0.10.7", + "rustversion", + "spin", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -768,6 +823,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctutils" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261af4dc7a06ea8a527a8717d8cbba3417530d6ea14e1edb3de6b67806be474f" +dependencies = [ + "cmov", +] + [[package]] name = "dashmap" version = "6.2.1" @@ -777,16 +841,15 @@ dependencies = [ "cfg-if", "crossbeam-utils", "hashbrown 0.14.5", - "lock_api", + "lock_api 0.4.14", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.12", ] [[package]] name = "datafusion" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd5317db77ba6bca25c761a8091b9d294a9f4188e4f9dcd0243e6a57de862ac" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-schema", @@ -826,21 +889,20 @@ dependencies = [ "liblzma", "log", "object_store", - "parking_lot", + "parking_lot 0.13.0", "parquet", "sqlparser", "tempfile", "tokio", "url", "uuid", - "zstd 0.13.3", + "zstd", ] [[package]] name = "datafusion-catalog" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f35d36794509b7bfcbf90e11a1b5e4d047123c34610105e2e80a2048351cdb2f" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "async-trait", @@ -857,15 +919,14 @@ dependencies = [ "itertools 0.15.0", "log", "object_store", - "parking_lot", + "parking_lot 0.13.0", "tokio", ] [[package]] name = "datafusion-catalog-listing" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6454bfb7f43b83942e2ca2576fd09342eedbd0d323e1eef72c3c8e4d57d7203" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "async-trait", @@ -887,9 +948,8 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26626e12b224b83351239465dcd00f72d47b02fb5e5b65a643e2dc8ed37f687" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-ipc", @@ -914,9 +974,8 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a106420cbfa980f242c22243ae9a6cce661d4d86ce96cff87dacabf7754a4470" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "futures", "log", @@ -925,9 +984,8 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "210aa1e9525e9454f2399dbb8b0b95510145bd8c1fdc35de74e9ccbf81bcb699" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "async-compression", @@ -952,19 +1010,19 @@ dependencies = [ "liblzma", "log", "object_store", - "parking_lot", + "parking_lot 0.13.0", + "pin-project-lite", "rand 0.9.5", "tokio", "tokio-util", "url", - "zstd 0.13.3", + "zstd", ] [[package]] name = "datafusion-datasource-arrow" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65b1b041688cad707fa103c3e7f4c6d89b3134ebf67da0129070dad3381851eb" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-ipc", @@ -987,28 +1045,31 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bac6aafe51a477bcf1972a82922ae8866d971716f2bf563b4714c07440da61fe" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-avro", "async-trait", "bytes", "datafusion-common", + "datafusion-common-runtime", "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", "object_store", + "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a7b34e253c4fa5251f174977af910a2f2ec831ec39dbbb56b29ce505cff278b" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "async-trait", @@ -1030,9 +1091,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1663338f43be6a5746473008e2d0fbde282577ddf0bae90d6c7386e988d71b29" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "async-trait", @@ -1054,9 +1114,8 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c521ae91b6d899d7190812881f45fe9463c222c11d90149f5332a2e56555904" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-schema", @@ -1080,22 +1139,20 @@ dependencies = [ "itertools 0.15.0", "log", "object_store", - "parking_lot", + "parking_lot 0.13.0", "parquet", "tokio", ] [[package]] name = "datafusion-doc" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2569c2e74a34fcae3a369ab3912d61e41a062592510f11e31b1dae2725ecdc" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" [[package]] name = "datafusion-execution" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cfdd5ac0769d1e74a885450e49fde8f63ca7ab41eb479a068564aaea80b837" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-buffer", @@ -1108,7 +1165,7 @@ dependencies = [ "futures", "log", "object_store", - "parking_lot", + "parking_lot 0.13.0", "pin-project-lite", "rand 0.9.5", "tempfile", @@ -1119,9 +1176,8 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86560fd7df4dda924f5474b588c56e7ae8e96dcf158c7327c9cfe9daff448ec5" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-schema", @@ -1145,9 +1201,8 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e82e313cf655cf6bc2dce6cbd23bcb0d20236a54a1b09c436615e4bac4d8c49f" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "datafusion-common", @@ -1157,9 +1212,8 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58a2da6b361eae9452cf8e2cafadb05b44501a4d965b26760327cb18e0d26c4f" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-schema", @@ -1229,12 +1283,12 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "939ae2771da8a3fdc23ab4de610ae8fef9515476626b49d96d0d5c782b105771" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-buffer", + "arrow-select", "base64 0.23.1", "blake2", "blake3", @@ -1250,7 +1304,7 @@ dependencies = [ "hex", "itertools 0.15.0", "log", - "md-5 0.11.0", + "md-5", "memchr", "num-traits", "rand 0.9.5", @@ -1261,11 +1315,11 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6c3aee9f2bc65716787d651e023743a90b4efbc57a2f679f036632e16fb110" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", + "arrow-select", "datafusion-common", "datafusion-doc", "datafusion-execution", @@ -1276,15 +1330,15 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.17.1", + "itertools 0.15.0", "log", "num-traits", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "988b5d3dfb4939d81c2f9fab267063610fe52a843620f2fa6cb2a1598e43369c" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "datafusion-common", @@ -1294,9 +1348,8 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08149d433a97c9ea3cdf80de06480ab5c7f5453485f1e452d5b3583c81d5c0dd" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-ord", @@ -1319,9 +1372,8 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c767567cce2782d4d0bdfa55fab010aaba0bdbe533509e9e0b9a6a3ee29ed59" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "async-trait", @@ -1330,14 +1382,15 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr", "datafusion-physical-plan", - "parking_lot", + "datafusion-proto-common", + "datafusion-proto-models", + "parking_lot 0.13.0", ] [[package]] name = "datafusion-functions-window" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a311c1f558d1f792f1f631c3670da12ee72a20fbf5d6ca151325d66191c03c1" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "datafusion-common", @@ -1352,9 +1405,8 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffd54b8acf6ecef284cc0677e01e1b8c71be64fec50612ba00a0ff0d9b41a52" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1362,9 +1414,8 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14d6df9763c8fb968b2417d797aef4c55ac2e4a895aaef38c72c16a287aa4a40" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "datafusion-doc", "quote", @@ -1373,9 +1424,8 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acfb8cea6e05e2e75bae183340cf7d53cccb3db0449e2f1fbc07654d980fd19f" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "chrono", @@ -1393,9 +1443,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f002cef849e10c0597a217ec7891c2796c0f8b42d739f5d24abc24496373406" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-schema", @@ -1409,17 +1458,17 @@ dependencies = [ "hashbrown 0.17.1", "indexmap", "itertools 0.15.0", - "parking_lot", + "parking_lot 0.13.0", "petgraph", "recursive", + "regex", "tokio", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82761522c4536e7059d3b185cd99f10525c2bf48f1bd26a0a57e73e19eadc0fb" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "datafusion-common", @@ -1432,9 +1481,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5c67aa252886f4d6b331c8114797f5b140464e774c4da36889269a1bf1bd8" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "chrono", @@ -1444,21 +1492,22 @@ dependencies = [ "hashbrown 0.17.1", "indexmap", "itertools 0.15.0", - "parking_lot", + "parking_lot 0.13.0", "pin-project", + "regex", ] [[package]] name = "datafusion-physical-optimizer" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a49dd3a76e4879012682c59dc3735282e60640c97483546a4113564a60bf8f" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "datafusion-common", "datafusion-execution", "datafusion-expr", "datafusion-expr-common", + "datafusion-functions-window", "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", @@ -1470,21 +1519,22 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "arrow-data", "arrow-ipc", "arrow-ord", "arrow-schema", + "arrow-select", "async-trait", "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", @@ -1499,7 +1549,7 @@ dependencies = [ "itertools 0.15.0", "log", "num-traits", - "parking_lot", + "parking_lot 0.13.0", "pin-project-lite", "serde", "serde_json", @@ -1508,9 +1558,8 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481881a3422072554c8c26a177a4f0a67f4d8c0e2a394a79e551e6bd9a5cec4b" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "datafusion-catalog", @@ -1535,9 +1584,8 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a95763d5fa64a88b658ba4ec7314d9c538e2aea749dc88d5ce2afaafbb4c69b8" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "datafusion-common", @@ -1546,9 +1594,8 @@ dependencies = [ [[package]] name = "datafusion-proto-models" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c4982d05dee1e0b90b7f3b39845dca7eee5635c550db09158d385b8181bdc6a" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "datafusion-common", "datafusion-proto-common", @@ -1557,9 +1604,8 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f11f0dbe2b5a642f6106e44de8bbd6654fa7220ca14f32313d4efd5514a86968" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", "datafusion-common", @@ -1590,7 +1636,7 @@ dependencies = [ "log", "mimalloc", "object_store", - "parking_lot", + "parking_lot 0.12.5", "prost", "prost-types", "pyo3", @@ -1617,9 +1663,8 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62dc5ce37a5ce1ff5362949776829aded2529a11dbaa13d24b0ffa6a059e106a" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow-schema", "async-trait", @@ -1627,16 +1672,17 @@ dependencies = [ "datafusion-execution", "datafusion-expr", "datafusion-physical-plan", - "parking_lot", + "futures", + "parking_lot 0.13.0", ] [[package]] name = "datafusion-spark" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d6f76463984c4ed8587c52ed4f21f142fd96fa55d8f128f673f2db4c46942" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", + "base64 0.23.1", "bigdecimal", "chrono", "crc32fast", @@ -1647,6 +1693,7 @@ dependencies = [ "datafusion-functions-aggregate", "datafusion-functions-aggregate-common", "datafusion-functions-nested", + "datafusion-physical-expr-common", "datafusion-session", "log", "num-traits", @@ -1661,11 +1708,11 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816c224a0ef5469a32a562b5d4830544f714f97c5f50142109286d40a288c7db" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "arrow", + "arrow-schema", "bigdecimal", "chrono", "datafusion-common", @@ -1681,9 +1728,8 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "55.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1192acc67dbeeea6f9a3234ef1243c5a80a028008d0959fd55e10fc49c3ab9c0" +version = "56.0.0-dev" +source = "git+https://github.com/apache/datafusion?branch=branch-56#0f3b5a08af513a8b2a1e316b4ed3bc53fb9d762f" dependencies = [ "async-recursion", "async-trait", @@ -1705,9 +1751,7 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", "crypto-common 0.1.7", - "subtle", ] [[package]] @@ -1716,9 +1760,10 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer 0.12.1", + "block-buffer", "const-oid", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -1732,6 +1777,12 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -1826,6 +1877,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.34" @@ -2134,7 +2191,6 @@ dependencies = [ "hyper", "hyper-util", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -2334,6 +2390,55 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.35" @@ -2420,9 +2525,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.190" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -2490,6 +2595,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lock_api" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65d91b22da179613dcf4d6ec8349563f52a363f6849e03b00dd0a185ec4a59af" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.34" @@ -2511,16 +2625,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest 0.10.7", -] - [[package]] name = "md-5" version = "0.11.0" @@ -2573,6 +2677,18 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "num-bigint" version = "0.4.8" @@ -2632,14 +2748,16 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.2" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" dependencies = [ "async-trait", - "base64 0.22.1", + "aws-lc-rs", + "base64 0.23.1", "bytes", "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -2649,14 +2767,14 @@ dependencies = [ "httparse", "humantime", "hyper", - "itertools 0.14.0", - "md-5 0.10.6", - "parking_lot", + "itertools 0.15.0", + "md-5", + "nix", + "parking_lot 0.12.5", "percent-encoding", "quick-xml", "rand 0.10.3", "reqwest", - "ring", "rustls-pki-types", "serde", "serde_json", @@ -2668,6 +2786,7 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] @@ -2688,8 +2807,18 @@ version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.4.14", + "parking_lot_core 0.9.12", +] + +[[package]] +name = "parking_lot" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9cf937ca4e1d0b7578784559063422fc39ae7c5a00f02471f2614d5486fd226" +dependencies = [ + "lock_api 0.5.0", + "parking_lot_core 0.10.0", ] [[package]] @@ -2700,16 +2829,28 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "parking_lot_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cfff060068cc18aa7815010fcaf80c68c7fb9e85fc7f5c0f9bf142c21d19826" +dependencies = [ + "libc", + "redox_syscall 0.9.5", "smallvec", "windows-link", ] [[package]] name = "parquet" -version = "59.3.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff322f54b1a0f9288e614ed1f2d329b380af5476420db19f46ffb865e1163d73" +checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4" dependencies = [ "ahash", "arrow-array", @@ -2736,7 +2877,7 @@ dependencies = [ "snap", "tokio", "twox-hash", - "zstd 0.13.3", + "zstd", ] [[package]] @@ -3050,9 +3191,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", "serde", @@ -3084,6 +3225,7 @@ version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ + "aws-lc-rs", "bytes", "getrandom 0.4.3", "lru-slab", @@ -3219,6 +3361,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "580c875845b41ecb1564ad1ae4703044cef774145e3d7a94dc2b3fa7c97868fd" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.13.1" @@ -3260,11 +3411,11 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.28" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures-core", "futures-util", @@ -3281,11 +3432,8 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", - "rustls-native-certs", "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", + "rustls-platform-verifier", "sync_wrapper", "tokio", "tokio-rustls", @@ -3348,8 +3496,8 @@ version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ + "aws-lc-rs", "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3378,12 +3526,40 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + [[package]] name = "rustls-webpki" version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3620,6 +3796,16 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -3660,11 +3846,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" + [[package]] name = "sqlparser" -version = "0.62.0" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +checksum = "3679862809bd1f92e563cf6fd820e28f39135303668001c10331d5027183e8b4" dependencies = [ "log", "recursive", @@ -3673,13 +3865,13 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +checksum = "2905ae8e8148d8f7f5748ea95d1e949fa4fc1d7f5161547b758ad118b2e4b0aa" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -4279,9 +4471,9 @@ dependencies = [ [[package]] name = "wasm-streams" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ "futures-util", "js-sys", @@ -4310,6 +4502,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "winapi-util" version = "0.1.11" @@ -4596,31 +4797,13 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe 7.3.0", -] - [[package]] name = "zstd" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" dependencies = [ - "zstd-safe 8.0.0", -] - -[[package]] -name = "zstd-safe" -version = "7.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" -dependencies = [ - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index e25cdf2a2..e2cc07c02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,21 +41,21 @@ pyo3 = { version = "0.29" } pyo3-async-runtimes = { version = "0.29" } pyo3-log = "0.13.3" chrono = { version = "0.4", default-features = false } -arrow = { version = "59" } -arrow-array = { version = "59" } -arrow-schema = { version = "59" } -arrow-select = { version = "59" } -datafusion = { version = "55.1.0" } -datafusion-substrait = { version = "55.1.0" } -datafusion-proto = { version = "55.1.0" } -datafusion-ffi = { version = "55.1.0" } -datafusion-catalog = { version = "55.1.0", default-features = false } -datafusion-common = { version = "55.1.0", default-features = false } -datafusion-functions-aggregate = { version = "55.1.0" } -datafusion-functions-window = { version = "55.1.0" } -datafusion-spark = { version = "55.1.0" } -datafusion-expr = { version = "55.1.0" } -datafusion-session = { version = "55.1.0" } +arrow = { version = "60" } +arrow-array = { version = "60" } +arrow-schema = { version = "60" } +arrow-select = { version = "60" } +datafusion = { version = "56.0.0-dev" } +datafusion-substrait = { version = "56.0.0-dev" } +datafusion-proto = { version = "56.0.0-dev" } +datafusion-ffi = { version = "56.0.0-dev" } +datafusion-catalog = { version = "56.0.0-dev", default-features = false } +datafusion-common = { version = "56.0.0-dev", default-features = false } +datafusion-functions-aggregate = { version = "56.0.0-dev" } +datafusion-functions-window = { version = "56.0.0-dev" } +datafusion-spark = { version = "56.0.0-dev" } +datafusion-expr = { version = "56.0.0-dev" } +datafusion-session = { version = "56.0.0-dev" } prost = "0.14.3" serde_json = "1" uuid = { version = "1.23" } @@ -63,7 +63,7 @@ mimalloc = { version = "0.1", default-features = false } async-trait = "0.1.89" futures = "0.3" cstr = "0.2" -object_store = { version = "0.13.1" } +object_store = { version = "0.14" } url = "2" log = "0.4.29" parking_lot = "0.12" @@ -78,3 +78,14 @@ codegen-units = 2 # We cannot publish to crates.io with any patches in the below section. Developers # must remove any entries in this section before creating a release candidate. [patch.crates-io] +datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion-substrait = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion-proto = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion-ffi = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion-catalog = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-common = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion-functions-window = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion-spark = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion-expr = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion-session = { git = "https://github.com/apache/datafusion", branch = "branch-56"} diff --git a/crates/core/src/context.rs b/crates/core/src/context.rs index fe75668fb..c862dd4da 100644 --- a/crates/core/src/context.rs +++ b/crates/core/src/context.rs @@ -22,7 +22,7 @@ use std::str::FromStr; use std::sync::Arc; use arrow::array::RecordBatchReader; -use arrow::ffi_stream::ArrowArrayStreamReader; +use arrow::ffi_stream::RecordBatchStreamReader; use arrow::pyarrow::FromPyArrow; use datafusion::arrow::datatypes::{DataType, Schema, SchemaRef}; use datafusion::arrow::pyarrow::PyArrowType; @@ -695,7 +695,7 @@ impl PySessionContext { py: Python, ) -> PyDataFusionResult { let (schema, batches) = - if let Ok(stream_reader) = ArrowArrayStreamReader::from_pyarrow_bound(&data) { + if let Ok(stream_reader) = RecordBatchStreamReader::from_pyarrow_bound(&data) { // Works for any object that implements __arrow_c_stream__ in pycapsule. let schema = stream_reader.schema().as_ref().to_owned(); diff --git a/crates/core/src/dataframe.rs b/crates/core/src/dataframe.rs index c8d907bde..bf7103266 100644 --- a/crates/core/src/dataframe.rs +++ b/crates/core/src/dataframe.rs @@ -33,6 +33,9 @@ use datafusion::arrow::pyarrow::{PyArrowType, ToPyArrow}; use datafusion::arrow::util::pretty; use datafusion::catalog::TableProvider; use datafusion::common::UnnestOptions; +use datafusion::common::parquet_config::{ + DFParquetCompression, DFParquetEncoding, DFParquetStatistics, DFParquetWriterVersion, +}; use datafusion::config::{CsvOptions, ParquetColumnOptions, ParquetOptions, TableParquetOptions}; use datafusion::dataframe::{DataFrame, DataFrameWriteOptions}; use datafusion::error::DataFusionError; @@ -229,8 +232,22 @@ impl PyParquetWriterOptions { maximum_buffered_record_batches_per_stream: usize, ) -> PyResult { let writer_version = - datafusion::common::parquet_config::DFParquetWriterVersion::from_str(writer_version) - .map_err(py_datafusion_err)?; + DFParquetWriterVersion::from_str(writer_version).map_err(py_datafusion_err)?; + let compression = compression + .as_deref() + .map(DFParquetCompression::from_str) + .transpose() + .map_err(py_datafusion_err)?; + let statistics_enabled = statistics_enabled + .as_deref() + .map(DFParquetStatistics::from_str) + .transpose() + .map_err(py_datafusion_err)?; + let encoding = encoding + .as_deref() + .map(DFParquetEncoding::from_str) + .transpose() + .map_err(py_datafusion_err)?; Ok(Self { options: ParquetOptions { data_pagesize_limit, @@ -1120,7 +1137,7 @@ impl PyDataFrame { } let mut options = TableParquetOptions::default(); - options.global.compression = Some(compression_string); + options.global.compression = Some(DFParquetCompression::from_str(&compression_string)?); let write_options = write_options .map(DataFrameWriteOptions::from) .unwrap_or_default(); diff --git a/crates/core/src/dataset.rs b/crates/core/src/dataset.rs index 2a5770338..17456963b 100644 --- a/crates/core/src/dataset.rs +++ b/crates/core/src/dataset.rs @@ -89,7 +89,7 @@ impl TableProvider for Dataset { async fn scan( &self, _ctx: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], // limit can be used to reduce the amount scanned // from the datasource as a performance optimization. @@ -99,7 +99,7 @@ impl TableProvider for Dataset { ) -> DFResult> { Python::attach(|py| { let plan: Arc = Arc::new( - DatasetExec::new(py, self.dataset.bind(py), projection.cloned(), filters) + DatasetExec::new(py, self.dataset.bind(py), projection, filters) .map_err(|err| DataFusionError::External(Box::new(err)))?, ); Ok(plan) diff --git a/crates/core/src/dataset_exec.rs b/crates/core/src/dataset_exec.rs index 963603c8b..3af2f5652 100644 --- a/crates/core/src/dataset_exec.rs +++ b/crates/core/src/dataset_exec.rs @@ -78,7 +78,7 @@ impl DatasetExec { pub fn new( py: Python, dataset: &Bound<'_, PyAny>, - projection: Option>, + projection: Option<&[usize]>, filters: &[Expr], ) -> PyDataFusionResult { let columns: Option>> = projection.map(|p| { diff --git a/crates/core/src/expr.rs b/crates/core/src/expr.rs index 11961521f..6e97cf9c5 100644 --- a/crates/core/src/expr.rs +++ b/crates/core/src/expr.rs @@ -63,6 +63,7 @@ pub mod aggregate; pub mod aggregate_expr; pub mod alias; pub mod analyze; +pub mod asof_join; pub mod between; pub mod binary_expr; pub mod bool_expr; @@ -73,14 +74,15 @@ pub mod conditional_expr; pub mod copy_to; pub mod create_catalog; pub mod create_catalog_schema; +pub mod create_external_catalog; pub mod create_external_table; pub mod create_function; pub mod create_index; pub mod create_memory_table; pub mod create_view; -pub mod describe_table; pub mod distinct; pub mod dml; +pub mod drop_catalog; pub mod drop_catalog_schema; pub mod drop_function; pub mod drop_table; @@ -101,6 +103,7 @@ pub mod like; pub mod limit; pub mod literal; pub mod logical_node; +pub mod materialized_cte; pub mod placeholder; pub mod projection; pub mod recursive_query; @@ -108,6 +111,7 @@ pub mod repartition; pub mod scalar_subquery; pub mod scalar_variable; pub mod set_comparison; +pub mod show; pub mod signature; pub mod sort; pub mod sort_expr; @@ -1091,13 +1095,19 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/crates/core/src/expr/asof_join.rs b/crates/core/src/expr/asof_join.rs new file mode 100644 index 000000000..34e1f8f10 --- /dev/null +++ b/crates/core/src/expr/asof_join.rs @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::{self, Display, Formatter}; + +use datafusion::logical_expr::logical_plan::{AsOfJoin, AsOfMatch}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::*; + +use crate::common::df_schema::PyDFSchema; +use crate::expr::PyExpr; +use crate::expr::join::PyJoinConstraint; +use crate::expr::logical_node::LogicalNode; +use crate::sql::logical::PyLogicalPlan; + +#[derive(Debug, Clone)] +#[pyclass(from_py_object, frozen, name = "AsOfMatch", module = "datafusion.expr")] +pub struct PyAsOfMatch { + match_condition: AsOfMatch, +} + +impl From for PyAsOfMatch { + fn from(match_condition: AsOfMatch) -> PyAsOfMatch { + PyAsOfMatch { match_condition } + } +} + +impl From for AsOfMatch { + fn from(match_condition: PyAsOfMatch) -> Self { + match_condition.match_condition + } +} + +#[pymethods] +impl PyAsOfMatch { + fn left(&self) -> PyResult { + Ok(self.match_condition.left.clone().into()) + } + + fn op(&self) -> PyResult { + Ok(self.match_condition.op.to_string()) + } + + fn right(&self) -> PyResult { + Ok(self.match_condition.right.clone().into()) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("{}", self.match_condition)) + } +} + +impl Display for PyAsOfMatch { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}", self.match_condition) + } +} + +#[pyclass( + from_py_object, + frozen, + name = "AsOfJoin", + module = "datafusion.expr", + subclass +)] +#[derive(Clone)] +pub struct PyAsOfJoin { + join: AsOfJoin, +} + +impl From for PyAsOfJoin { + fn from(join: AsOfJoin) -> PyAsOfJoin { + PyAsOfJoin { join } + } +} + +impl From for AsOfJoin { + fn from(join: PyAsOfJoin) -> Self { + join.join + } +} + +impl Display for PyAsOfJoin { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "AsOfJoin + Left: {:?} + Right: {:?} + On: {:?} + MatchCondition: {:?} + JoinConstraint: {:?} + Schema: {:?}", + self.join.left, + self.join.right, + self.join.on, + self.join.match_condition, + self.join.join_constraint, + self.join.schema, + ) + } +} + +#[pymethods] +impl PyAsOfJoin { + /// Retrieves the left input `LogicalPlan` to this `AsOfJoin` node + fn left(&self) -> PyResult { + Ok(self.join.left.as_ref().clone().into()) + } + + /// Retrieves the right input `LogicalPlan` to this `AsOfJoin` node + fn right(&self) -> PyResult { + Ok(self.join.right.as_ref().clone().into()) + } + + /// Retrieves the equality keys of this `AsOfJoin` node + fn on(&self) -> PyResult> { + Ok(self + .join + .on + .iter() + .map(|(l, r)| (PyExpr::from(l.clone()), PyExpr::from(r.clone()))) + .collect()) + } + + /// Retrieves the `AsOfMatch` condition of this `AsOfJoin` node + fn match_condition(&self) -> PyResult { + Ok(self.join.match_condition.as_ref().clone().into()) + } + + /// Retrieves the `JoinConstraint` to this `AsOfJoin` node + fn join_constraint(&self) -> PyResult { + Ok(self.join.join_constraint.into()) + } + + /// Resulting Schema for this `AsOfJoin` node instance + fn schema(&self) -> PyResult { + Ok(self.join.schema.as_ref().clone().into()) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("AsOfJoin({self})")) + } + + fn __name__(&self) -> PyResult { + Ok("AsOfJoin".to_string()) + } +} + +impl LogicalNode for PyAsOfJoin { + fn inputs(&self) -> Vec { + vec![ + PyLogicalPlan::from((*self.join.left).clone()), + PyLogicalPlan::from((*self.join.right).clone()), + ] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/crates/core/src/expr/create_external_catalog.rs b/crates/core/src/expr/create_external_catalog.rs new file mode 100644 index 000000000..bf251f697 --- /dev/null +++ b/crates/core/src/expr/create_external_catalog.rs @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::collections::HashMap; +use std::fmt::{self, Display, Formatter}; + +use datafusion::logical_expr::CreateExternalCatalog; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::*; + +use super::logical_node::LogicalNode; +use crate::common::df_schema::PyDFSchema; +use crate::sql::logical::PyLogicalPlan; + +#[pyclass( + from_py_object, + frozen, + name = "CreateExternalCatalog", + module = "datafusion.expr", + subclass +)] +#[derive(Clone)] +pub struct PyCreateExternalCatalog { + create: CreateExternalCatalog, +} + +impl From for CreateExternalCatalog { + fn from(create: PyCreateExternalCatalog) -> Self { + create.create + } +} + +impl From for PyCreateExternalCatalog { + fn from(create: CreateExternalCatalog) -> PyCreateExternalCatalog { + PyCreateExternalCatalog { create } + } +} + +impl Display for PyCreateExternalCatalog { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "CreateExternalCatalog: name={} catalog_type={}", + self.create.catalog_name, self.create.catalog_type + ) + } +} + +#[pymethods] +impl PyCreateExternalCatalog { + fn catalog_name(&self) -> String { + self.create.catalog_name.clone() + } + + fn catalog_type(&self) -> String { + self.create.catalog_type.clone() + } + + fn location(&self) -> Option { + self.create.location.clone() + } + + fn if_not_exists(&self) -> bool { + self.create.if_not_exists + } + + fn or_replace(&self) -> bool { + self.create.or_replace + } + + fn options(&self) -> HashMap { + self.create.options.clone() + } + + fn schema(&self) -> PyDFSchema { + (*self.create.schema).clone().into() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("CreateExternalCatalog({self})")) + } + + fn __name__(&self) -> PyResult { + Ok("CreateExternalCatalog".to_string()) + } +} + +impl LogicalNode for PyCreateExternalCatalog { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/crates/core/src/expr/drop_catalog.rs b/crates/core/src/expr/drop_catalog.rs new file mode 100644 index 000000000..557d78bf1 --- /dev/null +++ b/crates/core/src/expr/drop_catalog.rs @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::{self, Display, Formatter}; + +use datafusion::logical_expr::DropCatalog; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::*; + +use super::logical_node::LogicalNode; +use crate::common::df_schema::PyDFSchema; +use crate::sql::logical::PyLogicalPlan; + +#[pyclass( + from_py_object, + frozen, + name = "DropCatalog", + module = "datafusion.expr", + subclass +)] +#[derive(Clone)] +pub struct PyDropCatalog { + drop: DropCatalog, +} + +impl From for DropCatalog { + fn from(drop: PyDropCatalog) -> Self { + drop.drop + } +} + +impl From for PyDropCatalog { + fn from(drop: DropCatalog) -> PyDropCatalog { + PyDropCatalog { drop } + } +} + +impl Display for PyDropCatalog { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "DropCatalog: name={}", self.drop.name) + } +} + +#[pymethods] +impl PyDropCatalog { + fn name(&self) -> String { + self.drop.name.clone() + } + + fn if_exists(&self) -> bool { + self.drop.if_exists + } + + fn cascade(&self) -> bool { + self.drop.cascade + } + + fn schema(&self) -> PyDFSchema { + (*self.drop.schema).clone().into() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("DropCatalog({self})")) + } + + fn __name__(&self) -> PyResult { + Ok("DropCatalog".to_string()) + } +} + +impl LogicalNode for PyDropCatalog { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/crates/core/src/expr/join.rs b/crates/core/src/expr/join.rs index 488d9a11e..8b5b9ba47 100644 --- a/crates/core/src/expr/join.rs +++ b/crates/core/src/expr/join.rs @@ -156,7 +156,7 @@ impl PyJoin { Ok(self.join.right.as_ref().clone().into()) } - /// Retrieves the right input `LogicalPlan` to this `Join` node + /// Retrieves the equality keys of this `Join` node fn on(&self) -> PyResult> { Ok(self .join diff --git a/crates/core/src/expr/materialized_cte.rs b/crates/core/src/expr/materialized_cte.rs new file mode 100644 index 000000000..a38bec1ed --- /dev/null +++ b/crates/core/src/expr/materialized_cte.rs @@ -0,0 +1,167 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::{self, Display, Formatter}; + +use datafusion::logical_expr::logical_plan::{MaterializedCte, MaterializedCteScan}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::*; + +use crate::common::df_schema::PyDFSchema; +use crate::expr::logical_node::LogicalNode; +use crate::sql::logical::PyLogicalPlan; + +#[pyclass( + from_py_object, + frozen, + name = "MaterializedCte", + module = "datafusion.expr", + subclass +)] +#[derive(Clone)] +pub struct PyMaterializedCte { + cte: MaterializedCte, +} + +impl From for PyMaterializedCte { + fn from(cte: MaterializedCte) -> Self { + Self { cte } + } +} + +impl From for MaterializedCte { + fn from(cte: PyMaterializedCte) -> Self { + cte.cte + } +} + +impl Display for PyMaterializedCte { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "MaterializedCte: name={} id={}", + self.cte.name, self.cte.id + ) + } +} + +#[pymethods] +impl PyMaterializedCte { + fn name(&self) -> String { + self.cte.name.clone() + } + + fn id(&self) -> u64 { + self.cte.id.as_u64() + } + + fn cte(&self) -> PyLogicalPlan { + PyLogicalPlan::from((*self.cte.cte).clone()) + } + + fn continuation(&self) -> PyLogicalPlan { + PyLogicalPlan::from((*self.cte.continuation).clone()) + } + + fn __repr__(&self) -> PyResult { + Ok(format!("MaterializedCte({self})")) + } + + fn __name__(&self) -> PyResult { + Ok("MaterializedCte".to_string()) + } +} + +impl LogicalNode for PyMaterializedCte { + fn inputs(&self) -> Vec { + vec![ + PyLogicalPlan::from((*self.cte.cte).clone()), + PyLogicalPlan::from((*self.cte.continuation).clone()), + ] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} + +#[pyclass( + from_py_object, + frozen, + name = "MaterializedCteScan", + module = "datafusion.expr", + subclass +)] +#[derive(Clone)] +pub struct PyMaterializedCteScan { + scan: MaterializedCteScan, +} + +impl From for PyMaterializedCteScan { + fn from(scan: MaterializedCteScan) -> Self { + Self { scan } + } +} + +impl From for MaterializedCteScan { + fn from(scan: PyMaterializedCteScan) -> Self { + scan.scan + } +} + +impl Display for PyMaterializedCteScan { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "MaterializedCteScan: name={} id={}", + self.scan.name, self.scan.id + ) + } +} + +#[pymethods] +impl PyMaterializedCteScan { + fn name(&self) -> String { + self.scan.name.clone() + } + + fn id(&self) -> u64 { + self.scan.id.as_u64() + } + + fn schema(&self) -> PyDFSchema { + self.scan.schema.as_ref().clone().into() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("MaterializedCteScan({self})")) + } + + fn __name__(&self) -> PyResult { + Ok("MaterializedCteScan".to_string()) + } +} + +impl LogicalNode for PyMaterializedCteScan { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/crates/core/src/expr/show.rs b/crates/core/src/expr/show.rs new file mode 100644 index 000000000..0e848e2c4 --- /dev/null +++ b/crates/core/src/expr/show.rs @@ -0,0 +1,114 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fmt::{self, Display, Formatter}; +use std::sync::Arc; + +use arrow::datatypes::Schema; +use arrow::pyarrow::PyArrowType; +use datafusion::logical_expr::{Show, ShowKind}; +use pyo3::IntoPyObjectExt; +use pyo3::prelude::*; + +use super::logical_node::LogicalNode; +use crate::common::df_schema::PyDFSchema; +use crate::sql::logical::PyLogicalPlan; + +#[pyclass( + from_py_object, + frozen, + name = "Show", + module = "datafusion.expr", + subclass +)] +#[derive(Clone)] +pub struct PyShow { + show: Show, +} + +impl Display for PyShow { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "Show") + } +} + +#[pymethods] +impl PyShow { + #[new] + fn new(schema: PyArrowType, output_schema: PyDFSchema) -> Self { + Self { + show: Show { + kind: ShowKind::Describe { + schema: Arc::new(schema.0), + }, + output_schema: Arc::new(output_schema.into()), + }, + } + } + + pub fn kind(&self) -> &'static str { + match &self.show.kind { + ShowKind::Describe { .. } => "describe", + ShowKind::Tables => "tables", + ShowKind::Columns { .. } => "columns", + ShowKind::CreateTable { .. } => "create_table", + ShowKind::Variables { .. } => "variables", + ShowKind::Functions { .. } => "functions", + } + } + + pub fn schema(&self) -> Option> { + match &self.show.kind { + ShowKind::Describe { schema } => Some((**schema).clone().into()), + _ => None, + } + } + + pub fn output_schema(&self) -> PyDFSchema { + (*self.show.output_schema).clone().into() + } + + fn __repr__(&self) -> PyResult { + Ok(format!("Show({self})")) + } + + fn __name__(&self) -> PyResult { + Ok("Show".to_string()) + } +} + +impl From for Show { + fn from(show: PyShow) -> Self { + show.show + } +} + +impl From for PyShow { + fn from(show: Show) -> PyShow { + PyShow { show } + } +} + +impl LogicalNode for PyShow { + fn inputs(&self) -> Vec { + vec![] + } + + fn to_variant<'py>(&self, py: Python<'py>) -> PyResult> { + self.clone().into_bound_py_any(py) + } +} diff --git a/crates/core/src/sql/logical.rs b/crates/core/src/sql/logical.rs index 12fc43bdc..b859e7cce 100644 --- a/crates/core/src/sql/logical.rs +++ b/crates/core/src/sql/logical.rs @@ -17,7 +17,7 @@ use std::sync::Arc; -use datafusion::logical_expr::{DdlStatement, LogicalPlan, Statement}; +use datafusion::logical_expr::{DdlStatement, LogicalPlan, MaterializedCtePlan, Statement}; use datafusion_proto::logical_plan::AsLogicalPlan; use prost::Message; use pyo3::exceptions::PyRuntimeError; @@ -29,17 +29,19 @@ use crate::context::PySessionContext; use crate::errors::PyDataFusionResult; use crate::expr::aggregate::PyAggregate; use crate::expr::analyze::PyAnalyze; +use crate::expr::asof_join::PyAsOfJoin; use crate::expr::copy_to::PyCopyTo; use crate::expr::create_catalog::PyCreateCatalog; use crate::expr::create_catalog_schema::PyCreateCatalogSchema; +use crate::expr::create_external_catalog::PyCreateExternalCatalog; use crate::expr::create_external_table::PyCreateExternalTable; use crate::expr::create_function::PyCreateFunction; use crate::expr::create_index::PyCreateIndex; use crate::expr::create_memory_table::PyCreateMemoryTable; use crate::expr::create_view::PyCreateView; -use crate::expr::describe_table::PyDescribeTable; use crate::expr::distinct::PyDistinct; use crate::expr::dml::PyDmlStatement; +use crate::expr::drop_catalog::PyDropCatalog; use crate::expr::drop_catalog_schema::PyDropCatalogSchema; use crate::expr::drop_function::PyDropFunction; use crate::expr::drop_table::PyDropTable; @@ -51,9 +53,11 @@ use crate::expr::filter::PyFilter; use crate::expr::join::PyJoin; use crate::expr::limit::PyLimit; use crate::expr::logical_node::LogicalNode; +use crate::expr::materialized_cte::{PyMaterializedCte, PyMaterializedCteScan}; use crate::expr::projection::PyProjection; use crate::expr::recursive_query::PyRecursiveQuery; use crate::expr::repartition::PyRepartition; +use crate::expr::show::PyShow; use crate::expr::sort::PySort; use crate::expr::statement::{ PyDeallocate, PyExecute, PyPrepare, PyResetVariable, PySetVariable, PyTransactionEnd, @@ -137,6 +141,9 @@ impl PyLogicalPlan { DdlStatement::CreateExternalTable(plan) => { PyCreateExternalTable::from(plan.as_ref().clone()).to_variant(py) } + DdlStatement::CreateExternalCatalog(plan) => { + PyCreateExternalCatalog::from(plan.as_ref().clone()).to_variant(py) + } DdlStatement::CreateMemoryTable(plan) => { PyCreateMemoryTable::from(plan.clone()).to_variant(py) } @@ -150,6 +157,7 @@ impl PyLogicalPlan { DdlStatement::CreateIndex(plan) => PyCreateIndex::from(plan.clone()).to_variant(py), DdlStatement::DropTable(plan) => PyDropTable::from(plan.clone()).to_variant(py), DdlStatement::DropView(plan) => PyDropView::from(plan.clone()).to_variant(py), + DdlStatement::DropCatalog(plan) => PyDropCatalog::from(plan.clone()).to_variant(py), DdlStatement::DropCatalogSchema(plan) => { PyDropCatalogSchema::from(plan.clone()).to_variant(py) } @@ -161,10 +169,19 @@ impl PyLogicalPlan { } }, LogicalPlan::Copy(plan) => PyCopyTo::from(plan.clone()).to_variant(py), - LogicalPlan::DescribeTable(plan) => PyDescribeTable::from(plan.clone()).to_variant(py), + LogicalPlan::Show(plan) => PyShow::from(plan.clone()).to_variant(py), LogicalPlan::RecursiveQuery(plan) => { PyRecursiveQuery::from(plan.clone()).to_variant(py) } + LogicalPlan::AsOfJoin(plan) => PyAsOfJoin::from(plan.clone()).to_variant(py), + LogicalPlan::MaterializedCte(plan) => match plan { + MaterializedCtePlan::Cte(cte) => { + PyMaterializedCte::from(cte.clone()).to_variant(py) + } + MaterializedCtePlan::Scan(scan) => { + PyMaterializedCteScan::from(scan.clone()).to_variant(py) + } + }, } } diff --git a/crates/core/src/table.rs b/crates/core/src/table.rs index e0f0f0d13..dca6671a9 100644 --- a/crates/core/src/table.rs +++ b/crates/core/src/table.rs @@ -160,7 +160,7 @@ impl TableProvider for TempViewTable { async fn scan( &self, state: &dyn Session, - projection: Option<&Vec>, + projection: Option<&[usize]>, filters: &[Expr], limit: Option, ) -> datafusion::common::Result> { @@ -174,8 +174,7 @@ impl TableProvider for TempViewTable { let mut plan = if let Some(projection) = projection { // avoiding adding a redundant projection (e.g. SELECT * FROM view) - let current_projection = (0..plan.schema().fields().len()).collect::>(); - if projection == ¤t_projection { + if projection.iter().copied().eq(0..plan.schema().fields().len()) { plan } else { let fields: Vec = projection diff --git a/docs/source/user-guide/upgrade-guides.md b/docs/source/user-guide/upgrade-guides.md index 74318c83c..bd375e5f1 100644 --- a/docs/source/user-guide/upgrade-guides.md +++ b/docs/source/user-guide/upgrade-guides.md @@ -43,6 +43,28 @@ expr = f.first_value(column("a")).over( ) ``` +### `DescribeTable` logical node replaced by `Show` + +The `DescribeTable` logical node was replaced by the `Show` node, which +processes `DESCRIBE`, `SHOW TABLES`, `SHOW COLUMNS`, `SHOW CREATE TABLE`, +`SHOW FUNCTIONS`, and `SHOW `. Use the `Show.kind()` method to +determine the type. Example: + +```python +plan = ctx.sql("describe foo").logical_plan().to_variant() +assert isinstance(plan, Show) +plan.kind() # "columns" + +plan = ctx.sql("describe select a from foo").logical_plan().to_variant() +assert isinstance(plan, Show) +plan.kind() # "describe" + +plan = ctx.sql("show tables").logical_plan().to_variant() +assert isinstance(plan, Show) +plan.kind() # "tables" +``` + + ## DataFusion 55.0.0 This release extends the change made in 52.0.0 to the remaining diff --git a/python/datafusion/expr.py b/python/datafusion/expr.py index d6cf85bc1..9ca580c39 100644 --- a/python/datafusion/expr.py +++ b/python/datafusion/expr.py @@ -85,6 +85,8 @@ Alias = expr_internal.Alias Analyze = expr_internal.Analyze +AsOfJoin = expr_internal.AsOfJoin +AsOfMatch = expr_internal.AsOfMatch Aggregate = expr_internal.Aggregate AggregateFunction = expr_internal.AggregateFunction Between = expr_internal.Between @@ -95,6 +97,7 @@ CopyTo = expr_internal.CopyTo CreateCatalog = expr_internal.CreateCatalog CreateCatalogSchema = expr_internal.CreateCatalogSchema +CreateExternalCatalog = expr_internal.CreateExternalCatalog CreateExternalTable = expr_internal.CreateExternalTable @@ -123,9 +126,9 @@ def _create_external_table_location(self: Any) -> str: CreateMemoryTable = expr_internal.CreateMemoryTable CreateView = expr_internal.CreateView Deallocate = expr_internal.Deallocate -DescribeTable = expr_internal.DescribeTable Distinct = expr_internal.Distinct DmlStatement = expr_internal.DmlStatement +DropCatalog = expr_internal.DropCatalog DropCatalogSchema = expr_internal.DropCatalogSchema DropFunction = expr_internal.DropFunction DropTable = expr_internal.DropTable @@ -154,6 +157,8 @@ def _create_external_table_location(self: Any) -> str: Like = expr_internal.Like Limit = expr_internal.Limit Literal = expr_internal.Literal +MaterializedCte = expr_internal.MaterializedCte +MaterializedCteScan = expr_internal.MaterializedCteScan Negative = expr_internal.Negative Not = expr_internal.Not OperateFunctionArg = expr_internal.OperateFunctionArg @@ -166,6 +171,7 @@ def _create_external_table_location(self: Any) -> str: ScalarSubquery = expr_internal.ScalarSubquery ScalarVariable = expr_internal.ScalarVariable SetVariable = expr_internal.SetVariable +Show = expr_internal.Show SimilarTo = expr_internal.SimilarTo Sort = expr_internal.Sort Subquery = expr_internal.Subquery @@ -192,6 +198,8 @@ def _create_external_table_location(self: Any) -> str: "AggregateFunction", "Alias", "Analyze", + "AsOfJoin", + "AsOfMatch", "Between", "BinaryExpr", "Case", @@ -201,6 +209,7 @@ def _create_external_table_location(self: Any) -> str: "CopyTo", "CreateCatalog", "CreateCatalogSchema", + "CreateExternalCatalog", "CreateExternalTable", "CreateFunction", "CreateFunctionBody", @@ -208,9 +217,9 @@ def _create_external_table_location(self: Any) -> str: "CreateMemoryTable", "CreateView", "Deallocate", - "DescribeTable", "Distinct", "DmlStatement", + "DropCatalog", "DropCatalogSchema", "DropFunction", "DropTable", @@ -245,6 +254,8 @@ def _create_external_table_location(self: Any) -> str: "Limit", "Literal", "Literal", + "MaterializedCte", + "MaterializedCteScan", "Negative", "Not", "OperateFunctionArg", @@ -257,6 +268,7 @@ def _create_external_table_location(self: Any) -> str: "ScalarSubquery", "ScalarVariable", "SetVariable", + "Show", "SimilarTo", "Sort", "SortExpr", diff --git a/python/datafusion/functions/__init__.py b/python/datafusion/functions/__init__.py index c7d99f786..6e6c9bb13 100644 --- a/python/datafusion/functions/__init__.py +++ b/python/datafusion/functions/__init__.py @@ -5349,7 +5349,7 @@ def map_values(map: Expr) -> Expr: def map_extract(map: Expr, key: Expr) -> Expr: """Returns the value for a given key in the map. - Returns ``[None]`` if the key is absent. + Returns ``[]`` if the key is absent. Examples: >>> ctx = dfn.SessionContext() @@ -5385,7 +5385,7 @@ def map_entries(map: Expr) -> Expr: def element_at(map: Expr, key: Expr) -> Expr: """Returns the value for a given key in the map. - Returns ``[None]`` if the key is absent. + Returns ``[]`` if the key is absent. See Also: This is an alias for :py:func:`map_extract`. diff --git a/python/tests/test_context.py b/python/tests/test_context.py index 274f2a9b1..81abd5931 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -1669,7 +1669,7 @@ def test_sql_with_options_no_dml(ctx): def test_sql_with_options_no_statements(ctx): - sql = "SET time zone = 1;" + sql = "SET time zone = 'UTC';" ctx.sql(sql) options = SQLOptions().with_allow_statements(allow=False) with pytest.raises(Exception, match="SetVariable"): @@ -1789,7 +1789,7 @@ def test_read_csv_with_options(tmp_path, as_read, global_ctx): [ pa.array(["Alice", "Bob", "Phil", "Karin"]), pa.array([30, 25, 75, 50]), - pa.array(["New York; NY", None, "Detroit' MI", "Stockholm\nSweden"]), + pa.array(["New York; NY", "", "Detroit' MI", "Stockholm\nSweden"]), ], names=["column_1", "column_2", "column_3"], ) @@ -1811,7 +1811,7 @@ def test_read_csv_with_options(tmp_path, as_read, global_ctx): [ pa.array(["Alice", "Bob", "Phil", "Karin"]), pa.array([30.0, 25.0, 75.0, 50.0]), - pa.array(["New York; NY", None, "Detroit' MI", "Stockholm\nSweden"]), + pa.array(["New York; NY", "", "Detroit' MI", "Stockholm\nSweden"]), ], schema=schema, ) diff --git a/python/tests/test_dataframe.py b/python/tests/test_dataframe.py index 1b2bbfcd9..141298275 100644 --- a/python/tests/test_dataframe.py +++ b/python/tests/test_dataframe.py @@ -2719,7 +2719,9 @@ def test_write_parquet_with_options_compression(df, tmp_path, compression): def test_write_parquet_with_options_wrong_compression_level(df, tmp_path, compression): path = tmp_path - with pytest.raises(Exception, match=r"valid compression range .*? exceeded."): + with pytest.raises( + Exception, match=r"Invalid compression level \d+ for \w+\. Expected a level" + ): df.write_parquet_with_options( str(path), ParquetWriterOptions(compression=compression) ) diff --git a/python/tests/test_expr.py b/python/tests/test_expr.py index aca01f9b3..e963f9ea6 100644 --- a/python/tests/test_expr.py +++ b/python/tests/test_expr.py @@ -44,7 +44,6 @@ Column, CopyTo, CreateIndex, - DescribeTable, DmlStatement, DropCatalogSchema, Filter, @@ -52,6 +51,7 @@ Literal, Projection, RecursiveQuery, + Show, Sort, TableScan, TransactionEnd, @@ -383,12 +383,25 @@ def test_create_index(): assert isinstance(plan, CreateIndex) -def test_describe_table(): +@pytest.mark.parametrize( + ("sql", "kind"), + [ + ("describe foo", "columns"), + ("describe select a from foo", "describe"), + ("show tables", "tables"), + ("show columns from foo", "columns"), + ("show create table foo", "create_table"), + ("show datafusion.execution.batch_size", "variables"), + ("show functions", "functions"), + ], +) +def test_describe_table(sql, kind): ctx = SessionContext() ctx.sql("CREATE TABLE foo (a int, b int)").collect() - plan = ctx.sql("describe foo").logical_plan() + plan = ctx.sql(sql).logical_plan() plan = plan.to_variant() - assert isinstance(plan, DescribeTable) + assert isinstance(plan, Show) + assert plan.kind() == kind def test_dml_statement(): @@ -422,6 +435,65 @@ def test_recursive_query(): assert isinstance(plan, RecursiveQuery) +def test_materialized_cte(): + ctx = SessionContext() + ctx.sql("SET datafusion.sql_parser.dialect = 'PostgreSQL'") + df = ctx.sql( + """ + WITH t (a) AS MATERIALIZED ( + SELECT 1 + ) + SELECT * FROM t + UNION ALL + SELECT * FROM t + """ + ) + + assert df.optimized_logical_plan().display_indent() == ( + "MaterializedCte: name=t\n" + " SubqueryAlias: t\n" + " Projection: Int64(1) AS a\n" + " EmptyRelation: rows=1\n" + " Union\n" + " MaterializedCteScan: name=t\n" + " MaterializedCteScan: name=t" + ) + + result = pa.Table.from_batches(df.collect()) + assert result.column("a").to_pylist() == [1, 1] + + +def test_asof_join(): + ctx = SessionContext() + df = ctx.sql( + """ + WITH t1(k, ts, a) AS (VALUES (1, 1, 10), (1, 4, 20), (2, 2, 30)), + t2(k, ts, b) AS (VALUES (1, 2, 100), (1, 4, 200), (2, 1, 300)) + SELECT t1.a, t2.b + FROM t1 + ASOF JOIN t2 + MATCH_CONDITION (t1.ts <= t2.ts) + ON t1.k = t2.k + """ + ) + + assert df.optimized_logical_plan().display_indent() == ( + "Projection: t1.a, t2.b\n" + " AsOf Join: match=[t1.ts <= t2.ts], constraint=On, on=[t1.k = t2.k]\n" + " SubqueryAlias: t1\n" + " Projection: column1 AS k, column2 AS ts, column3 AS a\n" + " Values: (Int64(1), Int64(1), Int64(10)), " + "(Int64(1), Int64(4), Int64(20)), (Int64(2), Int64(2), Int64(30))\n" + " SubqueryAlias: t2\n" + " Projection: column1 AS k, column2 AS ts, column3 AS b\n" + " Values: (Int64(1), Int64(2), Int64(100)), " + "(Int64(1), Int64(4), Int64(200)), (Int64(2), Int64(1), Int64(300))" + ) + + result = pa.Table.from_batches(df.collect()).sort_by("a") + assert result.to_pydict() == {"a": [10, 20, 30], "b": [100, 200, None]} + + def test_values(): ctx = SessionContext() plan = ctx.sql("values (1, 'foo'), (2, 'bar')").logical_plan() diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 9cec531f9..e3442c59c 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -942,7 +942,7 @@ def test_make_map_mismatched_lengths(): ), pytest.param( lambda m: f.map_extract(m, literal("z")), - [None], + [], id="map_extract_missing_key", ), pytest.param( diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 5639a821f..d3483e9f7 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -19,5 +19,5 @@ # to compile this workspace and run CI jobs. [toolchain] -channel = "1.97.0" +channel = "1.99.0" components = ["rustfmt", "clippy"] From 0061384ced2631c2e5cbe2c037572f84e1c600c5 Mon Sep 17 00:00:00 2001 From: nuno-faria Date: Sat, 7 Nov 2026 16:33:12 +0000 Subject: [PATCH 2/3] Fix lint --- Cargo.toml | 20 ++++++++++---------- crates/core/src/table.rs | 6 +++++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e2cc07c02..ef1720340 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,7 +67,7 @@ object_store = { version = "0.14" } url = "2" log = "0.4.29" parking_lot = "0.12" -prost-types = "0.14.3" # keep in line with `datafusion-substrait` +prost-types = "0.14.3" # keep in line with `datafusion-substrait` pyo3-build-config = "0.29" datafusion-python-util = { path = "crates/util", version = "55.0.0" } @@ -78,14 +78,14 @@ codegen-units = 2 # We cannot publish to crates.io with any patches in the below section. Developers # must remove any entries in this section before creating a release candidate. [patch.crates-io] -datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-56"} -datafusion-substrait = { git = "https://github.com/apache/datafusion", branch = "branch-56"} -datafusion-proto = { git = "https://github.com/apache/datafusion", branch = "branch-56"} -datafusion-ffi = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-substrait = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-proto = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-ffi = { git = "https://github.com/apache/datafusion", branch = "branch-56" } datafusion-catalog = { git = "https://github.com/apache/datafusion", branch = "branch-56" } datafusion-common = { git = "https://github.com/apache/datafusion", branch = "branch-56" } -datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", branch = "branch-56"} -datafusion-functions-window = { git = "https://github.com/apache/datafusion", branch = "branch-56"} -datafusion-spark = { git = "https://github.com/apache/datafusion", branch = "branch-56"} -datafusion-expr = { git = "https://github.com/apache/datafusion", branch = "branch-56"} -datafusion-session = { git = "https://github.com/apache/datafusion", branch = "branch-56"} +datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-functions-window = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-spark = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-expr = { git = "https://github.com/apache/datafusion", branch = "branch-56" } +datafusion-session = { git = "https://github.com/apache/datafusion", branch = "branch-56" } diff --git a/crates/core/src/table.rs b/crates/core/src/table.rs index dca6671a9..a7f166d5b 100644 --- a/crates/core/src/table.rs +++ b/crates/core/src/table.rs @@ -174,7 +174,11 @@ impl TableProvider for TempViewTable { let mut plan = if let Some(projection) = projection { // avoiding adding a redundant projection (e.g. SELECT * FROM view) - if projection.iter().copied().eq(0..plan.schema().fields().len()) { + if projection + .iter() + .copied() + .eq(0..plan.schema().fields().len()) + { plan } else { let fields: Vec = projection From b1b413ac9927b094cfa304f92cb5d1235ee1ba66 Mon Sep 17 00:00:00 2001 From: nuno-faria Date: Sat, 7 Nov 2026 16:52:49 +0000 Subject: [PATCH 3/3] Pin macOS abi3 Python version to 3.14 --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23570ff70..e350ece55 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -336,7 +336,7 @@ jobs: - name: Install dependencies if: matrix.python-tag == 'abi3' - run: uv sync --dev --no-install-package datafusion + run: uv sync --python 3.14 --dev --no-install-package datafusion # Clippy is interpreter-agnostic; run once per OS (against the abi3 entry) # so the matrix doesn't pay the cost three times. @@ -419,7 +419,7 @@ jobs: - name: Install dependencies if: matrix.python-tag == 'abi3' - run: uv sync --dev --no-install-package datafusion + run: uv sync --python 3.14 --dev --no-install-package datafusion - name: Build wheel uses: ./.github/actions/build-wheel