Skip to content

Commit 03e39da

Browse files
rkrishn7rkrishn7
authored
fix: Implicitly plan UNNEST as lateral (#13695)
* plan implicit lateral if table factor is UNNEST * check for outer references in `create_relation_subquery` * add sqllogictest * fix lateral constant test to not expect a subquery node * replace sqllogictest in favor of logical plan test * update lateral join sqllogictests * add sqllogictests * fix logical plan test
1 parent e8314ab commit 03e39da

File tree

5 files changed

+77
-11
lines changed

5 files changed

+77
-11
lines changed

‎datafusion/sql/src/relation/join.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,7 @@ pub(crate) fn is_lateral(factor: &TableFactor) -> bool {
163163
match factor {
164164
TableFactor::Derived { lateral, .. } => *lateral,
165165
TableFactor::Function { lateral, .. } => *lateral,
166+
TableFactor::UNNEST { .. } => true,
166167
_ => false,
167168
}
168169
}

‎datafusion/sql/src/relation/mod.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,12 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
188188
planner_context.set_outer_query_schema(old_query_schema);
189189
planner_context.set_outer_from_schema(Some(old_from_schema));
190190

191+
// We can omit the subquery wrapper if there are no columns
192+
// referencing the outer scope.
193+
if outer_ref_columns.is_empty() {
194+
return Ok(plan);
195+
}
196+
191197
match plan {
192198
LogicalPlan::SubqueryAlias(SubqueryAlias { input, alias, .. }) => {
193199
subquery_alias(

‎datafusion/sql/tests/sql_integration.rs

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3129,9 +3129,8 @@ fn lateral_constant() {
31293129
\n Cross Join: \
31303130
\n TableScan: j1\
31313131
\n SubqueryAlias: j2\
3132-
\n Subquery:\
3133-
\n Projection: Int64(1)\
3134-
\n EmptyRelation";
3132+
\n Projection: Int64(1)\
3133+
\n EmptyRelation";
31353134
quick_test(sql, expected);
31363135
}
31373136

@@ -3230,6 +3229,21 @@ fn lateral_nested_left_join() {
32303229
quick_test(sql, expected);
32313230
}
32323231

3232+
#[test]
3233+
fn lateral_unnest() {
3234+
let sql = "SELECT * from unnest_table u, unnest(u.array_col)";
3235+
let expected = "Projection: *\
3236+
\n Cross Join: \
3237+
\n SubqueryAlias: u\
3238+
\n TableScan: unnest_table\
3239+
\n Subquery:\
3240+
\n Projection: __unnest_placeholder(outer_ref(u.array_col),depth=1) AS UNNEST(outer_ref(u.array_col))\
3241+
\n Unnest: lists[__unnest_placeholder(outer_ref(u.array_col))|depth=1] structs[]\
3242+
\n Projection: outer_ref(u.array_col) AS __unnest_placeholder(outer_ref(u.array_col))\
3243+
\n EmptyRelation";
3244+
quick_test(sql, expected);
3245+
}
3246+
32333247
#[test]
32343248
fn hive_aggregate_with_filter() -> Result<()> {
32353249
let dialect = &HiveDialect {};

‎datafusion/sqllogictest/test_files/joins.slt

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4058,10 +4058,12 @@ logical_plan
40584058
03)----TableScan: join_t1 projection=[t1_id, t1_name]
40594059
04)--SubqueryAlias: series
40604060
05)----Subquery:
4061-
06)------Projection: __unnest_placeholder(generate_series(Int64(1),outer_ref(t1.t1_int)),depth=1) AS i
4062-
07)--------Unnest: lists[__unnest_placeholder(generate_series(Int64(1),outer_ref(t1.t1_int)))|depth=1] structs[]
4063-
08)----------Projection: generate_series(Int64(1), CAST(outer_ref(t1.t1_int) AS Int64)) AS __unnest_placeholder(generate_series(Int64(1),outer_ref(t1.t1_int)))
4064-
09)------------EmptyRelation
4061+
06)------Projection: UNNEST(generate_series(Int64(1),outer_ref(t1.t1_int))) AS i
4062+
07)--------Subquery:
4063+
08)----------Projection: __unnest_placeholder(generate_series(Int64(1),outer_ref(t1.t1_int)),depth=1) AS UNNEST(generate_series(Int64(1),outer_ref(t1.t1_int)))
4064+
09)------------Unnest: lists[__unnest_placeholder(generate_series(Int64(1),outer_ref(t1.t1_int)))|depth=1] structs[]
4065+
10)--------------Projection: generate_series(Int64(1), CAST(outer_ref(t1.t1_int) AS Int64)) AS __unnest_placeholder(generate_series(Int64(1),outer_ref(t1.t1_int)))
4066+
11)----------------EmptyRelation
40654067
physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(UInt32, Column { relation: Some(Bare { table: "t1" }), name: "t1_int" })
40664068

40674069

@@ -4081,10 +4083,12 @@ logical_plan
40814083
03)----TableScan: join_t1 projection=[t1_id, t1_name]
40824084
04)--SubqueryAlias: series
40834085
05)----Subquery:
4084-
06)------Projection: __unnest_placeholder(generate_series(Int64(1),outer_ref(t2.t1_int)),depth=1) AS i
4085-
07)--------Unnest: lists[__unnest_placeholder(generate_series(Int64(1),outer_ref(t2.t1_int)))|depth=1] structs[]
4086-
08)----------Projection: generate_series(Int64(1), CAST(outer_ref(t2.t1_int) AS Int64)) AS __unnest_placeholder(generate_series(Int64(1),outer_ref(t2.t1_int)))
4087-
09)------------EmptyRelation
4086+
06)------Projection: UNNEST(generate_series(Int64(1),outer_ref(t2.t1_int))) AS i
4087+
07)--------Subquery:
4088+
08)----------Projection: __unnest_placeholder(generate_series(Int64(1),outer_ref(t2.t1_int)),depth=1) AS UNNEST(generate_series(Int64(1),outer_ref(t2.t1_int)))
4089+
09)------------Unnest: lists[__unnest_placeholder(generate_series(Int64(1),outer_ref(t2.t1_int)))|depth=1] structs[]
4090+
10)--------------Projection: generate_series(Int64(1), CAST(outer_ref(t2.t1_int) AS Int64)) AS __unnest_placeholder(generate_series(Int64(1),outer_ref(t2.t1_int)))
4091+
11)----------------EmptyRelation
40884092
physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(UInt32, Column { relation: Some(Bare { table: "t2" }), name: "t1_int" })
40894093

40904094

‎datafusion/sqllogictest/test_files/unnest.slt

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -860,6 +860,47 @@ select count(*) from (select unnest(range(0, 100000)) id) t inner join (select u
860860
----
861861
100000
862862

863+
# Test implicit LATERAL support for UNNEST
864+
# Issue: https://github.com/apache/datafusion/issues/13659
865+
# TODO: https://github.com/apache/datafusion/issues/10048
866+
query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(List\(Field \{ name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: \{\} \}\), Column \{ relation: Some\(Bare \{ table: "u" \}\), name: "column1" \}\)
867+
select * from unnest_table u, unnest(u.column1);
868+
869+
# Test implicit LATERAL support for UNNEST (INNER JOIN)
870+
query error DataFusion error: This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn\(List\(Field \{ name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: \{\} \}\), Column \{ relation: Some\(Bare \{ table: "u" \}\), name: "column1" \}\)
871+
select * from unnest_table u INNER JOIN unnest(u.column1) AS t(column1) ON u.column3 = t.column1;
872+
873+
# Test implicit LATERAL planning for UNNEST
874+
query TT
875+
explain select * from unnest_table u, unnest(u.column1);
876+
----
877+
logical_plan
878+
01)Cross Join:
879+
02)--SubqueryAlias: u
880+
03)----TableScan: unnest_table projection=[column1, column2, column3, column4, column5]
881+
04)--Subquery:
882+
05)----Projection: __unnest_placeholder(outer_ref(u.column1),depth=1) AS UNNEST(outer_ref(u.column1))
883+
06)------Unnest: lists[__unnest_placeholder(outer_ref(u.column1))|depth=1] structs[]
884+
07)--------Projection: outer_ref(u.column1) AS __unnest_placeholder(outer_ref(u.column1))
885+
08)----------EmptyRelation
886+
physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), Column { relation: Some(Bare { table: "u" }), name: "column1" })
887+
888+
# Test implicit LATERAL planning for UNNEST (INNER JOIN)
889+
query TT
890+
explain select * from unnest_table u INNER JOIN unnest(u.column1) AS t(column1) ON u.column3 = t.column1;
891+
----
892+
logical_plan
893+
01)Inner Join: u.column3 = t.column1
894+
02)--SubqueryAlias: u
895+
03)----TableScan: unnest_table projection=[column1, column2, column3, column4, column5]
896+
04)--SubqueryAlias: t
897+
05)----Subquery:
898+
06)------Projection: __unnest_placeholder(outer_ref(u.column1),depth=1) AS column1
899+
07)--------Unnest: lists[__unnest_placeholder(outer_ref(u.column1))|depth=1] structs[]
900+
08)----------Projection: outer_ref(u.column1) AS __unnest_placeholder(outer_ref(u.column1))
901+
09)------------EmptyRelation
902+
physical_plan_error This feature is not implemented: Physical plan does not support logical expression OuterReferenceColumn(List(Field { name: "item", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }), Column { relation: Some(Bare { table: "u" }), name: "column1" })
903+
863904

864905
## Unnest in subquery
865906
query IIII

0 commit comments

Comments
 (0)