Deprecate Expr::Wildcard (#14959) (#14976)

* Deprecate `Expr::Wildcard`

* Update



* Update

---------

Co-authored-by: Heran Lin <[email protected]>
Co-authored-by: Andrew Lamb <[email protected]>

Author: 3 people

datafusion/catalog-listing/src/helpers.rs

@@ -103,6 +103,8 @@ pub fn expr_applicable_for_cols(col_names: &[&str], expr: &Expr) -> bool {
         // - AGGREGATE and WINDOW should not end up in filter conditions, except maybe in some edge cases
         // - Can `Wildcard` be considered as a `Literal`?
         // - ScalarVariable could be `applicable`, but that would require access to the context
+        // TODO: remove the next line after `Expr::Wildcard` is removed
+        #[expect(deprecated)]
         Expr::AggregateFunction { .. }
         | Expr::WindowFunction { .. }
         | Expr::Wildcard { .. }

datafusion/expr/src/expr.rs

@@ -311,6 +311,10 @@ pub enum Expr {
     ///
     /// This expr has to be resolved to a list of columns before translating logical
     /// plan into physical plan.
+    #[deprecated(
+        since = "46.0.0",
+        note = "A wildcard needs to be resolved to concrete expressions when constructing the logical plan. See https://github.com/apache/datafusion/issues/7765"
+    )]
     Wildcard {
         qualifier: Option<TableReference>,
         options: Box<WildcardOptions>,
@@ -1175,6 +1179,7 @@ impl Expr {
             Expr::ScalarVariable(..) => "ScalarVariable",
             Expr::TryCast { .. } => "TryCast",
             Expr::WindowFunction { .. } => "WindowFunction",
+            #[expect(deprecated)]
             Expr::Wildcard { .. } => "Wildcard",
             Expr::Unnest { .. } => "Unnest",
         }
@@ -1648,6 +1653,8 @@ impl Expr {
             // Use explicit pattern match instead of a default
             // implementation, so that in the future if someone adds
             // new Expr types, they will check here as well
+            // TODO: remove the next line after `Expr::Wildcard` is removed
+            #[expect(deprecated)]
             Expr::AggregateFunction(..)
             | Expr::Alias(..)
             | Expr::Between(..)
@@ -2229,6 +2236,7 @@ impl HashNode for Expr {
             Expr::ScalarSubquery(subquery) => {
                 subquery.hash(state);
             }
+            #[expect(deprecated)]
             Expr::Wildcard { qualifier, options } => {
                 qualifier.hash(state);
                 options.hash(state);
@@ -2288,6 +2296,8 @@ impl Display for SchemaDisplay<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         match self.0 {
             // The same as Display
+            // TODO: remove the next line after `Expr::Wildcard` is removed
+            #[expect(deprecated)]
             Expr::Column(_)
             | Expr::Literal(_)
             | Expr::ScalarVariable(..)
@@ -2758,6 +2768,7 @@ impl Display for Expr {
                     write!(f, "{expr} IN ([{}])", expr_vec_fmt!(list))
                 }
             }
+            #[expect(deprecated)]
             Expr::Wildcard { qualifier, options } => match qualifier {
                 Some(qualifier) => write!(f, "{qualifier}.*{options}"),
                 None => write!(f, "*{options}"),

datafusion/expr/src/expr_fn.rs

@@ -121,6 +121,7 @@ pub fn placeholder(id: impl Into<String>) -> Expr {
 /// assert_eq!(p.to_string(), "*")
 /// ```
 pub fn wildcard() -> Expr {
+    #[expect(deprecated)]
     Expr::Wildcard {
         qualifier: None,
         options: Box::new(WildcardOptions::default()),
@@ -129,6 +130,7 @@ pub fn wildcard() -> Expr {
 
 /// Create an '*' [`Expr::Wildcard`] expression with the wildcard options
 pub fn wildcard_with_options(options: WildcardOptions) -> Expr {
+    #[expect(deprecated)]
     Expr::Wildcard {
         qualifier: None,
         options: Box::new(options),
@@ -146,6 +148,7 @@ pub fn wildcard_with_options(options: WildcardOptions) -> Expr {
 /// assert_eq!(p.to_string(), "t.*")
 /// ```
 pub fn qualified_wildcard(qualifier: impl Into<TableReference>) -> Expr {
+    #[expect(deprecated)]
     Expr::Wildcard {
         qualifier: Some(qualifier.into()),
         options: Box::new(WildcardOptions::default()),
@@ -157,6 +160,7 @@ pub fn qualified_wildcard_with_options(
     qualifier: impl Into<TableReference>,
     options: WildcardOptions,
 ) -> Expr {
+    #[expect(deprecated)]
     Expr::Wildcard {
         qualifier: Some(qualifier.into()),
         options: Box::new(options),

datafusion/expr/src/expr_rewriter/mod.rs

@@ -253,6 +253,7 @@ fn coerce_exprs_for_schema(
                     Expr::Alias(Alias { expr, name, .. }) => {
                         Ok(expr.cast_to(new_type, src_schema)?.alias(name))
                     }
+                    #[expect(deprecated)]
                     Expr::Wildcard { .. } => Ok(expr),
                     _ => expr.cast_to(new_type, src_schema),
                 }

datafusion/expr/src/expr_schema.rs

@@ -215,6 +215,7 @@ impl ExprSchemable for Expr {
                     Ok(DataType::Null)
                 }
             }
+            #[expect(deprecated)]
             Expr::Wildcard { .. } => Ok(DataType::Null),
             Expr::GroupingSet(_) => {
                 // Grouping sets do not really have a type and do not appear in projections
@@ -329,6 +330,7 @@ impl ExprSchemable for Expr {
             | Expr::SimilarTo(Like { expr, pattern, .. }) => {
                 Ok(expr.nullable(input_schema)? || pattern.nullable(input_schema)?)
             }
+            #[expect(deprecated)]
             Expr::Wildcard { .. } => Ok(false),
             Expr::GroupingSet(_) => {
                 // Grouping sets do not really have the concept of nullable and do not appear

datafusion/expr/src/logical_plan/builder.rs

@@ -1675,6 +1675,7 @@ fn project_with_validation(
     for (e, validate) in expr {
         let e = e.into();
         match e {
+            #[expect(deprecated)]
             Expr::Wildcard { .. } => projected_expr.push(e),
             _ => {
                 if validate {

datafusion/expr/src/logical_plan/plan.rs

@@ -2141,6 +2141,7 @@ impl Projection {
         input: Arc<LogicalPlan>,
         schema: DFSchemaRef,
     ) -> Result<Self> {
+        #[expect(deprecated)]
         if !expr.iter().any(|e| matches!(e, Expr::Wildcard { .. }))
             && expr.len() != schema.fields().len()
         {
@@ -3451,6 +3452,7 @@ fn calc_func_dependencies_for_project(
     let proj_indices = exprs
         .iter()
         .map(|expr| match expr {
+            #[expect(deprecated)]
             Expr::Wildcard { qualifier, options } => {
                 let wildcard_fields = exprlist_to_fields(
                     vec![&Expr::Wildcard {

datafusion/expr/src/tree_node.rs

@@ -67,6 +67,8 @@ impl TreeNode for Expr {
             Expr::GroupingSet(GroupingSet::GroupingSets(lists_of_exprs)) => {
                 lists_of_exprs.apply_elements(f)
             }
+            // TODO: remove the next line after `Expr::Wildcard` is removed
+            #[expect(deprecated)]
             Expr::Column(_)
             // Treat OuterReferenceColumn as a leaf expression
             | Expr::OuterReferenceColumn(_, _)
@@ -113,6 +115,8 @@ impl TreeNode for Expr {
         mut f: F,
     ) -> Result<Transformed<Self>> {
         Ok(match self {
+            // TODO: remove the next line after `Expr::Wildcard` is removed
+            #[expect(deprecated)]
             Expr::Column(_)
             | Expr::Wildcard { .. }
             | Expr::Placeholder(Placeholder { .. })

datafusion/expr/src/utils.rs

@@ -282,6 +282,8 @@ pub fn expr_to_columns(expr: &Expr, accum: &mut HashSet<Column>) -> Result<()> {
             // Use explicit pattern match instead of a default
             // implementation, so that in the future if someone adds
             // new Expr types, they will check here as well
+            // TODO: remove the next line after `Expr::Wildcard` is removed
+            #[expect(deprecated)]
             Expr::Unnest(_)
             | Expr::ScalarVariable(_, _)
             | Expr::Alias(_)
@@ -709,6 +711,7 @@ pub fn exprlist_to_fields<'a>(
     let result = exprs
         .into_iter()
         .map(|e| match e {
+            #[expect(deprecated)]
             Expr::Wildcard { qualifier, options } => match qualifier {
                 None => {
                     let mut excluded = exclude_using_columns(plan)?;
@@ -801,6 +804,7 @@ pub fn exprlist_len(
     exprs
         .iter()
         .map(|e| match e {
+            #[expect(deprecated)]
             Expr::Wildcard {
                 qualifier: None,
                 options,
@@ -818,6 +822,7 @@ pub fn exprlist_len(
                         .len(),
                 )
             }
+            #[expect(deprecated)]
             Expr::Wildcard {
                 qualifier: Some(qualifier),
                 options,

datafusion/functions-aggregate/src/planner.rs

@@ -82,6 +82,8 @@ impl ExprPlanner for AggregateFunctionPlanner {
         // handle count() and count(*) case
         // convert to count(1) as "count()"
         // or         count(1) as "count(*)"
+        // TODO: remove the next line after `Expr::Wildcard` is removed
+        #[expect(deprecated)]
         if raw_expr.func.name() == "count"
             && (raw_expr.args.len() == 1
                 && matches!(raw_expr.args[0], Expr::Wildcard { .. })

datafusion/functions-window/src/planner.rs

@@ -79,6 +79,8 @@ impl ExprPlanner for WindowFunctionPlanner {
             null_treatment,
         };
 
+        // TODO: remove the next line after `Expr::Wildcard` is removed
+        #[expect(deprecated)]
         if raw_expr.func_def.name() == "count"
             && (raw_expr.args.len() == 1
                 && matches!(raw_expr.args[0], Expr::Wildcard { .. })

datafusion/optimizer/src/analyzer/expand_wildcard_rule.rs

@@ -89,6 +89,7 @@ fn expand_exprlist(input: &LogicalPlan, expr: Vec<Expr>) -> Result<Vec<Expr>> {
     let input = find_base_plan(input);
     for e in expr {
         match e {
+            #[expect(deprecated)]
             Expr::Wildcard { qualifier, options } => {
                 if let Some(qualifier) = qualifier {
                     let expanded = expand_qualified_wildcard(

datafusion/optimizer/src/analyzer/type_coercion.rs

@@ -565,6 +565,8 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> {
                         .build()?,
                 ))
             }
+            // TODO: remove the next line after `Expr::Wildcard` is removed
+            #[expect(deprecated)]
             Expr::Alias(_)
             | Expr::Column(_)
             | Expr::ScalarVariable(_, _)
@@ -1021,6 +1023,7 @@ fn project_with_column_index(
                 spans: _,
             }) if name != schema.field(i).name() => Ok(e.alias(schema.field(i).name())),
             Expr::Alias { .. } | Expr::Column { .. } => Ok(e),
+            #[expect(deprecated)]
             Expr::Wildcard { .. } => {
                 plan_err!("Wildcard should be expanded before type coercion")
             }

datafusion/optimizer/src/common_subexpr_eliminate.rs

@@ -678,6 +678,8 @@ impl CSEController for ExprCSEController<'_> {
     }
 
     fn is_ignored(&self, node: &Expr) -> bool {
+        // TODO: remove the next line after `Expr::Wildcard` is removed
+        #[expect(deprecated)]
         let is_normal_minus_aggregates = matches!(
             node,
             Expr::Literal(..)

datafusion/optimizer/src/push_down_filter.rs

@@ -285,6 +285,8 @@ fn can_evaluate_as_join_condition(predicate: &Expr) -> Result<bool> {
         | Expr::TryCast(_)
         | Expr::InList { .. }
         | Expr::ScalarFunction(_) => Ok(TreeNodeRecursion::Continue),
+        // TODO: remove the next line after `Expr::Wildcard` is removed
+        #[expect(deprecated)]
         Expr::AggregateFunction(_)
         | Expr::WindowFunction(_)
         | Expr::Wildcard { .. }

datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs

@@ -582,6 +582,8 @@ impl<'a> ConstEvaluator<'a> {
         // added they can be checked for their ability to be evaluated
         // at plan time
         match expr {
+            // TODO: remove the next line after `Expr::Wildcard` is removed
+            #[expect(deprecated)]
             Expr::AggregateFunction { .. }
             | Expr::ScalarVariable(_, _)
             | Expr::Column(_)

datafusion/proto/src/logical_plan/from_proto.rs

@@ -527,6 +527,7 @@ pub fn parse_expr(
         ))),
         ExprType::Wildcard(protobuf::Wildcard { qualifier }) => {
             let qualifier = qualifier.to_owned().map(|x| x.try_into()).transpose()?;
+            #[expect(deprecated)]
             Ok(Expr::Wildcard {
                 qualifier,
                 options: Box::new(WildcardOptions::default()),

datafusion/proto/src/logical_plan/to_proto.rs

@@ -560,6 +560,7 @@ pub fn serialize_expr(
                 expr_type: Some(ExprType::InList(expr)),
             }
         }
+        #[expect(deprecated)]
         Expr::Wildcard { qualifier, .. } => protobuf::LogicalExprNode {
             expr_type: Some(ExprType::Wildcard(protobuf::Wildcard {
                 qualifier: qualifier.to_owned().map(|x| x.into()),

datafusion/sql/src/expr/mod.rs

@@ -593,10 +593,12 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 }
                 not_impl_err!("AnyOp not supported by ExprPlanner: {binary_expr:?}")
             }
+            #[expect(deprecated)]
             SQLExpr::Wildcard(_token) => Ok(Expr::Wildcard {
                 qualifier: None,
                 options: Box::new(WildcardOptions::default()),
             }),
+            #[expect(deprecated)]
             SQLExpr::QualifiedWildcard(object_name, _token) => Ok(Expr::Wildcard {
                 qualifier: Some(self.object_name_to_table_reference(object_name)?),
                 options: Box::new(WildcardOptions::default()),

datafusion/sql/src/unparser/expr.rs

@@ -429,6 +429,7 @@ impl Unparser<'_> {
                 })
             }
             // TODO: unparsing wildcard addition options
+            #[expect(deprecated)]
             Expr::Wildcard { qualifier, .. } => {
                 let attached_token = AttachedToken::empty();
                 if let Some(qualifier) = qualifier {
@@ -729,6 +730,7 @@ impl Unparser<'_> {
     ) -> Result<Vec<ast::FunctionArg>> {
         args.iter()
             .map(|e| {
+                #[expect(deprecated)]
                 if matches!(
                     e,
                     Expr::Wildcard {
@@ -1715,6 +1717,7 @@ mod tests {
     #[test]
     fn expr_to_sql_ok() -> Result<()> {
         let dummy_schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
+        #[expect(deprecated)]
         let dummy_logical_plan = table_scan(Some("t"), &dummy_schema, None)?
             .project(vec![Expr::Wildcard {
                 qualifier: None,

datafusion/sql/src/utils.rs

@@ -632,6 +632,8 @@ pub(crate) fn rewrite_recursive_unnest_bottom_up(
     } = original_expr.clone().rewrite(&mut rewriter)?;
 
     if !transformed {
+        // TODO: remove the next line after `Expr::Wildcard` is removed
+        #[expect(deprecated)]
         if matches!(&transformed_expr, Expr::Column(_))
             || matches!(&transformed_expr, Expr::Wildcard { .. })
         {

datafusion/substrait/src/logical_plan/producer.rs

@@ -1366,6 +1366,7 @@ pub fn to_substrait_rex(
         Expr::ScalarSubquery(expr) => {
             not_impl_err!("Cannot convert {expr:?} to Substrait")
         }
+        #[expect(deprecated)]
         Expr::Wildcard { .. } => not_impl_err!("Cannot convert {expr:?} to Substrait"),
         Expr::GroupingSet(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"),
         Expr::Placeholder(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"),