Fix UNION field nullability tracking

This commit fixes two bugs related to UNION handling

- when constructing union plan nullability of the other union branch was
  ignored, thus resulting field could easily have incorrect nullability
- when pruning/simplifying projects, in `recompute_schema` function
  there was similar logic, thus loosing nullability information even for
  correctly constructed Union plan node

As a result, other optimizer logic (e.g. `expr_simplifier.rs`) could
draw incorrect conclusions and thus lead to incorrect query results, as
demonstrated with the attached SLT test.

Author: findepi

datafusion/expr/src/logical_plan/builder.rs

@@ -1529,9 +1529,47 @@ pub fn union(left_plan: LogicalPlan, right_plan: LogicalPlan) -> Result<LogicalP
         );
     }
 
-    // Temporarily use the schema from the left input and later rely on the analyzer to
-    // coerce the two schemas into a common one.
-    let schema = (**left_schema).clone();
+    let schema = DFSchema::from_unqualified_fields(
+        left_schema
+            .fields()
+            .iter()
+            .zip(right_schema.fields()).enumerate()
+            .map(|(i, (left_field, right_field))| {
+                let name = left_field.name();
+                // TODO apply type coercion here, or document why it's better to defer
+                // temporarily use the data type from the left input and later rely on the analyzer to
+                // coerce the two schemas into a common one.
+                let data_type = left_field.data_type();
+                let nullable = left_field.is_nullable() || right_field.is_nullable();
+                let mut field = Field::new(
+                    name,
+                    data_type.clone(),
+                    nullable,
+                );
+                println!("Constructed union field {} nullable {}", field.name(), field.is_nullable());
+                if left_field.metadata() != right_field.metadata() {
+                    return plan_err!(
+                            "UNION field {i} have different metadata: left has {:?} whereas right has {:?}",
+                            left_field.metadata(),
+                            right_field.metadata()
+                        );
+                }
+                field.set_metadata(left_field.metadata().clone());
+                Ok(field)
+            })
+            .collect::<Result<_>>()?,
+    {
+        // Metadata is application specific. It's unclear how it should be merged/reconciled.
+        if left_schema.inner().metadata() != right_schema.inner().metadata() {
+            return plan_err!(
+                    "UNION queries have different metadata: left has {:?} whereas right has {:?}",
+                    left_schema.inner().metadata(),
+                    right_schema.inner().metadata()
+                );
+        }
+        left_schema.inner().metadata().clone()
+    },
+    ) ? ;
 
     // Functional Dependencies doesn't preserve after UNION operation
     let schema =

datafusion/expr/src/logical_plan/plan.rs

@@ -698,16 +698,8 @@ impl LogicalPlan {
                     node: node.with_exprs_and_inputs(expr, inputs)?,
                 }))
             }
-            LogicalPlan::Union(Union { inputs, schema }) => {
-                let input_schema = inputs[0].schema();
-                // If inputs are not pruned do not change schema
-                // TODO this seems wrong (shouldn't we always use the schema of the input?)
-                let schema = if schema.fields().len() == input_schema.fields().len() {
-                    Arc::clone(&schema)
-                } else {
-                    Arc::clone(input_schema)
-                };
-                Ok(LogicalPlan::Union(Union { inputs, schema }))
+            LogicalPlan::Union(_) => {
+                Ok(self)
             }
             LogicalPlan::Distinct(distinct) => {
                 let distinct = match distinct {

datafusion/sqllogictest/test_files/union.slt

@@ -836,3 +836,18 @@ physical_plan
 # Clean up after the test
 statement ok
 drop table aggregate_test_100;
+
+# test for https://github.com/apache/datafusion/issues/14352
+query TB rowsort
+SELECT
+    a,
+    a IS NOT NULL
+FROM (
+    -- second column, even though it's not selected, was necessary to reproduce the bug linked above
+    SELECT 'foo' AS a, 3 AS b
+    UNION ALL
+    SELECT NULL AS a, 4 AS b
+)
+----
+NULL false
+foo true