Skip to content

Commit 0506a80

Browse files
eliaperantonieliaperantoni
committed
feat: ignorable errors
1 parent 2d3e2a1 commit 0506a80

File tree

31 files changed

+276
-125
lines changed

31 files changed

+276
-125
lines changed

‎datafusion/common/src/column.rs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,8 @@ use std::fmt;
3333
use std::str::FromStr;
3434

3535
/// A named reference to a qualified field in a schema.
36-
#[derive(Debug, Clone, Derivative)]
37-
#[derivative(PartialEq, Eq, Hash, PartialOrd, Ord)]
36+
#[derive(Debug, Derivative)]
37+
#[derivative(PartialEq, Eq, Hash, PartialOrd, Ord, Clone)]
3838
pub struct Column {
3939
/// relation/table reference.
4040
pub relation: Option<TableReference>,
@@ -254,8 +254,7 @@ impl Column {
254254
.flat_map(|s| s.columns_with_unqualified_name(&self.name))
255255
.collect::<Vec<_>>();
256256
for using_col in using_columns {
257-
let all_matched =
258-
columns.iter().all(|c| using_col.contains(c));
257+
let all_matched = columns.iter().all(|c| using_col.contains(c));
259258
// All matched fields belong to the same using column set, in orther words
260259
// the same join clause. We simply pick the qualifier from the first match.
261260
if all_matched {

‎datafusion/common/src/dfschema/fields_spans.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ pub struct FieldsSpans(Vec<Vec<Span>>);
99

1010
impl FieldsSpans {
1111
pub fn empty(field_count: usize) -> Self {
12-
Self((0..field_count).into_iter().map(|_| Vec::new()).collect())
12+
Self((0..field_count).map(|_| Vec::new()).collect())
1313
}
1414

1515
pub fn iter(&self) -> impl Iterator<Item = &Vec<Span>> {
@@ -20,7 +20,7 @@ impl FieldsSpans {
2020
&self,
2121
other: &FieldsSpans,
2222
join_type: &JoinType,
23-
left_cols_len: usize,
23+
_left_cols_len: usize,
2424
) -> FieldsSpans {
2525
match join_type {
2626
JoinType::Inner | JoinType::Left | JoinType::Right | JoinType::Full => {

‎datafusion/common/src/dfschema/mod.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
mod fields_spans;
2121
pub use fields_spans::FieldsSpans;
2222

23-
use std::backtrace::Backtrace;
2423
use std::collections::{BTreeSet, HashMap, HashSet};
2524
use std::fmt::{Display, Formatter};
2625
use std::hash::Hash;

‎datafusion/common/src/error.rs

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ use std::sync::Arc;
2828

2929
use crate::diagnostic::Diagnostic;
3030
use crate::utils::quote_identifier;
31-
use crate::{Column, DFSchema, DiagnosticEntry, DiagnosticEntryKind, TableReference};
31+
use crate::{Column, DFSchema, TableReference};
3232
#[cfg(feature = "avro")]
3333
use apache_avro::Error as AvroError;
3434
use arrow::error::ArrowError;
@@ -514,8 +514,7 @@ impl DataFusionError {
514514
if let Some(source) = self
515515
.head
516516
.source()
517-
.map(|source| source.downcast_ref::<DataFusionError>())
518-
.flatten()
517+
.and_then(|source| source.downcast_ref::<DataFusionError>())
519518
{
520519
self.head = source;
521520
} else {
@@ -528,7 +527,9 @@ impl DataFusionError {
528527
DiagnosticsIterator { head: self }
529528
}
530529

531-
pub fn get_individual_errors(&self) -> impl Iterator<Item = &Self> + '_ {
530+
pub fn get_individual_errors(
531+
&self,
532+
) -> impl Iterator<Item = (Vec<Diagnostic>, &Self)> + '_ {
532533
fn contains_collection(err: &DataFusionError) -> bool {
533534
let mut head = err;
534535
loop {
@@ -538,8 +539,7 @@ impl DataFusionError {
538539

539540
if let Some(source) = head
540541
.source()
541-
.map(|source| source.downcast_ref::<DataFusionError>())
542-
.flatten()
542+
.and_then(|source| source.downcast_ref::<DataFusionError>())
543543
{
544544
head = source;
545545
} else {
@@ -549,37 +549,44 @@ impl DataFusionError {
549549
}
550550

551551
struct IndividualErrorsIterator<'a> {
552-
queue: Vec<&'a DataFusionError>,
552+
queue: Vec<(&'a DataFusionError, Vec<Diagnostic>)>,
553553
}
554554

555555
impl<'a> Iterator for IndividualErrorsIterator<'a> {
556-
type Item = &'a DataFusionError;
556+
type Item = (Vec<Diagnostic>, &'a DataFusionError);
557557

558558
fn next(&mut self) -> Option<Self::Item> {
559-
while let Some(err) = self.queue.pop() {
559+
while let Some((err, mut diagnostics_prefix)) = self.queue.pop() {
560560
if !contains_collection(err) {
561-
return Some(err);
561+
return Some((diagnostics_prefix, err));
562562
}
563563

564564
if let DataFusionError::Collection(errs) = err {
565-
self.queue.extend(errs.iter());
565+
self.queue.extend(
566+
errs.iter().map(|err| (err, diagnostics_prefix.clone())),
567+
);
566568
continue;
567569
}
568570

571+
if let DataFusionError::Diagnostic(diagnostics, _) = err {
572+
diagnostics_prefix.push(diagnostics.clone());
573+
}
574+
569575
if let Some(source) = err
570576
.source()
571-
.map(|source| source.downcast_ref::<DataFusionError>())
572-
.flatten()
577+
.and_then(|source| source.downcast_ref::<DataFusionError>())
573578
{
574-
self.queue.push(source);
579+
self.queue.push((source, diagnostics_prefix));
575580
}
576581
}
577582

578583
None
579584
}
580585
}
581586

582-
IndividualErrorsIterator { queue: vec![self] }
587+
IndividualErrorsIterator {
588+
queue: vec![(self, vec![])],
589+
}
583590
}
584591
}
585592

‎datafusion/common/src/lib.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ pub mod alias;
3333
pub mod cast;
3434
pub mod config;
3535
pub mod cse;
36+
pub mod diagnostic;
3637
pub mod display;
3738
pub mod error;
3839
pub mod file_options;
@@ -47,15 +48,15 @@ pub mod test_util;
4748
pub mod tree_node;
4849
pub mod types;
4950
pub mod utils;
50-
pub mod diagnostic;
5151
pub mod with_span;
5252

5353
/// Reexport arrow crate
5454
pub use arrow;
5555
pub use column::Column;
5656
pub use dfschema::{
57-
qualified_name, DFSchema, DFSchemaRef, ExprSchema, SchemaExt, ToDFSchema, FieldsSpans,
57+
qualified_name, DFSchema, DFSchemaRef, ExprSchema, FieldsSpans, SchemaExt, ToDFSchema,
5858
};
59+
pub use diagnostic::{Diagnostic, DiagnosticEntry, DiagnosticEntryKind};
5960
pub use error::{
6061
field_not_found, unqualified_field_not_found, DataFusionError, Result, SchemaError,
6162
SharedResult,
@@ -78,7 +79,6 @@ pub use stats::{ColumnStatistics, Statistics};
7879
pub use table_reference::{ResolvedTableReference, TableReference};
7980
pub use unnest::{RecursionUnnestOption, UnnestOptions};
8081
pub use utils::project_schema;
81-
pub use diagnostic::{Diagnostic, DiagnosticEntry, DiagnosticEntryKind};
8282
pub use with_span::WithSpans;
8383

8484
// These are hidden from docs purely to avoid polluting the public view of what this crate exports.

‎datafusion/common/src/with_span.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
use std::{
2-
cmp::Ordering,
32
fmt::{self, Debug, Display},
43
ops::{Deref, DerefMut},
54
};

‎datafusion/expr-common/src/type_coercion/binary.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ use datafusion_common::{
3434
Diagnostic, DiagnosticEntry, DiagnosticEntryKind, Result, WithSpans,
3535
};
3636
use itertools::Itertools;
37-
use sqlparser::tokenizer::Span;
3837

3938
/// The type signature of an instantiation of binary operator expression such as
4039
/// `lhs + rhs`

‎datafusion/expr/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ serde_json = { workspace = true }
5656
sqlparser = { workspace = true }
5757
strum = { version = "0.26.1", features = ["derive"] }
5858
strum_macros = "0.26.0"
59+
derivative = { workspace = true }
5960

6061
[dev-dependencies]
6162
ctor = { workspace = true }

‎datafusion/expr/src/expr.rs

Lines changed: 32 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ use datafusion_common::{
3838
plan_err, Column, DFSchema, HashMap, Result, ScalarValue, TableReference,
3939
};
4040
use datafusion_functions_window_common::field::WindowUDFFieldArgs;
41+
use derivative::Derivative;
4142
use sqlparser::ast::{
4243
display_comma_separated, ExceptSelectItem, ExcludeSelectItem, IlikeSelectItem,
4344
NullTreatment, RenameSelectItem, ReplaceSelectElement,
@@ -401,11 +402,19 @@ impl Unnest {
401402
}
402403

403404
/// Alias expression
404-
#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
405+
#[derive(Clone, Derivative, Debug)]
406+
#[derivative(PartialEq, Eq, PartialOrd, Hash)]
405407
pub struct Alias {
406408
pub expr: Box<Expr>,
407409
pub relation: Option<TableReference>,
408410
pub name: String,
411+
#[derivative(
412+
PartialEq = "ignore",
413+
Hash = "ignore",
414+
PartialOrd = "ignore",
415+
Ord = "ignore"
416+
)]
417+
pub span: Span,
409418
}
410419

411420
impl Alias {
@@ -419,8 +428,13 @@ impl Alias {
419428
expr: Box::new(expr),
420429
relation: relation.map(|r| r.into()),
421430
name: name.into(),
431+
span: Span::empty(),
422432
}
423433
}
434+
435+
pub fn with_span(self, span: Span) -> Self {
436+
Self { span, ..self }
437+
}
424438
}
425439

426440
/// Binary expression
@@ -1128,6 +1142,7 @@ impl Expr {
11281142
relation,
11291143
name,
11301144
spans: _,
1145+
..
11311146
}) => (relation.clone(), name.clone()),
11321147
Expr::Alias(Alias { relation, name, .. }) => (relation.clone(), name.clone()),
11331148
_ => (None, self.schema_name().to_string()),
@@ -1681,11 +1696,22 @@ impl Expr {
16811696
}
16821697
}
16831698

1684-
pub fn get_spans(&self) -> Option<&Vec<Span>> {
1699+
pub fn get_span(&self) -> Span {
16851700
match self {
1686-
Expr::Column(Column { spans, .. }) => Some(spans),
1687-
Expr::Alias(Alias { expr, .. }) => expr.get_spans(),
1688-
_ => None,
1701+
Expr::Column(Column { spans, .. }) => match spans.as_slice() {
1702+
[] => panic!("No spans for column expr"),
1703+
[span] => *span,
1704+
_ => panic!("Column expr has more than one span"),
1705+
},
1706+
Expr::Alias(Alias {
1707+
expr,
1708+
span: alias_span,
1709+
..
1710+
}) => {
1711+
let span = expr.get_span();
1712+
span.union(alias_span)
1713+
}
1714+
_ => Span::empty(),
16891715
}
16901716
}
16911717
}
@@ -1701,6 +1727,7 @@ impl HashNode for Expr {
17011727
expr: _expr,
17021728
relation,
17031729
name,
1730+
..
17041731
}) => {
17051732
relation.hash(state);
17061733
name.hash(state);

‎datafusion/expr/src/expr_rewriter/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,7 @@ pub fn create_col_from_scalar_expr(
186186
relation: _,
187187
name,
188188
spans,
189+
..
189190
}) => Ok(
190191
Column::new(Some::<TableReference>(subqry_alias.into()), name)
191192
.with_spans(spans.iter().copied()),

‎datafusion/expr/src/expr_rewriter/order_by.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ use crate::{expr::Sort, Cast, Expr, LogicalPlan, TryCast};
2323

2424
use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode};
2525
use datafusion_common::{Column, Result};
26-
use sqlparser::tokenizer::Span;
2726

2827
/// Rewrite sort on aggregate expressions to sort on the column of aggregate output
2928
/// For example, `max(x)` is written to `col("max(x)")`

‎datafusion/expr/src/expr_schema.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ use datafusion_common::{
3333
};
3434
use datafusion_functions_window_common::field::WindowUDFFieldArgs;
3535
use recursive::recursive;
36+
use sqlparser::tokenizer::Span;
3637
use std::collections::HashMap;
3738
use std::sync::Arc;
3839

@@ -405,16 +406,16 @@ impl ExprSchemable for Expr {
405406
}) => {
406407
let (left_type, left_is_nullable) =
407408
left.data_type_and_nullable(schema)?;
408-
let left_type = if let Some(spans) = left.get_spans() {
409-
WithSpans::new(&left_type, spans.iter().copied())
409+
let left_type = if left.get_span() != Span::empty() {
410+
WithSpans::new(&left_type, [left.get_span()])
410411
} else {
411412
(&left_type).into()
412413
};
413414

414415
let (right_type, right_is_nullable) =
415416
right.data_type_and_nullable(schema)?;
416-
let right_type = if let Some(spans) = right.get_spans() {
417-
WithSpans::new(&right_type, spans.iter().copied())
417+
let right_type = if right.get_span() != Span::empty() {
418+
WithSpans::new(&right_type, [right.get_span()])
418419
} else {
419420
(&right_type).into()
420421
};

‎datafusion/expr/src/logical_plan/builder.rs

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1543,14 +1543,20 @@ pub fn union(left_plan: LogicalPlan, right_plan: LogicalPlan) -> Result<LogicalP
15431543
Span::empty(),
15441544
),
15451545
DiagnosticEntry::new(
1546-
format!("This side has {} columns", left_plan.schema().fields().len()),
1546+
format!(
1547+
"This side has {} columns",
1548+
left_plan.schema().fields().len()
1549+
),
15471550
DiagnosticEntryKind::Note,
15481551
Span::union_iter(
15491552
left_plan.schema().fields_spans().iter().flatten().copied(),
15501553
),
15511554
),
15521555
DiagnosticEntry::new(
1553-
format!("This side has {} columns", right_plan.schema().fields().len()),
1556+
format!(
1557+
"This side has {} columns",
1558+
right_plan.schema().fields().len()
1559+
),
15541560
DiagnosticEntryKind::Note,
15551561
Span::union_iter(
15561562
right_plan.schema().fields_spans().iter().flatten().copied(),

‎datafusion/expr/src/logical_plan/plan.rs

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2163,12 +2163,8 @@ impl Projection {
21632163
/// produced by the projection operation. If the schema computation is successful,
21642164
/// the `Result` will contain the schema; otherwise, it will contain an error.
21652165
pub fn projection_schema(input: &LogicalPlan, exprs: &[Expr]) -> Result<Arc<DFSchema>> {
2166-
21672166
let metadata = input.schema().metadata().clone();
2168-
let fields_spans = exprs
2169-
.iter()
2170-
.map(|e| e.get_spans().cloned().unwrap_or_else(|| vec![]))
2171-
.collect();
2167+
let fields_spans = exprs.iter().map(|e| vec![e.get_span()]).collect();
21722168

21732169
let schema =
21742170
DFSchema::new_with_metadata(exprlist_to_fields(exprs, input)?, metadata)?

‎datafusion/expr/src/tree_node.rs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,16 @@ impl TreeNode for Expr {
127127
expr,
128128
relation,
129129
name,
130-
}) => f(*expr)?.update_data(|e| e.alias_qualified(relation, name)),
130+
span,
131+
..
132+
}) => f(*expr)?.update_data(|e| {
133+
let e = e.alias_qualified(relation, name);
134+
if let Expr::Alias(alias) = e {
135+
Expr::Alias(alias.with_span(span))
136+
} else {
137+
unreachable!();
138+
}
139+
}),
131140
Expr::InSubquery(InSubquery {
132141
expr,
133142
subquery,

0 commit comments

Comments
 (0)