Parse SUBSTRING FROM syntax in all dialects, reflect change in the AST (#1173)

Author: lovasoa

src/ast/mod.rs

@@ -559,13 +559,18 @@ pub enum Expr {
     /// ```sql
     /// SUBSTRING(<expr> [FROM <expr>] [FOR <expr>])
     /// ```
+    /// or
+    /// ```sql
+    /// SUBSTRING(<expr>, <expr>, <expr>)
+    /// ```
     Substring {
         expr: Box<Expr>,
         substring_from: Option<Box<Expr>>,
         substring_for: Option<Box<Expr>>,
 
-        // Some dialects use `SUBSTRING(expr [FROM start] [FOR len])` syntax while others omit FROM,
-        // FOR keywords (e.g. Microsoft SQL Server). This flags is used for formatting.
+        /// false if the expression is represented using the `SUBSTRING(expr [FROM start] [FOR len])` syntax
+        /// true if the expression is represented using the `SUBSTRING(expr, start, len)` syntax
+        /// This flag is used for formatting.
         special: bool,
     },
     /// ```sql

src/dialect/mod.rs

@@ -135,10 +135,6 @@ pub trait Dialect: Debug + Any {
     fn supports_group_by_expr(&self) -> bool {
         false
     }
-    /// Returns true if the dialect supports `SUBSTRING(expr [FROM start] [FOR len])` expressions
-    fn supports_substring_from_for_expr(&self) -> bool {
-        true
-    }
     /// Returns true if the dialect supports `(NOT) IN ()` expressions
     fn supports_in_empty_list(&self) -> bool {
         false
@@ -325,10 +321,6 @@ mod tests {
                 self.0.supports_group_by_expr()
             }
 
-            fn supports_substring_from_for_expr(&self) -> bool {
-                self.0.supports_substring_from_for_expr()
-            }
-
             fn supports_in_empty_list(&self) -> bool {
                 self.0.supports_in_empty_list()
             }

src/dialect/mssql.rs

@@ -40,8 +40,4 @@ impl Dialect for MsSqlDialect {
     fn convert_type_before_value(&self) -> bool {
         true
     }
-
-    fn supports_substring_from_for_expr(&self) -> bool {
-        false
-    }
 }

src/parser/mod.rs

@@ -1525,47 +1525,27 @@ impl<'a> Parser<'a> {
     }
 
     pub fn parse_substring_expr(&mut self) -> Result<Expr, ParserError> {
-        if self.dialect.supports_substring_from_for_expr() {
-            // PARSE SUBSTRING (EXPR [FROM 1] [FOR 3])
-            self.expect_token(&Token::LParen)?;
-            let expr = self.parse_expr()?;
-            let mut from_expr = None;
-            if self.parse_keyword(Keyword::FROM) || self.consume_token(&Token::Comma) {
-                from_expr = Some(self.parse_expr()?);
-            }
-
-            let mut to_expr = None;
-            if self.parse_keyword(Keyword::FOR) || self.consume_token(&Token::Comma) {
-                to_expr = Some(self.parse_expr()?);
-            }
-            self.expect_token(&Token::RParen)?;
-
-            Ok(Expr::Substring {
-                expr: Box::new(expr),
-                substring_from: from_expr.map(Box::new),
-                substring_for: to_expr.map(Box::new),
-                special: false,
-            })
-        } else {
-            // PARSE SUBSTRING(EXPR, start, length)
-            self.expect_token(&Token::LParen)?;
-            let expr = self.parse_expr()?;
-
-            self.expect_token(&Token::Comma)?;
-            let from_expr = Some(self.parse_expr()?);
-
-            self.expect_token(&Token::Comma)?;
-            let to_expr = Some(self.parse_expr()?);
-
-            self.expect_token(&Token::RParen)?;
+        // PARSE SUBSTRING (EXPR [FROM 1] [FOR 3])
+        self.expect_token(&Token::LParen)?;
+        let expr = self.parse_expr()?;
+        let mut from_expr = None;
+        let special = self.consume_token(&Token::Comma);
+        if special || self.parse_keyword(Keyword::FROM) {
+            from_expr = Some(self.parse_expr()?);
+        }
 
-            Ok(Expr::Substring {
-                expr: Box::new(expr),
-                substring_from: from_expr.map(Box::new),
-                substring_for: to_expr.map(Box::new),
-                special: true,
-            })
+        let mut to_expr = None;
+        if self.parse_keyword(Keyword::FOR) || self.consume_token(&Token::Comma) {
+            to_expr = Some(self.parse_expr()?);
         }
+        self.expect_token(&Token::RParen)?;
+
+        Ok(Expr::Substring {
+            expr: Box::new(expr),
+            substring_from: from_expr.map(Box::new),
+            substring_for: to_expr.map(Box::new),
+            special,
+        })
     }
 
     pub fn parse_overlay_expr(&mut self) -> Result<Expr, ParserError> {

tests/sqlparser_common.rs

@@ -5761,45 +5761,12 @@ fn parse_scalar_subqueries() {
 
 #[test]
 fn parse_substring() {
-    let from_for_supported_dialects = TestedDialects {
-        dialects: vec![
-            Box::new(GenericDialect {}),
-            Box::new(PostgreSqlDialect {}),
-            Box::new(AnsiDialect {}),
-            Box::new(SnowflakeDialect {}),
-            Box::new(HiveDialect {}),
-            Box::new(RedshiftSqlDialect {}),
-            Box::new(MySqlDialect {}),
-            Box::new(BigQueryDialect {}),
-            Box::new(SQLiteDialect {}),
-            Box::new(DuckDbDialect {}),
-        ],
-        options: None,
-    };
-
-    let from_for_unsupported_dialects = TestedDialects {
-        dialects: vec![Box::new(MsSqlDialect {})],
-        options: None,
-    };
-
-    from_for_supported_dialects
-        .one_statement_parses_to("SELECT SUBSTRING('1')", "SELECT SUBSTRING('1')");
-
-    from_for_supported_dialects.one_statement_parses_to(
-        "SELECT SUBSTRING('1' FROM 1)",
-        "SELECT SUBSTRING('1' FROM 1)",
-    );
-
-    from_for_supported_dialects.one_statement_parses_to(
-        "SELECT SUBSTRING('1' FROM 1 FOR 3)",
-        "SELECT SUBSTRING('1' FROM 1 FOR 3)",
-    );
-
-    from_for_unsupported_dialects
-        .one_statement_parses_to("SELECT SUBSTRING('1', 1, 3)", "SELECT SUBSTRING('1', 1, 3)");
-
-    from_for_supported_dialects
-        .one_statement_parses_to("SELECT SUBSTRING('1' FOR 3)", "SELECT SUBSTRING('1' FOR 3)");
+    verified_stmt("SELECT SUBSTRING('1')");
+    verified_stmt("SELECT SUBSTRING('1' FROM 1)");
+    verified_stmt("SELECT SUBSTRING('1' FROM 1 FOR 3)");
+    verified_stmt("SELECT SUBSTRING('1', 1, 3)");
+    verified_stmt("SELECT SUBSTRING('1', 1)");
+    verified_stmt("SELECT SUBSTRING('1' FOR 3)");
 }
 
 #[test]

tests/sqlparser_mysql.rs

@@ -1911,7 +1911,7 @@ fn parse_substring_in_select() {
     let sql = "SELECT DISTINCT SUBSTRING(description, 0, 1) FROM test";
     match mysql().one_statement_parses_to(
         sql,
-        "SELECT DISTINCT SUBSTRING(description FROM 0 FOR 1) FROM test",
+        "SELECT DISTINCT SUBSTRING(description, 0, 1) FROM test",
     ) {
         Statement::Query(query) => {
             assert_eq!(
@@ -1927,7 +1927,7 @@ fn parse_substring_in_select() {
                             })),
                             substring_from: Some(Box::new(Expr::Value(number("0")))),
                             substring_for: Some(Box::new(Expr::Value(number("1")))),
-                            special: false,
+                            special: true,
                         })],
                         into: None,
                         from: vec![TableWithJoins {

tests/sqlparser_sqlite.rs

@@ -413,6 +413,18 @@ fn parse_single_quoted_identified() {
     sqlite().verified_only_select("SELECT 't'.*, t.'x' FROM 't'");
     // TODO: add support for select 't'.x
 }
+
+#[test]
+fn parse_substring() {
+    // SQLite supports the SUBSTRING function since v3.34, but does not support the SQL standard
+    // SUBSTRING(expr FROM start FOR length) syntax.
+    // https://www.sqlite.org/lang_corefunc.html#substr
+    sqlite().verified_only_select("SELECT SUBSTRING('SQLITE', 3, 4)");
+    sqlite().verified_only_select("SELECT SUBSTR('SQLITE', 3, 4)");
+    sqlite().verified_only_select("SELECT SUBSTRING('SQLITE', 3)");
+    sqlite().verified_only_select("SELECT SUBSTR('SQLITE', 3)");
+}
+
 #[test]
 fn parse_window_function_with_filter() {
     for func_name in [