Redshift: Support implicit string concatenation using newline (apache#2167)

Author: yoavcloud

src/dialect/mod.rs

@@ -489,6 +489,19 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Returns true if the dialect supports concatenating string literals with a newline.
+    /// For example, the following statement would return `true`:
+    /// ```sql
+    /// SELECT 'abc' in (
+    ///   'a'
+    ///   'b'
+    ///   'c'
+    /// );
+    /// ```
+    fn supports_string_literal_concatenation_with_newline(&self) -> bool {
+        false
+    }
+
     /// Does the dialect support trailing commas in the projection list?
     fn supports_projection_trailing_commas(&self) -> bool {
         self.supports_trailing_commas()

src/dialect/redshift.rs

@@ -147,4 +147,8 @@ impl Dialect for RedshiftSqlDialect {
     fn supports_create_table_like_parenthesized(&self) -> bool {
         true
     }
+
+    fn supports_string_literal_concatenation_with_newline(&self) -> bool {
+        true
+    }
 }

src/parser/mod.rs

@@ -11356,7 +11356,34 @@ impl<'a> Parser<'a> {
                 str.push_str(s);
                 self.advance_token();
             }
+        } else if self
+            .dialect
+            .supports_string_literal_concatenation_with_newline()
+        {
+            // We are iterating over tokens including whitespaces, to identify
+            // string literals separated by newlines so we can concatenate them.
+            let mut after_newline = false;
+            loop {
+                match self.peek_token_no_skip().token {
+                    Token::Whitespace(Whitespace::Newline) => {
+                        after_newline = true;
+                        self.next_token_no_skip();
+                    }
+                    Token::Whitespace(_) => {
+                        self.next_token_no_skip();
+                    }
+                    Token::SingleQuotedString(ref s) | Token::DoubleQuotedString(ref s)
+                        if after_newline =>
+                    {
+                        str.push_str(s.clone().as_str());
+                        self.next_token_no_skip();
+                        after_newline = false;
+                    }
+                    _ => break,
+                }
+            }
         }
+
         str
     }
 

tests/sqlparser_common.rs

@@ -17738,6 +17738,25 @@ fn parse_adjacent_string_literal_concatenation() {
 
     let sql = "SELECT * FROM t WHERE col = 'Hello' \n ' ' \t 'World!'";
     dialects.one_statement_parses_to(sql, r"SELECT * FROM t WHERE col = 'Hello World!'");
+
+    let dialects = all_dialects_where(|d| d.supports_string_literal_concatenation_with_newline());
+    let sql = r#"
+    SELECT 'abc' in ('a'
+        'b'
+        'c',
+        'd'
+    )"#;
+    dialects.one_statement_parses_to(sql, "SELECT 'abc' IN ('abc', 'd')");
+
+    let sql = r#"
+    SELECT 'abc' in ('a'
+        'b'
+        -- COMMENT
+        'c',
+        -- COMMENT
+        'd'
+    )"#;
+    dialects.one_statement_parses_to(sql, "SELECT 'abc' IN ('abc', 'd')");
 }
 
 #[test]