Redshift utf8 idents (apache#1915)

Author: yoavcloud

src/dialect/redshift.rs

@@ -80,13 +80,15 @@ impl Dialect for RedshiftSqlDialect {
     }
 
     fn is_identifier_start(&self, ch: char) -> bool {
-        // Extends Postgres dialect with sharp
-        PostgreSqlDialect {}.is_identifier_start(ch) || ch == '#'
+        // Extends Postgres dialect with sharp and UTF-8 multibyte chars
+        // https://docs.aws.amazon.com/redshift/latest/dg/r_names.html
+        PostgreSqlDialect {}.is_identifier_start(ch) || ch == '#' || !ch.is_ascii()
     }
 
     fn is_identifier_part(&self, ch: char) -> bool {
-        // Extends Postgres dialect with sharp
-        PostgreSqlDialect {}.is_identifier_part(ch) || ch == '#'
+        // Extends Postgres dialect with sharp and UTF-8 multibyte chars
+        // https://docs.aws.amazon.com/redshift/latest/dg/r_names.html
+        PostgreSqlDialect {}.is_identifier_part(ch) || ch == '#' || !ch.is_ascii()
     }
 
     /// redshift has `CONVERT(type, value)` instead of `CONVERT(value, type)`

tests/sqlparser_common.rs

@@ -11071,10 +11071,17 @@ fn parse_non_latin_identifiers() {
         Box::new(RedshiftSqlDialect {}),
         Box::new(MySqlDialect {}),
     ]);
-
     supported_dialects.verified_stmt("SELECT a.説明 FROM test.public.inter01 AS a");
     supported_dialects.verified_stmt("SELECT a.説明 FROM inter01 AS a, inter01_transactions AS b WHERE a.説明 = b.取引 GROUP BY a.説明");
     supported_dialects.verified_stmt("SELECT 説明, hühnervögel, garçon, Москва, 東京 FROM inter01");
+
+    let supported_dialects = TestedDialects::new(vec![
+        Box::new(GenericDialect {}),
+        Box::new(DuckDbDialect {}),
+        Box::new(PostgreSqlDialect {}),
+        Box::new(MsSqlDialect {}),
+        Box::new(MySqlDialect {}),
+    ]);
     assert!(supported_dialects
         .parse_sql_statements("SELECT 💝 FROM table1")
         .is_err());

tests/sqlparser_redshift.rs

@@ -408,3 +408,8 @@ fn parse_extract_single_quotes() {
 fn parse_string_literal_backslash_escape() {
     redshift().one_statement_parses_to(r#"SELECT 'l\'auto'"#, "SELECT 'l''auto'");
 }
+
+#[test]
+fn parse_utf8_multibyte_idents() {
+    redshift().verified_stmt("SELECT 🚀.city AS 🎸 FROM customers AS 🚀");
+}