Merge sigma origin/main into upstream/main for sync

# Conflicts:
#	src/ast/mod.rs
#	src/ast/spans.rs
#	src/lib.rs
#	src/parser/mod.rs
#	tests/sqlparser_snowflake.rs

Author: ayman-sigma

.github/workflows/semgrep.yml

@@ -0,0 +1,19 @@
+name: Semgrep
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  semgrep:
+    name: Run Semgrep 
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    container:
+      # A Docker image with Semgrep installed. Do not change this.
+      image: returntocorp/semgrep
+    if: (github.actor != 'dependabot[bot]')
+    steps:
+      - uses: actions/checkout@v4
+      - run: semgrep ci
+        env:
+          SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN_PUBLIC }}

examples/cli.rs

@@ -48,6 +48,7 @@ $ cargo run --example cli - [--dialectname]
 
     let dialect: Box<dyn Dialect> = match std::env::args().nth(2).unwrap_or_default().as_ref() {
         "--ansi" => Box::new(AnsiDialect {}),
+        "--databricks" => Box::new(DatabricksDialect {}),
         "--bigquery" => Box::new(BigQueryDialect {}),
         "--postgres" => Box::new(PostgreSqlDialect {}),
         "--ms" => Box::new(MsSqlDialect {}),

src/ast/mod.rs

@@ -646,6 +646,9 @@ pub enum JsonPathElem {
     /// Accesses an object field or array element using bracket notation,
     /// e.g. `obj['foo']`.
     ///
+    /// Note that on Databricks this is *not* equivalent to dot notation; the
+    /// former is case-insensitive but the latter is not.
+    ///
     /// See <https://docs.snowflake.com/en/user-guide/querying-semistructured#bracket-notation>.
     Bracket {
         /// The expression used as the bracket key (string or numeric expression).
@@ -659,6 +662,11 @@ pub enum JsonPathElem {
         /// The expression used as the bracket key (string or numeric expression).
         key: Expr,
     },
+    /// Accesses all elements in the given (generally array) element. Used for
+    /// constructs like `foo:bar[*].baz`.
+    ///
+    /// See <https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-json-path-expression#extract-values-from-arrays>
+    AllElements,
 }
 
 /// A JSON path.
@@ -669,18 +677,23 @@ pub enum JsonPathElem {
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
 pub struct JsonPath {
+    /// True if the path should start with a colon. Some dialects (e.g. Snowflake) allow
+    /// `a['b']`, whereas others (e.g. Databricks) require the colon even in this case
+    /// (so `a:['b']`).
+    pub has_colon: bool,
     /// Sequence of path elements that form the JSON path.
     pub path: Vec<JsonPathElem>,
 }
 
 impl fmt::Display for JsonPath {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.has_colon {
+            write!(f, ":")?;
+        }
         for (i, elem) in self.path.iter().enumerate() {
             match elem {
                 JsonPathElem::Dot { key, quoted } => {
-                    if i == 0 {
-                        write!(f, ":")?;
-                    } else {
+                    if i != 0 {
                         write!(f, ".")?;
                     }
 
@@ -693,6 +706,9 @@ impl fmt::Display for JsonPath {
                 JsonPathElem::Bracket { key } => {
                     write!(f, "[{key}]")?;
                 }
+                JsonPathElem::AllElements => {
+                    write!(f, "[*]")?;
+                }
                 JsonPathElem::ColonBracket { key } => {
                     write!(f, ":[{key}]")?;
                 }
@@ -915,6 +931,13 @@ pub enum Expr {
         /// `true` when the `NOT` modifier is present.
         negated: bool,
     },
+    /// XXX not valid SQL syntax, this is a hack needed to support parameter substitution
+    /// `[ NOT ] IN <in_expr>`
+    InExpr {
+        expr: Box<Expr>,
+        in_expr: Box<Expr>,
+        negated: bool,
+    },
     /// `[ NOT ] IN UNNEST(array_expression)`
     InUnnest {
         /// Left-hand expression to test for membership.
@@ -1719,6 +1742,17 @@ impl fmt::Display for Expr {
                 if *negated { "NOT " } else { "" },
                 subquery
             ),
+            Expr::InExpr {
+                expr,
+                in_expr,
+                negated,
+            } => write!(
+                f,
+                "{} {}IN {}",
+                expr,
+                if *negated { "NOT " } else { "" },
+                in_expr,
+            ),
             Expr::InUnnest {
                 expr,
                 array_expr,

src/ast/query.rs

@@ -1492,6 +1492,12 @@ pub enum TableFactor {
         /// Optional table sample modifier
         sample: Option<TableSampleKind>,
     },
+    /// A pass-through query string that is not parsed.
+    /// This is useful while building/rewriting queries with a known valid SQL string and to avoid parsing it.
+    PassThroughQuery {
+        query: String,
+        alias: Option<TableAlias>,
+    },
     /// `TABLE(<expr>)[ AS <alias> ]`
     TableFunction {
         /// Expression representing the table function call.
@@ -2253,6 +2259,13 @@ impl fmt::Display for TableFactor {
                 if let Some(alias) = alias {
                     write!(f, " {alias}")?;
                 }
+                Ok(())
+            }
+            TableFactor::PassThroughQuery { query, alias } => {
+                write!(f, "({query})")?;
+                if let Some(alias) = alias {
+                    write!(f, " {alias}")?;
+                }
                 if let Some(TableSampleKind::AfterTableAlias(sample)) = sample {
                     write!(f, " {sample}")?;
                 }

src/ast/spans.rs

@@ -1479,6 +1479,11 @@ impl Spanned for Expr {
                 array_expr,
                 negated: _,
             } => expr.span().union(&array_expr.span()),
+            Expr::InExpr {
+                expr,
+                in_expr,
+                negated: _,
+            } => expr.span().union(&in_expr.span()),
             Expr::Between {
                 expr,
                 negated: _,
@@ -1787,7 +1792,7 @@ impl Spanned for FunctionArgumentClause {
 /// see Spanned impl for JsonPathElem for more information
 impl Spanned for JsonPath {
     fn span(&self) -> Span {
-        let JsonPath { path } = self;
+        let JsonPath { path, has_colon: _ } = self;
 
         union_spans(path.iter().map(|i| i.span()))
     }
@@ -1797,11 +1802,13 @@ impl Spanned for JsonPath {
 ///
 /// Missing spans:
 /// - [JsonPathElem::Dot]
+/// - [JsonPathElem::AllElements]
 impl Spanned for JsonPathElem {
     fn span(&self) -> Span {
         match self {
             JsonPathElem::Dot { .. } => Span::empty(),
             JsonPathElem::Bracket { key } => key.span(),
+            JsonPathElem::AllElements => Span::empty(),
             JsonPathElem::ColonBracket { key } => key.span(),
         }
     }
@@ -1952,6 +1959,8 @@ impl Spanned for TableFactor {
             } => subquery
                 .span()
                 .union_opt(&alias.as_ref().map(|alias| alias.span())),
+            // This is usually created at runtime, so we don't have a span for it
+            TableFactor::PassThroughQuery { query: _, alias: _ } => Span::empty(),
             TableFactor::TableFunction { expr, alias } => expr
                 .span()
                 .union_opt(&alias.as_ref().map(|alias| alias.span())),

src/dialect/databricks.rs

@@ -15,7 +15,9 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use crate::dialect::Dialect;
+use crate::dialect::{Dialect, Precedence};
+use crate::parser::{Parser, ParserError};
+use crate::tokenizer::Token;
 
 /// A [`Dialect`] for [Databricks SQL](https://www.databricks.com/)
 ///
@@ -39,6 +41,19 @@ impl Dialect for DatabricksDialect {
         matches!(ch, 'a'..='z' | 'A'..='Z' | '0'..='9' | '_')
     }
 
+    fn get_next_precedence(&self, parser: &Parser) -> Option<Result<u8, ParserError>> {
+        let token = parser.peek_token();
+        // : is used for JSON path access
+        match token.token {
+            Token::Colon => Some(Ok(self.prec_value(Precedence::Period))),
+            _ => None,
+        }
+    }
+
+    fn supports_semi_structured_array_all_elements(&self) -> bool {
+        true
+    }
+
     fn supports_filter_during_aggregation(&self) -> bool {
         true
     }
@@ -76,6 +91,11 @@ impl Dialect for DatabricksDialect {
         true
     }
 
+    // https://docs.databricks.com/aws/en/sql/language-manual/data-types/string-type#literals
+    fn supports_string_literal_backslash_escape(&self) -> bool {
+        true
+    }
+
     /// See <https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-qry-select-groupby.html>
     fn supports_group_by_with_modifier(&self) -> bool {
         true

src/dialect/mod.rs

@@ -1207,6 +1207,11 @@ pub trait Dialect: Debug + Any {
         false
     }
 
+    /// Returns true if the dialect supports writing `[*]` to select all elements in a JSON array.
+    fn supports_semi_structured_array_all_elements(&self) -> bool {
+        false
+    }
+
     /// Returns true if the specified keyword is reserved and cannot be
     /// used as an identifier without special handling like quoting.
     fn is_reserved_for_identifier(&self, kw: Keyword) -> bool {

src/lib.rs

@@ -153,6 +153,8 @@
 // Splitting complex nodes (expressions, statements, types) into separate types
 // would bloat the API and hide intent. Extra memory is a worthwhile tradeoff.
 #![allow(clippy::large_enum_variant)]
+// TODO: Fix and remove this.
+#![expect(clippy::unnecessary_unwrap)]
 #![forbid(clippy::unreachable)]
 #![forbid(missing_docs)]
 

src/parser/mod.rs

@@ -4224,8 +4224,10 @@ impl<'a> Parser<'a> {
         })
     }
 
+    // Parser is either looking at a : or a bracket expression.
     fn parse_json_path(&mut self) -> Result<JsonPath, ParserError> {
         let mut path = Vec::new();
+        let mut has_colon = false;
         loop {
             match self.next_token().token {
                 Token::Colon if path.is_empty() && self.peek_token_ref() == &Token::LBracket => {
@@ -4235,16 +4237,19 @@ impl<'a> Parser<'a> {
                     path.push(JsonPathElem::ColonBracket { key });
                 }
                 Token::Colon if path.is_empty() => {
-                    path.push(self.parse_json_path_object_key()?);
+                    has_colon = true;
+                    if *self.peek_token_ref() == Token::LBracket {
+                        path.push(self.parse_json_path_bracket_element()?);
+                    } else {
+                        path.push(self.parse_json_path_object_key()?);
+                    }
                 }
                 Token::Period if !path.is_empty() => {
                     path.push(self.parse_json_path_object_key()?);
                 }
                 Token::LBracket => {
-                    let key = self.parse_wildcard_expr()?;
-                    self.expect_token(&Token::RBracket)?;
-
-                    path.push(JsonPathElem::Bracket { key });
+                    self.prev_token();
+                    path.push(self.parse_json_path_bracket_element()?);
                 }
                 _ => {
                     self.prev_token();
@@ -4254,7 +4259,23 @@ impl<'a> Parser<'a> {
         }
 
         debug_assert!(!path.is_empty());
-        Ok(JsonPath { path })
+        Ok(JsonPath { has_colon, path })
+    }
+
+    /// Parses a single bracketed element in a JSON path expression, including both brackets.
+    fn parse_json_path_bracket_element(&mut self) -> Result<JsonPathElem, ParserError> {
+        self.expect_token(&Token::LBracket)?;
+        let elem = if *self.peek_token_ref() == Token::Mul
+            && self.dialect.supports_semi_structured_array_all_elements()
+        {
+            self.expect_token(&Token::Mul)?;
+            JsonPathElem::AllElements
+        } else {
+            let key = self.parse_expr()?;
+            JsonPathElem::Bracket { key }
+        };
+        self.expect_token(&Token::RBracket)?;
+        Ok(elem)
     }
 
     /// Parses the parens following the `[ NOT ] IN` operator.
@@ -4271,25 +4292,34 @@ impl<'a> Parser<'a> {
                 negated,
             });
         }
-        self.expect_token(&Token::LParen)?;
-        let in_op = match self.maybe_parse(|p| p.parse_query())? {
-            Some(subquery) => Expr::InSubquery {
-                expr: Box::new(expr),
-                subquery,
-                negated,
-            },
-            None => Expr::InList {
-                expr: Box::new(expr),
-                list: if self.dialect.supports_in_empty_list() {
-                    self.parse_comma_separated0(Parser::parse_expr, Token::RParen)?
-                } else {
-                    self.parse_comma_separated(Parser::parse_expr)?
+        if self.consume_token(&Token::LParen) {
+            let in_op = match self.maybe_parse(|p| p.parse_query())? {
+                Some(subquery) => Expr::InSubquery {
+                    expr: Box::new(expr),
+                    subquery,
+                    negated,
                 },
+                None => Expr::InList {
+                    expr: Box::new(expr),
+                    list: if self.dialect.supports_in_empty_list() {
+                        self.parse_comma_separated0(Parser::parse_expr, Token::RParen)?
+                    } else {
+                        self.parse_comma_separated(Parser::parse_expr)?
+                    },
+                    negated,
+                },
+            };
+            self.expect_token(&Token::RParen)?;
+            Ok(in_op)
+        } else {
+            // parse an expr
+            let in_expr = self.parse_expr()?;
+            Ok(Expr::InExpr {
+                expr: Box::new(expr),
+                in_expr: Box::new(in_expr),
                 negated,
-            },
-        };
-        self.expect_token(&Token::RParen)?;
-        Ok(in_op)
+            })
+        }
     }
 
     /// Parses `BETWEEN <low> AND <high>`, assuming the `BETWEEN` keyword was already consumed.
@@ -15586,7 +15616,8 @@ impl<'a> Parser<'a> {
                         | TableFactor::Unpivot { alias, .. }
                         | TableFactor::MatchRecognize { alias, .. }
                         | TableFactor::SemanticView { alias, .. }
-                        | TableFactor::NestedJoin { alias, .. } => {
+                        | TableFactor::NestedJoin { alias, .. }
+                        | TableFactor::PassThroughQuery { alias, .. } => {
                             // but not `FROM (mytable AS alias1) AS alias2`.
                             if let Some(inner_alias) = alias {
                                 return Err(ParserError::ParserError(format!(