Fix compilation errors and update tests after upstream sync

- Add missing `pos` field to State initializer in tokenizer hint parsing
- Remove spurious `sample` reference that leaked into PassThroughQuery Display
- Add `sample` AfterTableAlias check back to Derived Display
- Add missing doc comments to sigma types (InExpr, PassThroughQuery, TokenWithRange)
  required by upstream's new #![forbid(missing_docs)]
- Remove unfulfilled #![expect(clippy::unnecessary_unwrap)] from lib.rs
- Add has_colon: true to JsonPath initializers in Snowflake-style colon-path tests
- Update Databricks a:['b'] test to use new ColonBracket AST node (upstream change)
- Exclude DatabricksDialect from parse_array_subscript test since Databricks uses
  `:` for JSON paths, conflicting with array slice syntax arr[1:2]

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: ayman-sigma

src/ast/mod.rs

@@ -934,8 +934,11 @@ pub enum Expr {
     /// XXX not valid SQL syntax, this is a hack needed to support parameter substitution
     /// `[ NOT ] IN <in_expr>`
     InExpr {
+        /// Left-hand expression to test for membership.
         expr: Box<Expr>,
+        /// The expression providing the candidate values (used for parameter substitution).
         in_expr: Box<Expr>,
+        /// `true` when the `NOT` modifier is present.
         negated: bool,
     },
     /// `[ NOT ] IN UNNEST(array_expression)`

src/ast/query.rs

@@ -1495,7 +1495,9 @@ pub enum TableFactor {
     /// A pass-through query string that is not parsed.
     /// This is useful while building/rewriting queries with a known valid SQL string and to avoid parsing it.
     PassThroughQuery {
+        /// The raw SQL query string to pass through without parsing.
         query: String,
+        /// Optional alias for the pass-through query.
         alias: Option<TableAlias>,
     },
     /// `TABLE(<expr>)[ AS <alias> ]`
@@ -2259,16 +2261,16 @@ impl fmt::Display for TableFactor {
                 if let Some(alias) = alias {
                     write!(f, " {alias}")?;
                 }
+                if let Some(TableSampleKind::AfterTableAlias(sample)) = sample {
+                    write!(f, " {sample}")?;
+                }
                 Ok(())
             }
             TableFactor::PassThroughQuery { query, alias } => {
                 write!(f, "({query})")?;
                 if let Some(alias) = alias {
                     write!(f, " {alias}")?;
                 }
-                if let Some(TableSampleKind::AfterTableAlias(sample)) = sample {
-                    write!(f, " {sample}")?;
-                }
                 Ok(())
             }
             TableFactor::Function {

src/lib.rs

@@ -153,8 +153,6 @@
 // Splitting complex nodes (expressions, statements, types) into separate types
 // would bloat the API and hide intent. Extra memory is a worthwhile tradeoff.
 #![allow(clippy::large_enum_variant)]
-// TODO: Fix and remove this.
-#![expect(clippy::unnecessary_unwrap)]
 #![forbid(clippy::unreachable)]
 #![forbid(missing_docs)]
 

src/tokenizer.rs

@@ -788,13 +788,18 @@ impl fmt::Display for TokenWithSpan {
     }
 }
 
+/// A token together with its byte-offset range in the source string.
 pub struct TokenWithRange {
+    /// The token.
     pub token: Token,
+    /// The byte offset of the start of the token in the source string.
     pub start: usize,
+    /// The byte offset of the end of the token in the source string.
     pub end: usize,
 }
 
 impl TokenWithRange {
+    /// Creates a new [`TokenWithRange`] with the given token and byte-offset range.
     pub fn new(token: Token, start: usize, end: usize) -> Self {
         Self { token, start, end }
     }
@@ -957,6 +962,7 @@ impl<'a> Tokenizer<'a> {
         Ok(twl.into_iter().map(|t| t.token).collect())
     }
 
+    /// Tokenize the statement and produce a vector of tokens with their byte-offset ranges.
     pub fn tokenize_with_range(&mut self) -> Result<Vec<TokenWithRange>, TokenizerError> {
         let mut tokens = Vec::<TokenWithRange>::new();
         let mut state = State {
@@ -1057,6 +1063,7 @@ impl<'a> Tokenizer<'a> {
         // Create a state for tracking position within the hint
         let mut state = State {
             peekable: hint_content.chars().peekable(),
+            pos: 0,
             line: span.start.line,
             col: span.start.column,
         };

tests/sqlparser_common.rs

@@ -18647,6 +18647,7 @@ fn parse_semi_structured_data_traversal() {
         SelectItem::UnnamedExpr(Expr::JsonAccess {
             value: Box::new(Expr::Identifier(Ident::new("a"))),
             path: JsonPath {
+                has_colon: true,
                 path: vec![JsonPathElem::Dot {
                     key: "b".to_owned(),
                     quoted: false
@@ -18663,6 +18664,7 @@ fn parse_semi_structured_data_traversal() {
         SelectItem::UnnamedExpr(Expr::JsonAccess {
             value: Box::new(Expr::Identifier(Ident::new("a"))),
             path: JsonPath {
+                has_colon: true,
                 path: vec![JsonPathElem::Dot {
                     key: "my long object key name".to_owned(),
                     quoted: true
@@ -18682,6 +18684,7 @@ fn parse_semi_structured_data_traversal() {
             SelectItem::UnnamedExpr(Expr::JsonAccess {
                 value: Box::new(Expr::Identifier(Ident::new("a"))),
                 path: JsonPath {
+                    has_colon: true,
                     path: vec![JsonPathElem::Dot {
                         key: "select".to_owned(),
                         quoted: false
@@ -18691,6 +18694,7 @@ fn parse_semi_structured_data_traversal() {
             SelectItem::UnnamedExpr(Expr::JsonAccess {
                 value: Box::new(Expr::Identifier(Ident::new("a"))),
                 path: JsonPath {
+                    has_colon: true,
                     path: vec![JsonPathElem::Dot {
                         key: "from".to_owned(),
                         quoted: false
@@ -18709,6 +18713,7 @@ fn parse_semi_structured_data_traversal() {
         vec![SelectItem::UnnamedExpr(Expr::JsonAccess {
             value: Box::new(Expr::Identifier(Ident::new("a"))),
             path: JsonPath {
+                has_colon: true,
                 path: vec![
                     JsonPathElem::Dot {
                         key: "foo".to_owned(),
@@ -18736,6 +18741,7 @@ fn parse_semi_structured_data_traversal() {
         vec![SelectItem::UnnamedExpr(Expr::JsonAccess {
             value: Box::new(Expr::Identifier(Ident::new("a"))),
             path: JsonPath {
+                has_colon: true,
                 path: vec![
                     JsonPathElem::Dot {
                         key: "foo".to_owned(),
@@ -18762,6 +18768,9 @@ fn parse_array_subscript() {
             || d.is::<SnowflakeDialect>()
             || d.is::<SQLiteDialect>()
             || d.is::<RedshiftSqlDialect>()
+            // Databricks uses `:` for JSON path access (high precedence), which conflicts
+            // with array slice syntax `arr[1:2]`.
+            || d.is::<DatabricksDialect>()
     });
 
     dialects.verified_stmt("SELECT arr[1]");

tests/sqlparser_databricks.rs

@@ -718,16 +718,16 @@ fn parse_semi_structured_data_traversal() {
         select.projection[0]
     );
 
-    // asterisk for arrays
+    // colon bracket notation: a:['b'].c
     let sql = "SELECT a:['b'].c FROM t";
     let select = databricks().verified_only_select(sql);
     assert_eq!(
         SelectItem::UnnamedExpr(Expr::JsonAccess {
             value: Box::new(Expr::Identifier(Ident::new("a"))),
             path: JsonPath {
-                has_colon: true,
+                has_colon: false,
                 path: vec![
-                    JsonPathElem::Bracket {
+                    JsonPathElem::ColonBracket {
                         key: Expr::value(Value::SingleQuotedString("b".to_owned())),
                     },
                     JsonPathElem::Dot {