Update src/tokenizer.rs

xitep · iffyio · web-flow · commit a7e732b9b7ca · 2025-12-14T08:31:31.000+01:00
Co-authored-by: Ifeanyi Ubah &lt;ify1992@yahoo.com&gt;
diff --git a/src/tokenizer.rs b/src/tokenizer.rs
@@ -2079,7 +2079,6 @@ impl<'a> Tokenizer<'a> {
                 }
                 self.tokenizer_error(error_loc, "Unterminated string literal")
             }
-            // ~ not a literal introduced with _token_prefix_, assm
             _ => {
                 let s = self.tokenize_word(String::from_iter(word_prefix), chars);
                 Ok(Token::make_word(&s, None))

Original file line number	Diff line number	Diff line change
`@@ -2079,7 +2079,6 @@ impl<'a> Tokenizer<'a> {`
`2079`	`2079`	`}`
`2080`	`2080`	`self.tokenizer_error(error_loc, "Unterminated string literal")`
`2081`	`2081`	`}`
`2082`		`- // ~ not a literal introduced with _token_prefix_, assm`
`2083`	`2082`	`_ => {`
`2084`	`2083`	`let s = self.tokenize_word(String::from_iter(word_prefix), chars);`
`2085`	`2084`	`Ok(Token::make_word(&s, None))`