Fix nested comment parsing in literate scripts and add AddHtmlPrinter tests (#1138)

* Fix nested comment parsing in literate scripts and add AddHtmlPrinter tests

- Fix ParseScript.fs: the comment block parser used LastIndexOf("*)")
  to detect comment endings, which incorrectly matched nested
  (*** command ***) markers inside markdown text (e.g. backtick-quoted
  examples of include-it or include-it-raw). This caused content after
  such references to be silently dropped from HTML output. Replaced
  with a nesting-aware scanner (findOuterCommentEnd) that properly
  tracks (* ... *) depth.

- Add integration tests for:
  - Comment blocks containing nested (*** ***) markers
  - AddHtmlPrinter with include-it-raw
  - AddHtmlPrinter with CSS/JS resources
  - AddHtmlPrinter with base64 image pattern (as documented)
  - AddPrintTransformer chained with AddHtmlPrinter and include-it-raw

- Fix missing [<Test>] attribute on Can include-output-and-it test

* fix formatting

Author: dsyme

RELEASE_NOTES.md

@@ -3,6 +3,8 @@
 ## [Unreleased]
 
 ### Fixed
+* Fix literate script comment parser prematurely closing `(**` blocks when the markdown text contained nested `(*** ... ***)` references (e.g. in backtick-quoted command examples), causing subsequent content to be silently dropped from HTML output.
+* Add missing `[<Test>]` attribute on `Can include-output-and-it` test so it is executed by the test runner.
 * Add regression test confirming that types whose name matches their enclosing namespace are correctly included in generated API docs. [#944](https://github.com/fsprojects/FSharp.Formatting/issues/944)
 * Fix crash (`failwith "tbd - IndirectImage"`) when `Markdown.ToMd` is called on a document containing reference-style images with bracket syntax. The indirect image is now serialised as `![alt](url)` when the reference is resolved, or in bracket notation when it is not. [#1094](https://github.com/fsprojects/FSharp.Formatting/pull/1094)
 * Fix `Markdown.ToMd` serialising italic spans with asterisks incorrectly as bold spans. [#1102](https://github.com/fsprojects/FSharp.Formatting/pull/1102)

src/FSharp.Formatting.Literate/ParseScript.fs

@@ -59,11 +59,26 @@ module internal CodeBlockUtils =
     //  * collectSnippet - we're in a normal F# code and we're waiting for a comment
     //    (in both states, we also need to recognize (*** commands ***)
 
+    /// Find the position of the first `*)` at comment-nesting depth 0,
+    /// properly skipping nested `(* ... *)` pairs. Returns -1 if not found.
+    let private findOuterCommentEnd (comment: string) =
+        let rec scan depth i =
+            if i + 1 >= comment.Length then
+                -1
+            elif comment.[i] = '(' && comment.[i + 1] = '*' then
+                scan (depth + 1) (i + 2)
+            elif comment.[i] = '*' && comment.[i + 1] = ')' then
+                if depth > 0 then scan (depth - 1) (i + 2) else i
+            else
+                scan depth (i + 1)
+
+        scan 0 0
+
     /// Waiting for the end of a comment
     let rec private collectComment (comment: string) lines =
         seq {
             let findCommentEnd (comment: string) =
-                let cend = comment.LastIndexOf("*)", StringComparison.OrdinalIgnoreCase)
+                let cend = findOuterCommentEnd comment
 
                 if cend = -1 then
                     failwith "A (* comment was not closed"
@@ -79,7 +94,7 @@ module internal CodeBlockUtils =
                 yield! collectSnippet [] lines
 
             | (ConcatenatedComments text) :: _ when
-                comment.LastIndexOf("*)", StringComparison.Ordinal) <> -1
+                findOuterCommentEnd comment >= 0
                 && text.Trim().StartsWith("//", StringComparison.Ordinal)
                 ->
                 // Comment ended, but we found a code snippet starting with // comment

tests/FSharp.Literate.Tests/EvalTests.fs

@@ -507,6 +507,7 @@ let xxxx = 1+1
 
     html1 |> shouldNotContainText "2000"
 
+[<Test>]
 let ``Can include-output-and-it`` () =
     let content =
         """
@@ -593,3 +594,121 @@ $$$
     pynb |> shouldContainText """\begin{equation}"""
     pynb |> shouldContainText """\end{equation}"""
     pynb |> shouldContainText """30001"""
+
+[<Test>]
+let ``Comment with nested (*** ***) and following // line is not truncated`` () =
+    let content =
+        """
+(**
+Some text mentioning `(*** include-it ***)` or `(*** include-it-raw ***)`.
+
+A paragraph after nested comment markers.
+
+    [lang=fsharp]
+    // a comment inside an indented code block
+    let x = 1
+*)
+let a = 1
+"""
+
+    let doc = Literate.ParseScriptString(content, "." </> "A.fsx")
+
+    let html = Literate.ToHtml(doc)
+    // The paragraph after the nested markers must not be truncated
+    html |> shouldContainText "A paragraph after nested comment markers"
+    html |> shouldContainText "include-it-raw"
+
+[<Test>]
+let ``Can include-it-raw with AddHtmlPrinter`` () =
+    let content =
+        """
+type Html = Html of string
+fsi.AddHtmlPrinter(fun (Html h) -> seq [], h)
+Html "<div>RAW_HTML_OUTPUT</div>"
+(*** include-it-raw ***)
+"""
+
+    let fsie = getFsiEvaluator ()
+
+    let doc1 = Literate.ParseScriptString(content, "." </> "A.fsx", fsiEvaluator = fsie)
+
+    let html1 = Literate.ToHtml(doc1)
+    html1 |> shouldContainText "<div>RAW_HTML_OUTPUT</div>"
+
+[<Test>]
+let ``AddHtmlPrinter with CSS and JS resources`` () =
+    let content =
+        """
+type Widget = Widget of string
+fsi.AddHtmlPrinter(fun (Widget w) ->
+    seq [
+        "style", ".widget { color: red; }"
+        "script", "console.log('widget loaded')"
+    ], sprintf "<span class='widget'>%s</span>" w)
+Widget "test-widget"
+(*** include-it ***)
+"""
+
+    let fsie = getFsiEvaluator ()
+
+    let doc1 = Literate.ParseScriptString(content, "." </> "A.fsx", fsiEvaluator = fsie)
+
+    let html1 = Literate.ToHtml(doc1)
+    html1 |> shouldContainText "test-widget"
+    html1 |> shouldContainText ".widget { color: red; }"
+
+[<Test>]
+let ``AddHtmlPrinter with base64 image pattern`` () =
+    let content =
+        """
+type MyChart = { Data: int list }
+fsi.AddHtmlPrinter(fun (chart: MyChart) ->
+    let b64 = System.Convert.ToBase64String([| 0uy; 1uy; 2uy |])
+    seq [], sprintf "<img src=\"data:image/png;base64,%s\" />" b64)
+{ Data = [1;2;3] }
+(*** include-it ***)
+"""
+
+    let fsie = getFsiEvaluator ()
+
+    let doc1 = Literate.ParseScriptString(content, "." </> "A.fsx", fsiEvaluator = fsie)
+
+    let html1 = Literate.ToHtml(doc1)
+    html1 |> shouldContainText "data:image/png;base64,"
+    html1 |> shouldContainText "<img"
+
+[<Test>]
+let ``AddHtmlPrinter with include-it-raw emits unescaped HTML`` () =
+    let content =
+        """
+type Tag = Tag of string
+fsi.AddHtmlPrinter(fun (Tag t) -> seq [], sprintf "<b>%s</b>" t)
+Tag "BOLDTEXT"
+(*** include-it-raw ***)
+"""
+
+    let fsie = getFsiEvaluator ()
+
+    let doc1 = Literate.ParseScriptString(content, "." </> "A.fsx", fsiEvaluator = fsie)
+
+    let html1 = Literate.ToHtml(doc1)
+    html1 |> shouldContainText "<b>BOLDTEXT</b>"
+
+[<Test>]
+let ``AddPrintTransformer chained with AddHtmlPrinter and include-it-raw`` () =
+    let content =
+        """
+type Html = Html of string
+type Wrapped = Wrapped of string
+fsi.AddPrintTransformer(fun (Wrapped s) -> box (Html s))
+fsi.AddHtmlPrinter(fun (Html h) -> seq [], sprintf "<em>%s</em>" h)
+Wrapped "TRANSFORMED"
+(*** include-it-raw ***)
+"""
+
+    let fsie = getFsiEvaluator ()
+
+    let doc1 = Literate.ParseScriptString(content, "." </> "A.fsx", fsiEvaluator = fsie)
+
+    let html1 = Literate.ToHtml(doc1)
+    html1 |> shouldContainText "<em>TRANSFORMED</em>"