Improve docs-builder serve reliability and startup speed (#2982)

The serve command had several regressions that made the local authoring
workflow frustrating: editing docset.yml no longer reloaded properly,
adding/removing/renaming files required a full restart, OpenAPI generation
blocked startup, CTRL+C was slow to cancel, and the diagnostics HUD
disappeared after the first file change.

This fixes all of those issues and adds a quality-of-life improvement
that preserves navigation tree state across live reloads.

Configuration reload on file changes:
- Add BuildContext.ReloadConfiguration() to re-read docset.yml from disk
  when config files or file structure changes, while skipping the
  expensive re-parse for simple markdown content edits.
- Watch _docset.yml and toc.yml in addition to docset.yml and *.md so
  navigation structure changes are detected without a restart.
- Preserve serve-mode feature flags (e.g. diagnostics panel) across
  configuration reloads.

Lazy OpenAPI generation:
- Move OpenAPI document generation out of the startup path so the server
  reaches "Now listening on" immediately instead of blocking on spec
  parsing.
- Generate API references on-demand when the first /api/ request arrives,
  with change detection so specs are only regenerated when modified.

Faster CTRL+C cancellation:
- Wire a service-scoped CancellationTokenSource through the reload
  pipeline so in-flight reloads are cancelled promptly on shutdown,
  replacing the previous Cancel.None that caused hangs.

Navigation state preservation (dev mode only):
- Persist expanded/collapsed navigation tree state to sessionStorage so
  sections you had open stay open after a live reload, instead of
  collapsing back to only the current page's ancestors.
- Gated behind the diagnostics panel presence check so it has zero
  impact on production builds.

Made-with: Cursor

Author: Mpdreamz

docs/_docset.yml

@@ -266,4 +266,4 @@ toc:
           - file: known-issues.md
       - folder: release-notes-all
         children:
-          - file: index.md
+          - file: index.md

src/Elastic.Documentation.Configuration/BuildContext.cs

@@ -29,7 +29,7 @@ public record BuildContext : IDocumentationSetContext, IDocumentationConfigurati
 	public IDirectoryInfo DocumentationSourceDirectory { get; }
 	public IDirectoryInfo OutputDirectory { get; }
 
-	public ConfigurationFile Configuration { get; }
+	public ConfigurationFile Configuration { get; private set; }
 
 	public DocumentationSetFile ConfigurationYaml { get; set; }
 
@@ -134,4 +134,14 @@ public BuildContext(
 		};
 	}
 
+	/// <summary>Re-reads docset.yml from disk and rebuilds the configuration. Used by the serve command on file changes.</summary>
+	public void ReloadConfiguration()
+	{
+		var previousFeatures = Configuration.Features;
+		ConfigurationYaml = ConfigurationPath.Exists
+			? DocumentationSetFile.LoadAndResolve(Collector, ConfigurationPath, ReadFileSystem)
+			: new DocumentationSetFile();
+		Configuration = new ConfigurationFile(ConfigurationYaml, this, VersionsConfiguration, ProductsConfiguration);
+		Configuration.Features.DiagnosticsPanelEnabled = previousFeatures.DiagnosticsPanelEnabled;
+	}
 }

src/Elastic.Documentation.Site/Assets/pages-nav.ts

@@ -1,6 +1,35 @@
 import { throttle } from 'lodash'
 import { $, $$ } from 'select-dom'
 
+const NAV_STATE_KEY = 'nav-expanded'
+
+function isDevMode() {
+    return !!document.querySelector('diagnostics-panel')
+}
+
+function saveNavState(nav: HTMLElement) {
+    const expanded = $$('input[type="checkbox"]:checked', nav)
+        .map((el) => el.id)
+        .filter(Boolean)
+    sessionStorage.setItem(NAV_STATE_KEY, JSON.stringify(expanded))
+}
+
+function restoreNavState(nav: HTMLElement) {
+    const raw = sessionStorage.getItem(NAV_STATE_KEY)
+    if (!raw) return
+    try {
+        const ids: string[] = JSON.parse(raw)
+        for (const id of ids) {
+            const input = $(`#${CSS.escape(id)}`, nav)
+            if (input instanceof HTMLInputElement) {
+                input.checked = true
+            }
+        }
+    } catch {
+        /* ignore corrupt storage */
+    }
+}
+
 function expandAllParents(navItem: HTMLElement) {
     let parent: HTMLLIElement | null | undefined = navItem?.closest('li')
     while (parent) {
@@ -93,6 +122,10 @@ export function initNav() {
         preventFocusLossOnLinkClick(dropdownActiveAnchor)
     }
 
+    if (isDevMode()) {
+        restoreNavState(pagesNav)
+    }
+
     // Remove current class from all nav items before marking new ones
     const currentNavItems = $$('.current', pagesNav)
     currentNavItems.forEach((el) => {
@@ -109,4 +142,11 @@ export function initNav() {
         el.classList.add('current')
     })
     scrollCurrentNaviItemIntoView(pagesNav)
+
+    if (isDevMode()) {
+        saveNavState(pagesNav)
+        for (const cb of $$('input[type="checkbox"]', pagesNav)) {
+            cb.addEventListener('change', () => saveNavState(pagesNav))
+        }
+    }
 }

src/tooling/docs-builder/Http/DocumentationWebHost.cs

@@ -211,6 +211,8 @@ private static async Task WriteSSEEvent(HttpResponse response, string eventType,
 
 	private async Task<IResult> ServeApiFile(ReloadableGeneratorState holder, string slug, Cancel ctx)
 	{
+		await holder.EnsureApiReferencesAsync(ctx);
+
 		var path = Path.Combine(holder.ApiPath.FullName, slug.Trim('/'), "index.html");
 		var info = _writeFileSystem.FileInfo.New(path);
 		if (info.Exists)

src/tooling/docs-builder/Http/ReloadGeneratorService.cs

@@ -29,6 +29,7 @@ ILogger<ReloadGeneratorService> logger
 ) : IHostedService, IDisposable
 {
 	private FileSystemWatcher? _watcher;
+	private CancellationTokenSource? _serviceCts;
 	private ReloadableGeneratorState ReloadableGenerator { get; } = reloadableGenerator;
 	private InMemoryBuildState InMemoryBuildState { get; } = inMemoryBuildState;
 	private ILogger Logger { get; } = logger;
@@ -38,6 +39,8 @@ ILogger<ReloadGeneratorService> logger
 
 	public async Task StartAsync(Cancel cancellationToken)
 	{
+		_serviceCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+
 		// Run live reload and in-memory validation build in parallel
 		var sourcePath = ReloadableGenerator.Generator.Context.DocumentationSourceDirectory.FullName;
 		await Task.WhenAll(
@@ -73,12 +76,16 @@ await Task.WhenAll(
 #endif
 		watcher.Filters.Add("*.md");
 		watcher.Filters.Add("docset.yml");
+		watcher.Filters.Add("_docset.yml");
+		watcher.Filters.Add("toc.yml");
 		watcher.IncludeSubdirectories = true;
 		watcher.EnableRaisingEvents = true;
 		_watcher = watcher;
 	}
 
-	private void Reload() =>
+	private void Reload(bool reloadConfiguration = false)
+	{
+		var token = _serviceCts?.Token ?? Cancel.None;
 		_ = _debouncer.ExecuteAsync(async ctx =>
 		{
 			var sourcePath = ReloadableGenerator.Generator.Context.DocumentationSourceDirectory.FullName;
@@ -87,18 +94,24 @@ private void Reload() =>
 			var validationTask = InMemoryBuildState.StartBuildAsync(sourcePath, ctx);
 
 			// Wait for live reload to complete, then refresh the browser immediately
-			await ReloadableGenerator.ReloadAsync(ctx);
+			await ReloadableGenerator.ReloadAsync(ctx, reloadConfiguration);
 			Logger.LogInformation("Reload complete!");
 			_ = LiveReloadMiddleware.RefreshWebSocketRequest();
 
 			// Wait for validation build to complete
 			await validationTask;
-		}, Cancel.None);
+		}, token);
+	}
 
-	public Task StopAsync(Cancel cancellationToken)
+	public async Task StopAsync(Cancel cancellationToken)
 	{
+		if (_serviceCts is not null)
+		{
+			await _serviceCts.CancelAsync();
+			_serviceCts.Dispose();
+			_serviceCts = null;
+		}
 		_watcher?.Dispose();
-		return Task.CompletedTask;
 	}
 
 	// Check if a path should be ignored (output directories, hidden folders, etc.)
@@ -108,6 +121,9 @@ private static bool ShouldIgnorePath(string path) =>
 		path.Contains("/node_modules/") || path.Contains("\\node_modules\\") ||
 		path.Contains("/.git/") || path.Contains("\\.git\\");
 
+	private static bool IsConfigFile(string path) =>
+		path.EndsWith("docset.yml") || path.EndsWith("toc.yml");
+
 	private void OnChanged(object sender, FileSystemEventArgs e)
 	{
 		if (e.ChangeType != WatcherChangeTypes.Changed)
@@ -118,15 +134,14 @@ private void OnChanged(object sender, FileSystemEventArgs e)
 
 		Logger.LogInformation("Changed: {FullPath}", e.FullPath);
 
-		if (e.FullPath.EndsWith("docset.yml"))
-			Reload();
-		if (e.FullPath.EndsWith(".md"))
+		if (IsConfigFile(e.FullPath))
+			Reload(reloadConfiguration: true);
+		else if (e.FullPath.EndsWith(".md"))
 			Reload();
 #if DEBUG
 		if (e.FullPath.EndsWith(".cshtml"))
 			_ = LiveReloadMiddleware.RefreshWebSocketRequest();
 #endif
-
 	}
 
 	private void OnCreated(object sender, FileSystemEventArgs e)
@@ -135,8 +150,8 @@ private void OnCreated(object sender, FileSystemEventArgs e)
 			return;
 
 		Logger.LogInformation("Created: {FullPath}", e.FullPath);
-		if (e.FullPath.EndsWith(".md"))
-			Reload();
+		if (e.FullPath.EndsWith(".md") || IsConfigFile(e.FullPath))
+			Reload(reloadConfiguration: true);
 	}
 
 	private void OnDeleted(object sender, FileSystemEventArgs e)
@@ -145,8 +160,8 @@ private void OnDeleted(object sender, FileSystemEventArgs e)
 			return;
 
 		Logger.LogInformation("Deleted: {FullPath}", e.FullPath);
-		if (e.FullPath.EndsWith(".md"))
-			Reload();
+		if (e.FullPath.EndsWith(".md") || IsConfigFile(e.FullPath))
+			Reload(reloadConfiguration: true);
 	}
 
 	private void OnRenamed(object sender, RenamedEventArgs e)
@@ -157,8 +172,8 @@ private void OnRenamed(object sender, RenamedEventArgs e)
 		Logger.LogInformation("Renamed:");
 		Logger.LogInformation("    Old: {OldFullPath}", e.OldFullPath);
 		Logger.LogInformation("    New: {NewFullPath}", e.FullPath);
-		if (e.FullPath.EndsWith(".md"))
-			Reload();
+		if (e.FullPath.EndsWith(".md") || e.OldFullPath.EndsWith(".md") || IsConfigFile(e.FullPath) || IsConfigFile(e.OldFullPath))
+			Reload(reloadConfiguration: true);
 #if DEBUG
 		if (e.FullPath.EndsWith(".cshtml"))
 			_ = LiveReloadMiddleware.RefreshWebSocketRequest();
@@ -180,6 +195,7 @@ private void PrintException(Exception? ex)
 
 	public void Dispose()
 	{
+		_serviceCts?.Dispose();
 		_watcher?.Dispose();
 		_debouncer.Dispose();
 	}

src/tooling/docs-builder/Http/ReloadableGeneratorState.cs

@@ -56,11 +56,15 @@ bool isWatchBuild
 
 	// Track OpenAPI spec file modification times to detect changes
 	private readonly Dictionary<string, DateTimeOffset> _openApiSpecLastModified = [];
+	private volatile bool _apiReferencesStale = true;
+	private readonly SemaphoreSlim _apiSemaphore = new(1, 1);
 
-	public async Task ReloadAsync(Cancel ctx)
+	public async Task ReloadAsync(Cancel ctx, bool reloadConfiguration = true)
 	{
 		SourcePath.Refresh();
 		OutputPath.Refresh();
+		if (reloadConfiguration)
+			_context.ReloadConfiguration();
 		var crossLinks = await _crossLinkFetcher.FetchCrossLinks(ctx);
 		IUriEnvironmentResolver? uriResolver = crossLinks.CodexRepositories is not null
 			? new CodexAwareUriResolver(crossLinks.CodexRepositories)
@@ -76,12 +80,32 @@ public async Task ReloadAsync(Cancel ctx)
 		var generator = new DocumentationGenerator(docSet, _logFactory, markdownExporters: markdownExporters.ToArray());
 		await generator.ResolveDirectoryTree(ctx);
 		_ = Interlocked.Exchange(ref _generator, generator);
+		_apiReferencesStale = true;
+	}
+
+	/// <summary>Lazily generates OpenAPI references on the first /api/ request, and regenerates when spec files change.</summary>
+	public async Task EnsureApiReferencesAsync(Cancel ctx)
+	{
+		if (!_apiReferencesStale)
+			return;
 
-		// Only regenerate OpenAPI if spec files have changed
-		if (HaveOpenApiSpecsChanged(docSet.Configuration))
+		await _apiSemaphore.WaitAsync(ctx);
+		try
+		{
+			if (!_apiReferencesStale)
+				return;
+
+			var config = _generator.DocumentationSet.Configuration;
+			if (HaveOpenApiSpecsChanged(config))
+			{
+				await ReloadApiReferences(_generator.MarkdownStringRenderer, ctx);
+				UpdateOpenApiSpecTimestamps(config);
+			}
+			_apiReferencesStale = false;
+		}
+		finally
 		{
-			await ReloadApiReferences(generator.MarkdownStringRenderer, ctx);
-			UpdateOpenApiSpecTimestamps(docSet.Configuration);
+			_ = _apiSemaphore.Release();
 		}
 	}
 
@@ -137,6 +161,7 @@ private async Task ReloadApiReferences(IMarkdownStringRenderer markdownStringRen
 
 	public void Dispose()
 	{
+		_apiSemaphore.Dispose();
 		(_codexReader as IDisposable)?.Dispose();
 		GC.SuppressFinalize(this);
 	}