feat(assembler): on-demand assembler serve with live reload and improved frontend DX

.github/workflows/ci.yml

@@ -57,7 +57,7 @@ jobs:
           dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler config init --local 
           dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler clone -c local --skip-private-repositories
           dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler build -c local --skip-private-repositories
-          dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler serve &
+          dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler serve-static &
 
       - name: Wait for docs
         working-directory: src/Elastic.Documentation.Site

aspire/AppHost.cs

@@ -103,7 +103,7 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool assu
 		.WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl)
 		.WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath)
 		.WithHttpEndpoint(port: 4000, isProxied: false)
-		.WithArgs(["assembler", "serve", .. globalArguments])
+		.WithArgs(["assembler", "serve-static", .. globalArguments])
 		.WithHttpHealthCheck("/", 200)
 		.WaitForCompletion(buildAll)
 		.WithParentRelationship(cloneAll);

build/CommandLine.fs

@@ -43,6 +43,7 @@ type Build =
 
     | [<CliPrefix(CliPrefix.None);SubCommand>] Format of ParseResults<FormatArgs>
     | [<CliPrefix(CliPrefix.None);SubCommand>] Watch
+    | [<CliPrefix(CliPrefix.None);SubCommand>] Watch_Full
 
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] Lint of ParseResults<LintArgs>
     | [<CliPrefix(CliPrefix.None);Hidden;SubCommand>] PristineCheck
@@ -84,6 +85,7 @@ with
             | Format _ -> "runs dotnet format"
 
             | Watch -> "runs dotnet watch to continuous build code/templates and web assets on the fly"
+            | Watch_Full -> "runs assembler serve with dotnet watch — watches checkout dirs and live-reloads assembled docs"
 
             // steps
             | Lint _ -> "runs dotnet format --verify-no-changes"

build/Targets.fs

@@ -42,6 +42,8 @@ let private format (formatArgs: ParseResults<FormatArgs>) =
 
 let private watch _ = exec { run "dotnet" "watch" "--project" "src/tooling/docs-builder" "--configuration" "debug" "--" "serve" "--watch" }
 
+let private watchFull _ = exec { run "dotnet" "watch" "--project" "src/tooling/docs-builder" "--configuration" "debug" "--" "assembler" "serve" }
+
 let private lint (lintArgs: ParseResults<LintArgs>) =
     let includeFiles = lintArgs.TryGetResult LintArgs.Include |> Option.defaultValue []
     let includeArgs = 
@@ -256,6 +258,7 @@ let Setup (parsed:ParseResults<Build>) =
 
         | Format formatArgs -> Build.Step (fun _ -> format formatArgs)
         | Watch -> Build.Step watch
+        | Watch_Full -> Build.Step watchFull
 
         // steps
         | Lint lintArgs -> Build.Step (fun _ -> lint lintArgs)

src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs

@@ -9,7 +9,7 @@
 
 namespace Elastic.Documentation.LinkIndex;
 
-public class Aws3LinkIndexReader(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexReader
+public class Aws3LinkIndexReader(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexReader, IDisposable
 {
 
 	// <summary>
@@ -52,4 +52,10 @@ public async Task<RepositoryLinks> GetRepositoryLinks(string key, Cancel cancell
 	}
 
 	public string RegistryUrl { get; } = $"https://{bucketName}.s3.{s3Client.Config.RegionEndpoint.SystemName}.amazonaws.com/{registryKey}";
+
+	public void Dispose()
+	{
+		s3Client.Dispose();
+		GC.SuppressFinalize(this);
+	}
 }

src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs

@@ -44,9 +44,10 @@ public record FetchedCrossLinks
 	};
 }
 
-public abstract class CrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider, ScopedFileSystem? fileSystem = null) : IDisposable
+public abstract class CrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider, ScopedFileSystem? fileSystem = null, bool ownsReader = false) : IDisposable
 {
 	protected ILogger Logger { get; } = logFactory.CreateLogger(nameof(CrossLinkFetcher));
+	protected ILinkIndexReader LinkIndexProvider => linkIndexProvider;
 	private readonly IFileSystem _fileSystem = fileSystem ?? FileSystemFactory.AppData;
 	private LinkRegistry? _linkIndex;
 
@@ -192,7 +193,11 @@ private void WriteLinksJsonCachedFile(string repository, LinkRegistryEntry linkR
 
 	public void Dispose()
 	{
-		logFactory.Dispose();
+		// Only dispose linkIndexProvider when this fetcher created it (ownsReader = true).
+		// When the reader was injected by the caller, the caller retains ownership and must dispose it.
+		// logFactory is always injected — never disposed here.
+		if (ownsReader && linkIndexProvider is IDisposable disposableReader)
+			disposableReader.Dispose();
 		GC.SuppressFinalize(this);
 	}
 }

src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs

@@ -16,9 +16,11 @@ public class DocSetConfigurationCrossLinkFetcher(
 	ConfigurationFile configuration,
 	ILinkIndexReader? linkIndexProvider = null,
 	ILinkIndexReader? codexLinkIndexReader = null)
-	: CrossLinkFetcher(logFactory, linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous())
+	: CrossLinkFetcher(logFactory, linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous(), ownsReader: linkIndexProvider is null)
 {
 	private readonly ILogger _logger = logFactory.CreateLogger(nameof(DocSetConfigurationCrossLinkFetcher));
+	// _codexReader is injected by the caller who retains ownership and is responsible for disposal.
+	// ReloadableGeneratorState, the primary caller, disposes it directly in its own Dispose().
 	private readonly ILinkIndexReader? _codexReader = codexLinkIndexReader;
 
 	public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
@@ -30,7 +32,7 @@ public override async Task<FetchedCrossLinks> FetchCrossLinks(Cancel ctx)
 		var codexRepositories = new HashSet<string>();
 		var declaredRepositories = new HashSet<string>();
 
-		var publicReader = linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous();
+		var publicReader = LinkIndexProvider;
 		var useDualRegistry = configuration.Registry != DocSetRegistry.Public && _codexReader is not null;
 
 		foreach (var entry in configuration.CrossLinkEntries)

src/Elastic.Markdown/IO/DocumentationSet.cs

@@ -214,14 +214,26 @@ public INavigationItem FindNavigationByMarkdown(MarkdownFile markdown)
 	}
 
 	private bool _resolved;
+	private long _version;
+
+	public void InvalidateResolved()
+	{
+		_ = Interlocked.Increment(ref _version);
+		_resolved = false;
+	}
+
 	public async Task ResolveDirectoryTree(Cancel ctx)
 	{
 		if (_resolved)
 			return;
 
+		// Capture the version before parsing so that if InvalidateResolved() fires
+		// mid-flight we do not incorrectly mark the (now stale) result as resolved.
+		var capturedVersion = Interlocked.Read(ref _version);
 		await Parallel.ForEachAsync(MarkdownFiles, ctx, async (file, token) => await file.MinimalParseAsync(TryFindDocumentByRelativePath, token));
 
-		_resolved = true;
+		if (Interlocked.Read(ref _version) == capturedVersion)
+			_resolved = true;
 	}
 
 	public RepositoryLinks CreateLinkReference()

src/services/Elastic.Documentation.Assembler/AssembleSources.cs

@@ -42,13 +42,16 @@ Cancel ctx
 	{
 		var logger = logFactory.CreateLogger<AssembleSources>();
 
-		var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous();
 		var navigationTocMappings = GetTocMappings(context);
 		var uriResolver = new PublishEnvironmentUriResolver(navigationTocMappings, context.Environment);
 
 		var sw = System.Diagnostics.Stopwatch.StartNew();
-		var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexProvider);
-		var crossLinks = await crossLinkFetcher.FetchCrossLinks(ctx);
+		FetchedCrossLinks crossLinks;
+		// Use a separate using for the reader so ownership is explicit: the caller (this method)
+		// disposes it, not the fetcher (ownsReader stays false/default on AssemblerCrossLinkFetcher).
+		using var linkIndexReader = Aws3LinkIndexReader.CreateAnonymous();
+		using (var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexReader))
+			crossLinks = await crossLinkFetcher.FetchCrossLinks(ctx);
 		var crossLinkResolver = new CrossLinkResolver(crossLinks, uriResolver);
 		logger.LogInformation("  AssembleAsync: FetchCrossLinks in {Elapsed:mm\\:ss\\.fff}", sw.Elapsed);
 

src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs

@@ -62,14 +62,7 @@ public async Task BuildAllAsync(FrozenDictionary<string, AssemblerDocumentationS
 				continue;
 			}
 
-			// Create inferrer per-repository with git context
-			var documentInferrer = new DocumentInferrerService(
-				context.ProductsConfiguration,
-				context.VersionsConfiguration,
-				context.LegacyUrlMappings,
-				set.DocumentationSet.Configuration,
-				set.DocumentationSet.Context.Git
-			);
+			var documentInferrer = CreateInferrer(set);
 
 			var stopwatch = Stopwatch.StartNew();
 			try
@@ -157,6 +150,33 @@ string Resolve(string path)
 		}
 	}
 
+	private DocumentInferrerService CreateInferrer(AssemblerDocumentationSet set) =>
+		new(
+			context.ProductsConfiguration,
+			context.VersionsConfiguration,
+			context.LegacyUrlMappings,
+			set.DocumentationSet.Configuration,
+			set.DocumentationSet.Context.Git
+		);
+
+	public DocumentationGenerator CreateGenerator(AssemblerDocumentationSet set)
+	{
+		SetFeatureFlags(set);
+		return new DocumentationGenerator(
+			set.DocumentationSet,
+			logFactory, NavigationTraversable, HtmlWriter,
+			pathProvider,
+			legacyUrlMapper: LegacyUrlMapper,
+			documentInferrer: CreateInferrer(set)
+		);
+	}
+
+	public async Task BuildOneAsync(AssemblerDocumentationSet set, Cancel ctx)
+	{
+		await set.DocumentationSet.ResolveDirectoryTree(ctx);
+		_ = await BuildAsync(set, null, CreateInferrer(set), ctx);
+	}
+
 	private async Task<GenerationResult> BuildAsync(AssemblerDocumentationSet set, IMarkdownExporter[]? markdownExporters, IDocumentInferrerService documentInferrer, Cancel ctx)
 	{
 		SetFeatureFlags(set);

src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs

@@ -8,13 +8,19 @@
 using Documentation.Builder.Arguments;
 using Documentation.Builder.Http;
 using Elastic.Documentation;
+using Elastic.Documentation.Assembler;
 using Elastic.Documentation.Assembler.Building;
+using Elastic.Documentation.Assembler.Navigation;
 using Elastic.Documentation.Assembler.Sourcing;
 using Elastic.Documentation.Configuration;
 using Elastic.Documentation.Configuration.Assembler;
+using Elastic.Documentation.Configuration.Toc;
 using Elastic.Documentation.Diagnostics;
+using Elastic.Documentation.LegacyDocs;
+using Elastic.Documentation.Navigation.Assembler;
 using Elastic.Documentation.Services;
 using Microsoft.Extensions.Logging;
+using Nullean.ScopedFileSystem;
 
 namespace Documentation.Builder.Commands.Assembler;
 
@@ -153,17 +159,71 @@ await s.BuildAll(collector, state.strict, state.environment, state.metadataOnly,
 		return await serviceInvoker.InvokeAsync(ctx);
 	}
 
-	/// <summary> Serve the output of an assembler build</summary>
+	/// <summary>
+	/// Serve assembled documentation with live reload and on-demand per-request rendering.
+	/// Requires 'assembler clone' to have been run first. No prior build needed.
+	/// Pages are rendered on demand; file changes invalidate the repo and trigger a live reload.
+	/// </summary>
 	/// <param name="port">Port to serve the documentation.</param>
+	/// <param name="environment">The environment configuration to use.</param>
+	/// <param name="noWatchMd">Disable watching checkout directories for markdown changes. Static asset live reload still works. Useful when doing frontend (CSS/JS) work.</param>
 	/// <param name="ctx"></param>
 	[Command("serve")]
-	public async Task ServeAssemblerBuild(int port = 4000, string? path = null, Cancel ctx = default)
+	public async Task ServeAssemblerOnDemand(
+		int port = 4000,
+		string? environment = null,
+		bool noWatchMd = false,
+		Cancel ctx = default
+	)
+	{
+		environment ??= "dev";
+		var readFs = FileSystemFactory.RealRead;
+		var writeFs = FileSystemFactory.RealWrite;
+
+		var assembleContext = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, readFs, writeFs, null, null);
+
+		var cloner = new AssemblerRepositorySourcer(logFactory, assembleContext);
+		var checkoutResult = cloner.GetAll();
+		var checkouts = checkoutResult.Checkouts.ToArray();
+
+		if (checkouts.Length == 0)
+			throw new Exception("No checkouts found. Run 'assembler clone' first.");
+
+		var exporters = ExportOptions.Default
+			.Except([Exporter.DocumentationState])
+			.ToHashSet();
+
+		var assembleSources = await AssembleSources.AssembleAsync(logFactory, assembleContext, checkouts, configurationContext, exporters, ctx);
+
+		var navigationFileInfo = configurationContext.ConfigurationFileProvider.NavigationFile;
+		var siteNavigationFile = SiteNavigationFile.Deserialize(await readFs.File.ReadAllTextAsync(navigationFileInfo.FullName, ctx));
+		var documentationSets = assembleSources.AssembleSets.Values.Select(s => s.DocumentationSet.Navigation).ToArray();
+		var navigation = new SiteNavigation(siteNavigationFile, assembleContext, documentationSets, assembleContext.Environment.PathPrefix);
+
+		var pathProvider = new GlobalNavigationPathProvider(navigation, assembleSources, assembleContext);
+		using var htmlWriter = new GlobalNavigationHtmlWriter(logFactory, navigation, collector);
+		var legacyPageChecker = new LegacyPageService(logFactory);
+		var historyMapper = new PageLegacyUrlMapper(legacyPageChecker, assembleContext.VersionsConfiguration, assembleSources.LegacyUrlMappings);
+		var builder = new AssemblerBuilder(logFactory, assembleContext, navigation, htmlWriter, pathProvider, historyMapper);
+
+		var host = new AssemblerServeWebHost(port, assembleSources, builder, logFactory, watchMarkdown: !noWatchMd);
+		await host.RunAsync(ctx);
+		await host.StopAsync(ctx);
+		// since this command does not use ServiceInvoker, we stop the collector manually.
+		await collector.StopAsync(ctx);
+	}
+
+	/// <summary>Serve the static output of a prior 'assembler build' run.</summary>
+	/// <param name="port">Port to serve the documentation.</param>
+	/// <param name="path">Optional path to serve from, defaults to .artifacts/assembly.</param>
+	/// <param name="ctx"></param>
+	[Command("serve-static")]
+	public async Task ServeStaticAssemblerBuild(int port = 4000, string? path = null, Cancel ctx = default)
 	{
 		var host = new StaticWebHost(port, path);
 		await host.RunAsync(ctx);
 		await host.StopAsync(ctx);
 		// since this command does not use ServiceInvoker, we stop the collector manually.
-		// this should be an exception to the regular command pattern.
 		await collector.StopAsync(ctx);
 	}