diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 302edbc4f0..db7aa10b07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,7 +57,7 @@ jobs: dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler config init --local dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler clone -c local --skip-private-repositories dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler build -c local --skip-private-repositories - dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler serve & + dotnet run --project ../docs-builder/src/tooling/docs-builder -- assembler serve-static & - name: Wait for docs working-directory: src/Elastic.Documentation.Site diff --git a/aspire/AppHost.cs b/aspire/AppHost.cs index b5dd60527a..d2bd572f29 100644 --- a/aspire/AppHost.cs +++ b/aspire/AppHost.cs @@ -103,7 +103,7 @@ async Task BuildAspireHost(bool startElasticsearch, bool assumeCloned, bool assu .WithEnvironment("LLM_GATEWAY_FUNCTION_URL", llmUrl) .WithEnvironment("LLM_GATEWAY_SERVICE_ACCOUNT_KEY_PATH", llmServiceAccountPath) .WithHttpEndpoint(port: 4000, isProxied: false) - .WithArgs(["assembler", "serve", .. globalArguments]) + .WithArgs(["assembler", "serve-static", .. globalArguments]) .WithHttpHealthCheck("/", 200) .WaitForCompletion(buildAll) .WithParentRelationship(cloneAll); diff --git a/build/CommandLine.fs b/build/CommandLine.fs index 4ddfc1c587..f2267be9fe 100644 --- a/build/CommandLine.fs +++ b/build/CommandLine.fs @@ -43,6 +43,7 @@ type Build = | [] Format of ParseResults | [] Watch + | [] Watch_Full | [] Lint of ParseResults | [] PristineCheck @@ -84,6 +85,7 @@ with | Format _ -> "runs dotnet format" | Watch -> "runs dotnet watch to continuous build code/templates and web assets on the fly" + | Watch_Full -> "runs assembler serve with dotnet watch — watches checkout dirs and live-reloads assembled docs" // steps | Lint _ -> "runs dotnet format --verify-no-changes" diff --git a/build/Targets.fs b/build/Targets.fs index 7d3678484b..9ff2d8c93e 100644 --- a/build/Targets.fs +++ b/build/Targets.fs @@ -42,6 +42,8 @@ let private format (formatArgs: ParseResults) = let private watch _ = exec { run "dotnet" "watch" "--project" "src/tooling/docs-builder" "--configuration" "debug" "--" "serve" "--watch" } +let private watchFull _ = exec { run "dotnet" "watch" "--project" "src/tooling/docs-builder" "--configuration" "debug" "--" "assembler" "serve" } + let private lint (lintArgs: ParseResults) = let includeFiles = lintArgs.TryGetResult LintArgs.Include |> Option.defaultValue [] let includeArgs = @@ -256,6 +258,7 @@ let Setup (parsed:ParseResults) = | Format formatArgs -> Build.Step (fun _ -> format formatArgs) | Watch -> Build.Step watch + | Watch_Full -> Build.Step watchFull // steps | Lint lintArgs -> Build.Step (fun _ -> lint lintArgs) diff --git a/src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs b/src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs index b3a51192f3..1f4252925a 100644 --- a/src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs +++ b/src/Elastic.Documentation.LinkIndex/LinkIndexReader.cs @@ -9,7 +9,7 @@ namespace Elastic.Documentation.LinkIndex; -public class Aws3LinkIndexReader(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexReader +public class Aws3LinkIndexReader(IAmazonS3 s3Client, string bucketName = "elastic-docs-link-index", string registryKey = "link-index.json") : ILinkIndexReader, IDisposable { // @@ -52,4 +52,10 @@ public async Task GetRepositoryLinks(string key, Cancel cancell } public string RegistryUrl { get; } = $"https://{bucketName}.s3.{s3Client.Config.RegionEndpoint.SystemName}.amazonaws.com/{registryKey}"; + + public void Dispose() + { + s3Client.Dispose(); + GC.SuppressFinalize(this); + } } diff --git a/src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs b/src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs index c0c7d21e76..0234bb5074 100644 --- a/src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs +++ b/src/Elastic.Documentation.Links/CrossLinks/CrossLinkFetcher.cs @@ -44,9 +44,10 @@ public record FetchedCrossLinks }; } -public abstract class CrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider, ScopedFileSystem? fileSystem = null) : IDisposable +public abstract class CrossLinkFetcher(ILoggerFactory logFactory, ILinkIndexReader linkIndexProvider, ScopedFileSystem? fileSystem = null, bool ownsReader = false) : IDisposable { protected ILogger Logger { get; } = logFactory.CreateLogger(nameof(CrossLinkFetcher)); + protected ILinkIndexReader LinkIndexProvider => linkIndexProvider; private readonly IFileSystem _fileSystem = fileSystem ?? FileSystemFactory.AppData; private LinkRegistry? _linkIndex; @@ -192,7 +193,11 @@ private void WriteLinksJsonCachedFile(string repository, LinkRegistryEntry linkR public void Dispose() { - logFactory.Dispose(); + // Only dispose linkIndexProvider when this fetcher created it (ownsReader = true). + // When the reader was injected by the caller, the caller retains ownership and must dispose it. + // logFactory is always injected — never disposed here. + if (ownsReader && linkIndexProvider is IDisposable disposableReader) + disposableReader.Dispose(); GC.SuppressFinalize(this); } } diff --git a/src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs b/src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs index 0af5a856b4..0de23f6639 100644 --- a/src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs +++ b/src/Elastic.Documentation.Links/CrossLinks/DocSetConfigurationCrossLinkFetcher.cs @@ -16,9 +16,11 @@ public class DocSetConfigurationCrossLinkFetcher( ConfigurationFile configuration, ILinkIndexReader? linkIndexProvider = null, ILinkIndexReader? codexLinkIndexReader = null) - : CrossLinkFetcher(logFactory, linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous()) + : CrossLinkFetcher(logFactory, linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous(), ownsReader: linkIndexProvider is null) { private readonly ILogger _logger = logFactory.CreateLogger(nameof(DocSetConfigurationCrossLinkFetcher)); + // _codexReader is injected by the caller who retains ownership and is responsible for disposal. + // ReloadableGeneratorState, the primary caller, disposes it directly in its own Dispose(). private readonly ILinkIndexReader? _codexReader = codexLinkIndexReader; public override async Task FetchCrossLinks(Cancel ctx) @@ -30,7 +32,7 @@ public override async Task FetchCrossLinks(Cancel ctx) var codexRepositories = new HashSet(); var declaredRepositories = new HashSet(); - var publicReader = linkIndexProvider ?? Aws3LinkIndexReader.CreateAnonymous(); + var publicReader = LinkIndexProvider; var useDualRegistry = configuration.Registry != DocSetRegistry.Public && _codexReader is not null; foreach (var entry in configuration.CrossLinkEntries) diff --git a/src/Elastic.Markdown/IO/DocumentationSet.cs b/src/Elastic.Markdown/IO/DocumentationSet.cs index 910678741d..762d3a2627 100644 --- a/src/Elastic.Markdown/IO/DocumentationSet.cs +++ b/src/Elastic.Markdown/IO/DocumentationSet.cs @@ -214,14 +214,26 @@ public INavigationItem FindNavigationByMarkdown(MarkdownFile markdown) } private bool _resolved; + private long _version; + + public void InvalidateResolved() + { + _ = Interlocked.Increment(ref _version); + _resolved = false; + } + public async Task ResolveDirectoryTree(Cancel ctx) { if (_resolved) return; + // Capture the version before parsing so that if InvalidateResolved() fires + // mid-flight we do not incorrectly mark the (now stale) result as resolved. + var capturedVersion = Interlocked.Read(ref _version); await Parallel.ForEachAsync(MarkdownFiles, ctx, async (file, token) => await file.MinimalParseAsync(TryFindDocumentByRelativePath, token)); - _resolved = true; + if (Interlocked.Read(ref _version) == capturedVersion) + _resolved = true; } public RepositoryLinks CreateLinkReference() diff --git a/src/services/Elastic.Documentation.Assembler/AssembleSources.cs b/src/services/Elastic.Documentation.Assembler/AssembleSources.cs index e9693cb2e7..4ad95ed5e5 100644 --- a/src/services/Elastic.Documentation.Assembler/AssembleSources.cs +++ b/src/services/Elastic.Documentation.Assembler/AssembleSources.cs @@ -42,13 +42,16 @@ Cancel ctx { var logger = logFactory.CreateLogger(); - var linkIndexProvider = Aws3LinkIndexReader.CreateAnonymous(); var navigationTocMappings = GetTocMappings(context); var uriResolver = new PublishEnvironmentUriResolver(navigationTocMappings, context.Environment); var sw = System.Diagnostics.Stopwatch.StartNew(); - var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexProvider); - var crossLinks = await crossLinkFetcher.FetchCrossLinks(ctx); + FetchedCrossLinks crossLinks; + // Use a separate using for the reader so ownership is explicit: the caller (this method) + // disposes it, not the fetcher (ownsReader stays false/default on AssemblerCrossLinkFetcher). + using var linkIndexReader = Aws3LinkIndexReader.CreateAnonymous(); + using (var crossLinkFetcher = new AssemblerCrossLinkFetcher(logFactory, context.Configuration, context.Environment, linkIndexReader)) + crossLinks = await crossLinkFetcher.FetchCrossLinks(ctx); var crossLinkResolver = new CrossLinkResolver(crossLinks, uriResolver); logger.LogInformation(" AssembleAsync: FetchCrossLinks in {Elapsed:mm\\:ss\\.fff}", sw.Elapsed); diff --git a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs index 430abe212e..b6f68cccea 100644 --- a/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs +++ b/src/services/Elastic.Documentation.Assembler/Building/AssemblerBuilder.cs @@ -62,14 +62,7 @@ public async Task BuildAllAsync(FrozenDictionary + new( + context.ProductsConfiguration, + context.VersionsConfiguration, + context.LegacyUrlMappings, + set.DocumentationSet.Configuration, + set.DocumentationSet.Context.Git + ); + + public DocumentationGenerator CreateGenerator(AssemblerDocumentationSet set) + { + SetFeatureFlags(set); + return new DocumentationGenerator( + set.DocumentationSet, + logFactory, NavigationTraversable, HtmlWriter, + pathProvider, + legacyUrlMapper: LegacyUrlMapper, + documentInferrer: CreateInferrer(set) + ); + } + + public async Task BuildOneAsync(AssemblerDocumentationSet set, Cancel ctx) + { + await set.DocumentationSet.ResolveDirectoryTree(ctx); + _ = await BuildAsync(set, null, CreateInferrer(set), ctx); + } + private async Task BuildAsync(AssemblerDocumentationSet set, IMarkdownExporter[]? markdownExporters, IDocumentInferrerService documentInferrer, Cancel ctx) { SetFeatureFlags(set); diff --git a/src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs b/src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs index 5c3cf9f25b..c2702345ac 100644 --- a/src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs +++ b/src/tooling/docs-builder/Commands/Assembler/AssemblerCommands.cs @@ -8,13 +8,19 @@ using Documentation.Builder.Arguments; using Documentation.Builder.Http; using Elastic.Documentation; +using Elastic.Documentation.Assembler; using Elastic.Documentation.Assembler.Building; +using Elastic.Documentation.Assembler.Navigation; using Elastic.Documentation.Assembler.Sourcing; using Elastic.Documentation.Configuration; using Elastic.Documentation.Configuration.Assembler; +using Elastic.Documentation.Configuration.Toc; using Elastic.Documentation.Diagnostics; +using Elastic.Documentation.LegacyDocs; +using Elastic.Documentation.Navigation.Assembler; using Elastic.Documentation.Services; using Microsoft.Extensions.Logging; +using Nullean.ScopedFileSystem; namespace Documentation.Builder.Commands.Assembler; @@ -153,17 +159,71 @@ await s.BuildAll(collector, state.strict, state.environment, state.metadataOnly, return await serviceInvoker.InvokeAsync(ctx); } - /// Serve the output of an assembler build + /// + /// Serve assembled documentation with live reload and on-demand per-request rendering. + /// Requires 'assembler clone' to have been run first. No prior build needed. + /// Pages are rendered on demand; file changes invalidate the repo and trigger a live reload. + /// /// Port to serve the documentation. + /// The environment configuration to use. + /// Disable watching checkout directories for markdown changes. Static asset live reload still works. Useful when doing frontend (CSS/JS) work. /// [Command("serve")] - public async Task ServeAssemblerBuild(int port = 4000, string? path = null, Cancel ctx = default) + public async Task ServeAssemblerOnDemand( + int port = 4000, + string? environment = null, + bool noWatchMd = false, + Cancel ctx = default + ) + { + environment ??= "dev"; + var readFs = FileSystemFactory.RealRead; + var writeFs = FileSystemFactory.RealWrite; + + var assembleContext = new AssembleContext(assemblyConfiguration, configurationContext, environment, collector, readFs, writeFs, null, null); + + var cloner = new AssemblerRepositorySourcer(logFactory, assembleContext); + var checkoutResult = cloner.GetAll(); + var checkouts = checkoutResult.Checkouts.ToArray(); + + if (checkouts.Length == 0) + throw new Exception("No checkouts found. Run 'assembler clone' first."); + + var exporters = ExportOptions.Default + .Except([Exporter.DocumentationState]) + .ToHashSet(); + + var assembleSources = await AssembleSources.AssembleAsync(logFactory, assembleContext, checkouts, configurationContext, exporters, ctx); + + var navigationFileInfo = configurationContext.ConfigurationFileProvider.NavigationFile; + var siteNavigationFile = SiteNavigationFile.Deserialize(await readFs.File.ReadAllTextAsync(navigationFileInfo.FullName, ctx)); + var documentationSets = assembleSources.AssembleSets.Values.Select(s => s.DocumentationSet.Navigation).ToArray(); + var navigation = new SiteNavigation(siteNavigationFile, assembleContext, documentationSets, assembleContext.Environment.PathPrefix); + + var pathProvider = new GlobalNavigationPathProvider(navigation, assembleSources, assembleContext); + using var htmlWriter = new GlobalNavigationHtmlWriter(logFactory, navigation, collector); + var legacyPageChecker = new LegacyPageService(logFactory); + var historyMapper = new PageLegacyUrlMapper(legacyPageChecker, assembleContext.VersionsConfiguration, assembleSources.LegacyUrlMappings); + var builder = new AssemblerBuilder(logFactory, assembleContext, navigation, htmlWriter, pathProvider, historyMapper); + + var host = new AssemblerServeWebHost(port, assembleSources, builder, logFactory, watchMarkdown: !noWatchMd); + await host.RunAsync(ctx); + await host.StopAsync(ctx); + // since this command does not use ServiceInvoker, we stop the collector manually. + await collector.StopAsync(ctx); + } + + /// Serve the static output of a prior 'assembler build' run. + /// Port to serve the documentation. + /// Optional path to serve from, defaults to .artifacts/assembly. + /// + [Command("serve-static")] + public async Task ServeStaticAssemblerBuild(int port = 4000, string? path = null, Cancel ctx = default) { var host = new StaticWebHost(port, path); await host.RunAsync(ctx); await host.StopAsync(ctx); // since this command does not use ServiceInvoker, we stop the collector manually. - // this should be an exception to the regular command pattern. await collector.StopAsync(ctx); } diff --git a/src/tooling/docs-builder/Http/AssemblerReloadService.cs b/src/tooling/docs-builder/Http/AssemblerReloadService.cs new file mode 100644 index 0000000000..df9559beff --- /dev/null +++ b/src/tooling/docs-builder/Http/AssemblerReloadService.cs @@ -0,0 +1,177 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation; +using Elastic.Documentation.Assembler.Navigation; +using Elastic.Documentation.Configuration; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Westwind.AspNetCore.LiveReload; + +namespace Documentation.Builder.Http; + +public sealed class AssemblerReloadService( + IReadOnlyList assemblerSets, + bool watchMarkdown, + ILogger logger +) : IHostedService, IDisposable +{ + private readonly List _watchers = []; + private CancellationTokenSource? _serviceCts; + private readonly Debouncer _debouncer = new(TimeSpan.FromMilliseconds(200)); + + public Task StartAsync(Cancel cancellationToken) + { + _serviceCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + if (watchMarkdown) + StartMarkdownWatchers(); + + StartStaticAssetsWatcher(); + + return Task.CompletedTask; + } + + private void StartMarkdownWatchers() + { + foreach (var set in assemblerSets) + { + var checkoutDir = set.Checkout.Directory.FullName; + if (!Directory.Exists(checkoutDir)) + continue; + + logger.LogInformation("Start markdown watch on checkout: {Directory}", checkoutDir); + var watcher = new FileSystemWatcher(checkoutDir) + { + NotifyFilter = NotifyFilters.Attributes + | NotifyFilters.CreationTime + | NotifyFilters.DirectoryName + | NotifyFilters.FileName + | NotifyFilters.LastWrite + | NotifyFilters.Security + | NotifyFilters.Size + }; + watcher.Filters.Add("*.md"); + watcher.Filters.Add("docset.yml"); + watcher.Filters.Add("_docset.yml"); + watcher.Filters.Add("toc.yml"); + watcher.IncludeSubdirectories = true; + watcher.EnableRaisingEvents = true; + + watcher.Changed += (_, e) => OnMarkdownChanged(e.FullPath, set); + watcher.Created += (_, e) => OnMarkdownChanged(e.FullPath, set); + watcher.Deleted += (_, e) => OnMarkdownChanged(e.FullPath, set); + watcher.Renamed += (_, e) => OnMarkdownChanged(e.FullPath, set); + watcher.Error += (_, e) => logger.LogError(e.GetException(), "File watcher error in {Directory}", checkoutDir); + + _watchers.Add(watcher); + } + } + + private void StartStaticAssetsWatcher() + { + var solutionRoot = Paths.GetSolutionDirectory(); + if (solutionRoot is null) + return; + + var staticDir = Path.Join(solutionRoot.FullName, "src", "Elastic.Documentation.Site", "_static"); + if (!Directory.Exists(staticDir)) + return; + + logger.LogInformation("Start static assets watch on: {Directory}", staticDir); + var watcher = new FileSystemWatcher(staticDir) + { + NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.FileName | NotifyFilters.CreationTime, + IncludeSubdirectories = true, + EnableRaisingEvents = true + }; + + watcher.Changed += (_, e) => OnStaticAssetChanged(e.FullPath); + watcher.Created += (_, e) => OnStaticAssetChanged(e.FullPath); + watcher.Renamed += (_, e) => OnStaticAssetChanged(e.FullPath); + watcher.Error += (_, e) => logger.LogError(e.GetException(), "Static assets watcher error in {Directory}", staticDir); + + _watchers.Add(watcher); + } + + private static bool ShouldIgnorePath(string path) => + path.Contains("/.artifacts/") || path.Contains("\\.artifacts\\") || + path.Contains("/.git/") || path.Contains("\\.git\\") || + path.Contains("/node_modules/") || path.Contains("\\node_modules\\"); + + private void OnMarkdownChanged(string fullPath, AssemblerDocumentationSet set) + { + if (ShouldIgnorePath(fullPath)) + return; + + logger.LogInformation("Markdown changed: {FullPath}", fullPath); + + var token = _serviceCts?.Token ?? Cancel.None; + _ = _debouncer.ExecuteAsync(async _ => + { + set.DocumentationSet.InvalidateResolved(); + logger.LogInformation("Invalidated {RepositoryName}, triggering live reload", set.Checkout.Repository.Name); + await Task.Run(() => LiveReloadMiddleware.RefreshWebSocketRequest(), CancellationToken.None); + }, token); + } + + private void OnStaticAssetChanged(string fullPath) + { + logger.LogInformation("Static asset changed: {FullPath}", fullPath); + + var token = _serviceCts?.Token ?? Cancel.None; + _ = _debouncer.ExecuteAsync(async _ => + { + logger.LogInformation("Triggering live reload for static asset change"); + await Task.Run(() => LiveReloadMiddleware.RefreshWebSocketRequest(), CancellationToken.None); + }, token); + } + + public async Task StopAsync(Cancel cancellationToken) + { + if (_serviceCts is not null) + { + await _serviceCts.CancelAsync(); + _serviceCts.Dispose(); + _serviceCts = null; + } + foreach (var watcher in _watchers) + watcher.Dispose(); + _watchers.Clear(); + } + + public void Dispose() + { + _serviceCts?.Dispose(); + foreach (var watcher in _watchers) + watcher.Dispose(); + _debouncer.Dispose(); + } + + private sealed class Debouncer(TimeSpan window) : IDisposable + { + private readonly SemaphoreSlim _semaphore = new(1, 1); + private readonly long _windowInTicks = window.Ticks; + private long _nextRun; + + public async Task ExecuteAsync(Func innerAction, Cancel cancellationToken) + { + var requestStart = DateTime.UtcNow.Ticks; + try + { + await _semaphore.WaitAsync(cancellationToken); + if (requestStart <= _nextRun) + return; + await innerAction(cancellationToken); + _nextRun = requestStart + _windowInTicks; + } + finally + { + _ = _semaphore.Release(); + } + } + + public void Dispose() => _semaphore.Dispose(); + } +} diff --git a/src/tooling/docs-builder/Http/AssemblerServeWebHost.cs b/src/tooling/docs-builder/Http/AssemblerServeWebHost.cs new file mode 100644 index 0000000000..b2a2205b72 --- /dev/null +++ b/src/tooling/docs-builder/Http/AssemblerServeWebHost.cs @@ -0,0 +1,258 @@ +// Licensed to Elasticsearch B.V under one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using Elastic.Documentation.Assembler; +using Elastic.Documentation.Assembler.Building; +using Elastic.Documentation.Assembler.Navigation; +using Elastic.Documentation.Configuration; +using Elastic.Documentation.ServiceDefaults; +using Elastic.Documentation.Site.FileProviders; +using Elastic.Markdown; +using Elastic.Markdown.IO; +#if DEBUG +using Elastic.Documentation.Api.Infrastructure; +#endif +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Westwind.AspNetCore.LiveReload; + +namespace Documentation.Builder.Http; + +public class AssemblerServeWebHost +{ + private readonly WebApplication _webApplication; + + // Sorted longest-prefix-first for greedy matching + private readonly (string Prefix, string FileSlugBase, AssemblerDocumentationSet Set, DocumentationGenerator Generator)[] _prefixMap; + + // Shortest top-level prefix → first real content section (e.g. "/docs/get-started"). + // Null when no mappings exist and no path prefix is configured — callers must guard. + private readonly string? _rootRedirectUrl; + + public AssemblerServeWebHost( + int port, + AssembleSources assembleSources, + AssemblerBuilder assemblerBuilder, + ILoggerFactory logFactory, + bool watchMarkdown = true + ) + { + _prefixMap = BuildPrefixMap(assembleSources, assemblerBuilder); + var firstPrefix = _prefixMap.OrderBy(e => e.Prefix.Length).Select(e => e.Prefix).FirstOrDefault(); + var envPrefix = assembleSources.AssembleContext.Environment.PathPrefix; + _rootRedirectUrl = firstPrefix is not null + ? $"/{firstPrefix}" + : envPrefix is { Length: > 0 } + ? $"/{envPrefix}" + : null; + + var urlPathPrefix = assembleSources.AssembleSets.Values.FirstOrDefault()?.BuildContext.UrlPathPrefix ?? ""; + + var builder = WebApplication.CreateBuilder(new WebApplicationOptions + { + ContentRootPath = Paths.WorkingDirectoryRoot.FullName + }); + + _ = builder.AddDocumentationServiceDefaults(); +#if DEBUG + builder.Services.AddElasticDocsApiUsecases("dev"); +#endif + + _ = builder.Logging + .AddFilter("Microsoft.AspNetCore.Hosting.Diagnostics", LogLevel.Error) + .AddFilter("Microsoft.AspNetCore.StaticFiles.StaticFileMiddleware", LogLevel.Error) + .AddFilter("Microsoft.Hosting.Lifetime", LogLevel.Information); + + _ = builder.Services.AddAotLiveReload(s => + { + s.FolderToMonitor = assembleSources.AssembleContext.CheckoutDirectory.FullName; + s.ClientFileExtensions = watchMarkdown ? ".md,.yml" : ".css,.js"; + }); + + var sets = assembleSources.AssembleSets.Values.ToList(); + _ = builder.Services.AddHostedService(_ => + new AssemblerReloadService(sets, watchMarkdown, logFactory.CreateLogger()) + ); + + _ = builder.WebHost.UseUrls($"http://localhost:{port}"); + + _webApplication = builder.Build(); + SetUpRoutes(assembleSources, urlPathPrefix); + } + + public async Task RunAsync(Cancel ctx) => await _webApplication.RunAsync(ctx); + + public async Task StopAsync(Cancel ctx) => await _webApplication.StopAsync(ctx); + + private static (string Prefix, string FileSlugBase, AssemblerDocumentationSet Set, DocumentationGenerator Generator)[] BuildPrefixMap( + AssembleSources assembleSources, AssemblerBuilder assemblerBuilder) + { + var envPathPrefix = assembleSources.AssembleContext.Environment.PathPrefix; // e.g. "docs" + var entries = new List<(string, string, AssemblerDocumentationSet, DocumentationGenerator)>(); + var generatorCache = new Dictionary(); + + foreach (var (uri, mapping) in assembleSources.NavigationTocMappings) + { + var repoName = uri.Scheme; + if (!assembleSources.AssembleSets.TryGetValue(repoName, out var set)) + continue; + + // Reconstruct the path within SourceDirectory for this TOC mapping. + // The URI host is the subfolder of the repo's content root that this TOC covers. + var fileSlugBase = Path.Join(uri.Host, uri.AbsolutePath.Trim('/')).TrimStart('/'); + + // The URL slug includes the global path prefix (e.g. "docs/release-notes/elasticsearch"). + // SourcePathPrefix is just the suffix (e.g. "release-notes/elasticsearch"), so prepend. + var urlPrefix = string.IsNullOrEmpty(envPathPrefix) + ? mapping.SourcePathPrefix + : string.IsNullOrEmpty(mapping.SourcePathPrefix) + ? envPathPrefix + : $"{envPathPrefix}/{mapping.SourcePathPrefix}"; + + // Reuse the same generator for the same repo across multiple TOC entries + if (!generatorCache.TryGetValue(repoName, out var generator)) + { + generator = assemblerBuilder.CreateGenerator(set); + generatorCache[repoName] = generator; + } + + entries.Add((urlPrefix, fileSlugBase, set, generator)); + } + return [.. entries.OrderByDescending(e => e.Item1.Length)]; + } + + private void SetUpRoutes(AssembleSources assembleSources, string urlPathPrefix) + { + var firstBuildContext = assembleSources.AssembleSets.Values.FirstOrDefault()?.BuildContext; + + // Static assets MUST come before UseRouting so they're served before the {**slug} catch-all matches + var staticRequestPath = string.IsNullOrEmpty(urlPathPrefix) ? "/_static" : $"{urlPathPrefix}/_static"; + var pipeline = _webApplication + .UseLiveReloadWithManualScriptInjection(_webApplication.Lifetime) + .UseDeveloperExceptionPage(new DeveloperExceptionPageOptions()) + .Use(async (context, next) => + { + try + { + await next(context); + } + catch (Exception ex) + { + Console.WriteLine($"[UNHANDLED EXCEPTION] {ex.GetType().Name}: {ex.Message}"); + Console.WriteLine($"[STACK TRACE] {ex.StackTrace}"); + if (ex.InnerException != null) + Console.WriteLine($"[INNER EXCEPTION] {ex.InnerException.GetType().Name}: {ex.InnerException.Message}"); + throw; + } + }); + + if (firstBuildContext != null) + { + _ = pipeline.UseStaticFiles(new StaticFileOptions + { + FileProvider = new EmbeddedOrPhysicalFileProvider(firstBuildContext), + RequestPath = staticRequestPath + }); + } + + _ = pipeline.UseRouting(); + +#if DEBUG + var apiV1 = _webApplication.MapGroup($"{SystemEnvironmentVariables.Instance.ApiPrefix}/v1"); + var mapOtlpEndpoints = !string.IsNullOrWhiteSpace(_webApplication.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"]); + apiV1.MapElasticDocsApiEndpoints(mapOtlpEndpoints); +#endif + + _ = _webApplication.MapGet("/", (Cancel ctx) => ServeRoot(ctx)); + _ = _webApplication.MapGet("{**slug}", (string slug, Cancel ctx) => ServeDocumentationFile(slug, ctx)); + } + + private Task ServeRoot(Cancel _) => + Task.FromResult(_rootRedirectUrl is not null + ? Results.Redirect(_rootRedirectUrl) + : Results.NotFound()); + + private async Task ServeDocumentationFile(string slug, Cancel ctx) + { + if (slug == ".well-known/appspecific/com.chrome.devtools.json") + return Results.NotFound(); + + // Static asset requests that UseStaticFiles couldn't serve → 404, not a redirect + if (slug.Contains("/_static/") || slug.Contains("\\_static\\")) + return Results.NotFound(); + + var match = FindRepo(slug); + if (match is null) + return _rootRedirectUrl is not null ? Results.Redirect(_rootRedirectUrl) : Results.NotFound(); + + var (set, generator, relFileSlug) = match.Value; + + // Mirror the file lookup logic from DocumentationWebHost.ServeDocumentationFile + var s = Path.GetExtension(relFileSlug) == string.Empty + ? Path.Join(relFileSlug, "index.md") + : relFileSlug; + var fp = new FilePath(s, generator.DocumentationSet.SourceDirectory); + + if (!generator.DocumentationSet.Files.TryGetValue(fp, out var documentationFile)) + { + s = Path.GetExtension(relFileSlug) == string.Empty + ? relFileSlug + ".md" + : s.Replace($"{Path.DirectorySeparatorChar}index.md", ".md"); + fp = new FilePath(s, generator.DocumentationSet.SourceDirectory); + _ = generator.DocumentationSet.Files.TryGetValue(fp, out documentationFile); + } + + switch (documentationFile) + { + case MarkdownFile markdown: + var rendered = await generator.RenderLayout(markdown, ctx); + return LiveReloadHtml(rendered.Html); + case ImageFile image: + return Results.File(image.SourceFile.FullName, image.MimeType); + default: + // If this is a navigation node (index.md not found), try redirect to first child + var fp404 = new FilePath("404.md", generator.DocumentationSet.SourceDirectory); + if (generator.DocumentationSet.Files.TryGetValue(fp404, out var notFound) && notFound is MarkdownFile notFoundMd) + { + var renderedNotFound = await generator.RenderLayout(notFoundMd, ctx); + return Results.Content(renderedNotFound.Html, "text/html", null, 404); + } + return Results.NotFound(); + } + } + + private (AssemblerDocumentationSet Set, DocumentationGenerator Generator, string RelFileSlug)? FindRepo(string slug) + { + foreach (var (prefix, fileSlugBase, set, generator) in _prefixMap) + { + var matchesPrefix = + slug.Equals(prefix, StringComparison.OrdinalIgnoreCase) + || (slug.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + && slug.Length > prefix.Length + && slug[prefix.Length] == '/'); + if (!matchesPrefix) + continue; + var afterPrefix = slug[prefix.Length..].TrimStart('/'); + var relFileSlug = string.IsNullOrEmpty(fileSlugBase) + ? afterPrefix + : string.IsNullOrEmpty(afterPrefix) ? fileSlugBase : $"{fileSlugBase}/{afterPrefix}"; + return (set, generator, relFileSlug); + } + return null; + } + + private static IResult LiveReloadHtml(string content) + { + if (LiveReloadConfiguration.Current.LiveReloadEnabled) + { + var script = $"\n"; + content += script; + } + return Results.Content(content, "text/html"); + } +} diff --git a/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs b/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs index 70acd4d757..1693f79927 100644 --- a/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs +++ b/src/tooling/docs-builder/Http/ReloadableGeneratorState.cs @@ -249,6 +249,9 @@ public void Dispose() _apiGenerationCts?.Cancel(); _apiGenerationCts?.Dispose(); _apiSemaphore.Dispose(); + // _crossLinkFetcher owns its internally-created Aws3LinkIndexReader; dispose it. + // _codexReader is owned by this class (not by the fetcher); dispose it separately. + _crossLinkFetcher.Dispose(); (_codexReader as IDisposable)?.Dispose(); GC.SuppressFinalize(this); }