Skip to content

Commit fe65c40

Browse files
reakaleekreakaleek
authored
Add codex index command for Elasticsearch indexing (#2680)
* Add `codex index` command * Change the term 'environment' to 'namespace' to avoid confusion with prod/staging/qa envs * Revert "Change the term 'environment' to 'namespace'" This reverts commit c5c99e2.
1 parent 9b0165c commit fe65c40

10 files changed

Lines changed: 452 additions & 127 deletions

File tree

‎src/Elastic.Codex/Building/CodexBuildService.cs‎

Lines changed: 31 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
using Elastic.Documentation.Navigation.Isolated.Node;
1515
using Elastic.Documentation.Services;
1616
using Elastic.Documentation.Site.Navigation;
17+
using Elastic.Markdown.Exporters;
1718
using Elastic.Markdown.IO;
1819
using Microsoft.Extensions.Logging;
1920

@@ -31,12 +32,15 @@ public class CodexBuildService(
3132

3233
/// <summary>
3334
/// Builds all documentation sets from the cloned checkouts.
35+
/// When <paramref name="exporters"/> includes the Elasticsearch exporter, a single shared exporter
36+
/// is created and its lifecycle is managed across all documentation sets.
3437
/// </summary>
3538
public async Task<CodexBuildResult> BuildAll(
3639
CodexContext context,
3740
CodexCloneResult cloneResult,
3841
IFileSystem fileSystem,
39-
Cancel ctx)
42+
Cancel ctx,
43+
IReadOnlySet<Exporter>? exporters = null)
4044
{
4145
var outputDir = context.OutputDirectory;
4246
if (!outputDir.Exists)
@@ -66,8 +70,31 @@ public async Task<CodexBuildResult> BuildAll(
6670
documentationSets);
6771

6872
// Phase 3: Build each documentation set
73+
// When exporters are specified (e.g., Elasticsearch), create a single shared exporter
74+
// with one _batchIndexDate across all doc sets, mirroring AssemblerBuilder.BuildAllAsync
75+
IMarkdownExporter[]? sharedExporters = null;
76+
if (exporters is not null && buildContexts.Count > 0)
77+
{
78+
var firstContext = buildContexts[0].BuildContext;
79+
sharedExporters = exporters.CreateMarkdownExporters(logFactory, firstContext, context.IndexNamespace).ToArray();
80+
var startTasks = sharedExporters.Select(async e => await e.StartAsync(ctx));
81+
await Task.WhenAll(startTasks);
82+
}
83+
6984
foreach (var buildContext in buildContexts)
70-
await BuildDocumentationSet(context, buildContext, ctx);
85+
await BuildDocumentationSet(context, buildContext, sharedExporters, ctx);
86+
87+
if (sharedExporters is not null)
88+
{
89+
foreach (var exporter in sharedExporters)
90+
{
91+
_logger.LogInformation("Calling FinishExportAsync on {ExporterName}", exporter.GetType().Name);
92+
_ = await exporter.FinishExportAsync(context.OutputDirectory, ctx);
93+
}
94+
95+
var stopTasks = sharedExporters.Select(async e => await e.StopAsync(ctx));
96+
await Task.WhenAll(stopTasks);
97+
}
7198

7299
// Phase 4: Generate codex landing and category pages
73100
if (buildContexts.Count > 0)
@@ -148,6 +175,7 @@ public async Task<CodexBuildResult> BuildAll(
148175
private async Task BuildDocumentationSet(
149176
CodexContext context,
150177
CodexDocumentationSetBuildContext buildContext,
178+
IMarkdownExporter[]? sharedExporters,
151179
Cancel ctx)
152180
{
153181
_logger.LogInformation("Building documentation set: {Name}", buildContext.Checkout.Reference.Name);
@@ -159,6 +187,7 @@ private async Task BuildDocumentationSet(
159187
null, // Use doc set's navigation for traversal
160188
null, // Use default navigation HTML writer (doc set's navigation)
161189
ExportOptions.Default,
190+
sharedExporters,
162191
ctx);
163192
}
164193
catch (Exception ex)

‎src/Elastic.Codex/CodexContext.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,14 @@ public class CodexContext
2222
public IDirectoryInfo CheckoutDirectory { get; }
2323
public IDirectoryInfo OutputDirectory { get; }
2424

25+
/// <summary>
26+
/// The Elasticsearch index namespace for this codex, derived from the environment name.
27+
/// Falls back to "codex" when no environment is specified.
28+
/// </summary>
29+
public string IndexNamespace => string.IsNullOrEmpty(Configuration.Environment)
30+
? "codex"
31+
: $"codex-{Configuration.Environment}";
32+
2533
public CodexContext(
2634
CodexConfiguration configuration,
2735
IFileInfo configurationPath,

‎src/Elastic.Codex/Indexing/CodexIndexService.cs‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.IO.Abstractions;
6+
using Elastic.Codex.Building;
7+
using Elastic.Codex.Sourcing;
8+
using Elastic.Documentation;
9+
using Elastic.Documentation.Configuration;
10+
using Elastic.Documentation.Isolated;
11+
using Elastic.Documentation.Services;
12+
using Microsoft.Extensions.Logging;
13+
14+
namespace Elastic.Codex.Indexing;
15+
16+
/// <summary>
17+
/// Service for indexing codex documentation into Elasticsearch.
18+
/// Configures ES endpoint options using the shared <see cref="ElasticsearchEndpointConfigurator"/>
19+
/// and delegates to <see cref="CodexBuildService.BuildAll"/> with the Elasticsearch exporter.
20+
/// </summary>
21+
public class CodexIndexService(
22+
ILoggerFactory logFactory,
23+
IConfigurationContext configurationContext,
24+
IsolatedBuildService isolatedBuildService
25+
) : IService
26+
{
27+
/// <summary>
28+
/// Index codex documentation to Elasticsearch.
29+
/// </summary>
30+
public async Task<bool> Index(
31+
CodexContext codexContext,
32+
CodexCloneResult cloneResult,
33+
FileSystem fileSystem,
34+
ElasticsearchIndexOptions esOptions,
35+
Cancel ctx = default)
36+
{
37+
var cfg = configurationContext.Endpoints.Elasticsearch;
38+
await ElasticsearchEndpointConfigurator.ApplyAsync(cfg, esOptions, codexContext.Collector, fileSystem, ctx);
39+
40+
var exporters = new HashSet<Exporter> { Exporter.Elasticsearch };
41+
var buildService = new CodexBuildService(logFactory, configurationContext, isolatedBuildService);
42+
var result = await buildService.BuildAll(codexContext, cloneResult, fileSystem, ctx, exporters);
43+
return result.DocumentationSets.Count > 0;
44+
}
45+
}

‎src/Elastic.Documentation.Configuration/Codex/CodexConfiguration.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@ public record CodexConfiguration
2020
[YamlMember(Alias = "site_prefix")]
2121
public string SitePrefix { get; set; } = "/";
2222

23+
/// <summary>
24+
/// The environment name for this codex (e.g., "engineering", "security").
25+
/// Used as part of the Elasticsearch index namespace.
26+
/// </summary>
27+
[YamlMember(Alias = "environment")]
28+
public string? Environment { get; set; }
29+
2330
/// <summary>
2431
/// The title displayed on the codex index page.
2532
/// </summary>

‎src/Elastic.Documentation.Configuration/ElasticsearchEndpointConfigurator.cs‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.IO.Abstractions;
6+
using System.Security.Cryptography.X509Certificates;
7+
using Elastic.Documentation.Diagnostics;
8+
9+
namespace Elastic.Documentation.Configuration;
10+
11+
/// <summary>
12+
/// Options record for configuring an Elasticsearch endpoint from CLI arguments.
13+
/// Shared by all index commands (isolated, assembler, codex).
14+
/// </summary>
15+
public record ElasticsearchIndexOptions
16+
{
17+
// endpoint options
18+
public string? Endpoint { get; init; }
19+
public string? ApiKey { get; init; }
20+
public string? Username { get; init; }
21+
public string? Password { get; init; }
22+
23+
// inference options
24+
public bool? NoSemantic { get; init; }
25+
public bool? EnableAiEnrichment { get; init; }
26+
public int? SearchNumThreads { get; init; }
27+
public int? IndexNumThreads { get; init; }
28+
public bool? NoEis { get; init; }
29+
public int? BootstrapTimeout { get; init; }
30+
31+
// index options
32+
public string? IndexNamePrefix { get; init; }
33+
public bool? ForceReindex { get; init; }
34+
35+
// channel buffer options
36+
public int? BufferSize { get; init; }
37+
public int? MaxRetries { get; init; }
38+
39+
// connection options
40+
public bool? DebugMode { get; init; }
41+
public string? ProxyAddress { get; init; }
42+
public string? ProxyPassword { get; init; }
43+
public string? ProxyUsername { get; init; }
44+
45+
// certificate options
46+
public bool? DisableSslVerification { get; init; }
47+
public string? CertificateFingerprint { get; init; }
48+
public string? CertificatePath { get; init; }
49+
public bool? CertificateNotRoot { get; init; }
50+
}
51+
52+
/// <summary>
53+
/// Applies CLI options to an <see cref="ElasticsearchEndpoint"/>. Shared by all index commands.
54+
/// </summary>
55+
public static class ElasticsearchEndpointConfigurator
56+
{
57+
/// <summary>
58+
/// Applies the given options to the Elasticsearch endpoint configuration.
59+
/// </summary>
60+
public static async Task ApplyAsync(
61+
ElasticsearchEndpoint cfg,
62+
ElasticsearchIndexOptions options,
63+
IDiagnosticsCollector collector,
64+
IFileSystem fileSystem,
65+
Cancel ctx)
66+
{
67+
if (!string.IsNullOrEmpty(options.Endpoint))
68+
{
69+
if (!Uri.TryCreate(options.Endpoint, UriKind.Absolute, out var uri))
70+
collector.EmitGlobalError($"'{options.Endpoint}' is not a valid URI");
71+
else
72+
cfg.Uri = uri;
73+
}
74+
75+
if (!string.IsNullOrEmpty(options.ApiKey))
76+
cfg.ApiKey = options.ApiKey;
77+
if (!string.IsNullOrEmpty(options.Username))
78+
cfg.Username = options.Username;
79+
if (!string.IsNullOrEmpty(options.Password))
80+
cfg.Password = options.Password;
81+
82+
if (options.SearchNumThreads.HasValue)
83+
cfg.SearchNumThreads = options.SearchNumThreads.Value;
84+
if (options.IndexNumThreads.HasValue)
85+
cfg.IndexNumThreads = options.IndexNumThreads.Value;
86+
if (options.NoEis.HasValue)
87+
cfg.NoElasticInferenceService = options.NoEis.Value;
88+
if (!string.IsNullOrEmpty(options.IndexNamePrefix))
89+
cfg.IndexNamePrefix = options.IndexNamePrefix;
90+
if (options.BufferSize.HasValue)
91+
cfg.BufferSize = options.BufferSize.Value;
92+
if (options.MaxRetries.HasValue)
93+
cfg.MaxRetries = options.MaxRetries.Value;
94+
if (options.DebugMode.HasValue)
95+
cfg.DebugMode = options.DebugMode.Value;
96+
if (!string.IsNullOrEmpty(options.CertificateFingerprint))
97+
cfg.CertificateFingerprint = options.CertificateFingerprint;
98+
if (!string.IsNullOrEmpty(options.ProxyAddress))
99+
cfg.ProxyAddress = options.ProxyAddress;
100+
if (!string.IsNullOrEmpty(options.ProxyPassword))
101+
cfg.ProxyPassword = options.ProxyPassword;
102+
if (!string.IsNullOrEmpty(options.ProxyUsername))
103+
cfg.ProxyUsername = options.ProxyUsername;
104+
if (options.DisableSslVerification.HasValue)
105+
cfg.DisableSslVerification = options.DisableSslVerification.Value;
106+
if (!string.IsNullOrEmpty(options.CertificatePath))
107+
{
108+
if (!fileSystem.File.Exists(options.CertificatePath))
109+
collector.EmitGlobalError($"'{options.CertificatePath}' does not exist");
110+
var bytes = await fileSystem.File.ReadAllBytesAsync(options.CertificatePath, ctx);
111+
var loader = X509CertificateLoader.LoadCertificate(bytes);
112+
cfg.Certificate = loader;
113+
}
114+
115+
if (options.CertificateNotRoot.HasValue)
116+
cfg.CertificateIsNotRoot = options.CertificateNotRoot.Value;
117+
if (options.BootstrapTimeout.HasValue)
118+
cfg.BootstrapTimeout = options.BootstrapTimeout.Value;
119+
120+
if (options.NoSemantic.HasValue)
121+
cfg.NoSemantic = options.NoSemantic.Value;
122+
if (options.EnableAiEnrichment.HasValue)
123+
cfg.EnableAiEnrichment = options.EnableAiEnrichment.Value;
124+
if (options.ForceReindex.HasValue)
125+
cfg.ForceReindex = options.ForceReindex.Value;
126+
}
127+
}

‎src/services/Elastic.Documentation.Assembler/Indexing/AssemblerIndexService.cs‎

Lines changed: 25 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
// See the LICENSE file in the project root for more information
44

55
using System.IO.Abstractions;
6-
using System.Security.Cryptography.X509Certificates;
76
using Actions.Core.Services;
87
using Elastic.Documentation.Assembler.Building;
98
using Elastic.Documentation.Configuration;
@@ -86,65 +85,32 @@ public async Task<bool> Index(IDiagnosticsCollector collector,
8685
)
8786
{
8887
var cfg = _configurationContext.Endpoints.Elasticsearch;
89-
if (!string.IsNullOrEmpty(endpoint))
88+
var options = new ElasticsearchIndexOptions
9089
{
91-
if (!Uri.TryCreate(endpoint, UriKind.Absolute, out var uri))
92-
collector.EmitGlobalError($"'{endpoint}' is not a valid URI");
93-
else
94-
cfg.Uri = uri;
95-
}
96-
97-
if (!string.IsNullOrEmpty(apiKey))
98-
cfg.ApiKey = apiKey;
99-
if (!string.IsNullOrEmpty(username))
100-
cfg.Username = username;
101-
if (!string.IsNullOrEmpty(password))
102-
cfg.Password = password;
103-
104-
if (searchNumThreads.HasValue)
105-
cfg.SearchNumThreads = searchNumThreads.Value;
106-
if (indexNumThreads.HasValue)
107-
cfg.IndexNumThreads = indexNumThreads.Value;
108-
if (noEis.HasValue)
109-
cfg.NoElasticInferenceService = noEis.Value;
110-
if (!string.IsNullOrEmpty(indexNamePrefix))
111-
cfg.IndexNamePrefix = indexNamePrefix;
112-
if (bufferSize.HasValue)
113-
cfg.BufferSize = bufferSize.Value;
114-
if (maxRetries.HasValue)
115-
cfg.MaxRetries = maxRetries.Value;
116-
if (debugMode.HasValue)
117-
cfg.DebugMode = debugMode.Value;
118-
if (!string.IsNullOrEmpty(certificateFingerprint))
119-
cfg.CertificateFingerprint = certificateFingerprint;
120-
if (!string.IsNullOrEmpty(proxyAddress))
121-
cfg.ProxyAddress = proxyAddress;
122-
if (!string.IsNullOrEmpty(proxyPassword))
123-
cfg.ProxyPassword = proxyPassword;
124-
if (!string.IsNullOrEmpty(proxyUsername))
125-
cfg.ProxyUsername = proxyUsername;
126-
if (disableSslVerification.HasValue)
127-
cfg.DisableSslVerification = disableSslVerification.Value;
128-
if (!string.IsNullOrEmpty(certificatePath))
129-
{
130-
if (!fileSystem.File.Exists(certificatePath))
131-
collector.EmitGlobalError($"'{certificatePath}' does not exist");
132-
var bytes = await fileSystem.File.ReadAllBytesAsync(certificatePath, ctx);
133-
var loader = X509CertificateLoader.LoadCertificate(bytes);
134-
cfg.Certificate = loader;
135-
}
136-
137-
if (certificateNotRoot.HasValue)
138-
cfg.CertificateIsNotRoot = certificateNotRoot.Value;
139-
if (bootstrapTimeout.HasValue)
140-
cfg.BootstrapTimeout = bootstrapTimeout.Value;
141-
142-
if (noSemantic.HasValue)
143-
cfg.NoSemantic = noSemantic.Value;
144-
if (enableAiEnrichment.HasValue)
145-
cfg.EnableAiEnrichment = enableAiEnrichment.Value;
146-
if (forceReindex.HasValue)
147-
cfg.ForceReindex = forceReindex.Value;
90+
Endpoint = endpoint,
91+
ApiKey = apiKey,
92+
Username = username,
93+
Password = password,
94+
NoSemantic = noSemantic,
95+
EnableAiEnrichment = enableAiEnrichment,
96+
SearchNumThreads = searchNumThreads,
97+
IndexNumThreads = indexNumThreads,
98+
NoEis = noEis,
99+
BootstrapTimeout = bootstrapTimeout,
100+
IndexNamePrefix = indexNamePrefix,
101+
ForceReindex = forceReindex,
102+
BufferSize = bufferSize,
103+
MaxRetries = maxRetries,
104+
DebugMode = debugMode,
105+
ProxyAddress = proxyAddress,
106+
ProxyPassword = proxyPassword,
107+
ProxyUsername = proxyUsername,
108+
DisableSslVerification = disableSslVerification,
109+
CertificateFingerprint = certificateFingerprint,
110+
CertificatePath = certificatePath,
111+
CertificateNotRoot = certificateNotRoot
112+
};
113+
await ElasticsearchEndpointConfigurator.ApplyAsync(cfg, options, collector, fileSystem, ctx);
148114

149115
var exporters = new HashSet<Exporter> { Elasticsearch };
150116

0 commit comments

Comments
 (0)