Porting Reactive Scanning and Smart Categorization to All DLQ Commands (#9)

* Ported the reactive and smart categorization into the dump DLQ

* Ported the reactive and smart categorization into the purge DLQ

* Ported the reactive and smart categorization into the diagnose DLQ

* Combine shared code

* Extract common code into base class

* Fixed integration tests

* Added confirmation for purge DLQ

* Added test infra

* Telemetry fixes

* Increase the batch size

* Fixed diagnosing

* Updated the docs

Author: kyurkchyan

ServiceBusToolset.slnx

@@ -13,6 +13,7 @@
     <Folder Name="/src/">
         <Project Path="src/ServiceBusToolset.Application/ServiceBusToolset.Application.csproj"/>
         <Project Path="src/ServiceBusToolset.CLI/ServiceBusToolset.CLI.csproj"/>
+        <Project Path="src/ServiceBusToolset.TestHarness/ServiceBusToolset.TestHarness.csproj"/>
     </Folder>
     <Folder Name="/tests/">
         <Project Path="tests/ServiceBusToolset.Application.Tests/ServiceBusToolset.Application.Tests.csproj"/>

docs/articles/port-improvements-to-all-commands.md

@@ -19,8 +19,9 @@ dotnet run -- diagnose-dlq -n <namespace> (-q <queue> | -t <topic> -s <subscript
 | `--app-insights` | `-a` | **(Required)** Application Insights resource ID |
 | `--output` | `-o` | Output JSON file path (optional, prints summary to console) |
 | `--before` | | Only include messages enqueued before this UTC datetime (ISO 8601) |
-| `--max-messages` | | Maximum number of messages to diagnose (default: 100) |
+| `--max-messages` | | Maximum number of messages to diagnose (default: 1000) |
 | `--interactive` | `-i` | Interactive mode: view and select categories to diagnose |
+| `--merge-similar` | | Merge similar DLQ categories using LCS-based clustering (interactive mode only) |
 | `--verbose` | `-v` | Enable verbose output |
 
 ## Getting the App Insights Resource ID

docs/diagnose-dlq.md

@@ -20,6 +20,7 @@ dotnet run -- dump-dlq -n <namespace> (-q <queue> | -t <topic> -s <subscription>
 | `--before` | | Only include messages enqueued before this UTC datetime (ISO 8601) |
 | `--dry-run` | | Preview message count without writing to file |
 | `--interactive` | `-i` | Interactive mode: view and select categories to dump |
+| `--merge-similar` | | Merge similar DLQ categories using LCS-based clustering (interactive mode only) |
 | `--verbose` | `-v` | Enable verbose output |
 
 ## Examples

docs/dump-dlq.md

@@ -19,6 +19,7 @@ dotnet run -- purge-dlq -n <namespace> (-q <queue> | -t <topic> -s <subscription
 | `--before` | | Only purge messages enqueued before this UTC datetime (ISO 8601) |
 | `--dry-run` | | Preview message count without purging |
 | `--interactive` | `-i` | Interactive mode: view and select categories to purge |
+| `--merge-similar` | | Merge similar DLQ categories using LCS-based clustering (interactive mode only) |
 | `--verbose` | `-v` | Enable verbose output |
 
 ## Examples

docs/purge-dlq.md

@@ -21,6 +21,7 @@ dotnet run -- resubmit-dlq -n <namespace> (-q <queue> | -t <topic> -s <subscript
 | `--before` | | Only resubmit messages enqueued before this UTC datetime (ISO 8601) |
 | `--dry-run` | | Preview message count without resubmitting |
 | `--interactive` | `-i` | Interactive mode: view and select categories to resubmit |
+| `--merge-similar` | | Merge similar categories by replacing parameterized values (GUIDs, numbers) with wildcards |
 | `--verbose` | `-v` | Enable verbose output |
 
 ## Examples

docs/resubmit-dlq.md

@@ -0,0 +1,65 @@
+@description('Resource name prefix')
+param prefix string
+
+@description('Azure region')
+param location string
+
+@description('Service Bus SKU')
+@allowed(['Standard', 'Premium'])
+param serviceBusSku string
+
+var logAnalyticsName = '${prefix}-law'
+var appInsightsName = '${prefix}-ai'
+var serviceBusName = '${prefix}-sbns'
+var queueName = 'test-dlq-queue'
+
+resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2023-09-01' = {
+  name: logAnalyticsName
+  location: location
+  properties: {
+    sku: {
+      name: 'PerGB2018'
+    }
+    retentionInDays: 30
+  }
+}
+
+resource appInsights 'Microsoft.Insights/components@2020-02-02' = {
+  name: appInsightsName
+  location: location
+  kind: 'web'
+  properties: {
+    Application_Type: 'web'
+    WorkspaceResourceId: logAnalytics.id
+  }
+}
+
+resource serviceBusNamespace 'Microsoft.ServiceBus/namespaces@2024-01-01' = {
+  name: serviceBusName
+  location: location
+  sku: {
+    name: serviceBusSku
+    tier: serviceBusSku
+  }
+}
+
+resource testQueue 'Microsoft.ServiceBus/namespaces/queues@2024-01-01' = {
+  parent: serviceBusNamespace
+  name: queueName
+  properties: {
+    maxDeliveryCount: 1
+    deadLetteringOnMessageExpiration: true
+  }
+}
+
+@description('Fully qualified Service Bus namespace')
+output serviceBusFqdn string = '${serviceBusNamespace.name}.servicebus.windows.net'
+
+@description('Test queue name')
+output queueName string = testQueue.name
+
+@description('Application Insights resource ID')
+output appInsightsResourceId string = appInsights.id
+
+@description('Application Insights connection string')
+output appInsightsConnectionString string = appInsights.properties.ConnectionString

infra/test/main.bicep

@@ -0,0 +1,5 @@
+using 'main.bicep'
+
+param prefix = 'sbtools-test'
+param location = 'eastus2'
+param serviceBusSku = 'Standard'

infra/test/main.bicepparam

@@ -0,0 +1,73 @@
+[CmdletBinding()]
+param(
+    [Parameter()]
+    [string]$Prefix = 'sbtools-test',
+
+    [Parameter()]
+    [string]$Location = 'eastus2',
+
+    [Parameter()]
+    [string]$SubscriptionId
+)
+
+$ErrorActionPreference = 'Stop'
+
+$resourceGroup = "$Prefix-rg"
+
+# Set subscription if provided
+if ($SubscriptionId) {
+    Write-Host "Setting subscription to $SubscriptionId..."
+    az account set --subscription $SubscriptionId
+}
+
+# Create resource group
+Write-Host "Creating resource group '$resourceGroup' in '$Location'..."
+az group create --name $resourceGroup --location $Location --output none
+
+# Deploy Bicep template
+Write-Host "Deploying infrastructure..."
+$deployment = az deployment group create `
+    --resource-group $resourceGroup `
+    --template-file "$PSScriptRoot/main.bicep" `
+    --parameters "$PSScriptRoot/main.bicepparam" `
+    --parameters prefix=$Prefix location=$Location `
+    --output json | ConvertFrom-Json
+
+$fqdn = $deployment.properties.outputs.serviceBusFqdn.value
+$queueName = $deployment.properties.outputs.queueName.value
+$appInsightsConnectionString = $deployment.properties.outputs.appInsightsConnectionString.value
+$appInsightsResourceId = $deployment.properties.outputs.appInsightsResourceId.value
+
+# Assign Service Bus Data Owner role to current user
+Write-Host "Assigning 'Azure Service Bus Data Owner' role..."
+$currentUser = az ad signed-in-user show --query id --output tsv
+$serviceBusId = az servicebus namespace show --resource-group $resourceGroup --name "$Prefix-sbns" --query id --output tsv
+
+az role assignment create `
+    --assignee $currentUser `
+    --role "Azure Service Bus Data Owner" `
+    --scope $serviceBusId `
+    --output none 2>$null
+
+Write-Host ""
+Write-Host "======================================" -ForegroundColor Green
+Write-Host "  Infrastructure deployed successfully" -ForegroundColor Green
+Write-Host "======================================" -ForegroundColor Green
+Write-Host ""
+Write-Host "Service Bus FQDN              : $fqdn"
+Write-Host "Queue Name                    : $queueName"
+Write-Host "App Insights Connection String: $appInsightsConnectionString"
+Write-Host "App Insights Resource ID      : $appInsightsResourceId"
+Write-Host ""
+Write-Host "Example commands:" -ForegroundColor Cyan
+Write-Host "  # Generate DLQ messages with correlated App Insights telemetry:"
+Write-Host "  dotnet run --project src/ServiceBusToolset.TestHarness -- generate-dlq -n $fqdn -q $queueName -c 100 --app-insights-connection-string `"$appInsightsConnectionString`""
+Write-Host ""
+Write-Host "  # Generate DLQ messages without telemetry:"
+Write-Host "  dotnet run --project src/ServiceBusToolset.TestHarness -- generate-dlq -n $fqdn -q $queueName -c 100"
+Write-Host ""
+Write-Host "  # Diagnose DLQ messages (wait 2-5 min after generate-dlq for telemetry ingestion):"
+Write-Host "  dotnet run --project src/ServiceBusToolset.CLI -- diagnose-dlq -n $fqdn -q $queueName -a $appInsightsResourceId"
+Write-Host ""
+Write-Host "  # Dump DLQ messages:"
+Write-Host "  dotnet run --project src/ServiceBusToolset.CLI -- dump-dlq -n $fqdn -q $queueName -i --merge-similar -o dump.json"

infra/test/setup.ps1

@@ -0,0 +1,31 @@
+[CmdletBinding()]
+param(
+    [Parameter()]
+    [string]$Prefix = 'sbtools-test',
+
+    [Parameter()]
+    [string]$SubscriptionId
+)
+
+$ErrorActionPreference = 'Stop'
+
+$resourceGroup = "$Prefix-rg"
+
+# Set subscription if provided
+if ($SubscriptionId) {
+    Write-Host "Setting subscription to $SubscriptionId..."
+    az account set --subscription $SubscriptionId
+}
+
+# Confirm deletion
+$confirm = Read-Host "Are you sure you want to delete resource group '$resourceGroup'? (y/N)"
+if ($confirm -ne 'y') {
+    Write-Host "Cancelled."
+    return
+}
+
+# Delete resource group
+Write-Host "Deleting resource group '$resourceGroup' (no-wait)..."
+az group delete --name $resourceGroup --yes --no-wait
+
+Write-Host "Resource group deletion initiated. It may take a few minutes to complete." -ForegroundColor Yellow