Merge branch 'dev' into psl-smoketesting

Author: Vamshi-Microsoft

.github/workflows/azd-template-validation.yml

@@ -0,0 +1,40 @@
+name: AZD Template Validation
+on:
+  schedule:
+    - cron: '30 1 * * 4' # Every Thursday at 7:00 AM IST (1:30 AM UTC)
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+  pull-requests: write
+
+jobs:
+  template_validation:
+    runs-on: ubuntu-latest
+    name: azd template validation
+    environment: production
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set timestamp
+        run: echo "HHMM=$(date -u +'%H%M')" >> $GITHUB_ENV
+
+      - uses: microsoft/template-validation-action@v0.4.3
+        with:
+          validateAzd: ${{ vars.TEMPLATE_VALIDATE_AZD }}
+          validateTests: ${{ vars.TEMPLATE_VALIDATE_TESTS }}
+          useDevContainer: ${{ vars.TEMPLATE_USE_DEV_CONTAINER }}
+        id: validation
+        env:
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          AZURE_ENV_NAME: azd-${{ vars.AZURE_ENV_NAME }}-${{ env.HHMM }}
+          AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
+          AZURE_ENV_AI_SERVICE_LOCATION: ${{ vars.AZURE_LOCATION }}
+          AZURE_ENV_MODEL_CAPACITY: 1 # keep low to avoid potential quota issues
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: print result
+        run: cat ${{ steps.validation.outputs.resultFile }}

.github/workflows/azure-dev.yaml

@@ -1,38 +1,57 @@
-name: Azure Template Validation
+name: Azure Dev Deploy
+
 on:
   workflow_dispatch:
 
 permissions:
   contents: read
   id-token: write
-  pull-requests: write
 
 jobs:
-  template_validation_job:
-    environment: production
+  deploy:
     runs-on: ubuntu-latest
-    name: Template validation
-
+    environment: production
+    env:
+      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      AZURE_ENV_NAME: ${{ vars.AZURE_ENV_NAME }}
+      AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
+      AZURE_ENV_MODEL_CAPACITY: 1 # keep low to avoid potential quota issues
+      AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
     steps:
-      # Step 1: Checkout the code from your repository
-      - name: Checkout code
-        uses: actions/checkout@v5
-
-      # Step 2: Validate the Azure template using microsoft/template-validation-action
-      - name: Validate Azure Template
-        uses: microsoft/template-validation-action@v0.4.3
-        id: validation
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Set timestamp and env name
+        run: |
+          HHMM=$(date -u +'%H%M')
+          echo "AZURE_ENV_NAME=azd-${{ vars.AZURE_ENV_NAME }}-${HHMM}" >> $GITHUB_ENV
+
+      - name: Install azd
+        uses: Azure/setup-azd@v2
+
+      - name: Login to Azure
+        uses: azure/login@v2
         with:
-          useDevContainer: false
-        env:
-          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
-          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
-          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-          AZURE_ENV_NAME: ${{ secrets.AZURE_ENV_NAME }}
-          AZURE_LOCATION: ${{ secrets.AZURE_LOCATION }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
-
-      # Step 3: Print the result of the validation
-      - name: Print result
-        run: cat ${{ steps.validation.outputs.resultFile }}
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Login to AZD
+        shell: bash
+        run: |
+          azd auth login \
+            --client-id "$AZURE_CLIENT_ID" \
+            --federated-credential-provider "github" \
+            --tenant-id "$AZURE_TENANT_ID"
+
+      - name: Provision and Deploy
+        shell: bash
+        run: |
+          if ! azd env select "$AZURE_ENV_NAME"; then
+            azd env new "$AZURE_ENV_NAME" --subscription "$AZURE_SUBSCRIPTION_ID" --location "$AZURE_LOCATION" --no-prompt
+          fi
+          azd config set defaults.subscription "$AZURE_SUBSCRIPTION_ID"
+          azd env set AZURE_ENV_AI_SERVICE_LOCATION="$AZURE_LOCATION"
+          azd up --no-prompt

README.md

@@ -278,6 +278,8 @@ Follow the quick deploy steps on the deployment guide to deploy this solution
 
 <br/>
 
+> **Note**: Some tenants may have additional security restrictions that run periodically and could impact the application (e.g., blocking public network access). If you experience issues or the application stops working, check if these restrictions are the cause. In such cases, consider deploying the WAF-supported version to ensure compliance. To configure, [Click here](./docs/DeploymentGuide.md#31-choose-deployment-type-optional).
+
 > ⚠️ **Important: Check Azure OpenAI Quota Availability**
  <br/>To ensure sufficient quota is available in your subscription, please follow [quota check instructions guide](./docs/quota_check.md) before you deploy the solution.
 

azure.yaml

@@ -5,6 +5,7 @@ name: content-processing
 
 requiredVersions:
   azd: '>= 1.18.0 != 1.23.9'
+  bicep: '>= 0.33.0'
 
 metadata:
   template: content-processing@1.0

azure_custom.yaml

@@ -0,0 +1,76 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json
+# Custom AZD configuration for Content Processing Solution Accelerator.
+# Use this file to build and deploy your own modified code using AZD.
+# This file works with infra/main_custom.bicep which uses placeholder container images
+# that AZD replaces with your custom-built images from source.
+#
+# Usage:
+#   1. Copy this file to azure.yaml (or rename it)
+#   2. Ensure infra/main_custom.bicep is referenced (rename to main.bicep or update infra path)
+#   3. Run: azd up
+#
+# For more information, see the Deployment Guide in docs/DeploymentGuide.md
+name: content-processing
+
+requiredVersions:
+  azd: '>= 1.18.0 != 1.23.9'
+
+metadata:
+  template: content-processing@1.0
+  name: content-processing@1.0
+
+# infra:
+#   path: infra
+#   module: main_custom
+
+services:
+  contentprocessor:
+    project: ./src/ContentProcessor
+    language: py
+    host: containerapp
+    docker:
+      path: ./Dockerfile
+      image: contentprocessor
+      registry: ${AZURE_CONTAINER_REGISTRY_ENDPOINT}
+      remoteBuild: true
+
+  contentprocessorapi:
+    project: ./src/ContentProcessorAPI
+    language: py
+    host: containerapp
+    docker:
+      path: ./Dockerfile
+      image: contentprocessorapi
+      registry: ${AZURE_CONTAINER_REGISTRY_ENDPOINT}
+      remoteBuild: true
+
+  contentprocessorweb:
+    project: ./src/ContentProcessorWeb
+    language: js
+    host: containerapp
+    docker:
+      path: ./Dockerfile
+      image: contentprocessorweb
+      registry: ${AZURE_CONTAINER_REGISTRY_ENDPOINT}
+      remoteBuild: true
+
+  contentprocessorworkflow:
+    project: ./src/ContentProcessorWorkflow
+    language: py
+    host: containerapp
+    docker:
+      path: ./Dockerfile
+      image: contentprocessorworkflow
+      registry: ${AZURE_CONTAINER_REGISTRY_ENDPOINT}
+      remoteBuild: true
+
+hooks:
+  postprovision:
+    posix:
+      shell: sh
+      run: sed -i 's/\r$//' ./infra/scripts/post_deployment.sh; bash ./infra/scripts/post_deployment.sh
+      interactive: true
+    windows:
+      shell: pwsh
+      run: ./infra/scripts/post_deployment.ps1
+      interactive: true

docs/DeploymentGuide.md

@@ -6,6 +6,8 @@ This guide walks you through deploying the Content Processing Solution Accelerat
 
 🆘 **Need Help?** If you encounter any issues during deployment, check our [Troubleshooting Guide](./TroubleShootingSteps.md) for solutions to common problems.
 
+> **Note**: Some tenants may have additional security restrictions that run periodically and could impact the application (e.g., blocking public network access). If you experience issues or the application stops working, check if these restrictions are the cause. In such cases, consider deploying the WAF-supported version to ensure compliance. To configure, [Click here](#31-choose-deployment-type-optional).
+
 ## Step 1: Prerequisites & Setup
 
 ### 1.1 Azure Account Requirements
@@ -506,34 +508,29 @@ Now that your deployment is complete and tested, explore these resources:
 
 ---
 
-## Advanced: Deploy Local Code Changes
+## Advanced: Deploy Local Changes
 
-Use this method to quickly deploy code changes from your local machine to your existing Azure deployment without re-provisioning infrastructure.
+If you've made local modifications to the code and want to deploy them to Azure, follow these steps to swap the configuration files:
 
 > **Note:** To set up and run the application locally for development, see the [Local Development Setup Guide](./LocalDevelopmentSetup.md).
 
-### How it Works
-This process will:
-1. Rebuild the Docker containers locally using your modified source code.
-2. Push the new images to your Azure Container Registry (ACR).
-3. Restart the Azure Container Apps to pick up the new images.
+### Step 1: Rename Azure Configuration Files
 
-### Prerequisites
-- **Docker Desktop** must be installed and running.
-- You must have an active deployment environment selected (`azd env select <env-name>`).
+**In the root directory:**
+1. Rename `azure.yaml` to `azure_custom2.yaml`
+2. Rename `azure_custom.yaml` to `azure.yaml`
 
-### Deployment Steps
+### Step 2: Rename Infrastructure Files
 
-Run the build and push script for your operating system:
+**In the `infra` directory:**
+1. Rename `main.bicep` to `main_custom2.bicep`
+2. Rename `main_custom.bicep` to `main.bicep`
 
-**Linux/macOS:**
-```bash
-./infra/scripts/docker-build.sh
-```
+### Step 3: Deploy Changes
 
-**Windows (PowerShell):**
-```powershell
-./infra/scripts/docker-build.ps1
+Run the deployment command:
+```shell
+azd up
 ```
 
-> **Note:** These scripts will deploy your local code changes instead of pulling from the GitHub repository.
+> **Note:** These custom files are configured to deploy your local code changes instead of pulling from the GitHub repository.

docs/TroubleShootingSteps.md

@@ -27,6 +27,7 @@ Use these as quick reference guides to unblock your deployments.
 | **InternalSubscriptionIsOverQuotaForSku** | Subscription quota exceeded for the requested SKU | [View Solution](#quota--capacity-limitations) |
 | **InvalidResourceGroup** | Invalid resource group configuration | [View Solution](#resource-group--deployment-management) |
 | **RequestDisallowedByPolicy** | Azure Policy blocking the requested operation | [View Solution](#subscription--access-issues) |
+| **403 Forbidden - Content Understanding** | Content Understanding returns 403 in WAF/private networking deployment | [View Solution](#network--infrastructure-configuration) |
 
 ## 📖 Table of Contents
 
@@ -127,6 +128,7 @@ Use these as quick reference guides to unblock your deployments.
 | **RouteTableCannotBeAttachedForAzureBastionSubnet** | Route table attached to Azure Bastion subnet | This error occurs because Azure Bastion subnet (`AzureBastionSubnet`) has a platform restriction that prevents route tables from being attached.<br><br>**How to reproduce:**<br><ul><li>In `virtualNetwork.bicep`, add `attachRouteTable: true` to the `AzureBastionSubnet` configuration:<br>`{ name: 'AzureBastionSubnet', addressPrefixes: ['10.0.10.0/26'], attachRouteTable: true }`</li><li>Add a Route Table module to the template</li><li>Update subnet creation to attach route table conditionally:<br>`routeTableResourceId: subnet.?attachRouteTable == true ? routeTable.outputs.resourceId : null`</li><li>Deploy the template → Azure throws `RouteTableCannotBeAttachedForAzureBastionSubnet`</li></ul><br>**Resolution:**<br><ul><li>Remove the `attachRouteTable: true` flag from `AzureBastionSubnet` configuration</li><li>Ensure no route table is associated with `AzureBastionSubnet`</li><li>Route tables can only be attached to other subnets, not `AzureBastionSubnet`</li><li>For more details, refer to [Azure Bastion subnet requirements](https://learn.microsoft.com/en-us/azure/bastion/configuration-settings#subnet)</li></ul> |
 | **VMSizeIsNotPermittedToEnableAcceleratedNetworking** | VM size does not support accelerated networking | This error occurs when you attempt to enable accelerated networking on a VM size that does not support it. This deployment's jumpbox VM **requires** accelerated networking.<br><br>**Default VM size:** `Standard_D2s_v5` — supports accelerated networking.<br><br>**How this error happens:**<br><ul><li>You override the VM size (via `AZURE_ENV_VM_SIZE`) with a size that doesn't support accelerated networking (e.g., `Standard_A2m_v2`, A-series, or B-series VMs)</li><li>Azure rejects the deployment with `VMSizeIsNotPermittedToEnableAcceleratedNetworking`</li></ul><br>**Resolution:**<br><ul><li>Use the default `Standard_D2s_v5` (recommended)</li><li>If overriding VM size, choose one that supports accelerated networking:<br>`Standard_D2s_v4`, `Standard_D2as_v5` (AMD), `Standard_D2s_v3`</li><li>Verify VM size supports accelerated networking:<br>`az vm list-skus --location <region> --size <vm-size> --query "[?capabilities[?name=='AcceleratedNetworkingEnabled' && value=='True']]"`</li><li>Avoid A-series and B-series VMs — they do not support accelerated networking</li><li>See [VM sizes with accelerated networking](https://learn.microsoft.com/en-us/azure/virtual-network/accelerated-networking-overview)</li></ul> |
 | **NetworkSecurityGroupNotCompliantForAzureBastionSubnet** / **SecurityRuleParameterContainsUnsupportedValue** | NSG rules blocking required Azure Bastion ports | This error occurs when the Network Security Group (NSG) attached to `AzureBastionSubnet` explicitly denies inbound TCP ports 443 and/or 4443, which Azure Bastion requires for management and tunneling.<br><br>**How to reproduce:**<br><ul><li>Deploy the template with `enablePrivateNetworking=true` so the virtualNetwork module creates `AzureBastionSubnet` and a Network Security Group that denies ports 443 and 4443</li><li>Attempt to deploy Azure Bastion into that subnet</li><li>During validation, Bastion detects the deny rules and fails with `NetworkSecurityGroupNotCompliantForAzureBastionSubnet`</li></ul><br>**Resolution:**<br><ul><li>**Remove or modify deny rules** for ports 443 and 4443 in the NSG attached to `AzureBastionSubnet`</li><li>**Ensure required inbound rules** per [Azure Bastion NSG requirements](https://learn.microsoft.com/en-us/azure/bastion/bastion-nsg)</li><li>**Use Bicep conditions** to skip NSG attachments for `AzureBastionSubnet` if deploying Bastion</li><li>**Validate the NSG configuration** before deploying Bastion into the subnet</li></ul> |
+| **403 Forbidden - Content Understanding** | Azure AI Content Understanding returns 403 Forbidden in WAF (private networking) deployment | This error occurs when the **Azure AI Content Understanding** service returns a `403 Forbidden` response during document processing in a **WAF-enabled (private networking)** deployment.<br><br>**Why this happens:**<br>In WAF deployments (`enablePrivateNetworking=true`), the Content Understanding AI Services account (`aicu-<suffix>`) is configured with `publicNetworkAccess: Disabled`. All traffic must flow through the **private endpoint** (`pep-aicu-<suffix>`) and resolve via private DNS zones (`privatelink.cognitiveservices.azure.com`, `privatelink.services.ai.azure.com`, `privatelink.contentunderstanding.ai.azure.com`). If any part of this chain is misconfigured, the request either reaches the public endpoint (which is blocked) or fails to route entirely, resulting in a 403.<br><br>**Common causes:**<br><ul><li>Private DNS zones not linked to the VNet — DNS resolution falls back to the public IP, which is blocked</li><li>Private endpoint connection is not in **Approved** state</li><li>Content Understanding is deployed in a different region (`contentUnderstandingLocation`, defaults to `WestUS`) than the main deployment — the private endpoint still works cross-region, but DNS misconfiguration is more likely</li><li>Container Apps are not injected into the VNet or are on a subnet that cannot reach the private endpoint</li><li>Managed Identity used by the Container App does not have the required **Cognitive Services User** role on the Content Understanding resource</li></ul><br>**Resolution:**<br><ul><li>**Verify private endpoint status:**<br>`az network private-endpoint show --name pep-aicu-<suffix> --resource-group <rg-name> --query "privateLinkServiceConnections[0].privateLinkServiceConnectionState.status"`<br>Expected: `Approved`</li><li>**Verify private DNS zone VNet links:**<br>`az network private-dns zone list --resource-group <rg-name> -o table`<br>Ensure `privatelink.cognitiveservices.azure.com`, `privatelink.services.ai.azure.com`, and `privatelink.contentunderstanding.ai.azure.com` all have VNet links</li><li>**Test DNS resolution from the jumpbox VM** (inside the VNet):<br>`nslookup aicu-<suffix>.cognitiveservices.azure.com`<br>Should resolve to a private IP (e.g., `10.x.x.x`), NOT a public IP</li><li>**Verify RBAC role assignments:** Ensure the Container App managed identity has **Cognitive Services User** role on the Content Understanding resource:<br>`az role assignment list --scope /subscriptions/<sub-id>/resourceGroups/<rg-name>/providers/Microsoft.CognitiveServices/accounts/aicu-<suffix> --query "[?roleDefinitionName=='Cognitive Services User']" -o table`</li><li>**Check Container App VNet integration:** Confirm the Container App Environment is deployed into the VNet and can reach the backend subnet where the private endpoint resides</li><li>**Redeploy if needed:**<br>`azd up`</li></ul><br>**Reference:**<br><ul><li>[Configure private endpoints for Azure AI Services](https://learn.microsoft.com/en-us/azure/ai-services/cognitive-services-virtual-networks)</li><li>[Azure Private DNS zones](https://learn.microsoft.com/en-us/azure/dns/private-dns-overview)</li></ul> |
 
 ---------------------------------
 

infra/main.bicep

@@ -1890,5 +1890,8 @@ output CONTAINER_REGISTRY_NAME string = avmContainerRegistry.outputs.name
 @description('The login server of the Azure Container Registry.')
 output CONTAINER_REGISTRY_LOGIN_SERVER string = avmContainerRegistry.outputs.loginServer
 
+@description('The name of the Content Understanding AI Services account.')
+output CONTENT_UNDERSTANDING_ACCOUNT_NAME string = avmAiServices_cu.outputs.name
+
 @description('The resource group the resources were deployed into.')
 output AZURE_RESOURCE_GROUP string = resourceGroup().name