chore: down merge from dev

.github/CODEOWNERS

@@ -2,4 +2,4 @@
 # Each line is a file pattern followed by one or more owners.
 
 # These owners will be the default owners for everything in the repo.
-* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft @Vinay-Microsoft @aniaroramsft
+* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft @Vinay-Microsoft @aniaroramsft @toherman-msft @nchandhi @dgp10801

.github/workflows/azure-dev.yaml

@@ -22,13 +22,16 @@ jobs:
       - name: Validate Azure Template
         uses: microsoft/template-validation-action@v0.4.4
         id: validation
+        with:
+          useDevContainer: false
         env:
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
           AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
           AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
           AZURE_ENV_NAME: ${{ secrets.AZURE_ENV_NAME }}
           AZURE_LOCATION: ${{ secrets.AZURE_LOCATION }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
 
       # Step 3: Print the result of the validation
       - name: Print result

.github/workflows/broken-links-checker.yml

@@ -37,7 +37,7 @@ jobs:
         uses: lycheeverse/lychee-action@v2.7.0
         with:
           args: >
-            --verbose --exclude-mail --no-progress --exclude ^https?://
+            --verbose --no-progress --exclude ^https?://
             ${{ steps.changed-markdown-files.outputs.all_changed_files }}
           failIfEmpty: false
         env:
@@ -50,7 +50,7 @@ jobs:
         uses: lycheeverse/lychee-action@v2.7.0
         with:
           args: >
-            --verbose --exclude-mail --no-progress --exclude ^https?://
+            --verbose --no-progress --exclude ^https?://
             '**/*.md'
           failIfEmpty: false
         env:

.github/workflows/build-docker-image.yml

@@ -3,9 +3,46 @@ name: Build and Push Docker Images
 on:
   push:
     branches: [main, dev, demo, hotfix]
+    paths:
+      - 'src/ContentProcessor/src/**'
+      - 'src/**/Dockerfile'
+      - 'src/**/requirements.txt'
+      - 'src/**/pyproject.toml'
+      - 'src/ContentProcessorAPI/app/**'
+      - 'src/ContentProcessorAPI/helpers/**'
+      - 'src/ContentProcessorWeb/src/**'
+      - 'src/ContentProcessorWeb/public/**'
+      - 'src/ContentProcessorWeb/package.json'
+      - 'src/ContentProcessorWeb/package-lock.json'
+      - 'src/ContentProcessorWeb/yarn.lock'
+      - 'src/ContentProcessorWeb/tsconfig.json'
+      - 'src/ContentProcessorWeb/config-overrides.js'
+      - 'src/ContentProcessorWeb/nginx-custom.conf'
+      - 'src/ContentProcessorWeb/env.sh'
+      - '.github/workflows/build-docker-image.yml'
   pull_request:
     branches: [main, dev, demo, hotfix]
     types: [opened, ready_for_review, reopened, synchronize]
+    paths:
+      - 'src/ContentProcessor/src/**'
+      - 'src/**/Dockerfile'
+      - 'src/**/requirements.txt'
+      - 'src/**/pyproject.toml'
+      - 'src/ContentProcessorAPI/app/**'
+      - 'src/ContentProcessorAPI/helpers/**'
+      - 'src/ContentProcessorWeb/src/**'
+      - 'src/ContentProcessorWeb/public/**'
+      - 'src/ContentProcessorWeb/package.json'
+      - 'src/ContentProcessorWeb/package-lock.json'
+      - 'src/ContentProcessorWeb/yarn.lock'
+      - 'src/ContentProcessorWeb/tsconfig.json'
+      - 'src/ContentProcessorWeb/config-overrides.js'
+      - 'src/ContentProcessorWeb/nginx-custom.conf'
+      - 'src/ContentProcessorWeb/env.sh'
+      - 'infra/**/*.bicep'
+      - 'infra/**/*.json'
+      - 'azure.yaml'
+      - '.github/workflows/build-docker-image.yml'
   workflow_dispatch:
 
 jobs:
@@ -70,7 +107,7 @@ jobs:
         with:
           context: ./src/ContentProcessor
           file: ./src/ContentProcessor/Dockerfile
-          push: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix'|| github.ref_name == 'dependabotchanges' }}
+          push: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix' || github.ref_name == 'dependabotchanges' }}
           tags: |
             ${{ steps.registry.outputs.ext_registry }}/contentprocessor:${{ env.BASE_TAG }}
             ${{ steps.registry.outputs.ext_registry }}/contentprocessor:${{ env.DATE_TAG }}
@@ -80,7 +117,7 @@ jobs:
         with:
           context: ./src/ContentProcessorAPI
           file: ./src/ContentProcessorAPI/Dockerfile
-          push: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix'|| github.ref_name == 'dependabotchanges' }}
+          push: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix' || github.ref_name == 'dependabotchanges' }}
           tags: |
             ${{ steps.registry.outputs.ext_registry }}/contentprocessorapi:${{ env.BASE_TAG }}
             ${{ steps.registry.outputs.ext_registry }}/contentprocessorapi:${{ env.DATE_TAG }}
@@ -90,7 +127,7 @@ jobs:
         with:
           context: ./src/ContentProcessorWeb
           file: ./src/ContentProcessorWeb/Dockerfile
-          push: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix'|| github.ref_name == 'dependabotchanges' }}
+          push: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix' || github.ref_name == 'dependabotchanges' }}
           tags: |
             ${{ steps.registry.outputs.ext_registry }}/contentprocessorweb:${{ env.BASE_TAG }}
             ${{ steps.registry.outputs.ext_registry }}/contentprocessorweb:${{ env.DATE_TAG }}

.github/workflows/deploy-linux.yml

@@ -0,0 +1,112 @@
+name: Deploy-Test-Cleanup (v2) Linux
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'src/ContentProcessor/src/**'
+      - 'src/**/Dockerfile'
+      - 'src/**/requirements.txt' 
+      - 'src/**/pyproject.toml'
+      - 'src/ContentProcessorAPI/app/**'
+      - 'src/ContentProcessorAPI/helpers/**'
+      - 'src/ContentProcessorWeb/src/**'
+      - 'src/ContentProcessorWeb/public/**'
+      - 'src/ContentProcessorWeb/package.json'
+      - 'src/ContentProcessorWeb/package-lock.json'
+      - 'src/ContentProcessorWeb/yarn.lock'
+      - 'src/ContentProcessorWeb/tsconfig.json'
+      - 'src/ContentProcessorWeb/config-overrides.js'
+      - 'src/ContentProcessorWeb/nginx-custom.conf'
+      - 'src/ContentProcessorWeb/env.sh'
+      - '.github/workflows/deploy-linux.yml'
+  workflow_dispatch:
+    inputs:
+      azure_location:
+        description: 'Azure Location For Deployment'
+        required: false
+        default: 'australiaeast'
+        type: choice
+        options:
+          - 'australiaeast'
+          - 'centralus'
+          - 'eastasia'
+          - 'eastus2'
+          - 'japaneast'
+          - 'northeurope'
+          - 'southeastasia'
+          - 'uksouth'
+      resource_group_name:
+        description: 'Resource Group Name (Optional)'
+        required: false
+        default: ''
+        type: string
+
+      waf_enabled:
+        description: 'Enable WAF'
+        required: false
+        default: false
+        type: boolean
+      EXP:
+        description: 'Enable EXP'
+        required: false
+        default: false
+        type: boolean
+      build_docker_image:
+        description: 'Build And Push Docker Image (Optional)'
+        required: false
+        default: false
+        type: boolean
+
+      cleanup_resources:
+        description: 'Cleanup Deployed Resources'
+        required: false
+        default: false
+        type: boolean
+
+      run_e2e_tests:
+        description: 'Run End-to-End Tests'
+        required: false
+        default: 'GoldenPath-Testing'
+        type: choice
+        options:
+          - 'GoldenPath-Testing'
+          - 'Smoke-Testing'
+          - 'None'
+
+      AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID:
+        description: 'Log Analytics Workspace ID (Optional)'
+        required: false
+        default: ''
+        type: string
+      AZURE_EXISTING_AI_PROJECT_RESOURCE_ID:
+        description: 'AI Project Resource ID (Optional)'
+        required: false
+        default: ''
+        type: string
+      existing_webapp_url:
+        description: 'Existing Container WebApp URL (Skips Deployment)'
+        required: false
+        default: ''
+        type: string
+
+  schedule:
+    - cron: '0 9,21 * * *'  # Runs at 9:00 AM and 9:00 PM GMT
+
+jobs:
+  Run:
+    uses: ./.github/workflows/deploy-orchestrator.yml
+    with:
+      runner_os: ubuntu-latest
+      azure_location: ${{ github.event.inputs.azure_location || 'australiaeast' }}
+      resource_group_name: ${{ github.event.inputs.resource_group_name || '' }}
+      waf_enabled: ${{ github.event.inputs.waf_enabled == 'true' }}
+      EXP: ${{ github.event.inputs.EXP == 'true' }}
+      build_docker_image: ${{ github.event.inputs.build_docker_image == 'true' }}
+      cleanup_resources: ${{ github.event.inputs.cleanup_resources == 'true' }}
+      run_e2e_tests: ${{ github.event.inputs.run_e2e_tests || 'GoldenPath-Testing' }}
+      AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID: ${{ github.event.inputs.AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID || '' }}
+      AZURE_EXISTING_AI_PROJECT_RESOURCE_ID: ${{ github.event.inputs.AZURE_EXISTING_AI_PROJECT_RESOURCE_ID || '' }}
+      existing_webapp_url: ${{ github.event.inputs.existing_webapp_url || '' }}
+      trigger_type: ${{ github.event_name }}
+    secrets: inherit

.github/workflows/deploy-orchestrator.yml

@@ -0,0 +1,138 @@
+name: Deployment orchestrator
+
+on:
+  workflow_call:
+    inputs:
+      runner_os:
+        description: 'Runner OS (ubuntu-latest or windows-latest)'
+        required: true
+        type: string
+      azure_location:
+        description: 'Azure Location For Deployment'
+        required: false
+        default: 'australiaeast'
+        type: string
+      resource_group_name:
+        description: 'Resource Group Name (Optional)'
+        required: false
+        default: ''
+        type: string
+      waf_enabled:
+        description: 'Enable WAF'
+        required: false
+        default: false
+        type: boolean
+      EXP:
+        description: 'Enable EXP'
+        required: false
+        default: false
+        type: boolean
+      build_docker_image:
+        description: 'Build And Push Docker Image (Optional)'
+        required: false
+        default: false
+        type: boolean
+      cleanup_resources:
+        description: 'Cleanup Deployed Resources'
+        required: false
+        default: false
+        type: boolean
+      run_e2e_tests:
+        description: 'Run End-to-End Tests'
+        required: false
+        default: 'GoldenPath-Testing'
+        type: string
+      AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID:
+        description: 'Log Analytics Workspace ID (Optional)'
+        required: false
+        default: ''
+        type: string
+      AZURE_EXISTING_AI_PROJECT_RESOURCE_ID:
+        description: 'AI Project Resource ID (Optional)'
+        required: false
+        default: ''
+        type: string
+      existing_webapp_url:
+        description: 'Existing Container WebApp URL (Skips Deployment)'
+        required: false
+        default: ''
+        type: string
+      trigger_type:
+        description: 'Trigger type (workflow_dispatch, pull_request, schedule)'
+        required: true
+        type: string
+
+env:
+  AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
+
+jobs:
+  docker-build:
+    uses: ./.github/workflows/job-docker-build.yml
+    with:
+      trigger_type: ${{ inputs.trigger_type }}
+      build_docker_image: ${{ inputs.build_docker_image }}
+    secrets: inherit
+
+  deploy:
+    if: always() && (inputs.trigger_type != 'workflow_dispatch' || inputs.existing_webapp_url == '' || inputs.existing_webapp_url == null)
+    needs: docker-build
+    uses: ./.github/workflows/job-deploy.yml
+    with:
+      trigger_type: ${{ inputs.trigger_type }}
+      runner_os: ${{ inputs.runner_os }}
+      azure_location: ${{ inputs.azure_location }}
+      resource_group_name: ${{ inputs.resource_group_name }}
+      waf_enabled: ${{ inputs.waf_enabled }}
+      EXP: ${{ inputs.EXP }}
+      build_docker_image: ${{ inputs.build_docker_image }}
+      existing_webapp_url: ${{ inputs.existing_webapp_url }}
+      AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID: ${{ inputs.AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID }}
+      AZURE_EXISTING_AI_PROJECT_RESOURCE_ID: ${{ inputs.AZURE_EXISTING_AI_PROJECT_RESOURCE_ID }}
+      docker_image_tag: ${{ needs.docker-build.outputs.IMAGE_TAG }}
+      run_e2e_tests: ${{ inputs.run_e2e_tests }}
+      cleanup_resources: ${{ inputs.cleanup_resources }}
+    secrets: inherit
+
+  e2e-test:
+    if: always() && ((needs.deploy.result == 'success' && needs.deploy.outputs.CONTAINER_WEB_APPURL != '') || (inputs.existing_webapp_url != '' && inputs.existing_webapp_url != null)) && (inputs.trigger_type != 'workflow_dispatch' || (inputs.run_e2e_tests != 'None' && inputs.run_e2e_tests != '' && inputs.run_e2e_tests != null))
+    needs: [docker-build, deploy]
+    uses: ./.github/workflows/test-automation-v2.yml
+    with:
+      CP_WEB_URL: ${{ needs.deploy.outputs.CONTAINER_WEB_APPURL || inputs.existing_webapp_url }}
+      TEST_SUITE: ${{ inputs.trigger_type == 'workflow_dispatch' && inputs.run_e2e_tests || 'GoldenPath-Testing' }}
+    secrets: inherit
+
+  send-notification:
+    if: always()
+    needs: [docker-build, deploy, e2e-test]
+    uses: ./.github/workflows/job-send-notification.yml
+    with:
+      trigger_type: ${{ inputs.trigger_type }}
+      waf_enabled: ${{ inputs.waf_enabled }}
+      EXP: ${{ inputs.EXP }}
+      run_e2e_tests: ${{ inputs.run_e2e_tests }}
+      existing_webapp_url: ${{ inputs.existing_webapp_url }}
+      deploy_result: ${{ needs.deploy.result }}
+      e2e_test_result: ${{ needs.e2e-test.result }}
+      CONTAINER_WEB_APPURL: ${{ needs.deploy.outputs.CONTAINER_WEB_APPURL }}
+      RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
+      QUOTA_FAILED: ${{ needs.deploy.outputs.QUOTA_FAILED }}
+      TEST_SUCCESS: ${{ needs.e2e-test.outputs.TEST_SUCCESS }}
+      TEST_REPORT_URL: ${{ needs.e2e-test.outputs.TEST_REPORT_URL }}
+    secrets: inherit
+
+  cleanup-deployment:
+    if: always() && needs.deploy.result == 'success' && needs.deploy.outputs.RESOURCE_GROUP_NAME != '' && inputs.existing_webapp_url == '' && (inputs.trigger_type != 'workflow_dispatch' || inputs.cleanup_resources)
+    needs: [docker-build, deploy, e2e-test]
+    uses: ./.github/workflows/job-cleanup-deployment.yml
+    with:
+      runner_os: ${{ inputs.runner_os }}
+      trigger_type: ${{ inputs.trigger_type }}
+      cleanup_resources: ${{ inputs.cleanup_resources }}
+      existing_webapp_url: ${{ inputs.existing_webapp_url }}
+      RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
+      AZURE_LOCATION: ${{ needs.deploy.outputs.AZURE_LOCATION }}
+      AZURE_ENV_OPENAI_LOCATION: ${{ needs.deploy.outputs.AZURE_ENV_OPENAI_LOCATION }}
+      ENV_NAME: ${{ needs.deploy.outputs.ENV_NAME }}
+      IMAGE_TAG: ${{ needs.deploy.outputs.IMAGE_TAG }}
+    secrets: inherit