feat: Merge latest changes to demo

.github/ISSUE_TEMPLATE/subtask.md

@@ -0,0 +1,22 @@
+---
+name: Sub task
+about: A sub task
+title: ''
+labels: subtask
+assignees: ''
+
+---
+
+Required by <link to parent issue>
+
+# Description
+
+A clear and concise description of what this subtask is.
+
+# Tasks
+
+_To be filled in by the engineer picking up the subtask
+
+- [ ] Task 1
+- [ ] Task 2
+- [ ] ...

.github/workflows/Scheduled-Dependabot-PRs-Auto-Merge.yml

@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Install GitHub CLI
         run: |

.github/workflows/azure-dev.yml

@@ -15,11 +15,11 @@ jobs:
     steps:
       # Step 1: Checkout the code from your repository
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       # Step 2: Validate the Azure template using microsoft/template-validation-action
       - name: Validate Azure Template
-        uses: microsoft/template-validation-action@v0.3.5
+        uses: microsoft/template-validation-action@v0.4.3
         id: validation
         env:
           AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
@@ -28,6 +28,7 @@ jobs:
           AZURE_ENV_NAME: ${{ secrets.AZURE_ENV_NAME }}
           AZURE_LOCATION: ${{ secrets.AZURE_LOCATION }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
 
       # Step 3: Print the result of the validation
       - name: Print result

.github/workflows/broken-links-checker.yml

@@ -0,0 +1,57 @@
+name: Broken Link Checker
+
+on:
+  pull_request:
+    paths:
+      - '**/*.md'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  markdown-link-check:
+    name: Check Markdown Broken Links
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      # For PR : Get only changed markdown files
+      - name: Get changed markdown files (PR only)
+        id: changed-markdown-files
+        if: github.event_name == 'pull_request'
+        uses: tj-actions/changed-files@24d32ffd492484c1d75e0c0b894501ddb9d30d62 # v46
+        with:
+          files: |
+            **/*.md
+
+
+      # For PR: Check broken links only in changed files
+      - name: Check Broken Links in Changed Markdown Files
+        id: lychee-check-pr
+        if: github.event_name == 'pull_request' && steps.changed-markdown-files.outputs.any_changed == 'true'
+        uses: lycheeverse/lychee-action@v2.6.1
+        with:
+          args: >
+            --verbose --no-progress --exclude ^https?://
+            ${{ steps.changed-markdown-files.outputs.all_changed_files }}
+          failIfEmpty: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # For manual trigger: Check all markdown files in repo
+      - name: Check Broken Links in All Markdown Files in Entire Repo (Manual Trigger)
+        id: lychee-check-manual
+        if: github.event_name == 'workflow_dispatch'
+        uses: lycheeverse/lychee-action@v2.6.1
+        with:
+          args: >
+            --verbose --no-progress --exclude ^https?://
+            '**/*.md'
+          failIfEmpty: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/create-release.yml

@@ -14,11 +14,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           ref: ${{ github.event.workflow_run.head_sha }}
 
-      - uses: codfish/semantic-release-action@v3
+      - uses: codfish/semantic-release-action@v4
         id: semantic
         with:
           tag-format: 'v${version}'

.github/workflows/deploy.yml

@@ -1,22 +1,25 @@
 name: DocGen Deploy-Test-Cleanup Pipeline
 
 on:
-    workflow_run:
-      workflows: ["Build Docker and Optional Push"]
-      types:
-        - completed
-      branches:
-        - main
-        - dev
-        - demo
-
-    schedule:
-      - cron: '0 9,21 * * *'  # Runs at 9:00 AM and 9:00 PM GMT
+  pull_request:
+    branches:
+      - main
+  workflow_run:
+    workflows: ["Build Docker and Optional Push"]
+    types:
+      - completed
+    branches:
+      - main
+      - dev
+      - demo
+
+  schedule:
+    - cron: '0 9,21 * * *'  # Runs at 9:00 AM and 9:00 PM GMT
 
 env:
-  GPT_MIN_CAPACITY: 250
-  TEXT_EMBEDDING_MIN_CAPACITY: 40
-  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
+  GPT_MIN_CAPACITY: 150
+  TEXT_EMBEDDING_MIN_CAPACITY: 80
+  BRANCH_NAME: ${{ github.event.workflow_run.head_branch || github.head_ref || github.ref_name }}
 
 jobs:
   deploy:
@@ -26,7 +29,7 @@ jobs:
       WEBAPP_URL: ${{ steps.get_output.outputs.WEBAPP_URL }}
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v5
 
       - name: Setup Azure CLI
         run: |
@@ -83,7 +86,11 @@ jobs:
       - name: Set Deployment Region
         run: |
           echo "Selected Region: $VALID_REGION"
+          echo "AZURE_AI_LOCATION=$VALID_REGION" >> $GITHUB_ENV
           echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV
+          if [ "$VALID_REGION" == "eastus" ] || [ "$VALID_REGION" == "westus3" ]; then
+            echo "AZURE_LOCATION=uksouth" >> $GITHUB_ENV
+          fi
 
       - name: Generate Resource Group Name
         id: generate_rg_name
@@ -126,29 +133,32 @@ jobs:
           set -e
           # set image tag based on branch
           if [[ "${{ env.BRANCH_NAME }}" == "main" ]]; then
-            IMAGE_TAG="latest"
+            IMAGE_TAG="latest_waf"
           elif [[ "${{ env.BRANCH_NAME }}" == "dev" ]]; then
             IMAGE_TAG="dev"
           elif [[ "${{ env.BRANCH_NAME }}" == "demo" ]]; then
             IMAGE_TAG="demo"
           else
-            IMAGE_TAG="latest"
+            IMAGE_TAG="latest_waf"
           fi
 
+          # Generate current timestamp in desired format: YYYY-MM-DDTHH:MM:SS.SSSSSSSZ
+            current_date=$(date -u +"%Y-%m-%dT%H:%M:%S.%7NZ")
+
           az deployment group create \
             --name ${{ env.SOLUTION_PREFIX }}-deployment \
             --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
-            --template-file infra/main.json \
+            --template-file infra/main.bicep \
             --parameters \
-                environmentName="${{ env.SOLUTION_PREFIX }}" \
-                secondaryLocation="northcentralus" \
-                deploymentType="GlobalStandard" \
-                gptModelName="gpt-4.1" \
-                azureOpenaiAPIVersion="2024-05-01-preview" \
-                gptDeploymentCapacity=${{ env.GPT_MIN_CAPACITY }} \
-                embeddingModel="text-embedding-ada-002" \
+                solutionName="${{ env.SOLUTION_PREFIX }}" \
+                secondaryLocation="${{ env.AZURE_LOCATION }}" \
+                location="${{ env.AZURE_LOCATION }}" \
+                gptModelCapacity=${{ env.GPT_MIN_CAPACITY }} \
                 embeddingDeploymentCapacity=${{ env.TEXT_EMBEDDING_MIN_CAPACITY }} \
-                imageTag="${IMAGE_TAG}"
+                azureAiServiceLocation=${{ env.AZURE_AI_LOCATION }} \
+                imageTag="${IMAGE_TAG}"\
+                createdBy="Pipeline" \
+                tags="{'SecurityControl':'Ignore','Purpose':'Deploying and Cleaning Up Resources for Validation','CreatedDate':'$current_date'}"
 
       - name: Get Deployment Output and extract Values
         id: get_output
@@ -167,22 +177,31 @@ jobs:
           echo "KEY_VAULT_NAME=$KEY_VAULT_NAME" >> $GITHUB_ENV
           COSMOSDB_ACCOUNT_NAME=$(echo $BICEP_OUTPUT | jq -r '.cosmosdB_ACCOUNT_NAME.value')
           echo "COSMOSDB_ACCOUNT_NAME=$COSMOSDB_ACCOUNT_NAME" >> $GITHUB_ENV
+          AI_FOUNDRY_RESOURCE_ID=$(echo $BICEP_OUTPUT | jq -r '.aI_FOUNDRY_RESOURCE_ID.value')
+          echo "AI_FOUNDRY_RESOURCE_ID=$AI_FOUNDRY_RESOURCE_ID" >> $GITHUB_ENV
+          AI_SEARCH_SERVICE_NAME=$(echo $BICEP_OUTPUT | jq -r '.aI_SEARCH_SERVICE_NAME.value')
+          echo "AI_SEARCH_SERVICE_NAME=$AI_SEARCH_SERVICE_NAME" >> $GITHUB_ENV
           echo "Deployment output: $BICEP_OUTPUT"
 
       - name: Run Post-Deployment Script
         id: post_deploy
+        env: 
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
         run: |
           set -e
           az account set --subscription "${{ secrets.AZURE_SUBSCRIPTION_ID }}"
 
           echo "Running post-deployment script..."
-          bash ./infra/scripts/copy_kb_files.sh \
+
+          bash ./infra/scripts/process_sample_data.sh \
             "${{ env.STORAGE_ACCOUNT_NAME }}" \
             "${{ env.STORAGE_CONTAINER_NAME }}" \
-            "${{ secrets.AZURE_CLIENT_ID }}"
-          bash ./infra/scripts/run_create_index_scripts.sh \
             "${{ env.KEY_VAULT_NAME }}" \
-            "${{ secrets.AZURE_CLIENT_ID }}"
+            "${{ env.COSMOSDB_ACCOUNT_NAME }}" \
+            "${{ env.RESOURCE_GROUP_NAME }}" \
+            "${{ env.AI_SEARCH_SERVICE_NAME }}" \
+            "${{ secrets.AZURE_CLIENT_ID }}" \
+            "${{ env.AI_FOUNDRY_RESOURCE_ID }}"
 
       - name: Logout from Azure
         if: always()
@@ -395,4 +414,4 @@ jobs:
         if: always()
         run: |
           az logout
-          echo "Logged out from Azure."
+          echo "Logged out from Azure."

.github/workflows/docker-build-and-push.yml

@@ -19,10 +19,10 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v5
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Log in to Azure Container Registry - External Registry
         if: ${{ github.ref_name == 'main' }}
@@ -58,7 +58,7 @@ jobs:
         id: determine_tag
         run: |
           if [[ "${{ github.ref_name }}" == "main" ]]; then
-            echo "tagname=latest" >> $GITHUB_OUTPUT
+            echo "tagname=latest_waf" >> $GITHUB_OUTPUT
           elif [[ "${{ github.ref_name }}" == "dev" ]]; then
             echo "tagname=dev" >> $GITHUB_OUTPUT
           elif [[ "${{ github.ref_name }}" == "demo" ]]; then

.github/workflows/node.js.yml

@@ -22,9 +22,9 @@ jobs:
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v5
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v5
       with:
         node-version: ${{ matrix.node-version }}
         cache: 'npm'

.github/workflows/pr-title-checker.yml

@@ -17,6 +17,6 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ github.event_name != 'merge_group' }}
     steps:
-      - uses: amannn/action-semantic-pull-request@v5
+      - uses: amannn/action-semantic-pull-request@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pylint.yml

@@ -12,11 +12,11 @@ jobs:
     steps:
       # Step 1: Checkout code
       - name: Checkout code
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       # Step 2: Set up Python environment
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v3
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -29,5 +29,5 @@ jobs:
       - name: Pylint
         run: |
           echo "Running Pylint..."
-          python -m flake8 --config=.flake8 --verbose .
+          python -m flake8 --config=.flake8 --exclude=tests --verbose .
 

.github/workflows/python-app.yml

@@ -17,9 +17,9 @@ jobs:
     runs-on:
     - ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v5
     - name: Set up Python 3.11
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v6
       with:
         python-version: "3.11"
     - name: Install dependencies
@@ -35,9 +35,9 @@ jobs:
     runs-on:
     - windows-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v5
     - name: Set up Python 3.11
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v6
       with:
         python-version: "3.11"
     - name: Install dependencies

.github/workflows/stale-bot.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Mark Stale Issues and PRs
-        uses: actions/stale@v9
+        uses: actions/stale@v10
         with:
           stale-issue-message: "This issue is stale because it has been open 180 days with no activity. Remove stale label or comment, or it will be closed in 30 days."
           stale-pr-message: "This PR is stale because it has been open 180 days with no activity. Please update or it will be closed in 30 days."
@@ -24,7 +24,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           fetch-depth: 0  # Fetch full history for accurate branch checks
       - name: Fetch All Branches