chore: Dev to main (#2079)

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Roopan-Microsoft <168007406+Roopan-Microsoft@users.noreply.github.com>
Co-authored-by: Ajit Padhi <v-padhiajit@microsoft.com>
Co-authored-by: Roopan P M <v-roopanpm@microsoft.com>
Co-authored-by: Pavan-Microsoft <v-kupavan@microsoft.com>
Co-authored-by: Ross Smith <ross-p-smith@users.noreply.github.com>
Co-authored-by: gpickett <122489228+gpickett@users.noreply.github.com>
Co-authored-by: Francia Riesco <friesco@microsoft.com>
Co-authored-by: Francia Riesco <Fr4nc3@users.noreply.github.com>
Co-authored-by: Prajwal D C <v-dcprajwal@microsoft.com>
Co-authored-by: Harmanpreet-Microsoft <v-harmanprka@microsoft.com>
Co-authored-by: UtkarshMishra-Microsoft <v-utkamishra@microsoft.com>
Co-authored-by: Priyanka-Microsoft <v-prisinghal@microsoft.com>
Co-authored-by: Prasanjeet-Microsoft <v-singhprasa@microsoft.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Kiran-Siluveru-Microsoft <v-ksiluveru@microsoft.com>
Co-authored-by: Prashant-Microsoft <v-pmalusare@microsoft.com>
Co-authored-by: Rohini-Microsoft <v-rwalunj@microsoft.com>
Co-authored-by: Avijit-Microsoft <v-aghorui@microsoft.com>
Co-authored-by: RaviKiran-Microsoft <v-ravikirans@microsoft.com>
Co-authored-by: Somesh Joshi <v-somejoshi@microsoft.com>
Co-authored-by: Himanshi Agrawal <v-himagrawal@microsoft.com>
Co-authored-by: pradeepjha-microsoft <v-pradeepjha@microsoft.com>
Co-authored-by: Harmanpreet Kaur <v-harmanpkau@microsoft.com>
Co-authored-by: Bangarraju-Microsoft <v-golib@microsoft.com>
Co-authored-by: Harsh-Microsoft <v-hbangera@microsoft.com>
Co-authored-by: Kanchan-Microsoft <v-knagshetti@microsoft.com>
Co-authored-by: Cristopher Coronado <cristofima@hotmail.com>
Co-authored-by: Cristopher Coronado Moreira <crcorona@pichincha.com>
Co-authored-by: Vamshi-Microsoft <v-vamolla@microsoft.com>
Co-authored-by: Thanusree-Microsoft <168087422+Thanusree-Microsoft@users.noreply.github.com>
Co-authored-by: Rohini-Microsoft <168007985+Rohini-Microsoft@users.noreply.github.com>
Co-authored-by: Kingshuk-Microsoft <v-kidatta@microsoft.com>
Co-authored-by: Ayaz-Microsoft <v-ayazkhan@microsoft.com>
Co-authored-by: Abdul-Microsoft <v-amujeebta@microsoft.com>
Co-authored-by: Prekshith-Microsoft <v-pdj@microsoft.com>
Co-authored-by: Rafi-Microsoft <v-rafmd@microsoft.com>
Co-authored-by: Ragini-Microsoft <v-raginich@microsoft.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Atulku-Microsoft <v-atulku@microsoft.com>
Co-authored-by: Akhileswara-Microsoft <v-golnaidu@microsoft.com>
Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>

Author: 39 people

.github/workflows/bicep-audit.yml

@@ -19,7 +19,7 @@ jobs:
       security-events: write
     steps:
       - name: Checkout
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
 
       - name: Run Microsoft Security DevOps Analysis
         uses: microsoft/security-devops-action@preview

.github/workflows/broken-links-checker.yml

@@ -2,8 +2,6 @@ name: Broken Link Checker
 
 on:
   pull_request:
-    paths:
-      - '**/*.md'
   workflow_dispatch:
 
 permissions:
@@ -16,7 +14,7 @@ jobs:
 
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v5
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
@@ -28,14 +26,19 @@ jobs:
         with:
           files: |
             **/*.md
+
+      - name: Skip - No Markdown Files Changed
+        if: github.event_name == 'pull_request' && steps.changed-markdown-files.outputs.any_changed != 'true'
+        run: echo "No markdown files changed. Skipping broken link check."
+
       # For PR: Check broken links only in changed files
       - name: Check Broken Links in Changed Markdown Files
         id: lychee-check-pr
         if: github.event_name == 'pull_request' && steps.changed-markdown-files.outputs.any_changed == 'true'
         uses: lycheeverse/lychee-action@v2.7.0
         with:
           args: >
-            --verbose --no-progress --exclude ^https?://
+            --verbose --exclude-mail --no-progress --exclude ^https?://
             ${{ steps.changed-markdown-files.outputs.all_changed_files }}
           failIfEmpty: false
         env:
@@ -48,7 +51,7 @@ jobs:
         uses: lycheeverse/lychee-action@v2.7.0
         with:
           args: >
-            --verbose --no-progress --exclude ^https?://
+            --verbose --exclude-mail --no-progress --exclude ^https?://
             '**/*.md'
           failIfEmpty: false
         env:

.github/workflows/build-docker-images.yml

@@ -6,14 +6,6 @@ on:
       - main
       - dev
       - demo
-    paths:
-      - 'code/**'
-      - '!code/tests/**'
-      - 'docker/**'
-      - 'package.json'
-      - 'pyproject.toml'
-      - '.github/workflows/build-docker-images.yml'
-      - '.github/workflows/build-docker.yml'
   pull_request:
     branches:
       - main
@@ -28,7 +20,37 @@ on:
   workflow_dispatch:
 
 jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      should_build: ${{ steps.filter.outputs.docker_related }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Check for relevant changes
+        id: filter
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            docker_related:
+              - 'code/**'
+              - '!code/tests/**'
+              - 'docker/**'
+              - 'package.json'
+              - 'pyproject.toml'
+              - '.github/workflows/build-docker-images.yml'
+              - '.github/workflows/build-docker.yml'
+
+      - name: Skip - No Relevant Changes
+        if: steps.filter.outputs.docker_related != 'true' && github.event_name != 'workflow_dispatch'
+        run: echo "No relevant changes detected. Skipping docker build."
+
   docker-build:
+    needs: check-changes
+    if: needs.check-changes.outputs.should_build == 'true' || github.event_name == 'workflow_dispatch'
     strategy:
       matrix:
         include:

.github/workflows/ci.yml

@@ -6,13 +6,6 @@ on:
       - main
       - dev
       - demo
-    paths:
-      - 'infra/**'
-      - 'scripts/**'
-      - 'azure.yaml'
-      - 'pyproject.toml'
-      - 'Makefile'
-      - '.github/workflows/ci.yml'
   schedule:
     - cron: '0 8,20 * * *'  # Runs at 8:00 AM and 8:00 PM GMT
   workflow_dispatch:
@@ -25,7 +18,36 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
 
 jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      should_deploy: ${{ steps.filter.outputs.deploy_related }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Check for relevant changes
+        id: filter
+        uses: dorny/paths-filter@v3
+        with:
+          filters: |
+            deploy_related:
+              - 'infra/**'
+              - 'scripts/**'
+              - 'azure.yaml'
+              - 'pyproject.toml'
+              - 'Makefile'
+              - '.github/workflows/ci.yml'
+
+      - name: Skip - No Relevant Changes
+        if: steps.filter.outputs.deploy_related != 'true' && github.event_name != 'workflow_dispatch' && github.event_name != 'schedule'
+        run: echo "No relevant changes detected. Skipping deployment validation."
+
   deploy:
+    needs: check-changes
+    if: needs.check-changes.outputs.should_deploy == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
     runs-on: ubuntu-latest
     env:
       AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
@@ -387,17 +409,17 @@ jobs:
 
 
   e2e-test:
-    needs: deploy
-    if: needs.deploy.outputs.DEPLOYMENT_SUCCESS == 'true'
+    needs: [check-changes, deploy]
+    if: always() && needs.deploy.result == 'success' && needs.deploy.outputs.DEPLOYMENT_SUCCESS == 'true'
     uses: ./.github/workflows/test-automation.yml
     with:
       web_url: ${{ needs.deploy.outputs.web_url }}
       admin_url: ${{ needs.deploy.outputs.admin_url }}
 
 
   cleanup:
-    if: always()
-    needs: [deploy, e2e-test]
+    if: always() && needs.deploy.result != 'skipped'
+    needs: [check-changes, deploy, e2e-test]
     runs-on: ubuntu-latest
 
     env:

code/backend/batch/add_url_embeddings.py

@@ -59,11 +59,15 @@ def process_url_contents_directly(url: str, env_helper: EnvHelper):
 
 def download_url_and_upload_to_blob(url: str):
     try:
-        response = requests.get(url)
+        # Add User-Agent header to avoid being blocked by websites (e.g., Wikipedia)
+        headers = {
+            'User-Agent': 'cwyd-admin-user'
+        }
+        response = requests.get(url, headers=headers)
         parsed_data = BeautifulSoup(response.content, "html.parser")
         with io.BytesIO(parsed_data.get_text().encode("utf-8")) as stream:
             blob_client = AzureBlobStorageClient()
-            blob_client.upload_file(stream, url, metadata={"title": url})
+            blob_client.upload_file(stream, url)
         return func.HttpResponse(f"URL {url} added to knowledge base", status_code=200)
 
     except Exception: