Merge remote-tracking branch 'origin/main' into demo

Author: Prajwal-Microsoft

.coverage

@@ -7,6 +7,8 @@ FROM ghcr.io/astral-sh/uv:$UV_VERSION AS uv
 # Use Debian-based VS Code Dev Container as base
 FROM mcr.microsoft.com/vscode/devcontainers/base:$DEBIAN_VERSION
 
+ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0
+
 # Install dependencies and Node.js 20+ from NodeSource
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
@@ -15,13 +17,14 @@ RUN apt-get update \
     && apt-get install -y nodejs \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* \
-    && curl -fsSL https://aka.ms/install-azd.sh | bash
+    && curl -fsSL https://aka.ms/install-azd.sh | bash \
+    && ln -sf /usr/bin/python3 /usr/bin/python
 
 # Copy UV binaries
 COPY --from=uv --chown=vscode: /uv /uvx /bin/
 
 # Install global NPM packages
-RUN npm install -g tslint-to-eslint-config typescript yarn react-app-rewired
+RUN npm install -g tslint-to-eslint-config typescript react-app-rewired pnpm@10.28.2
 
 # Set default shell
 SHELL ["/bin/bash", "-c"]

.devcontainer/Dockerfile

@@ -18,7 +18,8 @@ cd ../../
 
 echo "Installing dependencies for ContentProcessorWeb..."
 cd ./src/ContentProcessorWeb
-yarn install
+export HOME="${HOME:-/home/vscode}"
+pnpm install --store-dir "$HOME/.local/share/pnpm/store"
 
 cd ../../
 
@@ -27,6 +28,6 @@ sed -i 's/\r$//' infra/scripts/post_deployment.sh
 sudo chmod +x infra/scripts/docker-build.sh
 sudo chmod +x infra/scripts/post_deployment.sh
 sudo chmod +x src/ContentProcessorAPI/samples/upload_files.sh
-sudo chmod +x src/ContentProcessorAPI/samples/schemas/register_schema.sh
+# register_schema.py is cross-platform and does not need chmod
 
 echo "Setup complete! 🎉"

.devcontainer/setupEnv.sh

@@ -3,18 +3,59 @@ name: Build and Push Docker Images
 on:
   push:
     branches: [main, dev, demo, hotfix]
+    paths:
+      - 'src/ContentProcessor/src/**'
+      - 'src/**/Dockerfile'
+      - 'src/**/requirements.txt'
+      - 'src/**/pyproject.toml'
+      - 'src/ContentProcessorAPI/app/**'
+      - 'src/ContentProcessorAPI/helpers/**'
+      - 'src/ContentProcessorWeb/src/**'
+      - 'src/ContentProcessorWeb/public/**'
+      - 'src/ContentProcessorWeb/package.json'
+      - 'src/ContentProcessorWeb/package-lock.json'
+      - 'src/ContentProcessorWeb/yarn.lock'
+      - 'src/ContentProcessorWeb/tsconfig.json'
+      - 'src/ContentProcessorWeb/config-overrides.js'
+      - 'src/ContentProcessorWeb/nginx-custom.conf'
+      - 'src/ContentProcessorWeb/env.sh'
+      - '.github/workflows/build-docker-image.yml'
   pull_request:
     branches: [main, dev, demo, hotfix]
     types: [opened, ready_for_review, reopened, synchronize]
+    paths:
+      - 'src/ContentProcessor/src/**'
+      - 'src/**/Dockerfile'
+      - 'src/**/requirements.txt'
+      - 'src/**/pyproject.toml'
+      - 'src/ContentProcessorAPI/app/**'
+      - 'src/ContentProcessorAPI/helpers/**'
+      - 'src/ContentProcessorWeb/src/**'
+      - 'src/ContentProcessorWeb/public/**'
+      - 'src/ContentProcessorWeb/package.json'
+      - 'src/ContentProcessorWeb/package-lock.json'
+      - 'src/ContentProcessorWeb/yarn.lock'
+      - 'src/ContentProcessorWeb/tsconfig.json'
+      - 'src/ContentProcessorWeb/config-overrides.js'
+      - 'src/ContentProcessorWeb/nginx-custom.conf'
+      - 'src/ContentProcessorWeb/env.sh'
+      - 'infra/**/*.bicep'
+      - 'infra/**/*.json'
+      - 'azure.yaml'
+      - '.github/workflows/build-docker-image.yml'
   workflow_dispatch:
 
+permissions:
+  contents: read
+  actions: read
+  id-token: write
+  
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
+    environment: production
     env:
       ACR_LOGIN_SERVER: ${{ secrets.ACR_LOGIN_SERVER }}
-      ACR_USERNAME: ${{ secrets.ACR_USERNAME }}
-      ACR_PASSWORD: ${{ secrets.ACR_PASSWORD }}
 
     steps:
       - name: Checkout repository
@@ -27,13 +68,17 @@ jobs:
         id: date
         run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
 
-      - name: Log in to Azure Container Registry
+      - name: Login to Azure
         if: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix' || github.ref_name == 'dependabotchanges' }}
-        uses: azure/docker-login@v2
+        uses: azure/login@v2
         with:
-          login-server: ${{ env.ACR_LOGIN_SERVER }}
-          username: ${{ env.ACR_USERNAME }}
-          password: ${{ env.ACR_PASSWORD }}
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Log in to Azure Container Registry
+        if: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix' || github.ref_name == 'dependabotchanges' }}
+        run: az acr login --name ${{ env.ACR_LOGIN_SERVER }}
 
       - name: Get registry
         id: registry
@@ -47,7 +92,7 @@ jobs:
           DATE="${{ steps.date.outputs.date }}"
           GITHUB_RUN_NUMBER="${{ github.run_number }}"
           if [[ "$BRANCH" == "main" ]]; then
-            BASE_TAG="latest"
+            BASE_TAG="latest_v2"
           elif [[ "$BRANCH" == "dev" ]]; then
             BASE_TAG="dev"
           elif [[ "$BRANCH" == "demo" ]]; then
@@ -56,7 +101,6 @@ jobs:
             BASE_TAG="hotfix"
           elif [[ "$BRANCH" == "dependabotchanges" ]]; then
             BASE_TAG="dependabotchanges"
-            
           else
             BASE_TAG="pullrequest-ignore"
           fi
@@ -94,3 +138,13 @@ jobs:
           tags: |
             ${{ steps.registry.outputs.ext_registry }}/contentprocessorweb:${{ env.BASE_TAG }}
             ${{ steps.registry.outputs.ext_registry }}/contentprocessorweb:${{ env.DATE_TAG }}
+
+      - name: Build and Push ContentProcessorWorkflow Docker image
+        uses: docker/build-push-action@v6
+        with:
+          context: ./src/ContentProcessorWorkflow
+          file: ./src/ContentProcessorWorkflow/Dockerfile
+          push: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix' || github.ref_name == 'dependabotchanges' }}
+          tags: |
+            ${{ steps.registry.outputs.ext_registry }}/contentprocessorworkflow:${{ env.BASE_TAG }}
+            ${{ steps.registry.outputs.ext_registry }}/contentprocessorworkflow:${{ env.DATE_TAG }}

.github/workflows/build-docker-image.yml

@@ -3,6 +3,23 @@ on:
   push:
     branches:
       - main
+    paths:
+      - 'src/ContentProcessor/src/**'
+      - 'src/**/Dockerfile'
+      - 'src/**/requirements.txt' 
+      - 'src/**/pyproject.toml'
+      - 'src/ContentProcessorAPI/app/**'
+      - 'src/ContentProcessorAPI/helpers/**'
+      - 'src/ContentProcessorWeb/src/**'
+      - 'src/ContentProcessorWeb/public/**'
+      - 'src/ContentProcessorWeb/package.json'
+      - 'src/ContentProcessorWeb/package-lock.json'
+      - 'src/ContentProcessorWeb/yarn.lock'
+      - 'src/ContentProcessorWeb/tsconfig.json'
+      - 'src/ContentProcessorWeb/config-overrides.js'
+      - 'src/ContentProcessorWeb/nginx-custom.conf'
+      - 'src/ContentProcessorWeb/env.sh'
+      - '.github/workflows/deploy-linux.yml'
   workflow_dispatch:
     inputs:
       azure_location:
@@ -75,21 +92,191 @@ on:
 
   schedule:
     - cron: '0 9,21 * * *'  # Runs at 9:00 AM and 9:00 PM GMT
-
+permissions:
+  contents: read
+  actions: read
 jobs:
+  validate-inputs:
+    runs-on: ubuntu-latest
+    outputs:
+      validation_passed: ${{ steps.validate.outputs.passed }}
+      azure_location: ${{ steps.validate.outputs.azure_location }}
+      resource_group_name: ${{ steps.validate.outputs.resource_group_name }}
+      waf_enabled: ${{ steps.validate.outputs.waf_enabled }}
+      exp: ${{ steps.validate.outputs.exp }}
+      build_docker_image: ${{ steps.validate.outputs.build_docker_image }}
+      cleanup_resources: ${{ steps.validate.outputs.cleanup_resources }}
+      run_e2e_tests: ${{ steps.validate.outputs.run_e2e_tests }}
+      azure_env_log_analytics_workspace_id: ${{ steps.validate.outputs.azure_env_log_analytics_workspace_id }}
+      azure_existing_ai_project_resource_id: ${{ steps.validate.outputs.azure_existing_ai_project_resource_id }}
+      existing_webapp_url: ${{ steps.validate.outputs.existing_webapp_url }}
+    steps:
+      - name: Validate Workflow Input Parameters
+        id: validate
+        shell: bash
+        env:
+          INPUT_AZURE_LOCATION: ${{ github.event.inputs.azure_location }}
+          INPUT_RESOURCE_GROUP_NAME: ${{ github.event.inputs.resource_group_name }}
+          INPUT_WAF_ENABLED: ${{ github.event.inputs.waf_enabled }}
+          INPUT_EXP: ${{ github.event.inputs.EXP }}
+          INPUT_BUILD_DOCKER_IMAGE: ${{ github.event.inputs.build_docker_image }}
+          INPUT_CLEANUP_RESOURCES: ${{ github.event.inputs.cleanup_resources }}
+          INPUT_RUN_E2E_TESTS: ${{ github.event.inputs.run_e2e_tests }}
+          INPUT_AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID: ${{ github.event.inputs.AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID }}
+          INPUT_AZURE_EXISTING_AI_PROJECT_RESOURCE_ID: ${{ github.event.inputs.AZURE_EXISTING_AI_PROJECT_RESOURCE_ID }}
+          INPUT_EXISTING_WEBAPP_URL: ${{ github.event.inputs.existing_webapp_url }}
+        run: |
+          echo "🔍 Validating workflow input parameters..."
+          VALIDATION_FAILED=false
+          
+          # Validate azure_location (Azure region format)
+          LOCATION="${INPUT_AZURE_LOCATION:-australiaeast}"
+          
+          if [[ ! "$LOCATION" =~ ^[a-z0-9]+$ ]]; then
+            echo "❌ ERROR: azure_location '$LOCATION' is invalid. Must contain only lowercase letters and numbers"
+            VALIDATION_FAILED=true
+          else
+            echo "✅ azure_location: '$LOCATION' is valid"
+          fi
+          
+          # Validate resource_group_name (Azure naming convention, optional)
+          if [[ -n "$INPUT_RESOURCE_GROUP_NAME" ]]; then
+            if [[ ! "$INPUT_RESOURCE_GROUP_NAME" =~ ^[a-zA-Z0-9._\(\)-]+$ ]] || [[ "$INPUT_RESOURCE_GROUP_NAME" =~ \.$ ]]; then
+              echo "❌ ERROR: resource_group_name '$INPUT_RESOURCE_GROUP_NAME' is invalid. Must contain only alphanumerics, periods, underscores, hyphens, and parentheses. Cannot end with period."
+              VALIDATION_FAILED=true
+            elif [[ ${#INPUT_RESOURCE_GROUP_NAME} -gt 90 ]]; then
+              echo "❌ ERROR: resource_group_name '$INPUT_RESOURCE_GROUP_NAME' exceeds 90 characters (length: ${#INPUT_RESOURCE_GROUP_NAME})"
+              VALIDATION_FAILED=true
+            else
+              echo "✅ resource_group_name: '$INPUT_RESOURCE_GROUP_NAME' is valid"
+            fi
+          else
+            echo "✅ resource_group_name: Not provided (will be auto-generated)"
+          fi
+          
+          # Validate waf_enabled (boolean)
+          WAF_ENABLED="${INPUT_WAF_ENABLED:-false}"
+          if [[ "$WAF_ENABLED" != "true" && "$WAF_ENABLED" != "false" ]]; then
+            echo "❌ ERROR: waf_enabled must be 'true' or 'false', got: '$WAF_ENABLED'"
+            VALIDATION_FAILED=true
+          else
+            echo "✅ waf_enabled: '$WAF_ENABLED' is valid"
+          fi
+          
+          # Validate EXP (boolean)
+          EXP_ENABLED="${INPUT_EXP:-false}"
+          if [[ "$EXP_ENABLED" != "true" && "$EXP_ENABLED" != "false" ]]; then
+            echo "❌ ERROR: EXP must be 'true' or 'false', got: '$EXP_ENABLED'"
+            VALIDATION_FAILED=true
+          else
+            echo "✅ EXP: '$EXP_ENABLED' is valid"
+          fi
+          
+          # Validate build_docker_image (boolean)
+          BUILD_DOCKER="${INPUT_BUILD_DOCKER_IMAGE:-false}"
+          if [[ "$BUILD_DOCKER" != "true" && "$BUILD_DOCKER" != "false" ]]; then
+            echo "❌ ERROR: build_docker_image must be 'true' or 'false', got: '$BUILD_DOCKER'"
+            VALIDATION_FAILED=true
+          else
+            echo "✅ build_docker_image: '$BUILD_DOCKER' is valid"
+          fi
+          
+          # Validate cleanup_resources (boolean)
+          CLEANUP_RESOURCES="${INPUT_CLEANUP_RESOURCES:-false}"
+          if [[ "$CLEANUP_RESOURCES" != "true" && "$CLEANUP_RESOURCES" != "false" ]]; then
+            echo "❌ ERROR: cleanup_resources must be 'true' or 'false', got: '$CLEANUP_RESOURCES'"
+            VALIDATION_FAILED=true
+          else
+            echo "✅ cleanup_resources: '$CLEANUP_RESOURCES' is valid"
+          fi
+          
+          # Validate run_e2e_tests (specific allowed values)
+          TEST_OPTION="${INPUT_RUN_E2E_TESTS:-GoldenPath-Testing}"
+          if [[ "$TEST_OPTION" != "GoldenPath-Testing" && "$TEST_OPTION" != "Smoke-Testing" && "$TEST_OPTION" != "None" ]]; then
+            echo "❌ ERROR: run_e2e_tests must be one of: GoldenPath-Testing, Smoke-Testing, None, got: '$TEST_OPTION'"
+            VALIDATION_FAILED=true
+          else
+            echo "✅ run_e2e_tests: '$TEST_OPTION' is valid"
+          fi
+          
+          # Validate AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID (optional, Azure Resource ID format)
+          if [[ -n "$INPUT_AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID" ]]; then
+            if [[ ! "$INPUT_AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID" =~ ^/subscriptions/[a-fA-F0-9-]+/[Rr]esource[Gg]roups/[^/]+/providers/[Mm]icrosoft\.[Oo]perational[Ii]nsights/[Ww]orkspaces/[^/]+$ ]]; then
+              echo "❌ ERROR: AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID is invalid. Must be a valid Azure Resource ID format:"
+              echo "   /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.OperationalInsights/workspaces/{workspaceName}"
+              echo "   Got: '$INPUT_AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID'"
+              VALIDATION_FAILED=true
+            else
+              echo "✅ AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID: Valid Resource ID format"
+            fi
+          else
+            echo "✅ AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID: Not provided (optional)"
+          fi
+          
+          # Validate AZURE_EXISTING_AI_PROJECT_RESOURCE_ID (optional, Azure Resource ID format)
+          if [[ -n "$INPUT_AZURE_EXISTING_AI_PROJECT_RESOURCE_ID" ]]; then
+            if [[ ! "$INPUT_AZURE_EXISTING_AI_PROJECT_RESOURCE_ID" =~ ^/subscriptions/[a-fA-F0-9-]+/[Rr]esource[Gg]roups/[^/]+/providers/([Mm]icrosoft\.[Mm]achine[Ll]earning[Ss]ervices/([Ww]orkspaces|[Pp]rojects)/[^/]+|[Mm]icrosoft\.[Cc]ognitive[Ss]ervices/[Aa]ccounts/[^/]+/[Pp]rojects/[^/]+)$ ]]; then
+              echo "❌ ERROR: AZURE_EXISTING_AI_PROJECT_RESOURCE_ID is invalid. Must be a valid Azure Resource ID format:"
+              echo "   /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/providers/Microsoft.CognitiveServices/accounts/{accountName}/projects/{projectName}"
+              echo "   Got: '$INPUT_AZURE_EXISTING_AI_PROJECT_RESOURCE_ID'"
+              VALIDATION_FAILED=true
+            else
+              echo "✅ AZURE_EXISTING_AI_PROJECT_RESOURCE_ID: Valid Resource ID format"
+            fi
+          else
+            echo "✅ AZURE_EXISTING_AI_PROJECT_RESOURCE_ID: Not provided (optional)"
+          fi
+          
+          # Validate existing_webapp_url (optional, must start with https)
+          if [[ -n "$INPUT_EXISTING_WEBAPP_URL" ]]; then
+            if [[ ! "$INPUT_EXISTING_WEBAPP_URL" =~ ^https:// ]]; then
+              echo "❌ ERROR: existing_webapp_url must start with 'https://', got: '$INPUT_EXISTING_WEBAPP_URL'"
+              VALIDATION_FAILED=true
+            else
+              echo "✅ existing_webapp_url: '$INPUT_EXISTING_WEBAPP_URL' is valid"
+            fi
+          else
+            echo "✅ existing_webapp_url: Not provided (will perform deployment)"
+          fi
+          
+          # Fail workflow if any validation failed
+          if [[ "$VALIDATION_FAILED" == "true" ]]; then
+            echo ""
+            echo "❌ Parameter validation failed. Please correct the errors above and try again."
+            exit 1
+          fi
+          
+          echo ""
+          echo "✅ All input parameters validated successfully!"
+          
+          # Output validated values
+          echo "passed=true" >> $GITHUB_OUTPUT
+          echo "azure_location=$LOCATION" >> $GITHUB_OUTPUT
+          echo "resource_group_name=$INPUT_RESOURCE_GROUP_NAME" >> $GITHUB_OUTPUT
+          echo "waf_enabled=$WAF_ENABLED" >> $GITHUB_OUTPUT
+          echo "exp=$EXP_ENABLED" >> $GITHUB_OUTPUT
+          echo "build_docker_image=$BUILD_DOCKER" >> $GITHUB_OUTPUT
+          echo "cleanup_resources=$CLEANUP_RESOURCES" >> $GITHUB_OUTPUT
+          echo "run_e2e_tests=$TEST_OPTION" >> $GITHUB_OUTPUT
+          echo "azure_env_log_analytics_workspace_id=$INPUT_AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID" >> $GITHUB_OUTPUT
+          echo "azure_existing_ai_project_resource_id=$INPUT_AZURE_EXISTING_AI_PROJECT_RESOURCE_ID" >> $GITHUB_OUTPUT
+          echo "existing_webapp_url=$INPUT_EXISTING_WEBAPP_URL" >> $GITHUB_OUTPUT
+
   Run:
+    needs: validate-inputs
+    if: needs.validate-inputs.outputs.validation_passed == 'true'
     uses: ./.github/workflows/deploy-orchestrator.yml
     with:
       runner_os: ubuntu-latest
-      azure_location: ${{ github.event.inputs.azure_location || 'australiaeast' }}
-      resource_group_name: ${{ github.event.inputs.resource_group_name || '' }}
-      waf_enabled: ${{ github.event.inputs.waf_enabled == 'true' }}
-      EXP: ${{ github.event.inputs.EXP == 'true' }}
-      build_docker_image: ${{ github.event.inputs.build_docker_image == 'true' }}
-      cleanup_resources: ${{ github.event.inputs.cleanup_resources == 'true' }}
-      run_e2e_tests: ${{ github.event.inputs.run_e2e_tests || 'GoldenPath-Testing' }}
-      AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID: ${{ github.event.inputs.AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID || '' }}
-      AZURE_EXISTING_AI_PROJECT_RESOURCE_ID: ${{ github.event.inputs.AZURE_EXISTING_AI_PROJECT_RESOURCE_ID || '' }}
-      existing_webapp_url: ${{ github.event.inputs.existing_webapp_url || '' }}
+      azure_location: ${{ needs.validate-inputs.outputs.azure_location || 'australiaeast' }}
+      resource_group_name: ${{ needs.validate-inputs.outputs.resource_group_name || '' }}
+      waf_enabled: ${{ needs.validate-inputs.outputs.waf_enabled == 'true' }}
+      EXP: ${{ needs.validate-inputs.outputs.exp == 'true' }}
+      build_docker_image: ${{ needs.validate-inputs.outputs.build_docker_image == 'true' }}
+      cleanup_resources: ${{ needs.validate-inputs.outputs.cleanup_resources == 'true' }}
+      run_e2e_tests: ${{ needs.validate-inputs.outputs.run_e2e_tests || 'GoldenPath-Testing' }}
+      AZURE_ENV_LOG_ANALYTICS_WORKSPACE_ID: ${{ needs.validate-inputs.outputs.azure_env_log_analytics_workspace_id || '' }}
+      AZURE_EXISTING_AI_PROJECT_RESOURCE_ID: ${{ needs.validate-inputs.outputs.azure_existing_ai_project_resource_id || '' }}
+      existing_webapp_url: ${{ needs.validate-inputs.outputs.existing_webapp_url || '' }}
       trigger_type: ${{ github.event_name }}
     secrets: inherit