Merge pull request #140 from microsoft/dev

feat: Enhance deployment pipeline, modularize test workflows, simplify infra, and update docs

Author: Roopan-Microsoft

.github/workflows/deploy.yml

@@ -1,16 +1,28 @@
-name: Validate Deployment
+name: Deploy-Test-Cleanup Pipeline
 
 on:
-  push:
-    branches:
-      - main
-  schedule:
-    - cron: '0 5,17 * * *'  # Runs at 5:00 AM and 5:00 PM GMT
-  workflow_dispatch:
+    workflow_run:
+      workflows: ["Build Docker and Optional Push"]
+      types:
+        - completed
+      branches:
+        - main
+        - dev
+        - demo
+    schedule:
+      - cron: '0 5,17 * * *'  # Runs at 5:00 AM and 5:00 PM GMT
+    workflow_dispatch:
+
+env:
+  GPT_MIN_CAPACITY: 200
+  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
+    outputs:
+      RESOURCE_GROUP_NAME: ${{ steps.check_create_rg.outputs.RESOURCE_GROUP_NAME }}
+      WEBAPP_URL: ${{ steps.get_output.outputs.WEBAPP_URL }}
     steps:
       - name: Checkout Code
         uses: actions/checkout@v3
@@ -36,7 +48,6 @@ jobs:
           UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}"
           echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV
           echo "Generated RESOURCE_GROUP_NAME: ${UNIQUE_RG_NAME}"
-  
 
       - name: Check and Create Resource Group
         id: check_create_rg
@@ -50,8 +61,8 @@ jobs:
           else
             echo "Resource group already exists."
           fi
+          echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT
 
-      
       - name: Generate Unique Solution Prefix
         id: generate_solution_prefix
         run: |
@@ -63,17 +74,42 @@ jobs:
           echo "SOLUTION_PREFIX=${UNIQUE_SOLUTION_PREFIX}" >> $GITHUB_ENV
           echo "Generated SOLUTION_PREFIX: ${UNIQUE_SOLUTION_PREFIX}"
 
-
       - name: Deploy Bicep Template
         id: deploy
         run: |
           set -e
+          # set image tag based on branch
+          if [[ "${{ env.BRANCH_NAME }}" == "main" ]]; then
+            IMAGE_TAG="latest"
+          elif [[ "${{ env.BRANCH_NAME }}" == "dev" ]]; then
+            IMAGE_TAG="dev"
+          elif [[ "${{ env.BRANCH_NAME }}" == "demo" ]]; then
+            IMAGE_TAG="demo"
+          else
+            IMAGE_TAG="latest"
+          fi
+
           az deployment group create \
+            --name ${{ env.SOLUTION_PREFIX }}-deployment \
             --resource-group ${{ env.RESOURCE_GROUP_NAME }} \
             --template-file infra/main.bicep \
-            --parameters AzureAiServiceLocation=swedencentral Prefix=${{ env.SOLUTION_PREFIX }}\
+            --parameters \
+                Prefix="${{ env.SOLUTION_PREFIX }}" \
+                AzureAiServiceLocation="eastus" \
+                capacity=${{ env.GPT_MIN_CAPACITY }} \
+                imageVersion="${IMAGE_TAG}"\
             --debug
 
+      - name: Get Deployment Output and extract Values
+        id: get_output
+        run: |
+          set -e
+          echo "Fetching deployment output..."
+          BICEP_OUTPUT=$(az deployment group show --name ${{ env.SOLUTION_PREFIX }}-deployment --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "properties.outputs" -o json)
+          echo "Extracting deployment output..."
+          WEBAPP_URL=$(echo $BICEP_OUTPUT | jq -r '.weB_APP_URL.value')
+          echo "WEBAPP_URL=$WEBAPP_URL" >> $GITHUB_OUTPUT
+          echo "Deployment output: $BICEP_OUTPUT"
 
       - name: Send Notification on Failure
         if: failure()
@@ -93,6 +129,34 @@ jobs:
             -H "Content-Type: application/json" \
             -d "$EMAIL_BODY" || echo "Failed to send notification"
 
+      - name: Logout from Azure
+        if: always()
+        run: |
+          az logout
+          echo "Logged out from Azure."
+
+  e2e-test:
+    needs: deploy
+    uses: ./.github/workflows/test-automation.yml
+    with:
+      CODEMOD_WEB_URL: ${{ needs.deploy.outputs.WEBAPP_URL }}
+    secrets: inherit
+
+  cleanup-deployment:
+    if: always() && needs.deploy.outputs.RESOURCE_GROUP_NAME != ''
+    needs: [deploy, e2e-test]
+    runs-on: ubuntu-latest
+    env:
+      RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }}
+    steps:
+      - name: Setup Azure CLI
+        run: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+          az --version  # Verify installation
+
+      - name: Login to Azure
+        run: |
+          az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }}
 
       - name: Get Log Analytics Workspace and OpenAI from Resource Group
         if: always()
@@ -126,7 +190,6 @@ jobs:
             echo "OpenAI resource name: ${openai_resource_name}" 
           fi
 
-
       - name: List KeyVaults and Store in Array
         if: always()
         id: list_keyvaults
@@ -177,7 +240,6 @@ jobs:
 
           echo "Log analytics workspace resource purging completed successfully"    
       
-
       - name: Delete Bicep Deployment
         if: always()
         run: |
@@ -195,7 +257,6 @@ jobs:
             echo "Resource group does not exists."
           fi
 
-
       - name: Wait for resource deletion to complete
         if: always()
         run: |
@@ -255,7 +316,6 @@ jobs:
               break
             fi
           done
-
           
       - name: Purging the Resources
         if: always()
@@ -302,3 +362,9 @@ jobs:
             fi
           done
           echo "Resource purging completed successfully"
+
+      - name: Logout from Azure
+        if: always()
+        run: |
+          az logout
+          echo "Logged out from Azure."

.github/workflows/test-automation.yml

@@ -1,23 +1,23 @@
 name: Test Automation Code Modernization
 
 on:
-  push:
-    branches:
-      - main
-      - dev
-    paths:
-      - 'tests/e2e-test/**'
-  schedule:
-      - cron: '0 13 * * *'  # Runs at 1 PM UTC
-  workflow_dispatch:
+  workflow_call:
+    inputs:
+      CODEMOD_WEB_URL:
+        required: true
+        type: string
+        description: "Web URL for Code Modernization"
+    secrets:
+      EMAILNOTIFICATION_LOGICAPP_URL_TA:
+        required: false
+        description: "Logic App URL for email notifications"
 
 env:
-    url: ${{ vars.CODEMOD_WEB_URL }}
-    accelerator_name: "Code Modernization"
+  url: ${{ inputs.CODEMOD_WEB_URL }}
+  accelerator_name: "Code Modernization"
 
 jobs:
   test:
-
     runs-on: ubuntu-latest
     steps:      
       - name: Checkout repository
@@ -33,15 +33,6 @@ jobs:
         with:
           creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}'
 
-      - name: Start Container App
-        id: start-container-app
-        uses: azure/cli@v2
-        with:
-          azcliversion: 'latest'
-          inlineScript: |
-            az rest -m post -u "/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ vars.CODEMOD_RG }}/providers/Microsoft.App/containerApps/${{ vars.CODEMOD_FRONTEND_CONTAINER_NAME }}/start?api-version=2025-01-01"
-            az rest -m post -u "/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ vars.CODEMOD_RG }}/providers/Microsoft.App/containerApps/${{ vars.CODEMOD_BACKEND_CONTAINER_NAME }}/start?api-version=2025-01-01"
-
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
@@ -50,6 +41,11 @@ jobs:
       - name: Ensure browsers are installed
         run: python -m playwright install --with-deps chromium
 
+      - name: Open URL
+        run: |
+          echo "Opening URL: ${{ env.url }}"
+          python -m webbrowser "${{ env.url }}"
+
       - name: Run tests(1)
         id: test1
         run: |
@@ -118,14 +114,4 @@ jobs:
           # Send the notification
           curl -X POST "${{ secrets.EMAILNOTIFICATION_LOGICAPP_URL_TA }}" \
             -H "Content-Type: application/json" \
-            -d "$EMAIL_BODY" || echo "Failed to send notification"
-
-      - name: Stop Container App
-        if: always()
-        uses: azure/cli@v2
-        with:
-          azcliversion: 'latest'
-          inlineScript: |
-            az rest -m post -u "/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ vars.CODEMOD_RG }}/providers/Microsoft.App/containerApps/${{ vars.CODEMOD_FRONTEND_CONTAINER_NAME }}/stop?api-version=2025-01-01"
-            az rest -m post -u "/subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ vars.CODEMOD_RG }}/providers/Microsoft.App/containerApps/${{ vars.CODEMOD_BACKEND_CONTAINER_NAME }}/stop?api-version=2025-01-01"
-            az logout
+            -d "$EMAIL_BODY" || echo "Failed to send notification"

azure.yaml

@@ -26,14 +26,14 @@ hooks:
     posix:
       shell: sh
       run: >
-        chmod u+r+x ./scripts/validate_model_deployment_quota.sh; chmod u+r+x ./scripts/validate_model_quota.sh; ./scripts/validate_model_deployment_quota.sh --subscription "$AZURE_SUBSCRIPTION_ID" --location "${AZURE_AISERVICE_LOCATION:-japaneast}" --models-parameter "aiModelDeployments"
-      interactive: false
+        chmod u+r+x ./scripts/validate_model_deployment_quota.sh; chmod u+r+x ./scripts/validate_model_quota.sh; ./scripts/validate_model_deployment_quota.sh --SubscriptionId "$AZURE_SUBSCRIPTION_ID" --Location "${AZURE_AISERVICE_LOCATION:-japaneast}" --ModelsParameter "aiModelDeployments"
+      interactive: true
       continueOnError: false
 
     windows:
       shell: pwsh
       run: >
         $location = if ($env:AZURE_AISERVICE_LOCATION) { $env:AZURE_AISERVICE_LOCATION } else { "japaneast" };
         ./scripts/validate_model_deployment_quota.ps1 -SubscriptionId $env:AZURE_SUBSCRIPTION_ID -Location $location -ModelsParameter "aiModelDeployments"
-      interactive: false
+      interactive: true
       continueOnError: false

infra/abbreviations.json

@@ -2,6 +2,8 @@
     "ai": {
       "aiSearch": "srch-",
       "aiServices": "aisa-",
+      "aiFoundry": "aif-",
+      "aiFoundryProject": "aifp-",
       "aiVideoIndexer": "avi-",
       "machineLearningWorkspace": "mlw-",
       "openAIService": "oai-",