Deploy-Test-Cleanup Pipeline #441
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy-Test-Cleanup Pipeline | |
| on: | |
| workflow_run: | |
| workflows: ["Build Docker and Optional Push"] | |
| types: | |
| - completed | |
| branches: | |
| - vee-pipeline-fixes | |
| schedule: | |
| - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT | |
| workflow_dispatch: | |
| env: | |
| GPT_MIN_CAPACITY: 150 | |
| BRANCH_NAME: ${{ github.head_ref || github.ref_name }} | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| RESOURCE_GROUP_NAME: ${{ steps.check_create_rg.outputs.RESOURCE_GROUP_NAME }} | |
| WEBAPP_URL: ${{ steps.get_output.outputs.WEBAPP_URL }} | |
| SELECTED_AI_REGION: ${{ steps.deploy.outputs.selected_ai_region || env.VALID_REGION }} | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| - name: Setup Azure CLI | |
| run: | | |
| curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash | |
| az --version # Verify installation | |
| - name: Login to Azure | |
| run: | | |
| az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Run Quota Check | |
| id: quota-check | |
| run: | | |
| export AZURE_CLIENT_ID=${{ secrets.AZURE_CLIENT_ID }} | |
| export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }} | |
| export AZURE_CLIENT_SECRET=${{ secrets.AZURE_CLIENT_SECRET }} | |
| export AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" | |
| export GPT_MIN_CAPACITY="${{ env.GPT_MIN_CAPACITY }}" | |
| export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}" | |
| chmod +x scripts/checkquota.sh | |
| if ! scripts/checkquota.sh; then | |
| # If quota check fails due to insufficient quota, set the flag | |
| if grep -q "No region with sufficient quota found" scripts/checkquota.sh; then | |
| echo "QUOTA_FAILED=true" >> $GITHUB_ENV | |
| fi | |
| exit 1 # Fail the pipeline if any other failure occurs | |
| fi | |
| - name: Send Notification on Quota Failure | |
| if: env.QUOTA_FAILED == 'true' | |
| run: | | |
| RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| EMAIL_BODY=$(cat <<EOF | |
| { | |
| "body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a></p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>" | |
| } | |
| EOF | |
| ) | |
| curl -X POST "${{ secrets.LOGIC_APP_URL }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$EMAIL_BODY" || echo "Failed to send notification" | |
| - name: Fail Pipeline if Quota Check Fails | |
| if: env.QUOTA_FAILED == 'true' | |
| run: exit 1 | |
| - name: Install Bicep CLI | |
| run: az bicep install | |
| - name: Generate Resource Group Name | |
| id: generate_rg_name | |
| run: | | |
| echo "Generating a unique resource group name..." | |
| ACCL_NAME="codemod" | |
| SHORT_UUID=$(uuidgen | cut -d'-' -f1) | |
| UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}" | |
| echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV | |
| echo "Generated RESOURCE_GROUP_NAME: ${UNIQUE_RG_NAME}" | |
| - name: Check and Create Resource Group | |
| id: check_create_rg | |
| run: | | |
| set -e | |
| echo "Checking if resource group exists..." | |
| rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) | |
| if [ "$rg_exists" = "false" ]; then | |
| echo "Resource group does not exist. Creating..." | |
| az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location northcentralus || { echo "Error creating resource group"; exit 1; } | |
| else | |
| echo "Resource group already exists." | |
| fi | |
| echo "RESOURCE_GROUP_NAME=${{ env.RESOURCE_GROUP_NAME }}" >> $GITHUB_OUTPUT | |
| - name: Generate Unique Solution Prefix | |
| id: generate_solution_prefix | |
| run: | | |
| set -e | |
| COMMON_PART="pslcod" | |
| TIMESTAMP=$(date +%s) | |
| UPDATED_TIMESTAMP=$(echo $TIMESTAMP | tail -c 3) | |
| UNIQUE_SOLUTION_PREFIX="${COMMON_PART}${UPDATED_TIMESTAMP}" | |
| echo "SOLUTION_PREFIX=${UNIQUE_SOLUTION_PREFIX}" >> $GITHUB_ENV | |
| echo "Generated SOLUTION_PREFIX: ${UNIQUE_SOLUTION_PREFIX}" | |
| - name: Deploy Bicep Template | |
| id: deploy | |
| run: | | |
| set -e | |
| # set image tag based on branch | |
| if [[ "${{ env.BRANCH_NAME }}" == "main" ]]; then | |
| IMAGE_TAG="latest" | |
| elif [[ "${{ env.BRANCH_NAME }}" == "dev" ]]; then | |
| IMAGE_TAG="dev" | |
| elif [[ "${{ env.BRANCH_NAME }}" == "demo" ]]; then | |
| IMAGE_TAG="demo" | |
| elif [[ "${{ env.BRANCH_NAME }}" == "vee-pipeline-fixes" ]]; then | |
| IMAGE_TAG="vee-pipeline-fixes" | |
| else | |
| IMAGE_TAG="latest" | |
| fi | |
| EFFECTIVE_AI_REGION="${VALID_REGION:-eastus}" | |
| echo "Using AI Deployments Region: $EFFECTIVE_AI_REGION" | |
| echo "selected_ai_region=$EFFECTIVE_AI_REGION" >> $GITHUB_OUTPUT | |
| az deployment group create \ | |
| --name ${{ env.SOLUTION_PREFIX }}-deployment \ | |
| --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ | |
| --template-file infra/main.bicep \ | |
| --parameters \ | |
| solutionName="${{ env.SOLUTION_PREFIX }}" \ | |
| aiDeploymentsLocation="$EFFECTIVE_AI_REGION" \ | |
| useWafAlignedArchitecture=false \ | |
| capacity=${{ env.GPT_MIN_CAPACITY }} \ | |
| imageVersion="${IMAGE_TAG}" \ | |
| createdBy="Pipeline" | |
| - name: Assign Contributor role to Service Principal | |
| if: always() | |
| run: | | |
| echo "Assigning Contributor role to SPN for RG: ${{ env.RESOURCE_GROUP_NAME }}" | |
| az role assignment create \ | |
| --assignee ${{ secrets.AZURE_CLIENT_ID }} \ | |
| --role "Contributor" \ | |
| --scope /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }} | |
| - name: Get Deployment Output and extract Values | |
| id: get_output | |
| run: | | |
| set -e | |
| echo "Fetching deployment output..." | |
| BICEP_OUTPUT=$(az deployment group show --name ${{ env.SOLUTION_PREFIX }}-deployment --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "properties.outputs" -o json) | |
| echo "Extracting deployment output..." | |
| WEBAPP_URL=$(echo $BICEP_OUTPUT | jq -r '.weB_APP_URL.value') | |
| echo "WEBAPP_URL=$WEBAPP_URL" >> $GITHUB_OUTPUT | |
| echo "Deployment output: $BICEP_OUTPUT" | |
| - name: Logout from Azure | |
| if: always() | |
| run: | | |
| az logout | |
| echo "Logged out from Azure." | |
| e2e-test: | |
| needs: deploy | |
| uses: ./.github/workflows/test-automation.yml | |
| with: | |
| CODEMOD_WEB_URL: ${{ needs.deploy.outputs.WEBAPP_URL }} | |
| secrets: inherit | |
| cleanup-deployment: | |
| if: always() && needs.deploy.outputs.RESOURCE_GROUP_NAME != '' | |
| needs: [deploy, e2e-test] | |
| runs-on: ubuntu-latest | |
| env: | |
| RESOURCE_GROUP_NAME: ${{ needs.deploy.outputs.RESOURCE_GROUP_NAME }} | |
| AI_REGION: ${{ needs.deploy.outputs.SELECTED_AI_REGION }} | |
| steps: | |
| - name: Setup Azure CLI | |
| run: | | |
| curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash | |
| az --version # Verify installation | |
| - name: Login to Azure | |
| run: | | |
| az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Get Log Analytics Workspace and OpenAI from Resource Group | |
| if: always() | |
| id: get_azure_resources | |
| run: | | |
| set -e | |
| echo "Collecting resource identifiers from RG ${{ env.RESOURCE_GROUP_NAME }} before deletion..." | |
| log_analytics_workspace_name=$(az monitor log-analytics workspace list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[0].name" -o tsv || true) | |
| if [ -n "$log_analytics_workspace_name" ]; then | |
| echo "LOG_ANALYTICS_WORKSPACE_NAME=${log_analytics_workspace_name}" >> $GITHUB_ENV | |
| fi | |
| openai_resource_name=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --resource-type "Microsoft.CognitiveServices/accounts" --query "[0].name" -o tsv || true) | |
| if [ -n "$openai_resource_name" ]; then | |
| echo "OPENAI_RESOURCE_NAME=${openai_resource_name}" >> $GITHUB_ENV | |
| fi | |
| keyvaults=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[?type=='Microsoft.KeyVault/vaults'].name" -o tsv || true) | |
| if [ -n "$keyvaults" ]; then | |
| keyvault_array="["; first=true; for kv in $keyvaults; do if [ "$first" = true ]; then keyvault_array="$keyvault_array\"$kv\""; first=false; else keyvault_array="$keyvault_array,\"$kv\""; fi; done; keyvault_array="$keyvault_array]"; echo "KEYVAULTS=$keyvault_array" >> $GITHUB_ENV; fi | |
| echo "Snapshot complete." | |
| - name: Delete Resource Group (primary deletion) | |
| if: always() | |
| run: | | |
| set -e | |
| echo "Initiating deletion of RG ${{ env.RESOURCE_GROUP_NAME }}..." | |
| az group delete --name ${{ env.RESOURCE_GROUP_NAME }} --yes --no-wait || echo "Delete command issued." | |
| - name: Wait for RG deletion propagation | |
| if: always() | |
| run: | | |
| set -e | |
| echo "Polling for RG deletion..." | |
| max_retries=15 | |
| sleep_interval=20 | |
| for i in $(seq 1 $max_retries); do | |
| exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }} || echo true) | |
| if [ "$exists" = false ]; then | |
| echo "Resource group deleted (confirmed)." | |
| break | |
| fi | |
| echo "RG still exists (attempt $i). Waiting $sleep_interval s..." | |
| sleep $sleep_interval | |
| done | |
| - name: Purge soft-deleted OpenAI (if any) | |
| if: always() | |
| run: | | |
| set -e | |
| if [ -n "${{ env.OPENAI_RESOURCE_NAME }}" ]; then | |
| echo "Attempting purge of soft-deleted OpenAI: ${{ env.OPENAI_RESOURCE_NAME }} in region ${{ env.AI_REGION }}" | |
| az resource delete --ids /subscriptions/${{ secrets.AZURE_SUBSCRIPTION_ID }}/providers/Microsoft.CognitiveServices/locations/${{ env.AI_REGION || 'eastus' }}/resourceGroups/${{ env.RESOURCE_GROUP_NAME }}/deletedAccounts/${{ env.OPENAI_RESOURCE_NAME }} --verbose || echo "OpenAI purge not available yet or failed (non-fatal)." | |
| else | |
| echo "No OpenAI resource recorded to purge." | |
| fi | |
| - name: Purge soft-deleted KeyVaults (if any) | |
| if: always() | |
| run: | | |
| set -e | |
| KEYVAULTS="${{ env.KEYVAULTS }}" | |
| stripped=$(echo "$KEYVAULTS" | sed 's/\[\|\]//g') | |
| IFS=',' read -r -a kvs <<< "$stripped" | |
| for raw in "${kvs[@]}"; do | |
| kv=$(echo "$raw" | sed 's/\"//g' | xargs) | |
| [ -z "$kv" ] && continue | |
| echo "Checking soft-delete state for KeyVault: $kv" | |
| deleted=$(az keyvault list-deleted --query "[?name=='$kv']" -o json --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }} || echo '[]') | |
| if [ "$(echo "$deleted" | jq length)" -gt 0 ]; then | |
| echo "Purging KeyVault $kv ..." | |
| az keyvault purge --name "$kv" --no-wait || echo "Failed to purge $kv (non-fatal)" | |
| else | |
| echo "KeyVault $kv not soft-deleted; skipping." | |
| fi | |
| done | |
| - name: Purge Log Analytics Workspace (best-effort) | |
| if: always() | |
| run: | | |
| set -e | |
| if [ -n "${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}" ]; then | |
| echo "Attempting to purge Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }} (best-effort)" | |
| az monitor log-analytics workspace delete --force --workspace-name ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }} --resource-group ${{ env.RESOURCE_GROUP_NAME }} --yes --verbose || echo "Workspace purge not applicable or already gone." | |
| else | |
| echo "No Log Analytics workspace recorded." | |
| fi | |
| - name: Send Notification on Failure | |
| if: failure() || needs.deploy.result == 'failure' | |
| run: | | |
| RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| EMAIL_BODY=$(cat <<EOF | |
| { | |
| "body": "<p>Dear Team,</p><p>We would like to inform you that the DocGen Deployment Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> <a href=\"${RUN_URL}\">${RUN_URL}</a><br></p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>" | |
| } | |
| EOF | |
| ) | |
| curl -X POST "${{ secrets.LOGIC_APP_URL }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$EMAIL_BODY" || echo "Failed to send notification" | |
| - name: Final Resource Group Deletion | |
| if: always() | |
| run: | | |
| echo "Final RG delete step retained for idempotency; primary deletion already initiated." | |
| - name: Logout from Azure | |
| if: always() | |
| run: | | |
| az logout | |
| echo "Logged out from Azure." |