Deployment Lifecycle Automation #163
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deployment Lifecycle Automation | |
| on: | |
| push: | |
| branches: | |
| - main | |
| schedule: | |
| - cron: "0 9,21 * * *" # Runs at 9:00 AM and 9:00 PM GMT | |
| workflow_dispatch: | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Code | |
| uses: actions/checkout@v3 | |
| - name: Setup Azure CLI | |
| run: | | |
| curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash | |
| az --version # Verify installation | |
| - name: Login to Azure | |
| run: | | |
| az login --service-principal -u ${{ secrets.AZURE_MAINTENANCE_CLIENT_ID }} -p ${{ secrets.AZURE_MAINTENANCE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Run Quota Check | |
| id: quota-check | |
| run: | | |
| export AZURE_MAINTENANCE_CLIENT_ID=${{ secrets.AZURE_MAINTENANCE_CLIENT_ID }} | |
| export AZURE_TENANT_ID=${{ secrets.AZURE_TENANT_ID }} | |
| export AZURE_MAINTENANCE_CLIENT_SECRET=${{ secrets.AZURE_MAINTENANCE_CLIENT_SECRET }} | |
| export AZURE_MAINTENANCE_SUBSCRIPTION_ID="${{ secrets.AZURE_MAINTENANCE_SUBSCRIPTION_ID }}" | |
| export GPT_MIN_CAPACITY="100" | |
| export AZURE_REGIONS="${{ vars.AZURE_REGIONS }}" | |
| chmod +x infra/scripts/checkquota.sh | |
| if ! infra/scripts/checkquota.sh; then | |
| # If quota check fails due to insufficient quota, set the flag | |
| if grep -q "No region with sufficient quota found" infra/scripts/checkquota.sh; then | |
| echo "QUOTA_FAILED=true" >> $GITHUB_ENV | |
| fi | |
| exit 1 # Fail the pipeline if any other failure occurs | |
| fi | |
| - name: Send Notification on Quota Failure | |
| if: env.QUOTA_FAILED == 'true' | |
| run: | | |
| RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| EMAIL_BODY=$(cat <<EOF | |
| { | |
| "body": "<p>Dear Team,</p><p>The quota check has failed, and the pipeline cannot proceed.</p><p><strong>Build URL:</strong> ${RUN_URL}</p><p>Please take necessary action.</p><p>Best regards,<br>Your Automation Team</p>" | |
| } | |
| EOF | |
| ) | |
| curl -X POST "${{ secrets.LOGIC_APP_URL }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$EMAIL_BODY" || echo "Failed to send notification" | |
| - name: Fail Pipeline if Quota Check Fails | |
| if: env.QUOTA_FAILED == 'true' | |
| run: exit 1 | |
| - name: Install Bicep CLI | |
| run: az bicep install | |
| - name: Set Deployment Region | |
| run: | | |
| echo "Selected Region: $VALID_REGION" | |
| echo "AZURE_LOCATION=$VALID_REGION" >> $GITHUB_ENV | |
| - name: Generate Resource Group Name | |
| id: generate_rg_name | |
| run: | | |
| echo "Generating a unique resource group name..." | |
| ACCL_NAME="cpc" # Account name as specified | |
| SHORT_UUID=$(uuidgen | cut -d'-' -f1) | |
| UNIQUE_RG_NAME="arg-${ACCL_NAME}-${SHORT_UUID}" | |
| echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV | |
| echo "Generated Resource_GROUP_PREFIX: ${UNIQUE_RG_NAME}" | |
| - name: Check and Create Resource Group | |
| id: check_create_rg | |
| run: | | |
| set -e | |
| echo "Checking if resource group exists..." | |
| rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) | |
| if [ "$rg_exists" = "false" ]; then | |
| echo "Resource group does not exist. Creating..." | |
| # Generate current timestamp in desired format: YYYY-MM-DDTHH:MM:SS.SSSSSSSZ | |
| current_date=$(date -u +"%Y-%m-%dT%H:%M:%S.%7NZ") | |
| az group create --name ${{ env.RESOURCE_GROUP_NAME }} \ | |
| --location ${{ env.AZURE_LOCATION }} \ | |
| --tags "CreatedBy=Deployment Lifecycle Automation Pipeline" \ | |
| "Purpose=Deploying and Cleaning Up Resources for Validation" \ | |
| "CreatedDate=$current_date" \ | |
| "ApplicationName=Content Processing Accelerator" \ | |
| || { echo "Error creating resource group"; exit 1; } | |
| else | |
| echo "Resource group already exists." | |
| fi | |
| - name: Generate Environment Name | |
| id: generate_environment_name | |
| run: | | |
| set -e | |
| TIMESTAMP_SHORT=$(date +%s | tail -c 5) # Last 4-5 digits of epoch seconds | |
| RANDOM_SUFFIX=$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 8) # 8 random alphanum chars | |
| UNIQUE_ENV_NAME="${TIMESTAMP_SHORT}${RANDOM_SUFFIX}" # Usually ~12-13 chars | |
| echo "ENVIRONMENT_NAME=${UNIQUE_ENV_NAME}" >> $GITHUB_ENV | |
| echo "Generated ENVIRONMENT_NAME: ${UNIQUE_ENV_NAME}" | |
| - name: Deploy Bicep Template | |
| id: deploy | |
| run: | | |
| set -e | |
| az deployment group create \ | |
| --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ | |
| --template-file infra/main.json \ | |
| --parameters \ | |
| environmentName="${{ env.ENVIRONMENT_NAME }}" \ | |
| secondaryLocation="EastUs2" \ | |
| contentUnderstandingLocation="WestUS" \ | |
| deploymentType="GlobalStandard" \ | |
| gptModelName="gpt-4o" \ | |
| gptModelVersion="2024-08-06" \ | |
| gptDeploymentCapacity="30" \ | |
| minReplicaContainerApp="1" \ | |
| maxReplicaContainerApp="1" \ | |
| minReplicaContainerApi="1" \ | |
| maxReplicaContainerApi="1" \ | |
| minReplicaContainerWeb="1" \ | |
| maxReplicaContainerWeb="1" \ | |
| useLocalBuild="false" | |
| - name: Delete Bicep Deployment | |
| if: always() # This ensures that resource group deletion happens regardless of success or failure | |
| run: | | |
| set -e | |
| echo "Checking if resource group exists..." | |
| rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) | |
| if [ "$rg_exists" = "true" ]; then | |
| echo "Resource group exists. Cleaning..." | |
| az group delete \ | |
| --name ${{ env.RESOURCE_GROUP_NAME }} \ | |
| --yes \ | |
| --no-wait | |
| echo "Resource group deleted... ${{ env.RESOURCE_GROUP_NAME }}" | |
| else | |
| echo "Resource group does not exist." | |
| fi | |
| - name: Wait for Resource Deletion to Complete | |
| if: always() | |
| run: | | |
| echo "Fetching resources in the resource group: ${{ env.RESOURCE_GROUP_NAME }}" | |
| # Ensure correct subscription is set | |
| az account set --subscription "${{ secrets.AZURE_MAINTENANCE_SUBSCRIPTION_ID }}" | |
| # Fetch all resource IDs dynamically (instead of names) | |
| resources_to_check=($(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[].id" -o tsv)) | |
| # Exit early if no resources found | |
| if [ ${#resources_to_check[@]} -eq 0 ]; then | |
| echo "No resources found in the resource group. Skipping deletion check." | |
| exit 0 | |
| fi | |
| echo "Resources to check: ${resources_to_check[@]}" | |
| # Extract only resource names and store them in a space-separated string | |
| resources_to_purge="" | |
| for resource_id in "${resources_to_check[@]}"; do | |
| resource_name=$(basename "$resource_id") # Extract the last part of the ID as the name | |
| resources_to_purge+="$resource_name " | |
| done | |
| # Save the list for later use | |
| echo "RESOURCES_TO_PURGE=$resources_to_purge" >> "$GITHUB_ENV" | |
| echo "Waiting for resources to be fully deleted..." | |
| # Maximum retries & retry intervals | |
| max_retries=10 | |
| retry_intervals=(150 180 210 240 270 300) # increased intervals for each retry for potentially long deletion times | |
| retries=0 | |
| while true; do | |
| all_deleted=true | |
| for resource_id in "${resources_to_check[@]}"; do | |
| echo "Checking if resource '$resource_id' is deleted..." | |
| # Check resource existence using full ID | |
| resource_status=$(az resource show --ids "$resource_id" --query "id" -o tsv 2>/dev/null || echo "NotFound") | |
| if [[ "$resource_status" != "NotFound" ]]; then | |
| echo "Resource '$resource_id' is still present." | |
| all_deleted=false | |
| else | |
| echo "Resource '$resource_id' is fully deleted." | |
| fi | |
| done | |
| # Break loop if all resources are deleted | |
| if [ "$all_deleted" = true ]; then | |
| echo "All resources are fully deleted. Proceeding with purging..." | |
| break | |
| fi | |
| # Stop retrying if max retries are reached | |
| if [ $retries -ge $max_retries ]; then | |
| echo "Some resources were not deleted after $max_retries retries. Failing the pipeline." | |
| exit 1 | |
| fi | |
| echo "Some resources are still present. Retrying in ${retry_intervals[$retries]} seconds..." | |
| sleep ${retry_intervals[$retries]} | |
| retries=$((retries + 1)) | |
| done | |
| - name: Purging the Resources | |
| if: always() | |
| run: | | |
| set -e | |
| echo "Using saved list of deleted resources from previous step..." | |
| # Ensure the correct subscription is set | |
| az account set --subscription "${{ secrets.AZURE_MAINTENANCE_SUBSCRIPTION_ID }}" | |
| # Iterate over each deleted resource | |
| for resource_name in $RESOURCES_TO_PURGE; do | |
| echo "Checking for deleted resource: $resource_name" | |
| # Query Azure for deleted resources based on type | |
| case "$resource_name" in | |
| *"kv-cps"*) | |
| deleted_resource=$(az keyvault list-deleted --query "[?name=='$resource_name'].{name:name, type:type, id:id}" -o json) | |
| ;; | |
| *"stcps"*) | |
| deleted_resource=$(az storage account list --query "[?name=='$resource_name']" -o json || echo "{}") | |
| ;; | |
| *"cosmos-cps"*) | |
| deleted_resource=$(az cosmosdb show --name "$resource_name" --query "{name:name, type:type, id:id}" -o json 2>/dev/null || echo "{}") | |
| ;; | |
| *"aisa-cps"*) | |
| deleted_resource=$(az cognitiveservices account list-deleted --query "[?name=='$resource_name'].{name:name, type:type, id:id}" -o json) | |
| ;; | |
| *"appcs-cps"*) | |
| deleted_resource=$(az resource list --query "[?starts_with(name, 'appcs') && type=='Microsoft.Insights/components'].{name:name, type:type, id:id}" -o json) | |
| ;; | |
| *"appi-cps"*) | |
| deleted_resource=$(az resource list --query "[?starts_with(name, 'appi') && type=='Microsoft.Insights/components'].{name:name, type:type, id:id}" -o json) | |
| ;; | |
| *"ca-cps"*) | |
| deleted_resource=$(az resource list --query "[?starts_with(name, 'ca') && type=='Microsoft.Web/containerApps'].{name:name, type:type, id:id}" -o json) | |
| ;; | |
| *) | |
| deleted_resource=$(az resource list --query "[?name=='$resource_name'].{name:name, type:type, id:id}" -o json) | |
| ;; | |
| esac | |
| if [[ -z "$deleted_resource" || "$deleted_resource" == "[]" || "$deleted_resource" == "{}" ]]; then | |
| echo "Resource $resource_name not found in deleted list. Skipping..." | |
| continue | |
| fi | |
| # Extract name, type, and ID from the JSON response | |
| name=$(echo "$deleted_resource" | jq -r '.[0].name') | |
| type=$(echo "$deleted_resource" | jq -r '.[0].type') | |
| id=$(echo "$deleted_resource" | jq -r '.[0].id') | |
| echo "Purging resource: $name (Type: $type)" | |
| case "$type" in | |
| "Microsoft.KeyVault/deletedVaults") | |
| echo "Purging Key Vault: $name" | |
| purge_output=$(az keyvault purge --name "$name" 2>&1 || true) | |
| if echo "$purge_output" | grep -q "MethodNotAllowed"; then | |
| echo "WARNING: Soft Delete Protection is enabled for $name. Purge is not allowed. Skipping..." | |
| else | |
| echo "Key Vault $name purged successfully." | |
| fi | |
| ;; | |
| "Microsoft.ContainerRegistry/registries") | |
| echo "Deleting Azure Container Registry (ACR): $name" | |
| az acr delete --name "$name" --yes || echo "Failed to delete Azure Container Registry: $name" | |
| ;; | |
| "Microsoft.Storage/storageAccounts") | |
| echo "Purging Storage Account: $name" | |
| az storage account delete --name "$name" --yes || echo "Failed to delete Storage Account: $name" | |
| ;; | |
| "Microsoft.DocumentDB/databaseAccounts") | |
| echo "Purging Cosmos DB: $name" | |
| az cosmosdb delete --name "$name" --yes || echo "Failed to delete Cosmos DB Account: $name" | |
| ;; | |
| "Microsoft.CognitiveServices/deletedAccounts") | |
| echo "Purging Cognitive Services Account: $name" | |
| az cognitiveservices account purge --location "${{ env.AZURE_LOCATION }}" --resource-group "${{ env.RESOURCE_GROUP_NAME }}" --name "$name" || echo "Failed to purge Cognitive Services Account: $name" | |
| ;; | |
| "Microsoft.AppConfiguration/configurationStores") | |
| echo "Deleting App Configuration: $name" | |
| az appconfig delete --name "$name" --yes || echo "Failed to delete App Configuration: $name" | |
| ;; | |
| "Microsoft.Insights/components") | |
| echo "Deleting Application Insights: $name" | |
| az monitor app-insights component delete --ids "$id" || echo "Failed to delete Application Insights: $name" | |
| ;; | |
| "Microsoft.Web/containerApps") | |
| echo "Deleting Container App: $name" | |
| az containerapp delete --name "$name" --yes || echo "Failed to delete Container App: $name" | |
| ;; | |
| *) | |
| echo "Purging General Resource: $name" | |
| if [[ -n "$id" && "$id" != "null" ]]; then | |
| az resource delete --ids "$id" --verbose || echo "Failed to delete $name" | |
| else | |
| echo "Resource ID not found for $name. Skipping purge." | |
| fi | |
| ;; | |
| esac | |
| done | |
| echo "Resource purging completed successfully" | |
| - name: Send Notification on Failure | |
| if: failure() | |
| run: | | |
| RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| EMAIL_BODY=$(cat <<EOF | |
| { | |
| "body": "<p>Dear Team,</p><p>We would like to inform you that the Content Processing Automation process has encountered an issue and has failed to complete successfully.</p><p><strong>Build URL:</strong> ${RUN_URL}<br> ${OUTPUT}</p><p>Please investigate the matter at your earliest convenience.</p><p>Best regards,<br>Your Automation Team</p>" | |
| } | |
| EOF | |
| ) | |
| curl -X POST "${{ secrets.LOGIC_APP_URL }}" \ | |
| -H "Content-Type: application/json" \ | |
| -d "$EMAIL_BODY" || echo "Failed to send notification" |