Skip to content

Commit 35149fa

Browse files
Priyanka-MicrosoftPriyanka-Microsoft
committed
checking for user region only at first place then fallback region
1 parent efeb617 commit 35149fa

4 files changed

Lines changed: 185 additions & 103 deletions

File tree

‎scripts/validate_model_deployment_quota.ps1‎

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,19 @@ param (
44
[string]$ModelsParameter
55
)
66

7-
# Verify all required parameters are provided
7+
# Read from environment variables (do not pass in azure.yaml)
8+
$AiServiceName = $env:AZURE_AISERVICE_NAME
9+
$ResourceGroup = $env:AZURE_RESOURCE_GROUP
10+
11+
# Validate required parameters
812
$MissingParams = @()
913

1014
if (-not $SubscriptionId) {
1115
$MissingParams += "subscription"
1216
}
13-
1417
if (-not $Location) {
1518
$MissingParams += "location"
1619
}
17-
1820
if (-not $ModelsParameter) {
1921
$MissingParams += "models-parameter"
2022
}
@@ -25,20 +27,56 @@ if ($MissingParams.Count -gt 0) {
2527
exit 1
2628
}
2729

30+
# Load main.parameters.json
2831
$JsonContent = Get-Content -Path "./infra/main.parameters.json" -Raw | ConvertFrom-Json
29-
3032
if (-not $JsonContent) {
3133
Write-Error "❌ ERROR: Failed to parse main.parameters.json. Ensure the JSON file is valid."
3234
exit 1
3335
}
3436

3537
$aiModelDeployments = $JsonContent.parameters.$ModelsParameter.value
36-
3738
if (-not $aiModelDeployments -or -not ($aiModelDeployments -is [System.Collections.IEnumerable])) {
38-
Write-Error "❌ ERROR: The specified property $ModelsParameter does not exist or is not an array."
39+
Write-Error "❌ ERROR: The specified property '$ModelsParameter' does not exist or is not an array."
3940
exit 1
4041
}
4142

43+
# Check if AI resource + all deployments already exist
44+
if ($AiServiceName -and $ResourceGroup) {
45+
$existing = az cognitiveservices account show `
46+
--name $AiServiceName `
47+
--resource-group $ResourceGroup `
48+
--query "name" --output tsv 2>$null
49+
50+
if ($existing) {
51+
$deployedModels = az cognitiveservices account deployment list `
52+
--name $AiServiceName `
53+
--resource-group $ResourceGroup `
54+
--query "[].name" --output tsv 2>$null
55+
56+
$requiredDeployments = @()
57+
foreach ($deployment in $aiModelDeployments) {
58+
$requiredDeployments += $deployment.name
59+
}
60+
61+
$missingDeployments = @()
62+
foreach ($required in $requiredDeployments) {
63+
if ($deployedModels -notcontains $required) {
64+
$missingDeployments += $required
65+
}
66+
}
67+
68+
if ($missingDeployments.Count -eq 0) {
69+
Write-Host "ℹ️ Azure AI service '$AiServiceName' exists and all required model deployments are provisioned."
70+
Write-Host "⏭️ Skipping quota validation."
71+
exit 0
72+
} else {
73+
Write-Host "🔍 AI service exists, but the following model deployments are missing: $($missingDeployments -join ', ')"
74+
Write-Host "➡️ Proceeding with quota validation for missing models..."
75+
}
76+
}
77+
}
78+
79+
# Start quota validation
4280
az account set --subscription $SubscriptionId
4381
Write-Host "🎯 Active Subscription: $(az account show --query '[name, id]' --output tsv)"
4482

@@ -56,8 +94,7 @@ foreach ($deployment in $aiModelDeployments) {
5694

5795
if ($exitCode -ne 0) {
5896
if ($exitCode -eq 2) {
59-
# Quota error already printed inside the script, exit gracefully without reprinting
60-
exit 1
97+
exit 1 # already printed, graceful
6198
}
6299
Write-Error "❌ ERROR: Quota validation failed for model deployment: $name"
63100
$QuotaAvailable = $false

‎scripts/validate_model_deployment_quota.sh‎

Lines changed: 63 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -19,37 +19,66 @@ while [[ $# -gt 0 ]]; do
1919
shift 2
2020
;;
2121
*)
22-
echo "Unknown option: $1"
22+
echo "❌ ERROR: Unknown option: $1"
2323
exit 1
2424
;;
2525
esac
2626
done
2727

28-
# Verify all required parameters are provided and echo missing ones
28+
# Validate required parameters
2929
MISSING_PARAMS=()
30+
[[ -z "$SUBSCRIPTION_ID" ]] && MISSING_PARAMS+=("subscription")
31+
[[ -z "$LOCATION" ]] && MISSING_PARAMS+=("location")
32+
[[ -z "$MODELS_PARAMETER" ]] && MISSING_PARAMS+=("models-parameter")
3033

31-
if [[ -z "$SUBSCRIPTION_ID" ]]; then
32-
MISSING_PARAMS+=("subscription")
34+
if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then
35+
echo "❌ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}"
36+
echo "Usage: $0 --subscription <SUBSCRIPTION_ID> --location <LOCATION> --models-parameter <MODELS_PARAMETER>"
37+
exit 1
3338
fi
3439

35-
if [[ -z "$LOCATION" ]]; then
36-
MISSING_PARAMS+=("location")
37-
fi
40+
# Read from environment
41+
AISERVICE_NAME="${AZURE_AISERVICE_NAME}"
42+
RESOURCE_GROUP="${AZURE_RESOURCE_GROUP}"
3843

39-
if [[ -z "$MODELS_PARAMETER" ]]; then
40-
MISSING_PARAMS+=("models-parameter")
41-
fi
44+
# Check service and deployment existence
45+
if [[ -n "$AISERVICE_NAME" && -n "$RESOURCE_GROUP" ]]; then
46+
existing=$(az cognitiveservices account show --name "$AISERVICE_NAME" --resource-group "$RESOURCE_GROUP" --query "name" --output tsv 2>/dev/null)
47+
if [[ -n "$existing" ]]; then
48+
echo "ℹ️ Found Azure AI service: $AISERVICE_NAME"
4249

43-
if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then
44-
echo "❌ ERROR: Missing required parameters: ${MISSING_PARAMS[*]}"
45-
echo "Usage: $0 --subscription <SUBSCRIPTION_ID> --location <LOCATION> --models-parameter <MODELS_PARAMETER>"
46-
exit 1
50+
existing_deployments=$(az cognitiveservices account deployment list --name "$AISERVICE_NAME" --resource-group "$RESOURCE_GROUP" --query "[].name" --output tsv 2>/dev/null)
51+
52+
# Extract required model names
53+
required_models=$(jq -r ".parameters.$MODELS_PARAMETER.value[].name" ./infra/main.parameters.json 2>/dev/null)
54+
55+
if [[ -z "$required_models" ]]; then
56+
echo "❌ ERROR: Failed to extract required model names from main.parameters.json"
57+
exit 1
58+
fi
59+
60+
all_present=true
61+
for model in $required_models; do
62+
if ! grep -q -w "$model" <<< "$existing_deployments"; then
63+
all_present=false
64+
break
65+
fi
66+
done
67+
68+
if [[ "$all_present" == "true" ]]; then
69+
echo "✅ All required model deployments already exist in AI service '$AISERVICE_NAME'."
70+
echo "⏭️ Skipping quota validation."
71+
exit 0
72+
else
73+
echo "🔍 AI service exists but some model deployments are missing — proceeding with quota validation."
74+
fi
75+
fi
4776
fi
4877

78+
# If we reach here, continue with normal quota checks
4979
aiModelDeployments=$(jq -c ".parameters.$MODELS_PARAMETER.value[]" ./infra/main.parameters.json)
50-
51-
if [ $? -ne 0 ]; then
52-
echo "Error: Failed to parse main.parameters.json. Ensure jq is installed and the JSON file is valid."
80+
if [[ $? -ne 0 ]]; then
81+
echo "❌ ERROR: Failed to parse main.parameters.json. Ensure jq is installed and the JSON is valid."
5382
exit 1
5483
fi
5584

@@ -59,30 +88,29 @@ echo "🎯 Active Subscription: $(az account show --query '[name, id]' --output
5988
quotaAvailable=true
6089

6190
while IFS= read -r deployment; do
62-
name=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.name')}
63-
model=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.model.name')}
64-
type=${AZURE_ENV_MODEL_DEPLOYMENT_TYPE:-$(echo "$deployment" | jq -r '.sku.name')}
65-
capacity=${AZURE_ENV_MODEL_CAPACITY:-$(echo "$deployment" | jq -r '.sku.capacity')}
91+
name=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.name')}
92+
model=${AZURE_ENV_MODEL_NAME:-$(echo "$deployment" | jq -r '.model.name')}
93+
type=${AZURE_ENV_MODEL_DEPLOYMENT_TYPE:-$(echo "$deployment" | jq -r '.sku.name')}
94+
capacity=${AZURE_ENV_MODEL_CAPACITY:-$(echo "$deployment" | jq -r '.sku.capacity')}
6695

6796
echo "🔍 Validating model deployment: $name ..."
68-
./scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity $capacity --deployment-type $type
97+
./scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity "$capacity" --deployment-type "$type"
6998

70-
# Check if the script failed
7199
exit_code=$?
72-
if [ $exit_code -ne 0 ]; then
73-
if [ $exit_code -eq 2 ]; then
74-
# Skip printing any quota validation error — already handled inside the validation script
75-
exit 1
76-
fi
77-
echo "❌ ERROR: Quota validation failed for model deployment: $name"
78-
quotaAvailable=false
100+
if [[ $exit_code -ne 0 ]]; then
101+
if [[ $exit_code -eq 2 ]]; then
102+
# Quota validation handled inside script — stop immediately
103+
exit 1
104+
fi
105+
echo "❌ ERROR: Quota validation failed for model deployment: $name"
106+
quotaAvailable=false
79107
fi
80108
done <<< "$(echo "$aiModelDeployments")"
81109

82-
if [ "$quotaAvailable" = false ]; then
83-
echo "❌ ERROR: One or more model deployments failed validation."
84-
exit 1
110+
if [[ "$quotaAvailable" = false ]]; then
111+
echo "❌ ERROR: One or more model deployments failed validation."
112+
exit 1
85113
else
86-
echo "✅ All model deployments passed quota validation successfully."
87-
exit 0
114+
echo "✅ All model deployments passed quota validation successfully."
115+
exit 0
88116
fi

‎scripts/validate_model_quota.ps1‎

Lines changed: 47 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ param (
55
[int]$Capacity
66
)
77

8-
# Verify required parameters
8+
# Validate parameters
99
$MissingParams = @()
1010
if (-not $Location) { $MissingParams += "location" }
1111
if (-not $Model) { $MissingParams += "model" }
@@ -24,7 +24,6 @@ if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") {
2424
}
2525

2626
$ModelType = "OpenAI.$DeploymentType.$Model"
27-
2827
$PreferredRegions = @('australiaeast', 'eastus', 'eastus2', 'francecentral', 'japaneast', 'norwayeast', 'southindia', 'swedencentral', 'uksouth', 'westus', 'westus3')
2928
$AllResults = @()
3029

@@ -33,66 +32,76 @@ function Check-Quota {
3332
[string]$Region
3433
)
3534

36-
$ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json
37-
$ModelInfo = $null
38-
3935
try {
36+
$ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json
4037
$ModelInfo = $ModelInfoRaw | ConvertFrom-Json
38+
if (-not $ModelInfo) { return }
39+
40+
$CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue
41+
$Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit
42+
43+
$CurrentValue = [int]($CurrentValue -replace '\.0+$', '')
44+
$Limit = [int]($Limit -replace '\.0+$', '')
45+
$Available = $Limit - $CurrentValue
46+
47+
return [PSCustomObject]@{
48+
Region = $Region
49+
Model = $ModelType
50+
Limit = $Limit
51+
Used = $CurrentValue
52+
Available = $Available
53+
}
4154
} catch {
4255
return
4356
}
57+
}
4458

45-
if (-not $ModelInfo) {
46-
return
59+
# First, check the user-specified region
60+
Write-Host "`n🔍 Checking quota in the requested region '$Location'..."
61+
$PrimaryResult = Check-Quota -Region $Location
62+
63+
if ($PrimaryResult) {
64+
$AllResults += $PrimaryResult
65+
if ($PrimaryResult.Available -ge $Capacity) {
66+
Write-Host "`n✅ Sufficient quota found in original region '$Location'."
67+
exit 0
68+
} else {
69+
Write-Host "`n⚠️ Insufficient quota in '$Location' (Available: $($PrimaryResult.Available), Required: $Capacity). Checking fallback regions..."
4770
}
71+
} else {
72+
Write-Host "`n⚠️ Could not retrieve quota info for region '$Location'. Checking fallback regions..."
73+
}
4874

49-
$CurrentValue = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).currentValue
50-
$Limit = ($ModelInfo | Where-Object { $_.name.value -eq $ModelType }).limit
51-
52-
$CurrentValue = [int]($CurrentValue -replace '\.0+$', '')
53-
$Limit = [int]($Limit -replace '\.0+$', '')
54-
$Available = $Limit - $CurrentValue
75+
# Remove primary region from fallback list
76+
$FallbackRegions = $PreferredRegions | Where-Object { $_ -ne $Location }
5577

56-
$script:AllResults += [PSCustomObject]@{
57-
Region = $Region
58-
Model = $ModelType
59-
Limit = $Limit
60-
Used = $CurrentValue
61-
Available = $Available
78+
foreach ($region in $FallbackRegions) {
79+
$result = Check-Quota -Region $region
80+
if ($result) {
81+
$AllResults += $result
6282
}
6383
}
6484

65-
foreach ($region in $PreferredRegions) {
66-
Check-Quota -Region $region
67-
}
68-
6985
# Display Results Table
70-
Write-Host "\n-------------------------------------------------------------------------------------------------------------"
86+
Write-Host "`n-------------------------------------------------------------------------------------------------------------"
7187
Write-Host "| No. | Region | Model Name | Limit | Used | Available |"
7288
Write-Host "-------------------------------------------------------------------------------------------------------------"
7389

7490
$count = 1
7591
foreach ($entry in $AllResults) {
76-
$index = $PreferredRegions.IndexOf($entry.Region) + 1
7792
$modelShort = $entry.Model.Substring($entry.Model.LastIndexOf(".") + 1)
78-
Write-Host ("| {0,-4} | {1,-16} | {2,-35} | {3,-7} | {4,-7} | {5,-9} |" -f $index, $entry.Region, $entry.Model, $entry.Limit, $entry.Used, $entry.Available)
93+
Write-Host ("| {0,-4} | {1,-16} | {2,-35} | {3,-7} | {4,-7} | {5,-9} |" -f $count, $entry.Region, $entry.Model, $entry.Limit, $entry.Used, $entry.Available)
7994
$count++
8095
}
8196
Write-Host "-------------------------------------------------------------------------------------------------------------"
8297

83-
$EligibleRegion = $AllResults | Where-Object { $_.Region -eq $Location -and $_.Available -ge $Capacity }
84-
if ($EligibleRegion) {
85-
Write-Host "\n✅ Sufficient quota found in original region '$Location'."
86-
exit 0
87-
}
98+
# Suggest fallback regions
99+
$EligibleFallbacks = $AllResults | Where-Object { $_.Region -ne $Location -and $_.Available -ge $Capacity }
88100

89-
$FallbackRegions = $AllResults | Where-Object { $_.Region -ne $Location -and $_.Available -ge $Capacity }
90-
91-
if ($FallbackRegions.Count -gt 0) {
92-
Write-Host "`n❌ Deployment cannot proceed because the original region '$Location' lacks sufficient quota."
101+
if ($EligibleFallbacks.Count -gt 0) {
102+
Write-Host "`n❌ Deployment cannot proceed in '$Location'."
93103
Write-Host "➡️ You can retry using one of the following regions with sufficient quota:`n"
94-
95-
foreach ($region in $FallbackRegions) {
104+
foreach ($region in $EligibleFallbacks) {
96105
Write-Host " • $($region.Region) (Available: $($region.Available))"
97106
}
98107

@@ -104,5 +113,5 @@ if ($FallbackRegions.Count -gt 0) {
104113
exit 2
105114
}
106115

107-
Write-Error "❌ ERROR: No available quota found in any region."
116+
Write-Error "`n❌ ERROR: No available quota found in any region."
108117
exit 1

0 commit comments

Comments
 (0)