Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions scripts/validate_model_deployment_quota.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,18 @@ if ($MissingParams.Count -gt 0) {
exit 1
}

# Check Azure login
try {
$accountCheck = az account show --output none 2>$null
if ($LASTEXITCODE -ne 0) {
Write-Error "❌ ERROR: You are not logged in to Azure CLI. Please run 'az login' to continue."
exit 1
}
} catch {
Write-Error "❌ ERROR: Failed to verify Azure login. Please run 'az login' to continue."
exit 1
}

# Load model deployments from parameter file
$JsonContent = Get-Content -Path "./infra/main.parameters.json" -Raw | ConvertFrom-Json
$aiModelDeployments = $JsonContent.parameters.$ModelsParameter.value
Expand Down
15 changes: 11 additions & 4 deletions scripts/validate_model_deployment_quota.sh
100644 → 100755
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,13 @@ if [[ ${#MISSING_PARAMS[@]} -ne 0 ]]; then
exit 1
fi

# Check if user is logged in to Azure
if ! az account show > /dev/null 2>&1; then
echo "❌ ERROR: You are not logged in to Azure CLI."
echo "👉 Please run 'az login' to continue."
exit 1
fi

# Load model definitions
aiModelDeployments=$(jq -c ".parameters.$MODELS_PARAMETER.value[]" ./infra/main.parameters.json 2>/dev/null)
if [[ $? -ne 0 || -z "$aiModelDeployments" ]]; then
Expand Down Expand Up @@ -100,22 +107,22 @@ while IFS= read -r deployment; do
type=${AZURE_ENV_MODEL_DEPLOYMENT_TYPE:-$(echo "$deployment" | jq -r '.sku.name')}
capacity=${AZURE_ENV_MODEL_CAPACITY:-$(echo "$deployment" | jq -r '.sku.capacity')}

echo ""
echo "🔍 Validating model deployment: $name ..."
echo -e "🔍 Validation started for model deployment: \e[1m$name\e[0m in region \e[1m$LOCATION\e[0m with capacity \e[1m$capacity\e[0m tokens..."
./scripts/validate_model_quota.sh --location "$LOCATION" --model "$model" --capacity "$capacity" --deployment-type "$type"
exit_code=$?

if [[ $exit_code -ne 0 ]]; then
if [[ $exit_code -eq 2 ]]; then
exit 1
fi
echo "❌ ERROR: Quota validation failed for model deployment: $name"
echo -e "\n❌ ERROR: Quota validation failed for model deployment: \033[1m$name\033[0m"
quotaAvailable=false
fi
done <<< "$(echo "$aiModelDeployments")"

if [[ "$quotaAvailable" = false ]]; then
echo "❌ ERROR: One or more model deployments failed quota validation."
echo -e "\n❌ ERROR: Quota validation failed — insufficient quota in all regions. Deployment cannot proceed."
echo -e "\nℹ️ Please request a quota increase at https://portal.azure.com/#blade/Microsoft_Azure_Capacity/UsageAndQuota and try again."
exit 1
else
echo "✅ All model deployments passed quota validation successfully."
Expand Down
199 changes: 102 additions & 97 deletions scripts/validate_model_quota.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -14,34 +14,42 @@ $RecommendedRegions = @()
$NotRecommendedRegions = @()
$EligibleFallbacks = @()

# ------------------ Validate Inputs ------------------
$MissingParams = @()
if (-not $Location) { $MissingParams += "location" }
if (-not $Model) { $MissingParams += "model" }
if (-not $Capacity -or $Capacity -le 0) { $MissingParams += "capacity" }

if ($MissingParams.Count -gt 0) {
Write-Error "❌ ERROR: Missing or invalid parameters: $($MissingParams -join ', ')"
Write-Host "Usage: .\validate_model_quota.ps1 -Location <LOCATION> -Model <MODEL> -Capacity <CAPACITY> [-DeploymentType <DEPLOYMENT_TYPE>]"
exit 1
}
function Validate-Inputs {
$MissingParams = @()
if (-not $Location) { $MissingParams += "location" }
if (-not $Model) { $MissingParams += "model" }
if (-not $Capacity -or $Capacity -le 0) { $MissingParams += "capacity" }

if ($MissingParams.Count -gt 0) {
Write-Error "❌ ERROR: Missing or invalid parameters: $($MissingParams -join ', ')"
Write-Host "Usage: .\validate_model_quota.ps1 -Location <LOCATION> -Model <MODEL> -Capacity <CAPACITY> [-DeploymentType <DEPLOYMENT_TYPE>]"
exit 1
}

if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") {
Write-Error "❌ ERROR: Invalid deployment type: $DeploymentType. Allowed: 'Standard', 'GlobalStandard'"
exit 1
if ($DeploymentType -ne "Standard" -and $DeploymentType -ne "GlobalStandard") {
Write-Error "❌ ERROR: Invalid deployment type: $DeploymentType. Allowed: 'Standard', 'GlobalStandard'"
exit 1
}
}

$ModelType = "OpenAI.$DeploymentType.$Model"
function Confirm-Action ($message) {
do {
$response = Read-Host "$message (y/n)"
if ($response -notmatch "^[YyNn]$") {
Write-Host "❌ Invalid input. Please enter 'y' or 'n'."
}
} while ($response -notmatch "^[YyNn]$")
return $response -match "^[Yy]$"
}

function Check-Quota {
param ([string]$Region)

try {
$ModelType = "OpenAI.$DeploymentType.$Model"
$ModelInfoRaw = az cognitiveservices usage list --location $Region --query "[?name.value=='$ModelType']" --output json 2>$null
$ModelInfo = $ModelInfoRaw | ConvertFrom-Json
if (-not $ModelInfo -or $ModelInfo.Count -eq 0) {
return $null
}
if (-not $ModelInfo -or $ModelInfo.Count -eq 0) { return $null }

$Current = [int]$ModelInfo[0].currentValue
$Limit = [int]$ModelInfo[0].limit
Expand Down Expand Up @@ -99,105 +107,102 @@ function Set-DeploymentValues($Region, $Capacity) {
}
}

# ------------------ Check Primary Region ------------------
Write-Host "`n🔍 Checking quota in the requested region '$Location'..."
$PrimaryResult = Check-Quota -Region $Location
function Manual-Prompt {
while ($true) {
Write-Host "`n📍 Recommended regions (≥ $RECOMMENDED_TOKENS tokens available): $($RecommendedRegions -join ', ')"
$ManualRegion = Read-Host "Please enter a region you want to try manually"
if (-not $ManualRegion) {
Write-Host "❌ ERROR: No region entered. Exiting."
exit 1
}

if ($PrimaryResult) {
$AllResults += $PrimaryResult
if ($PrimaryResult.Available -ge $Capacity) {
if ($RecommendedRegions -notcontains $Location -and $RecommendedRegions.Count -gt 0) {
Write-Host "`n⚠️ Selected region '$Location' has sufficient quota but is not among the recommended regions (≥ $RECOMMENDED_TOKENS tokens)."
Write-Host "🚨 Your application may not work as expected due to limited quota."
Write-Host "`nℹ️ Recommended regions: $($RecommendedRegions -join ', ')"
Write-Host "👉 It's advisable to deploy in one of these regions for optimal app performance."
$choice = Read-Host "❓ Do you want to choose a recommended region instead? (y/n)"
if ($choice -match "^[Yy]$") {
Show-Table
break
} else {
if ($Capacity -gt 200) {
Write-Host "`n⚠️ Reducing capacity to 200 in '$BicepParamsFile' for safer deployment..."
(Get-Content $BicepParamsFile) -replace "capacity\s*:\s*\d+", "capacity: 200" | Set-Content $BicepParamsFile
Write-Host "✅ Updated '$BicepParamsFile' with capacity 200."
}
Set-DeploymentValues $Location $Capacity
Write-Host "✅ Proceeding with '$Location' as selected."
exit 0
$ManualCapacityStr = Read-Host "Enter the capacity you want to use (numeric value)"
if (-not ($ManualCapacityStr -as [int]) -or [int]$ManualCapacityStr -le 0) {
Write-Host "❌ Invalid capacity value. Try again."
continue
}

$ManualCapacity = [int]$ManualCapacityStr

if ($ManualCapacity -lt $RECOMMENDED_TOKENS) {
Write-Host "`n⚠️ You have entered a capacity of $ManualCapacity, which is less than the recommended minimum ($RECOMMENDED_TOKENS)."
Write-Host "🚨 This may cause performance issues or unexpected behavior."
Write-Host "ℹ️ Recommended regions (≥ $RECOMMENDED_TOKENS tokens available): $($RecommendedRegions -join ', ')"
if (-not (Confirm-Action "❓ Proceed anyway?")) { continue }
}

Write-Host "`n🔍 Checking quota in region '$ManualRegion' for requested capacity: $ManualCapacity..."
$ManualResult = Check-Quota -Region $ManualRegion

if (-not $ManualResult) {
Write-Host "⚠️ Could not retrieve quota info for region '$ManualRegion'. Try again."
continue
}

if ($ManualResult.Available -ge $ManualCapacity) {
if ($ManualResult.Available -lt $RECOMMENDED_TOKENS) {
if (-not (Confirm-Action "❓ Proceed anyway?")) { continue }
}
} else {
Write-Host "`n✅ Sufficient quota found in original region '$Location'."
Set-DeploymentValues $Location $Capacity
Set-DeploymentValues $ManualRegion $ManualCapacity
Write-Host "✅ Deployment values set. Exiting."
exit 0
} else {
Write-Host "❌ Quota in region '$ManualRegion' is insufficient. Available: $($ManualResult.Available), Required: $ManualCapacity"
}
} else {
Write-Host "`n⚠️ Insufficient quota in '$Location'. Checking fallback regions..."
}
} else {
Write-Host "`n⚠️ Could not retrieve quota info for region '$Location'."
}

# ------------------ Check Fallback Regions ------------------
# Start validation and quota checks
Validate-Inputs

Write-Host "`n🔍 Checking quota in the requested region '$Location'..."
$PrimaryResult = Check-Quota -Region $Location
if ($PrimaryResult) {
$AllResults = @($AllResults) + $PrimaryResult
}

foreach ($region in $PreferredRegions) {
if ($region -eq $Location) { continue }
$result = Check-Quota -Region $region
if ($result) {
$AllResults += $result
if ($result.Available -ge $Capacity) {
$EligibleFallbacks += $region
if ($region -ne $Location) {
$fallbackResult = Check-Quota -Region $region
if ($fallbackResult) {
$AllResults = @($AllResults) + $fallbackResult
if ($fallbackResult.Available -ge $Capacity) {
$EligibleFallbacks += $region
}
}
}
}

Show-Table

if ($EligibleFallbacks.Count -gt 0) {
Write-Host "`n➡️ Found fallback regions with sufficient quota."
if ($Capacity -lt $RECOMMENDED_TOKENS) {
Write-Host "`n⚠️ You have entered a capacity of $Capacity, which is less than the recommended minimum ($RECOMMENDED_TOKENS)."
Write-Host "🚨 This may cause performance issues or unexpected behavior."
Write-Host "ℹ️ Recommended regions (≥ $RECOMMENDED_TOKENS tokens available):"
if ($RecommendedRegions.Count -gt 0) {
Write-Host "`nℹ️ Recommended regions (≥ $RECOMMENDED_TOKENS tokens available):"
foreach ($region in $RecommendedRegions) {
Write-Host " - $region"
}
} else {
Write-Host " ❌ No recommended regions currently available."
}
}

# ------------------ Manual Prompt if No Quota Found ------------------
Write-Host "`n❌ ERROR: No region has sufficient quota."

while ($true) {
$ManualRegion = Read-Host "`nPlease enter a region you want to try manually"
if (-not $ManualRegion) {
Write-Host "❌ ERROR: No region entered. Exiting."
exit 1
}

$ManualCapacityStr = Read-Host "Enter the capacity you want to use (numeric value)"
if (-not ($ManualCapacityStr -as [int]) -or [int]$ManualCapacityStr -le 0) {
Write-Host "❌ Invalid capacity value. Try again."
continue
}

$ManualCapacity = [int]$ManualCapacityStr
$ManualResult = Check-Quota -Region $ManualRegion

if (-not $ManualResult) {
Write-Host "⚠️ Could not retrieve quota info for region '$ManualRegion'. Try again."
continue
if (-not (Confirm-Action "❓ Proceed anyway?")) {
Manual-Prompt
exit 0
}
}

if ($ManualResult.Available -ge $ManualCapacity) {
if ($ManualResult.Available -lt $RECOMMENDED_TOKENS) {
Write-Host "`n⚠️ Region '$ManualRegion' has less than recommended ($RECOMMENDED_TOKENS) tokens."
$proceed = Read-Host "❓ Proceed anyway? (y/n)"
if ($proceed -notmatch "^[Yy]$") {
continue
}
}
if ($PrimaryResult -and $PrimaryResult.Available -ge $Capacity) {
Set-DeploymentValues $Location $Capacity
Write-Host "✅ Proceeding with '$Location' as selected."
exit 0
}

Set-DeploymentValues $ManualRegion $ManualCapacity
Write-Host "✅ Deployment values set. Exiting."
exit 0
} else {
Write-Host "❌ Quota in region '$ManualRegion' is insufficient. Available: $($ManualResult.Available), Required: $ManualCapacity"
}
Write-Host "`n❗ The originally selected region '$Location' does not have enough quota."
if ($EligibleFallbacks.Count -gt 0) {
Write-Host "👉 You can manually choose one of the recommended fallback regions for deployment."
} else {
Write-Host "❌ ERROR: No region has sufficient quota."
}

Manual-Prompt
Loading