Merge remote-tracking branch 'origin/dev' into psl-bicep-param

Author: Harmanpreet-Microsoft

.github/workflows/azd-template-validation.yml

@@ -0,0 +1,46 @@
+name: AZD Template Validation
+
+on:
+  schedule:
+    - cron: '30 1 * * 4' # Every Thursday at 7:00 AM IST (1:30 AM UTC)
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+  pull-requests: write
+
+jobs:
+  template_validation:
+    runs-on: ubuntu-latest
+    name: azd template validation
+    environment: production
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set timestamp
+        run: echo "HHMM=$(date -u +'%H%M')" >> $GITHUB_ENV
+
+      - name: Validate Azure Template
+        id: validation
+        uses: microsoft/template-validation-action@v0.4.3
+        with:
+          validateAzd: ${{ vars.TEMPLATE_VALIDATE_AZD }}
+          validateTests: ${{ vars.TEMPLATE_VALIDATE_TESTS }}
+          useDevContainer: ${{ vars.TEMPLATE_USE_DEV_CONTAINER }}
+          
+        env:
+          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          AZURE_ENV_NAME:  azd-${{ secrets.AZURE_ENV_NAME }}-${{ env.HHMM }}
+          AZURE_LOCATION: ${{ secrets.AZURE_LOCATION }}
+          AZURE_ENV_AI_SERVICE_LOCATION: ${{ secrets.AZURE_AI_DEPLOYMENT_LOCATION || secrets.AZURE_LOCATION }}
+          AZURE_ENV_MODEL_CAPACITY: 1
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
+
+      - name: Print result
+        shell: bash
+        run: cat "${{ steps.validation.outputs.resultFile }}"

.github/workflows/azure-dev.yml

@@ -1,37 +1,65 @@
-name: Azure Template Validation 
-on: 
-    workflow_dispatch:
-    
-permissions: 
-  contents: read 
-  id-token: write 
-  pull-requests: write 
-jobs: 
-  template_validation_job: 
-    runs-on: ubuntu-latest 
+name: Azure Dev Deploy
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
     environment: production
-    name: Template validation 
-    steps: 
-      # Step 1: Checkout the code from your repository 
-      - name: Checkout code 
-        uses: actions/checkout@v6 
-      # Step 2: Validate the Azure template using microsoft/template-validation-action 
-      - name: Validate Azure Template 
-        uses: microsoft/template-validation-action@v0.4.3
-        with: 
-          validateAzd: true
-          useDevContainer: false
-        id: validation 
-        env: 
-          AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} 
-          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} 
-          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 
-          AZURE_ENV_NAME: ${{ secrets.AZURE_ENV_NAME }} 
-          AZURE_LOCATION: ${{ secrets.AZURE_LOCATION }} 
-          AZURE_AI_DEPLOYMENT_LOCATION : ${{ secrets.AZURE_AI_DEPLOYMENT_LOCATION }}
-          AZURE_ENV_GPT_MODEL_CAPACITY : 1
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
-      # Step 3: Print the result of the validation 
-      - name: Print result 
-        run: cat ${{ steps.validation.outputs.resultFile }} 
+    env:
+      AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
+      AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
+      AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      AZURE_ENV_NAME: ${{ secrets.AZURE_ENV_NAME }}
+      AZURE_LOCATION: ${{ secrets.AZURE_LOCATION }}
+      AZURE_AI_DEPLOYMENT_LOCATION: ${{ secrets.AZURE_AI_DEPLOYMENT_LOCATION || secrets.AZURE_LOCATION }}
+      AZURE_ENV_GPT_MODEL_CAPACITY: 1
+      AZURE_DEV_COLLECT_TELEMETRY: ${{ vars.AZURE_DEV_COLLECT_TELEMETRY }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v6
+
+      - name: Set timestamp and env name
+        run: |
+          HHMM=$(date -u +'%H%M')
+          echo "AZURE_ENV_NAME=azd-${{ vars.AZURE_ENV_NAME }}-${HHMM}" >> $GITHUB_ENV
+
+      - name: Install azd
+        uses: Azure/setup-azd@v2
+
+      - name: Login to Azure
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Login to AZD
+        shell: bash
+        run: |
+          azd auth login \
+            --client-id "$AZURE_CLIENT_ID" \
+            --federated-credential-provider "github" \
+            --tenant-id "$AZURE_TENANT_ID"
+
+      - name: Provision and deploy
+        shell: bash
+        run: |
+          set -e
+
+          if ! azd env select "$AZURE_ENV_NAME"; then
+            azd env new "$AZURE_ENV_NAME" --subscription "$AZURE_SUBSCRIPTION_ID" --location "$AZURE_LOCATION" --no-prompt
+          fi
+
+          azd config set defaults.subscription "$AZURE_SUBSCRIPTION_ID"
+          azd env set AZURE_SUBSCRIPTION_ID "$AZURE_SUBSCRIPTION_ID"
+          azd env set AZURE_LOCATION "$AZURE_LOCATION"
+          azd env set AZURE_ENV_AI_SERVICE_LOCATION "${AZURE_AI_DEPLOYMENT_LOCATION:-$AZURE_LOCATION}"
+          azd env set AZURE_ENV_GPT_MODEL_CAPACITY "$AZURE_ENV_GPT_MODEL_CAPACITY"
+
+          azd up --no-prompt

azure.yaml

@@ -4,6 +4,7 @@ metadata:
 
 requiredVersions:
   azd: '>= 1.18.0 != 1.23.9'
+  bicep: '>= 0.33.0'
 
 parameters:
   AzureAiServiceLocation:

infra/main.json

@@ -6,7 +6,7 @@
     "_generator": {
       "name": "bicep",
       "version": "0.40.2.10011",
-      "templateHash": "12712615740947825780"
+      "templateHash": "13589960712112840698"
     },
     "name": "Modernize Your Code Solution Accelerator",
     "description": "CSA CTO Gold Standard Solution Accelerator for Modernize Your Code. \r\n"
@@ -12895,11 +12895,11 @@
       },
       "dependsOn": [
         "applicationInsights",
+        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').monitor)]",
+        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').oms)]",
         "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageBlob)]",
         "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').ods)]",
         "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').agentSvc)]",
-        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').monitor)]",
-        "[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').oms)]",
         "dataCollectionEndpoint",
         "logAnalyticsWorkspace",
         "virtualNetwork"
@@ -25612,7 +25612,7 @@
             "_generator": {
               "name": "bicep",
               "version": "0.40.2.10011",
-              "templateHash": "15268521456197740686"
+              "templateHash": "665208465907096971"
             },
             "name": "AI Services and Project Module",
             "description": "This module creates an AI Services resource and an AI Foundry project within it. It supports private networking, OpenAI deployments, and role assignments."
@@ -26953,7 +26953,7 @@
                     "_generator": {
                       "name": "bicep",
                       "version": "0.40.2.10011",
-                      "templateHash": "6664885964502172426"
+                      "templateHash": "7604365129625921085"
                     }
                   },
                   "definitions": {
@@ -27910,7 +27910,10 @@
                         "raiPolicyName": "[tryGet(coalesce(parameters('deployments'), createArray())[copyIndex()], 'raiPolicyName')]",
                         "versionUpgradeOption": "[tryGet(coalesce(parameters('deployments'), createArray())[copyIndex()], 'versionUpgradeOption')]"
                       },
-                      "sku": "[coalesce(tryGet(coalesce(parameters('deployments'), createArray())[copyIndex()], 'sku'), createObject('name', parameters('sku'), 'capacity', tryGet(parameters('sku'), 'capacity'), 'tier', tryGet(parameters('sku'), 'tier'), 'size', tryGet(parameters('sku'), 'size'), 'family', tryGet(parameters('sku'), 'family')))]"
+                      "sku": "[coalesce(tryGet(coalesce(parameters('deployments'), createArray())[copyIndex()], 'sku'), createObject('name', parameters('sku'), 'capacity', tryGet(parameters('sku'), 'capacity'), 'tier', tryGet(parameters('sku'), 'tier'), 'size', tryGet(parameters('sku'), 'size'), 'family', tryGet(parameters('sku'), 'family')))]",
+                      "dependsOn": [
+                        "aiProject"
+                      ]
                     },
                     "cognitiveService_lock": {
                       "condition": "[and(not(empty(coalesce(parameters('lock'), createObject()))), not(equals(tryGet(parameters('lock'), 'kind'), 'None')))]",
@@ -29037,7 +29040,7 @@
                     "_generator": {
                       "name": "bicep",
                       "version": "0.40.2.10011",
-                      "templateHash": "6664885964502172426"
+                      "templateHash": "7604365129625921085"
                     }
                   },
                   "definitions": {
@@ -29994,7 +29997,10 @@
                         "raiPolicyName": "[tryGet(coalesce(parameters('deployments'), createArray())[copyIndex()], 'raiPolicyName')]",
                         "versionUpgradeOption": "[tryGet(coalesce(parameters('deployments'), createArray())[copyIndex()], 'versionUpgradeOption')]"
                       },
-                      "sku": "[coalesce(tryGet(coalesce(parameters('deployments'), createArray())[copyIndex()], 'sku'), createObject('name', parameters('sku'), 'capacity', tryGet(parameters('sku'), 'capacity'), 'tier', tryGet(parameters('sku'), 'tier'), 'size', tryGet(parameters('sku'), 'size'), 'family', tryGet(parameters('sku'), 'family')))]"
+                      "sku": "[coalesce(tryGet(coalesce(parameters('deployments'), createArray())[copyIndex()], 'sku'), createObject('name', parameters('sku'), 'capacity', tryGet(parameters('sku'), 'capacity'), 'tier', tryGet(parameters('sku'), 'tier'), 'size', tryGet(parameters('sku'), 'size'), 'family', tryGet(parameters('sku'), 'family')))]",
+                      "dependsOn": [
+                        "aiProject"
+                      ]
                     },
                     "cognitiveService_lock": {
                       "condition": "[and(not(empty(coalesce(parameters('lock'), createObject()))), not(equals(tryGet(parameters('lock'), 'kind'), 'None')))]",

infra/modules/ai-foundry/dependencies.bicep

@@ -208,6 +208,9 @@ resource cognitiveService_deployments 'Microsoft.CognitiveServices/accounts/depl
       size: sku.?size
       family: sku.?family
     }
+    dependsOn: [
+      aiProject
+    ]
   }
 ]
 

src/backend/common/database/cosmosdb.py

@@ -1,11 +1,13 @@
+import asyncio
 from datetime import datetime, timezone
 from typing import Dict, List, Optional
 from uuid import UUID, uuid4
 
 from azure.cosmos.aio import CosmosClient
 from azure.cosmos.aio._database import DatabaseProxy
 from azure.cosmos.exceptions import (
-    CosmosResourceExistsError
+    CosmosResourceExistsError,
+    CosmosResourceNotFoundError,
 )
 
 from common.database.database_base import DatabaseBase
@@ -85,9 +87,28 @@ async def create_batch(self, user_id: str, batch_id: UUID) -> BatchRecord:
                 await self.batch_container.create_item(body=batch.dict())
                 return batch
             except CosmosResourceExistsError:
-                self.logger.info(f"Batch with ID {batch_id} already exists")
-                batchexists = await self.get_batch(user_id, str(batch_id))
-                return batchexists
+                self.logger.info("Batch already exists, reading existing record", batch_id=str(batch_id))
+                # Retry read with backoff to handle replication lag after 409 conflict
+                for attempt in range(3):
+                    try:
+                        batchexists = await self.batch_container.read_item(
+                            item=str(batch_id), partition_key=str(batch_id)
+                        )
+                    except CosmosResourceNotFoundError:
+                        if attempt < 2:
+                            self.logger.info("Batch read returned 404 after conflict, retrying", batch_id=str(batch_id), attempt=attempt + 1)
+                            await asyncio.sleep(0.5 * (attempt + 1))
+                            continue
+                        raise RuntimeError(
+                            f"Batch {batch_id} already exists but could not be read after retries"
+                        )
+
+                    if batchexists.get("user_id") != user_id:
+                        self.logger.error("Batch belongs to a different user", batch_id=str(batch_id))
+                        raise PermissionError("Batch not found")
+
+                    self.logger.info("Returning existing batch record", batch_id=str(batch_id))
+                    return BatchRecord.fromdb(batchexists)
 
         except Exception as e:
             self.logger.error("Failed to create batch", error=str(e))
@@ -158,7 +179,7 @@ async def get_batch(self, user_id: str, batch_id: str) -> Optional[Dict]:
             ]
             batch = None
             async for item in self.batch_container.query_items(
-                query=query, parameters=params
+                query=query, parameters=params, partition_key=batch_id
             ):
                 batch = item
 
@@ -173,7 +194,7 @@ async def get_file(self, file_id: str) -> Optional[Dict]:
             params = [{"name": "@file_id", "value": file_id}]
             file_entry = None
             async for item in self.file_container.query_items(
-                query=query, parameters=params
+                query=query, parameters=params, partition_key=file_id
             ):
                 file_entry = item
             return file_entry
@@ -209,7 +230,7 @@ async def get_batch_from_id(self, batch_id: str) -> Dict:
 
             batch = None  # Store the batch
             async for item in self.batch_container.query_items(
-                query=query, parameters=params
+                query=query, parameters=params, partition_key=batch_id
             ):
                 batch = item  # Assign the batch to the variable
 
@@ -335,11 +356,14 @@ async def add_file_log(
             raise
 
     async def update_batch_entry(
-        self, batch_id: str, user_id: str, status: ProcessStatus, file_count: int
+        self, batch_id: str, user_id: str, status: ProcessStatus, file_count: int,
+        existing_batch: Optional[Dict] = None
     ):
-        """Update batch status."""
+        """Update batch status. If existing_batch is provided, skip the re-fetch."""
         try:
-            batch = await self.get_batch(user_id, batch_id)
+            batch = existing_batch
+            if batch is None:
+                batch = await self.get_batch(user_id, batch_id)
             if not batch:
                 raise ValueError("Batch not found")
 

src/backend/common/database/database_factory.py

@@ -9,25 +9,40 @@
 
 class DatabaseFactory:
     _instance: Optional[DatabaseBase] = None
+    _lock: Optional[asyncio.Lock] = None
     _logger = AppLogger("DatabaseFactory")
 
+    @staticmethod
+    def _get_lock() -> asyncio.Lock:
+        if DatabaseFactory._lock is None:
+            DatabaseFactory._lock = asyncio.Lock()
+        return DatabaseFactory._lock
+
     @staticmethod
     async def get_database():
+        if DatabaseFactory._instance is not None:
+            return DatabaseFactory._instance
+
+        async with DatabaseFactory._get_lock():
+            # Double-check after acquiring the lock
+            if DatabaseFactory._instance is not None:
+                return DatabaseFactory._instance
 
-        config = Config()  # Create an instance of Config
+            config = Config()  # Create an instance of Config
 
-        cosmos_db_client = CosmosDBClient(
-            endpoint=config.cosmosdb_endpoint,
-            credential=config.get_azure_credentials(),
-            database_name=config.cosmosdb_database,
-            batch_container=config.cosmosdb_batch_container,
-            file_container=config.cosmosdb_file_container,
-            log_container=config.cosmosdb_log_container,
-        )
+            cosmos_db_client = CosmosDBClient(
+                endpoint=config.cosmosdb_endpoint,
+                credential=config.get_azure_credentials(),
+                database_name=config.cosmosdb_database,
+                batch_container=config.cosmosdb_batch_container,
+                file_container=config.cosmosdb_file_container,
+                log_container=config.cosmosdb_log_container,
+            )
 
-        await cosmos_db_client.initialize_cosmos()
+            await cosmos_db_client.initialize_cosmos()
 
-        return cosmos_db_client
+            DatabaseFactory._instance = cosmos_db_client
+            return cosmos_db_client
 
 
 # Local testing of config and code