Skip to content

Commit 08cbd5e

Browse files
Prajwal-MicrosoftPrajwal-Microsoft
authored
Merge pull request #21 from microsoft/psl-raju-dev
feat: added delete option for uploaded files
2 parents f45fc8c + 43d24b3 commit 08cbd5e

File tree

37 files changed

+1134
-1545
lines changed

37 files changed

+1134
-1545
lines changed

‎infra/container_app/deploy_container_app_api_web.bicep‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,10 @@ module containerAppWeb 'deploy_container_app.bicep' = {
146146
name: 'APP_MSAL_TOKEN_SCOPE'
147147
value: '<BACKEND_API_SCOPE>'
148148
}
149+
{
150+
name: 'APP_ISLOGS_ENABLED'
151+
value: 'false'
152+
}
149153
]
150154
minReplicas: minReplicaContainerWeb
151155
maxReplicas: maxReplicaContainerWeb

‎infra/main.json‎

Lines changed: 18 additions & 150 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"_generator": {
66
"name": "bicep",
77
"version": "0.34.44.8038",
8-
"templateHash": "14103938519671400586"
8+
"templateHash": "17985360808056860425"
99
}
1010
},
1111
"parameters": {
@@ -21,7 +21,7 @@
2121
"type": "string",
2222
"defaultValue": "EastUs2",
2323
"metadata": {
24-
"description": "Location used for Cosmos DB, Container App deployment"
24+
"description": "Location used for Azure Cosmos DB, Azure Container App deployment"
2525
}
2626
},
2727
"contentUnderstandingLocation": {
@@ -35,7 +35,7 @@
3535
"azd": {
3636
"type": "location"
3737
},
38-
"description": "Location for the Content Understanding service deployment:"
38+
"description": "Location for the Azure AI Content Understanding service deployment:"
3939
},
4040
"minLength": 1
4141
},
@@ -362,7 +362,8 @@
362362
"solutionPrefix": "[format('cps-{0}', padLeft(take(variables('uniqueId'), 12), 12, '0'))]",
363363
"containerImageEndPoint": "cpscontainerreg.azurecr.io",
364364
"resourceGroupLocation": "[resourceGroup().location]",
365-
"abbrs": "[variables('$fxv#0')]"
365+
"abbrs": "[variables('$fxv#0')]",
366+
"useLocalBuildLower": "[toLower(parameters('useLocalBuild'))]"
366367
},
367368
"resources": [
368369
{
@@ -934,7 +935,7 @@
934935
"_generator": {
935936
"name": "bicep",
936937
"version": "0.34.44.8038",
937-
"templateHash": "13228064894943437182"
938+
"templateHash": "6622167858340258597"
938939
}
939940
},
940941
"parameters": {
@@ -1432,147 +1433,6 @@
14321433
"dependsOn": [
14331434
"[resourceId('Microsoft.MachineLearningServices/workspaces', variables('aiHubName'))]"
14341435
]
1435-
},
1436-
{
1437-
"type": "Microsoft.KeyVault/vaults/secrets",
1438-
"apiVersion": "2021-11-01-preview",
1439-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'TENANT-ID')]",
1440-
"properties": {
1441-
"value": "[subscription().tenantId]"
1442-
}
1443-
},
1444-
{
1445-
"type": "Microsoft.KeyVault/vaults/secrets",
1446-
"apiVersion": "2021-11-01-preview",
1447-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-KEY')]",
1448-
"properties": {
1449-
"value": "[listKeys(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName')), '2021-10-01').key1]"
1450-
},
1451-
"dependsOn": [
1452-
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName'))]"
1453-
]
1454-
},
1455-
{
1456-
"type": "Microsoft.KeyVault/vaults/secrets",
1457-
"apiVersion": "2021-11-01-preview",
1458-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPEN-AI-DEPLOYMENT-MODEL')]",
1459-
"properties": {
1460-
"value": "[parameters('gptModelName')]"
1461-
}
1462-
},
1463-
{
1464-
"type": "Microsoft.KeyVault/vaults/secrets",
1465-
"apiVersion": "2021-11-01-preview",
1466-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-PREVIEW-API-VERSION')]",
1467-
"properties": {
1468-
"value": "[parameters('gptModelVersion')]"
1469-
}
1470-
},
1471-
{
1472-
"type": "Microsoft.KeyVault/vaults/secrets",
1473-
"apiVersion": "2021-11-01-preview",
1474-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-ENDPOINT')]",
1475-
"properties": {
1476-
"value": "[reference(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName')), '2021-10-01').endpoint]"
1477-
},
1478-
"dependsOn": [
1479-
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName'))]"
1480-
]
1481-
},
1482-
{
1483-
"type": "Microsoft.KeyVault/vaults/secrets",
1484-
"apiVersion": "2021-11-01-preview",
1485-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-AI-PROJECT-CONN-STRING')]",
1486-
"properties": {
1487-
"value": "[format('{0};{1};{2};{3}', split(reference(resourceId('Microsoft.MachineLearningServices/workspaces', variables('aiProjectName')), '2024-01-01-preview').discoveryUrl, '/')[2], subscription().subscriptionId, resourceGroup().name, variables('aiProjectName'))]"
1488-
},
1489-
"dependsOn": [
1490-
"[resourceId('Microsoft.MachineLearningServices/workspaces', variables('aiProjectName'))]"
1491-
]
1492-
},
1493-
{
1494-
"type": "Microsoft.KeyVault/vaults/secrets",
1495-
"apiVersion": "2021-11-01-preview",
1496-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-CU-ENDPOINT')]",
1497-
"properties": {
1498-
"value": "[reference(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName_cu')), '2021-10-01').endpoint]"
1499-
},
1500-
"dependsOn": [
1501-
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName_cu'))]"
1502-
]
1503-
},
1504-
{
1505-
"type": "Microsoft.KeyVault/vaults/secrets",
1506-
"apiVersion": "2021-11-01-preview",
1507-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-CU-KEY')]",
1508-
"properties": {
1509-
"value": "[listKeys(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName_cu')), '2021-10-01').key1]"
1510-
},
1511-
"dependsOn": [
1512-
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName_cu'))]"
1513-
]
1514-
},
1515-
{
1516-
"type": "Microsoft.KeyVault/vaults/secrets",
1517-
"apiVersion": "2021-11-01-preview",
1518-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-CU-VERSION')]",
1519-
"properties": {
1520-
"value": "?api-version=2024-12-01-preview"
1521-
}
1522-
},
1523-
{
1524-
"type": "Microsoft.KeyVault/vaults/secrets",
1525-
"apiVersion": "2021-11-01-preview",
1526-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'COG-SERVICES-ENDPOINT')]",
1527-
"properties": {
1528-
"value": "[reference(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName')), '2021-10-01').endpoint]"
1529-
},
1530-
"dependsOn": [
1531-
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName'))]"
1532-
]
1533-
},
1534-
{
1535-
"type": "Microsoft.KeyVault/vaults/secrets",
1536-
"apiVersion": "2021-11-01-preview",
1537-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'COG-SERVICES-KEY')]",
1538-
"properties": {
1539-
"value": "[listKeys(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName')), '2021-10-01').key1]"
1540-
},
1541-
"dependsOn": [
1542-
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName'))]"
1543-
]
1544-
},
1545-
{
1546-
"type": "Microsoft.KeyVault/vaults/secrets",
1547-
"apiVersion": "2021-11-01-preview",
1548-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'COG-SERVICES-NAME')]",
1549-
"properties": {
1550-
"value": "[variables('aiServicesName')]"
1551-
}
1552-
},
1553-
{
1554-
"type": "Microsoft.KeyVault/vaults/secrets",
1555-
"apiVersion": "2021-11-01-preview",
1556-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-SUBSCRIPTION-ID')]",
1557-
"properties": {
1558-
"value": "[subscription().subscriptionId]"
1559-
}
1560-
},
1561-
{
1562-
"type": "Microsoft.KeyVault/vaults/secrets",
1563-
"apiVersion": "2021-11-01-preview",
1564-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-RESOURCE-GROUP')]",
1565-
"properties": {
1566-
"value": "[resourceGroup().name]"
1567-
}
1568-
},
1569-
{
1570-
"type": "Microsoft.KeyVault/vaults/secrets",
1571-
"apiVersion": "2021-11-01-preview",
1572-
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-LOCATION')]",
1573-
"properties": {
1574-
"value": "[parameters('solutionLocation')]"
1575-
}
15761436
}
15771437
],
15781438
"outputs": {
@@ -1795,7 +1655,7 @@
17951655
"_generator": {
17961656
"name": "bicep",
17971657
"version": "0.34.44.8038",
1798-
"templateHash": "15388289119319771934"
1658+
"templateHash": "15815884747026956332"
17991659
}
18001660
},
18011661
"parameters": {
@@ -2546,6 +2406,10 @@
25462406
{
25472407
"name": "APP_MSAL_TOKEN_SCOPE",
25482408
"value": "<BACKEND_API_SCOPE>"
2409+
},
2410+
{
2411+
"name": "APP_ISLOGS_ENABLED",
2412+
"value": "false"
25492413
}
25502414
]
25512415
},
@@ -3269,7 +3133,7 @@
32693133
"location": {
32703134
"value": "[parameters('secondaryLocation')]"
32713135
},
3272-
"azureContainerRegistry": "[if(equals(parameters('useLocalBuild'), 'true'), createObject('value', reference(resourceId('Microsoft.Resources/deployments', 'deploy_container_registry'), '2022-09-01').outputs.acrEndpoint.value), createObject('value', variables('containerImageEndPoint')))]",
3136+
"azureContainerRegistry": "[if(equals(variables('useLocalBuildLower'), 'true'), createObject('value', reference(resourceId('Microsoft.Resources/deployments', 'deploy_container_registry'), '2022-09-01').outputs.acrEndpoint.value), createObject('value', variables('containerImageEndPoint')))]",
32733137
"appConfigEndPoint": {
32743138
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_app_config_service'), '2022-09-01').outputs.appConfigEndpoint.value]"
32753139
},
@@ -3304,7 +3168,7 @@
33043168
"value": "[parameters('maxReplicaContainerWeb')]"
33053169
},
33063170
"useLocalBuild": {
3307-
"value": "[parameters('useLocalBuild')]"
3171+
"value": "[variables('useLocalBuildLower')]"
33083172
}
33093173
},
33103174
"template": {
@@ -3314,7 +3178,7 @@
33143178
"_generator": {
33153179
"name": "bicep",
33163180
"version": "0.34.44.8038",
3317-
"templateHash": "15388289119319771934"
3181+
"templateHash": "15815884747026956332"
33183182
}
33193183
},
33203184
"parameters": {
@@ -4065,6 +3929,10 @@
40653929
{
40663930
"name": "APP_MSAL_TOKEN_SCOPE",
40673931
"value": "<BACKEND_API_SCOPE>"
3932+
},
3933+
{
3934+
"name": "APP_ISLOGS_ENABLED",
3935+
"value": "false"
40683936
}
40693937
]
40703938
},

‎src/ContentProcessorAPI/app/libs/cosmos_db/helper.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ def update_document_by_query(self, query: Dict[str, Any], update: Dict[str, Any]
102102
result = self.container.update_one(query, {"$set": update})
103103
return result
104104

105-
def delete_document(self, item_id: str):
106-
result = self.container.delete_one({"Id": item_id})
105+
def delete_document(self, item_id: str, field_name: str = None):
106+
field_name = field_name or "Id" # Use "Id" if field_name is empty or None
107+
result = self.container.delete_one({field_name: item_id})
107108
return result

‎src/ContentProcessorAPI/app/libs/storage_blob/helper.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,3 +90,24 @@ def delete_blob_and_cleanup(self, blob_name, container_name=None):
9090
# Delete the (virtual) folder in the Container
9191
blob_client = container_client.get_blob_client(container_name)
9292
blob_client.delete_blob()
93+
94+
def delete_folder(self, folder_name, container_name=None):
95+
container_client = self._get_container_client(container_name)
96+
97+
# List all blobs inside the folder
98+
blobs_to_delete = container_client.list_blobs(name_starts_with=folder_name + "/")
99+
100+
# Delete each blob
101+
for blob in blobs_to_delete:
102+
blob_client = container_client.get_blob_client(blob.name)
103+
blob_client.delete_blob()
104+
105+
blobs_to_delete = container_client.list_blobs()
106+
if not blobs_to_delete:
107+
108+
# Get Parent Container
109+
container_client = self._get_container_client()
110+
111+
# Delete the (virtual) folder in the Container
112+
blob_client = container_client.get_blob_client(folder_name)
113+
blob_client.delete_blob()

‎src/ContentProcessorAPI/app/routers/contentprocessor.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
import urllib.parse
77
import uuid
88

9-
from fastapi import APIRouter, Body, Depends, File, UploadFile
9+
from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
1010
from fastapi.responses import JSONResponse, StreamingResponse
1111
from pymongo.results import UpdateResult
1212

@@ -28,6 +28,7 @@
2828
ContentProcess,
2929
ContentProcessorRequest,
3030
ContentResultUpdate,
31+
ContentResultDelete,
3132
Paging,
3233
ProcessFile,
3334
Status,
@@ -490,3 +491,31 @@ async def get_original_file(
490491
return StreamingResponse(
491492
file_stream, media_type=content_type_string, headers=headers
492493
)
494+
495+
@router.delete(
496+
"/processed/{process_id}",
497+
response_model=ContentResultDelete,
498+
summary="Delete the processed content result",
499+
description="""
500+
Returns the deleted record for a given process ID.
501+
""",
502+
)
503+
async def delete_processed_file(
504+
process_id: str, app_config: AppConfiguration = Depends(get_app_config)
505+
) -> ContentResultDelete:
506+
try:
507+
deleted_file = CosmosContentProcess(process_id=process_id).delete_processed_file(
508+
connection_string=app_config.app_cosmos_connstr,
509+
database_name=app_config.app_cosmos_database,
510+
collection_name=app_config.app_cosmos_container_process,
511+
storage_connection_string=app_config.app_storage_blob_url,
512+
container_name=app_config.app_cps_processes,
513+
)
514+
515+
except Exception as e:
516+
raise HTTPException(status_code=500, detail=str(e))
517+
return ContentResultDelete(
518+
status="Success" if deleted_file else "Failed",
519+
process_id=deleted_file.process_id if deleted_file else "",
520+
message="" if deleted_file else "This record no longer exists. Please refresh the page."
521+
)

‎src/ContentProcessorAPI/app/routers/models/contentprocessor/content_process.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,41 @@ def get_status_from_cosmos(
189189
return ContentProcess(**existing_process[0])
190190
else:
191191
return None
192+
193+
def delete_processed_file(
194+
self,
195+
connection_string: str,
196+
database_name: str,
197+
collection_name: str,
198+
storage_connection_string: str,
199+
container_name: str,
200+
):
201+
"""
202+
Delete the processed file from Cosmos DB & Storage account.
203+
"""
204+
mongo_helper = CosmosMongDBHelper(
205+
connection_string=connection_string,
206+
db_name=database_name,
207+
container_name=collection_name,
208+
indexes=[("process_id", 1)],
209+
)
210+
211+
blob_helper = StorageBlobHelper(
212+
account_url=storage_connection_string, container_name=container_name
213+
)
214+
215+
# Check if the process_id already exists in the database
216+
existing_process = mongo_helper.find_document(
217+
query={"process_id": self.process_id}
218+
)
219+
220+
blob_helper.delete_folder(folder_name=self.process_id)
221+
222+
if existing_process:
223+
mongo_helper.delete_document(item_id=self.process_id, field_name="process_id")
224+
return ContentProcess(**existing_process[0])
225+
else:
226+
return None
192227

193228
def update_process_result(
194229
self,

0 commit comments

Comments
 (0)