Skip to content

feat: added delete option for uploaded files #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Prajwal-Microsoft merged 3 commits into from
Apr 9, 2025
Merged

feat: added delete option for uploaded files #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7105506
feat: Added API for deletion of processed files
Prajwal-Microsoft Apr 3, 2025
62712a9
CP - US(#16438) story implementation and bug fix(#16072) and code cle…
Bangarraju-Microsoft Apr 8, 2025
43d24b3
UI - Changes and added flag for logs
Bangarraju-Microsoft Apr 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions infra/container_app/deploy_container_app_api_web.bicep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,10 @@ module containerAppWeb 'deploy_container_app.bicep' = {
name: 'APP_MSAL_TOKEN_SCOPE'
value: '<BACKEND_API_SCOPE>'
}
{
name: 'APP_ISLOGS_ENABLED'
value: 'false'
}
]
minReplicas: minReplicaContainerWeb
maxReplicas: maxReplicaContainerWeb
Expand Down
168 changes: 18 additions & 150 deletions infra/main.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"_generator": {
"name": "bicep",
"version": "0.34.44.8038",
"templateHash": "14103938519671400586"
"templateHash": "17985360808056860425"
}
},
"parameters": {
Expand All @@ -21,7 +21,7 @@
"type": "string",
"defaultValue": "EastUs2",
"metadata": {
"description": "Location used for Cosmos DB, Container App deployment"
"description": "Location used for Azure Cosmos DB, Azure Container App deployment"
}
},
"contentUnderstandingLocation": {
Expand All @@ -35,7 +35,7 @@
"azd": {
"type": "location"
},
"description": "Location for the Content Understanding service deployment:"
"description": "Location for the Azure AI Content Understanding service deployment:"
},
"minLength": 1
},
Expand Down Expand Up @@ -362,7 +362,8 @@
"solutionPrefix": "[format('cps-{0}', padLeft(take(variables('uniqueId'), 12), 12, '0'))]",
"containerImageEndPoint": "cpscontainerreg.azurecr.io",
"resourceGroupLocation": "[resourceGroup().location]",
"abbrs": "[variables('$fxv#0')]"
"abbrs": "[variables('$fxv#0')]",
"useLocalBuildLower": "[toLower(parameters('useLocalBuild'))]"
},
"resources": [
{
Expand Down Expand Up @@ -934,7 +935,7 @@
"_generator": {
"name": "bicep",
"version": "0.34.44.8038",
"templateHash": "13228064894943437182"
"templateHash": "6622167858340258597"
}
},
"parameters": {
Expand Down Expand Up @@ -1432,147 +1433,6 @@
"dependsOn": [
"[resourceId('Microsoft.MachineLearningServices/workspaces', variables('aiHubName'))]"
]
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'TENANT-ID')]",
"properties": {
"value": "[subscription().tenantId]"
}
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-KEY')]",
"properties": {
"value": "[listKeys(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName')), '2021-10-01').key1]"
},
"dependsOn": [
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName'))]"
]
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPEN-AI-DEPLOYMENT-MODEL')]",
"properties": {
"value": "[parameters('gptModelName')]"
}
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-PREVIEW-API-VERSION')]",
"properties": {
"value": "[parameters('gptModelVersion')]"
}
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-ENDPOINT')]",
"properties": {
"value": "[reference(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName')), '2021-10-01').endpoint]"
},
"dependsOn": [
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName'))]"
]
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-AI-PROJECT-CONN-STRING')]",
"properties": {
"value": "[format('{0};{1};{2};{3}', split(reference(resourceId('Microsoft.MachineLearningServices/workspaces', variables('aiProjectName')), '2024-01-01-preview').discoveryUrl, '/')[2], subscription().subscriptionId, resourceGroup().name, variables('aiProjectName'))]"
},
"dependsOn": [
"[resourceId('Microsoft.MachineLearningServices/workspaces', variables('aiProjectName'))]"
]
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-CU-ENDPOINT')]",
"properties": {
"value": "[reference(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName_cu')), '2021-10-01').endpoint]"
},
"dependsOn": [
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName_cu'))]"
]
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-CU-KEY')]",
"properties": {
"value": "[listKeys(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName_cu')), '2021-10-01').key1]"
},
"dependsOn": [
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName_cu'))]"
]
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-OPENAI-CU-VERSION')]",
"properties": {
"value": "?api-version=2024-12-01-preview"
}
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'COG-SERVICES-ENDPOINT')]",
"properties": {
"value": "[reference(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName')), '2021-10-01').endpoint]"
},
"dependsOn": [
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName'))]"
]
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'COG-SERVICES-KEY')]",
"properties": {
"value": "[listKeys(resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName')), '2021-10-01').key1]"
},
"dependsOn": [
"[resourceId('Microsoft.CognitiveServices/accounts', variables('aiServicesName'))]"
]
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'COG-SERVICES-NAME')]",
"properties": {
"value": "[variables('aiServicesName')]"
}
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-SUBSCRIPTION-ID')]",
"properties": {
"value": "[subscription().subscriptionId]"
}
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-RESOURCE-GROUP')]",
"properties": {
"value": "[resourceGroup().name]"
}
},
{
"type": "Microsoft.KeyVault/vaults/secrets",
"apiVersion": "2021-11-01-preview",
"name": "[format('{0}/{1}', parameters('keyVaultName'), 'AZURE-LOCATION')]",
"properties": {
"value": "[parameters('solutionLocation')]"
}
}
],
"outputs": {
Expand Down Expand Up @@ -1795,7 +1655,7 @@
"_generator": {
"name": "bicep",
"version": "0.34.44.8038",
"templateHash": "15388289119319771934"
"templateHash": "15815884747026956332"
}
},
"parameters": {
Expand Down Expand Up @@ -2546,6 +2406,10 @@
{
"name": "APP_MSAL_TOKEN_SCOPE",
"value": "<BACKEND_API_SCOPE>"
},
{
"name": "APP_ISLOGS_ENABLED",
"value": "false"
}
]
},
Expand Down Expand Up @@ -3269,7 +3133,7 @@
"location": {
"value": "[parameters('secondaryLocation')]"
},
"azureContainerRegistry": "[if(equals(parameters('useLocalBuild'), 'true'), createObject('value', reference(resourceId('Microsoft.Resources/deployments', 'deploy_container_registry'), '2022-09-01').outputs.acrEndpoint.value), createObject('value', variables('containerImageEndPoint')))]",
"azureContainerRegistry": "[if(equals(variables('useLocalBuildLower'), 'true'), createObject('value', reference(resourceId('Microsoft.Resources/deployments', 'deploy_container_registry'), '2022-09-01').outputs.acrEndpoint.value), createObject('value', variables('containerImageEndPoint')))]",
"appConfigEndPoint": {
"value": "[reference(extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_app_config_service'), '2022-09-01').outputs.appConfigEndpoint.value]"
},
Expand Down Expand Up @@ -3304,7 +3168,7 @@
"value": "[parameters('maxReplicaContainerWeb')]"
},
"useLocalBuild": {
"value": "[parameters('useLocalBuild')]"
"value": "[variables('useLocalBuildLower')]"
}
},
"template": {
Expand All @@ -3314,7 +3178,7 @@
"_generator": {
"name": "bicep",
"version": "0.34.44.8038",
"templateHash": "15388289119319771934"
"templateHash": "15815884747026956332"
}
},
"parameters": {
Expand Down Expand Up @@ -4065,6 +3929,10 @@
{
"name": "APP_MSAL_TOKEN_SCOPE",
"value": "<BACKEND_API_SCOPE>"
},
{
"name": "APP_ISLOGS_ENABLED",
"value": "false"
}
]
},
Expand Down
5 changes: 3 additions & 2 deletions src/ContentProcessorAPI/app/libs/cosmos_db/helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ def update_document_by_query(self, query: Dict[str, Any], update: Dict[str, Any]
result = self.container.update_one(query, {"$set": update})
return result

def delete_document(self, item_id: str):
result = self.container.delete_one({"Id": item_id})
def delete_document(self, item_id: str, field_name: str = None):
field_name = field_name or "Id" # Use "Id" if field_name is empty or None
result = self.container.delete_one({field_name: item_id})
return result
21 changes: 21 additions & 0 deletions src/ContentProcessorAPI/app/libs/storage_blob/helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,3 +90,24 @@ def delete_blob_and_cleanup(self, blob_name, container_name=None):
# Delete the (virtual) folder in the Container
blob_client = container_client.get_blob_client(container_name)
blob_client.delete_blob()

def delete_folder(self, folder_name, container_name=None):
container_client = self._get_container_client(container_name)

# List all blobs inside the folder
blobs_to_delete = container_client.list_blobs(name_starts_with=folder_name + "/")

# Delete each blob
for blob in blobs_to_delete:
blob_client = container_client.get_blob_client(blob.name)
blob_client.delete_blob()

blobs_to_delete = container_client.list_blobs()
if not blobs_to_delete:

# Get Parent Container
container_client = self._get_container_client()

# Delete the (virtual) folder in the Container
blob_client = container_client.get_blob_client(folder_name)
blob_client.delete_blob()
31 changes: 30 additions & 1 deletion src/ContentProcessorAPI/app/routers/contentprocessor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import urllib.parse
import uuid

from fastapi import APIRouter, Body, Depends, File, UploadFile
from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
from fastapi.responses import JSONResponse, StreamingResponse
from pymongo.results import UpdateResult

Expand All @@ -28,6 +28,7 @@
ContentProcess,
ContentProcessorRequest,
ContentResultUpdate,
ContentResultDelete,
Paging,
ProcessFile,
Status,
Expand Down Expand Up @@ -490,3 +491,31 @@ async def get_original_file(
return StreamingResponse(
file_stream, media_type=content_type_string, headers=headers
)

@router.delete(
"/processed/{process_id}",
response_model=ContentResultDelete,
summary="Delete the processed content result",
description="""
Returns the deleted record for a given process ID.
""",
)
async def delete_processed_file(
process_id: str, app_config: AppConfiguration = Depends(get_app_config)
) -> ContentResultDelete:
try:
deleted_file = CosmosContentProcess(process_id=process_id).delete_processed_file(
connection_string=app_config.app_cosmos_connstr,
database_name=app_config.app_cosmos_database,
collection_name=app_config.app_cosmos_container_process,
storage_connection_string=app_config.app_storage_blob_url,
container_name=app_config.app_cps_processes,
)

except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
return ContentResultDelete(
status="Success" if deleted_file else "Failed",
process_id=deleted_file.process_id if deleted_file else "",
message="" if deleted_file else "This record no longer exists. Please refresh the page."
)
35 changes: 35 additions & 0 deletions src/ContentProcessorAPI/app/routers/models/contentprocessor/content_process.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,41 @@ def get_status_from_cosmos(
return ContentProcess(**existing_process[0])
else:
return None

def delete_processed_file(
self,
connection_string: str,
database_name: str,
collection_name: str,
storage_connection_string: str,
container_name: str,
):
"""
Delete the processed file from Cosmos DB & Storage account.
"""
mongo_helper = CosmosMongDBHelper(
connection_string=connection_string,
db_name=database_name,
container_name=collection_name,
indexes=[("process_id", 1)],
)

blob_helper = StorageBlobHelper(
account_url=storage_connection_string, container_name=container_name
)

# Check if the process_id already exists in the database
existing_process = mongo_helper.find_document(
query={"process_id": self.process_id}
)

blob_helper.delete_folder(folder_name=self.process_id)

if existing_process:
mongo_helper.delete_document(item_id=self.process_id, field_name="process_id")
return ContentProcess(**existing_process[0])
else:
return None

def update_process_result(
self,
Expand Down
Loading
Loading