Skip to content

Commit 9b182ab

Browse files
tanya-borisovatanya-borisova
authored
Fix the batch processing RBAC bug (#579)
* Fix the batch processing RBAC bug * Add some clarifications to the docs * Create storage queue role for keys as well * Initialise queue client differently for rbac and keys * Fix bicep rule * Fix msg policy
1 parent 5a205d2 commit 9b182ab

5 files changed

Lines changed: 201 additions & 23 deletions

File tree

‎code/backend/batch/BatchStartProcessing.py‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,11 @@
22
import json
33
import azure.functions as func
44
import sys
5-
from azure.storage.queue import QueueClient, BinaryBase64EncodePolicy
65
from utilities.helpers.EnvHelper import EnvHelper
7-
from utilities.helpers.AzureBlobStorageHelper import AzureBlobStorageClient
6+
from utilities.helpers.AzureBlobStorageHelper import (
7+
AzureBlobStorageClient,
8+
create_queue_client,
9+
)
810

911
sys.path.append("..")
1012
bp_batch_start_processing = func.Blueprint()
@@ -25,13 +27,9 @@ def batch_start_processing(req: func.HttpRequest) -> func.HttpResponse:
2527
else files_data
2628
)
2729
files_data = list(map(lambda x: {"filename": x["filename"]}, files_data))
28-
# Create the QueueClient object
29-
queue_client = QueueClient.from_connection_string(
30-
azure_blob_storage_client.connect_str,
31-
env_helper.DOCUMENT_PROCESSING_QUEUE_NAME,
32-
message_encode_policy=BinaryBase64EncodePolicy(),
33-
)
30+
3431
# Send a message to the queue for each file
32+
queue_client = create_queue_client()
3533
for fd in files_data:
3634
queue_client.send_message(json.dumps(fd).encode("utf-8"))
3735

‎code/backend/batch/utilities/helpers/AzureBlobStorageHelper.py‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,35 @@
77
ContentSettings,
88
UserDelegationKey,
99
)
10+
from azure.storage.queue import QueueClient, BinaryBase64EncodePolicy
1011
from .EnvHelper import EnvHelper
1112
from azure.identity import DefaultAzureCredential
1213

1314

15+
def connection_string(account_name: str, account_key: str):
16+
return f"DefaultEndpointsProtocol=https;AccountName={account_name};AccountKey={account_key};EndpointSuffix=core.windows.net"
17+
18+
19+
def create_queue_client():
20+
env_helper: EnvHelper = EnvHelper()
21+
if env_helper.AZURE_AUTH_TYPE == "rbac":
22+
return QueueClient(
23+
account_url=f"https://{env_helper.AZURE_BLOB_ACCOUNT_NAME}.queue.core.windows.net/",
24+
queue_name=env_helper.DOCUMENT_PROCESSING_QUEUE_NAME,
25+
credential=DefaultAzureCredential(),
26+
message_encode_policy=BinaryBase64EncodePolicy(),
27+
)
28+
29+
else:
30+
return QueueClient.from_connection_string(
31+
conn_str=connection_string(
32+
env_helper.AZURE_BLOB_ACCOUNT_NAME, env_helper.AZURE_BLOB_ACCOUNT_KEY
33+
),
34+
queue_name=env_helper.DOCUMENT_PROCESSING_QUEUE_NAME,
35+
message_encode_policy=BinaryBase64EncodePolicy(),
36+
)
37+
38+
1439
class AzureBlobStorageClient:
1540
def __init__(
1641
self,
@@ -25,29 +50,27 @@ def __init__(
2550
self.account_name = (
2651
account_name if account_name else env_helper.AZURE_BLOB_ACCOUNT_NAME
2752
)
53+
self.account_key = None
2854
self.container_name: str = (
2955
container_name
3056
if container_name
3157
else env_helper.AZURE_BLOB_CONTAINER_NAME
3258
)
33-
credential = DefaultAzureCredential()
34-
account_url = f"https://{self.account_name}.blob.core.windows.net/"
3559
self.blob_service_client = BlobServiceClient(
36-
account_url=account_url, credential=credential
60+
account_url=f"https://{self.account_name}.blob.core.windows.net/",
61+
credential=DefaultAzureCredential(),
3762
)
3863
self.user_delegation_key = self.request_user_delegation_key(
3964
blob_service_client=self.blob_service_client
4065
)
41-
self.account_key = None
4266
else:
4367
self.account_name = (
4468
account_name if account_name else env_helper.AZURE_BLOB_ACCOUNT_NAME
4569
)
4670
self.account_key = (
4771
account_key if account_key else env_helper.AZURE_BLOB_ACCOUNT_KEY
4872
)
49-
self.user_delegation_key = None
50-
self.connect_str = f"DefaultEndpointsProtocol=https;AccountName={self.account_name};AccountKey={self.account_key};EndpointSuffix=core.windows.net"
73+
self.connect_str = connection_string(self.account_name, self.account_key)
5174
self.container_name: str = (
5275
container_name
5376
if container_name
@@ -56,6 +79,7 @@ def __init__(
5679
self.blob_service_client: BlobServiceClient = (
5780
BlobServiceClient.from_connection_string(self.connect_str)
5881
)
82+
self.user_delegation_key = None
5983

6084
def request_user_delegation_key(
6185
self, blob_service_client: BlobServiceClient

‎docs/LOCAL_DEPLOYMENT.md‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,14 @@ The easiest way to run this accelerator is in a VS Code Dev Containers, which wi
1919

2020
> NOTE: It may take up to an hour for the application to be fully deployed. If you see a "Python Developer" welcome screen or an error page, then wait a bit and refresh the page.
2121
22-
**Notes:** the default auth type uses keys, if you want to switch to rbac, please run `azd env set AUTH_TYPE rbac`.
22+
> NOTE: The default auth type uses keys that are stored in the Azure Keyvault. If you want to use RBAC-based auth (more secure), please run before deploying:
23+
24+
```bash
25+
azd env set AZURE_AUTH_TYPE rbac
26+
azd env set USE_KEY_VAULT false
27+
```
28+
29+
Also please refer to the section on [setting up RBAC auth](#authenticate-using-rbac).
2330

2431
## Detailed Development Container setup instructions
2532

@@ -142,8 +149,9 @@ Or use the [Azure Functions VS Code extension](https://marketplace.visualstudio.
142149
#### Debugging the batch processing functions locally
143150
Rename the file `local.settings.json.sample` in the `batch` folder to `local.settings.json` and update the `AzureWebJobsStorage` value with the storage account connection string.
144151

145-
Execute the above [shell command](#L81) to run the function locally. You may need to stop the deployed function on the portal so that all requests are debugged locally.
152+
Copy the .env file from [previous section](#local-debugging) to the `batch` folder.
146153

154+
Execute the above [shell command](#L81) to run the function locally. You may need to stop the deployed function on the portal so that all requests are debugged locally. To trigger the function, you can click on the corresponding URL that will be printed to the terminal.
147155

148156
## Environment variables
149157

‎infra/app/function.bicep‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -113,15 +113,25 @@ module searchRoleFunction '../core/security/role.bicep' = if (authType == 'rbac'
113113
}
114114

115115
// Storage Blob Data Contributor
116-
module storageRoleFunction '../core/security/role.bicep' = if (authType == 'rbac') {
117-
name: 'storage-role-function'
116+
module storageBlobRoleFunction '../core/security/role.bicep' = if (authType == 'rbac') {
117+
name: 'storage-blob-role-function'
118118
params: {
119119
principalId: function.outputs.identityPrincipalId
120120
roleDefinitionId: 'ba92f5b4-2d11-453d-a403-e96b0029c9fe'
121121
principalType: 'ServicePrincipal'
122122
}
123123
}
124124

125+
// Storage Queue Data Contributor
126+
module storageQueueRoleFunction '../core/security/role.bicep' = if (authType == 'rbac') {
127+
name: 'storage-queue-role-function'
128+
params: {
129+
principalId: function.outputs.identityPrincipalId
130+
roleDefinitionId: '974c5e8b-45b9-4653-ba55-5f855dd0fb88'
131+
principalType: 'ServicePrincipal'
132+
}
133+
}
134+
125135
module functionaccess '../core/security/keyvault-access.bicep' = if (useKeyVault) {
126136
name: 'function-keyvault-access'
127137
params: {

‎infra/main.json‎

Lines changed: 143 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"_generator": {
66
"name": "bicep",
77
"version": "0.26.54.24096",
8-
"templateHash": "14235243275465943137"
8+
"templateHash": "4295550392354123736"
99
}
1010
},
1111
"parameters": {
@@ -6551,7 +6551,7 @@
65516551
"_generator": {
65526552
"name": "bicep",
65536553
"version": "0.26.54.24096",
6554-
"templateHash": "12912745910813883723"
6554+
"templateHash": "17960689149478457729"
65556555
}
65566556
},
65576557
"parameters": {
@@ -7446,7 +7446,7 @@
74467446
"condition": "[equals(parameters('authType'), 'rbac')]",
74477447
"type": "Microsoft.Resources/deployments",
74487448
"apiVersion": "2022-09-01",
7449-
"name": "storage-role-function",
7449+
"name": "storage-blob-role-function",
74507450
"properties": {
74517451
"expressionEvaluationOptions": {
74527452
"scope": "inner"
@@ -7511,6 +7511,75 @@
75117511
"[resourceId('Microsoft.Resources/deployments', format('{0}-app-module', parameters('name')))]"
75127512
]
75137513
},
7514+
{
7515+
"condition": "[equals(parameters('authType'), 'rbac')]",
7516+
"type": "Microsoft.Resources/deployments",
7517+
"apiVersion": "2022-09-01",
7518+
"name": "storage-queue-role-function",
7519+
"properties": {
7520+
"expressionEvaluationOptions": {
7521+
"scope": "inner"
7522+
},
7523+
"mode": "Incremental",
7524+
"parameters": {
7525+
"principalId": {
7526+
"value": "[reference(resourceId('Microsoft.Resources/deployments', format('{0}-app-module', parameters('name'))), '2022-09-01').outputs.identityPrincipalId.value]"
7527+
},
7528+
"roleDefinitionId": {
7529+
"value": "974c5e8b-45b9-4653-ba55-5f855dd0fb88"
7530+
},
7531+
"principalType": {
7532+
"value": "ServicePrincipal"
7533+
}
7534+
},
7535+
"template": {
7536+
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
7537+
"contentVersion": "1.0.0.0",
7538+
"metadata": {
7539+
"_generator": {
7540+
"name": "bicep",
7541+
"version": "0.26.54.24096",
7542+
"templateHash": "5795525499710207356"
7543+
},
7544+
"description": "Creates a role assignment for a service principal."
7545+
},
7546+
"parameters": {
7547+
"principalId": {
7548+
"type": "string"
7549+
},
7550+
"principalType": {
7551+
"type": "string",
7552+
"defaultValue": "ServicePrincipal",
7553+
"allowedValues": [
7554+
"Device",
7555+
"ForeignGroup",
7556+
"Group",
7557+
"ServicePrincipal",
7558+
"User"
7559+
]
7560+
},
7561+
"roleDefinitionId": {
7562+
"type": "string"
7563+
}
7564+
},
7565+
"resources": [
7566+
{
7567+
"type": "Microsoft.Authorization/roleAssignments",
7568+
"apiVersion": "2022-04-01",
7569+
"name": "[guid(subscription().id, resourceGroup().id, parameters('principalId'), parameters('roleDefinitionId'))]",
7570+
"properties": {
7571+
"principalId": "[parameters('principalId')]",
7572+
"principalType": "[parameters('principalType')]",
7573+
"roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', parameters('roleDefinitionId'))]"
7574+
}
7575+
}
7576+
]
7577+
}
7578+
},
7579+
"dependsOn": [
7580+
"[resourceId('Microsoft.Resources/deployments', format('{0}-app-module', parameters('name')))]"
7581+
]
7582+
},
75147583
{
75157584
"condition": "[parameters('useKeyVault')]",
75167585
"type": "Microsoft.Resources/deployments",
@@ -7694,7 +7763,7 @@
76947763
"_generator": {
76957764
"name": "bicep",
76967765
"version": "0.26.54.24096",
7697-
"templateHash": "12912745910813883723"
7766+
"templateHash": "17960689149478457729"
76987767
}
76997768
},
77007769
"parameters": {
@@ -8589,7 +8658,7 @@
85898658
"condition": "[equals(parameters('authType'), 'rbac')]",
85908659
"type": "Microsoft.Resources/deployments",
85918660
"apiVersion": "2022-09-01",
8592-
"name": "storage-role-function",
8661+
"name": "storage-blob-role-function",
85938662
"properties": {
85948663
"expressionEvaluationOptions": {
85958664
"scope": "inner"
@@ -8654,6 +8723,75 @@
86548723
"[resourceId('Microsoft.Resources/deployments', format('{0}-app-module', parameters('name')))]"
86558724
]
86568725
},
8726+
{
8727+
"condition": "[equals(parameters('authType'), 'rbac')]",
8728+
"type": "Microsoft.Resources/deployments",
8729+
"apiVersion": "2022-09-01",
8730+
"name": "storage-queue-role-function",
8731+
"properties": {
8732+
"expressionEvaluationOptions": {
8733+
"scope": "inner"
8734+
},
8735+
"mode": "Incremental",
8736+
"parameters": {
8737+
"principalId": {
8738+
"value": "[reference(resourceId('Microsoft.Resources/deployments', format('{0}-app-module', parameters('name'))), '2022-09-01').outputs.identityPrincipalId.value]"
8739+
},
8740+
"roleDefinitionId": {
8741+
"value": "974c5e8b-45b9-4653-ba55-5f855dd0fb88"
8742+
},
8743+
"principalType": {
8744+
"value": "ServicePrincipal"
8745+
}
8746+
},
8747+
"template": {
8748+
"$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
8749+
"contentVersion": "1.0.0.0",
8750+
"metadata": {
8751+
"_generator": {
8752+
"name": "bicep",
8753+
"version": "0.26.54.24096",
8754+
"templateHash": "5795525499710207356"
8755+
},
8756+
"description": "Creates a role assignment for a service principal."
8757+
},
8758+
"parameters": {
8759+
"principalId": {
8760+
"type": "string"
8761+
},
8762+
"principalType": {
8763+
"type": "string",
8764+
"defaultValue": "ServicePrincipal",
8765+
"allowedValues": [
8766+
"Device",
8767+
"ForeignGroup",
8768+
"Group",
8769+
"ServicePrincipal",
8770+
"User"
8771+
]
8772+
},
8773+
"roleDefinitionId": {
8774+
"type": "string"
8775+
}
8776+
},
8777+
"resources": [
8778+
{
8779+
"type": "Microsoft.Authorization/roleAssignments",
8780+
"apiVersion": "2022-04-01",
8781+
"name": "[guid(subscription().id, resourceGroup().id, parameters('principalId'), parameters('roleDefinitionId'))]",
8782+
"properties": {
8783+
"principalId": "[parameters('principalId')]",
8784+
"principalType": "[parameters('principalType')]",
8785+
"roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', parameters('roleDefinitionId'))]"
8786+
}
8787+
}
8788+
]
8789+
}
8790+
},
8791+
"dependsOn": [
8792+
"[resourceId('Microsoft.Resources/deployments', format('{0}-app-module', parameters('name')))]"
8793+
]
8794+
},
86578795
{
86588796
"condition": "[parameters('useKeyVault')]",
86598797
"type": "Microsoft.Resources/deployments",

0 commit comments

Comments
 (0)