Skip to content

Commit cdf98fc

Browse files
AjitPadhi-MicrosoftAjitPadhi-Microsoft
committed
Fixed telemetary logging
1 parent b2b65e2 commit cdf98fc

16 files changed

Lines changed: 2811 additions & 1923 deletions

File tree

‎infra/main.bicep‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1009,6 +1009,10 @@ module avmContainerApp 'br/public:avm/res/app/container-app:0.19.0' = {
10091009
name: 'APPLICATIONINSIGHTS_CONNECTION_STRING'
10101010
value: enableMonitoring ? applicationInsights.outputs.connectionString : ''
10111011
}
1012+
{
1013+
name: 'OTEL_SERVICE_NAME'
1014+
value: 'ContentProcessor'
1015+
}
10121016
]
10131017
}
10141018
]
@@ -1073,6 +1077,10 @@ module avmContainerApp_API 'br/public:avm/res/app/container-app:0.19.0' = {
10731077
name: 'APPLICATIONINSIGHTS_CONNECTION_STRING'
10741078
value: enableMonitoring ? applicationInsights.outputs.connectionString : ''
10751079
}
1080+
{
1081+
name: 'OTEL_SERVICE_NAME'
1082+
value: 'ContentProcessorAPI'
1083+
}
10761084
]
10771085
probes: [
10781086
// Liveness Probe - Checks if the app is still running
@@ -1282,6 +1290,10 @@ module avmContainerApp_Workflow 'br/public:avm/res/app/container-app:0.19.0' = {
12821290
name: 'APPLICATIONINSIGHTS_CONNECTION_STRING'
12831291
value: enableMonitoring ? applicationInsights.outputs.connectionString : ''
12841292
}
1293+
{
1294+
name: 'OTEL_SERVICE_NAME'
1295+
value: 'ContentProcessorWorkflow'
1296+
}
12851297
]
12861298
}
12871299
]
@@ -1658,6 +1670,10 @@ module avmContainerApp_update 'br/public:avm/res/app/container-app:0.19.0' = {
16581670
name: 'APPLICATIONINSIGHTS_CONNECTION_STRING'
16591671
value: enableMonitoring ? applicationInsights.outputs.connectionString : ''
16601672
}
1673+
{
1674+
name: 'OTEL_SERVICE_NAME'
1675+
value: 'ContentProcessor'
1676+
}
16611677
]
16621678
}
16631679
]
@@ -1737,6 +1753,10 @@ module avmContainerApp_API_update 'br/public:avm/res/app/container-app:0.19.0' =
17371753
name: 'APPLICATIONINSIGHTS_CONNECTION_STRING'
17381754
value: enableMonitoring ? applicationInsights.outputs.connectionString : ''
17391755
}
1756+
{
1757+
name: 'OTEL_SERVICE_NAME'
1758+
value: 'ContentProcessorAPI'
1759+
}
17401760
]
17411761
probes: [
17421762
// Liveness Probe - Checks if the app is still running
@@ -1867,6 +1887,10 @@ module avmContainerApp_Workflow_update 'br/public:avm/res/app/container-app:0.19
18671887
name: 'APPLICATIONINSIGHTS_CONNECTION_STRING'
18681888
value: enableMonitoring ? applicationInsights.outputs.connectionString : ''
18691889
}
1890+
{
1891+
name: 'OTEL_SERVICE_NAME'
1892+
value: 'ContentProcessorWorkflow'
1893+
}
18701894
]
18711895
}
18721896
]

‎infra/main.json‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"_generator": {
77
"name": "bicep",
88
"version": "0.41.2.15936",
9-
"templateHash": "5358772599129171911"
9+
"templateHash": "5718783598601616074"
1010
},
1111
"name": "Content Processing Solution Accelerator",
1212
"description": "Bicep template to deploy the Content Processing Solution Accelerator with AVM compliance."
@@ -35187,8 +35187,8 @@
3518735187
"avmContainerApp_API",
3518835188
"avmContainerApp_Workflow",
3518935189
"avmManagedIdentity",
35190-
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageQueue)]",
3519135190
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageBlob)]",
35191+
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').storageQueue)]",
3519235192
"virtualNetwork"
3519335193
]
3519435194
},
@@ -45022,9 +45022,9 @@
4502245022
},
4502345023
"dependsOn": [
4502445024
"avmAiServices_cu",
45025+
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').contentUnderstanding)]",
4502545026
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').aiServices)]",
4502645027
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').cognitiveServices)]",
45027-
"[format('avmPrivateDnsZones[{0}]', variables('dnsZoneIndex').contentUnderstanding)]",
4502845028
"virtualNetwork"
4502945029
]
4503045030
},
@@ -46521,6 +46521,14 @@
4652146521
{
4652246522
"name": "AZURE_LOGGING_PACKAGES",
4652346523
"value": ""
46524+
},
46525+
{
46526+
"name": "APPLICATIONINSIGHTS_CONNECTION_STRING",
46527+
"value": "[if(parameters('enableMonitoring'), reference('applicationInsights').outputs.connectionString.value, '')]"
46528+
},
46529+
{
46530+
"name": "OTEL_SERVICE_NAME",
46531+
"value": "ContentProcessor"
4652446532
}
4652546533
]
4652646534
}
@@ -48061,6 +48069,7 @@
4806148069
}
4806248070
},
4806348071
"dependsOn": [
48072+
"applicationInsights",
4806448073
"avmContainerAppEnv",
4806548074
"avmContainerRegistryReader"
4806648075
]
@@ -48133,6 +48142,14 @@
4813348142
{
4813448143
"name": "AZURE_LOGGING_PACKAGES",
4813548144
"value": ""
48145+
},
48146+
{
48147+
"name": "APPLICATIONINSIGHTS_CONNECTION_STRING",
48148+
"value": "[if(parameters('enableMonitoring'), reference('applicationInsights').outputs.connectionString.value, '')]"
48149+
},
48150+
{
48151+
"name": "OTEL_SERVICE_NAME",
48152+
"value": "ContentProcessorAPI"
4813648153
}
4813748154
],
4813848155
"probes": [
@@ -49734,6 +49751,7 @@
4973449751
}
4973549752
},
4973649753
"dependsOn": [
49754+
"applicationInsights",
4973749755
"avmContainerAppEnv",
4973849756
"avmContainerRegistryReader"
4973949757
]
@@ -51441,6 +51459,14 @@
5144151459
{
5144251460
"name": "AZURE_LOGGING_PACKAGES",
5144351461
"value": ""
51462+
},
51463+
{
51464+
"name": "APPLICATIONINSIGHTS_CONNECTION_STRING",
51465+
"value": "[if(parameters('enableMonitoring'), reference('applicationInsights').outputs.connectionString.value, '')]"
51466+
},
51467+
{
51468+
"name": "OTEL_SERVICE_NAME",
51469+
"value": "ContentProcessorWorkflow"
5144451470
}
5144551471
]
5144651472
}
@@ -52978,6 +53004,7 @@
5297853004
}
5297953005
},
5298053006
"dependsOn": [
53007+
"applicationInsights",
5298153008
"avmContainerAppEnv",
5298253009
"avmContainerRegistryReader"
5298353010
]
@@ -63656,6 +63683,14 @@
6365663683
{
6365763684
"name": "AZURE_LOGGING_PACKAGES",
6365863685
"value": ""
63686+
},
63687+
{
63688+
"name": "APPLICATIONINSIGHTS_CONNECTION_STRING",
63689+
"value": "[if(parameters('enableMonitoring'), reference('applicationInsights').outputs.connectionString.value, '')]"
63690+
},
63691+
{
63692+
"name": "OTEL_SERVICE_NAME",
63693+
"value": "ContentProcessor"
6365963694
}
6366063695
]
6366163696
}
@@ -65194,6 +65229,7 @@
6519465229
}
6519565230
},
6519665231
"dependsOn": [
65232+
"applicationInsights",
6519765233
"avmAppConfig",
6519865234
"avmContainerAppEnv",
6519965235
"avmContainerRegistryReader",
@@ -65269,6 +65305,14 @@
6526965305
{
6527065306
"name": "AZURE_LOGGING_PACKAGES",
6527165307
"value": ""
65308+
},
65309+
{
65310+
"name": "APPLICATIONINSIGHTS_CONNECTION_STRING",
65311+
"value": "[if(parameters('enableMonitoring'), reference('applicationInsights').outputs.connectionString.value, '')]"
65312+
},
65313+
{
65314+
"name": "OTEL_SERVICE_NAME",
65315+
"value": "ContentProcessorAPI"
6527265316
}
6527365317
],
6527465318
"probes": [
@@ -66870,6 +66914,7 @@
6687066914
}
6687166915
},
6687266916
"dependsOn": [
66917+
"applicationInsights",
6687366918
"avmAppConfig",
6687466919
"avmContainerAppEnv",
6687566920
"avmContainerRegistryReader",
@@ -66944,6 +66989,14 @@
6694466989
{
6694566990
"name": "AZURE_LOGGING_PACKAGES",
6694666991
"value": ""
66992+
},
66993+
{
66994+
"name": "APPLICATIONINSIGHTS_CONNECTION_STRING",
66995+
"value": "[if(parameters('enableMonitoring'), reference('applicationInsights').outputs.connectionString.value, '')]"
66996+
},
66997+
{
66998+
"name": "OTEL_SERVICE_NAME",
66999+
"value": "ContentProcessorWorkflow"
6694767000
}
6694867001
]
6694967002
}
@@ -68481,6 +68534,7 @@
6848168534
}
6848268535
},
6848368536
"dependsOn": [
68537+
"applicationInsights",
6848468538
"avmAppConfig",
6848568539
"avmContainerAppEnv",
6848668540
"avmContainerRegistryReader"
@@ -68558,6 +68612,13 @@
6855868612
},
6855968613
"value": "[reference('avmContainerRegistry').outputs.loginServer.value]"
6856068614
},
68615+
"CONTENT_UNDERSTANDING_ACCOUNT_NAME": {
68616+
"type": "string",
68617+
"metadata": {
68618+
"description": "The name of the Content Understanding AI Services account."
68619+
},
68620+
"value": "[reference('avmAiServices_cu').outputs.name.value]"
68621+
},
6856168622
"AZURE_RESOURCE_GROUP": {
6856268623
"type": "string",
6856368624
"metadata": {

‎src/ContentProcessor/pyproject.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@ description = "Content Process Gold Standard Solution Accelerator - Content Proc
55
readme = "README.md"
66
requires-python = ">=3.12"
77
dependencies = [
8-
"agent-framework>=1.0.0b260127",
8+
"agent-framework==1.0.0b260127",
99
"azure-ai-inference>=1.0.0b9",
1010
"azure-appconfiguration>=1.7.2",
1111
"azure-identity>=1.25.1",
12+
"azure-monitor-opentelemetry>=1.8.7",
1213
"azure-storage-blob>=12.28.0",
1314
"azure-storage-queue>=12.15.0",
1415
"certifi>=2026.1.4",

‎src/ContentProcessor/src/libs/application/application_configuration.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ class AppConfiguration(_configuration_base):
7474
app_cosmos_database: str
7575
app_cosmos_container_process: str
7676
app_cosmos_container_schema: str
77+
applicationinsights_connection_string: str = ""
7778

7879
@field_validator("app_process_steps", mode="before")
7980
@classmethod

‎src/ContentProcessor/src/libs/pipeline/queue_handler_base.py‎

Lines changed: 52 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
from abc import ABC, abstractmethod
1616

1717
from azure.storage.queue import QueueClient
18+
from opentelemetry import trace
1819
from libs.application.application_context import AppContext
1920
from libs.base.application_models import AppModelBase
2021
from libs.models.content_process import ContentProcess, Step_Outputs
@@ -122,13 +123,28 @@ async def _connect_async(
122123

123124
self._current_message_context.data_pipeline.pipeline_status.active_step = self.handler_name
124125

126+
process_id = self._current_message_context.data_pipeline.pipeline_status.process_id
127+
document_name = self._current_message_context.data_pipeline.files[0].name
128+
129+
# Add process_id and document tracking to the current span
130+
current_span = trace.get_current_span()
131+
if current_span.is_recording():
132+
current_span.set_attribute("process_id", process_id)
133+
current_span.set_attribute("document_name", document_name)
134+
current_span.set_attribute("pipeline_stage", self.handler_name)
135+
136+
logging.info(
137+
"Pipeline stage started: process_id=%s, document=%s, stage=%s",
138+
process_id,
139+
document_name,
140+
self.handler_name,
141+
)
142+
125143
# Update status to the currently running step BEFORE execution
126144
# so the UI reflects real-time progress.
127145
ContentProcess(
128-
process_id=self._current_message_context.data_pipeline.pipeline_status.process_id,
129-
processed_file_name=self._current_message_context.data_pipeline.files[
130-
0
131-
].name,
146+
process_id=process_id,
147+
processed_file_name=document_name,
132148
processed_file_mime_type=self._current_message_context.data_pipeline.files[
133149
0
134150
].mime_type,
@@ -148,13 +164,30 @@ async def _connect_async(
148164
print(
149165
f"Start Processing : {self.handler_name}"
150166
) if show_information else None
151-
with stopwatch.Stopwatch() as timer:
152-
step_result = await self.execute(
153-
self._current_message_context
154-
)
167+
tracer = trace.get_tracer(__name__)
168+
with tracer.start_as_current_span(
169+
f"pipeline.{self.handler_name}",
170+
attributes={
171+
"process_id": process_id,
172+
"document_name": document_name,
173+
"pipeline_stage": self.handler_name,
174+
},
175+
):
176+
with stopwatch.Stopwatch() as timer:
177+
step_result = await self.execute(
178+
self._current_message_context
179+
)
155180
print(
156181
f"Completed : {self.handler_name} - Elapsed :{timer.elapsed_string}"
157182
) if show_information else None
183+
184+
logging.info(
185+
"Pipeline stage completed: process_id=%s, document=%s, stage=%s, elapsed=%s",
186+
process_id,
187+
document_name,
188+
self.handler_name,
189+
timer.elapsed_string,
190+
)
158191
step_result.elapsed = timer.elapsed_string
159192

160193
step_result.save_to_persistent_storage(
@@ -208,7 +241,17 @@ async def _connect_async(
208241
logging.error("Message is not a valid model.")
209242
self._move_to_dead_letter_queue(queue_message)
210243
except Exception as e:
211-
logging.error(f"Error Occurred: {e}")
244+
logging.error(
245+
"Pipeline error: process_id=%s, stage=%s, error=%s",
246+
data_pipeline.pipeline_status.process_id if data_pipeline else "unknown",
247+
self.handler_name,
248+
e,
249+
)
250+
error_span = trace.get_current_span()
251+
if error_span.is_recording():
252+
error_span.set_attribute("process_id", data_pipeline.pipeline_status.process_id if data_pipeline else "unknown")
253+
error_span.set_attribute("pipeline_stage", self.handler_name)
254+
error_span.set_attribute("error", True)
212255

213256
def _get_artifact_type(step_name: str) -> ArtifactType:
214257
if step_name == "extract":

0 commit comments

Comments
 (0)