Skip to content

Commit 468fed9

Browse files
MarcosNicolauJuArceentropidelic
authored
feat: broken sockets metric in batcher (#1255)
Co-authored-by: Julian Arce <52429267+JuArce@users.noreply.github.com> Co-authored-by: Mariano Nicolini <mariano.nicolini.91@gmail.com>
1 parent 2c11cf1 commit 468fed9

3 files changed

Lines changed: 130 additions & 37 deletions

File tree

‎batcher/aligned-batcher/src/lib.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,10 @@ impl Batcher {
332332
.try_for_each(|msg| self.clone().handle_message(msg, outgoing.clone()))
333333
.await
334334
{
335-
Err(e) => error!("Unexpected error: {}", e),
335+
Err(e) => {
336+
self.metrics.broken_ws_connections.inc();
337+
error!("Unexpected error: {}", e)
338+
}
336339
Ok(_) => info!("{} disconnected", &addr),
337340
}
338341

‎batcher/aligned-batcher/src/metrics.rs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ pub struct BatcherMetrics {
1313
pub reverted_batches: IntCounter,
1414
pub batcher_started: IntCounter,
1515
pub gas_price_used_on_latest_batch: IntGauge,
16+
pub broken_ws_connections: IntCounter,
1617
}
1718

1819
impl BatcherMetrics {
@@ -27,12 +28,18 @@ impl BatcherMetrics {
2728
let batcher_started = register_int_counter!(opts!("batcher_started", "Batcher Started"))?;
2829
let gas_price_used_on_latest_batch =
2930
register_int_gauge!(opts!("gas_price_used_on_latest_batch", "Gas Price"))?;
31+
let broken_ws_connections = register_int_counter!(opts!(
32+
"broken_ws_connections",
33+
"Broken websocket connections"
34+
))?;
3035

3136
registry.register(Box::new(open_connections.clone()))?;
3237
registry.register(Box::new(received_proofs.clone()))?;
3338
registry.register(Box::new(sent_batches.clone()))?;
3439
registry.register(Box::new(reverted_batches.clone()))?;
40+
registry.register(Box::new(gas_price_used_on_latest_batch.clone()))?;
3541
registry.register(Box::new(batcher_started.clone()))?;
42+
registry.register(Box::new(broken_ws_connections.clone()))?;
3643

3744
let metrics_route = warp::path!("metrics")
3845
.and(warp::any().map(move || registry.clone()))
@@ -51,6 +58,7 @@ impl BatcherMetrics {
5158
reverted_batches,
5259
batcher_started,
5360
gas_price_used_on_latest_batch,
61+
broken_ws_connections,
5462
})
5563
}
5664

‎grafana/provisioning/dashboards/aligned/aggregator_batcher.json‎

Lines changed: 118 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
"editable": true,
1919
"fiscalYearStartMonth": 0,
2020
"graphTooltip": 0,
21-
"id": 4,
21+
"id": 2,
2222
"links": [],
2323
"liveNow": false,
2424
"panels": [
@@ -71,9 +71,7 @@
7171
"justifyMode": "auto",
7272
"orientation": "auto",
7373
"reduceOptions": {
74-
"calcs": [
75-
"lastNotNull"
76-
],
74+
"calcs": ["lastNotNull"],
7775
"fields": "",
7876
"values": false
7977
},
@@ -138,9 +136,7 @@
138136
"options": {
139137
"orientation": "auto",
140138
"reduceOptions": {
141-
"calcs": [
142-
"lastNotNull"
143-
],
139+
"calcs": ["lastNotNull"],
144140
"fields": "",
145141
"values": false
146142
},
@@ -312,9 +308,7 @@
312308
"justifyMode": "auto",
313309
"orientation": "auto",
314310
"reduceOptions": {
315-
"calcs": [
316-
"lastNotNull"
317-
],
311+
"calcs": ["lastNotNull"],
318312
"fields": "",
319313
"values": false
320314
},
@@ -582,9 +576,7 @@
582576
"justifyMode": "auto",
583577
"orientation": "auto",
584578
"reduceOptions": {
585-
"calcs": [
586-
"lastNotNull"
587-
],
579+
"calcs": ["lastNotNull"],
588580
"fields": "",
589581
"values": false
590582
},
@@ -710,13 +702,113 @@
710702
"title": "Batches Reverted",
711703
"type": "timeseries"
712704
},
705+
{
706+
"datasource": {
707+
"type": "prometheus",
708+
"uid": "prometheus"
709+
},
710+
"description": "Measures websocket connections that were abnormally disconnected.",
711+
"fieldConfig": {
712+
"defaults": {
713+
"color": {
714+
"mode": "palette-classic"
715+
},
716+
"custom": {
717+
"axisCenteredZero": false,
718+
"axisColorMode": "text",
719+
"axisLabel": "",
720+
"axisPlacement": "auto",
721+
"barAlignment": 0,
722+
"drawStyle": "line",
723+
"fillOpacity": 0,
724+
"gradientMode": "none",
725+
"hideFrom": {
726+
"legend": false,
727+
"tooltip": false,
728+
"viz": false
729+
},
730+
"insertNulls": false,
731+
"lineInterpolation": "linear",
732+
"lineWidth": 1,
733+
"pointSize": 5,
734+
"scaleDistribution": {
735+
"type": "linear"
736+
},
737+
"showPoints": "auto",
738+
"spanNulls": false,
739+
"stacking": {
740+
"group": "A",
741+
"mode": "none"
742+
},
743+
"thresholdsStyle": {
744+
"mode": "off"
745+
}
746+
},
747+
"mappings": [],
748+
"thresholds": {
749+
"mode": "absolute",
750+
"steps": [
751+
{
752+
"color": "green",
753+
"value": null
754+
},
755+
{
756+
"color": "red",
757+
"value": 80
758+
}
759+
]
760+
}
761+
},
762+
"overrides": []
763+
},
764+
"gridPos": {
765+
"h": 8,
766+
"w": 12,
767+
"x": 0,
768+
"y": 19
769+
},
770+
"id": 20,
771+
"interval": "1m",
772+
"options": {
773+
"legend": {
774+
"calcs": [],
775+
"displayMode": "list",
776+
"placement": "bottom",
777+
"showLegend": true
778+
},
779+
"tooltip": {
780+
"mode": "single",
781+
"sort": "none"
782+
}
783+
},
784+
"targets": [
785+
{
786+
"datasource": {
787+
"type": "prometheus",
788+
"uid": "prometheus"
789+
},
790+
"disableTextWrap": false,
791+
"editorMode": "builder",
792+
"expr": "broken_ws_connections{job=\"aligned-batcher\"}",
793+
"fullMetaSearch": false,
794+
"includeNullMetadata": true,
795+
"instant": false,
796+
"legendFormat": "__auto",
797+
"range": true,
798+
"refId": "A",
799+
"useBackend": false
800+
}
801+
],
802+
"title": "Broken websocket connections",
803+
"type": "timeseries"
804+
},
713805
{
714806
"collapsed": true,
715807
"gridPos": {
716808
"h": 1,
717809
"w": 24,
718810
"x": 0,
719-
"y": 19
811+
"y": 27
720812
},
721813
"id": 10,
722814
"panels": [
@@ -758,7 +850,7 @@
758850
"h": 7,
759851
"w": 10,
760852
"x": 0,
761-
"y": 34
853+
"y": 42
762854
},
763855
"id": 9,
764856
"options": {
@@ -767,9 +859,7 @@
767859
"justifyMode": "auto",
768860
"orientation": "auto",
769861
"reduceOptions": {
770-
"calcs": [
771-
"lastNotNull"
772-
],
862+
"calcs": ["lastNotNull"],
773863
"fields": "",
774864
"values": false
775865
},
@@ -910,7 +1000,7 @@
9101000
"h": 7,
9111001
"w": 10,
9121002
"x": 10,
913-
"y": 34
1003+
"y": 42
9141004
},
9151005
"id": 1,
9161006
"options": {
@@ -974,7 +1064,7 @@
9741064
"h": 7,
9751065
"w": 5,
9761066
"x": 0,
977-
"y": 41
1067+
"y": 49
9781068
},
9791069
"id": 8,
9801070
"options": {
@@ -983,9 +1073,7 @@
9831073
"justifyMode": "auto",
9841074
"orientation": "auto",
9851075
"reduceOptions": {
986-
"calcs": [
987-
"lastNotNull"
988-
],
1076+
"calcs": ["lastNotNull"],
9891077
"fields": "",
9901078
"values": false
9911079
},
@@ -1044,7 +1132,7 @@
10441132
"h": 7,
10451133
"w": 5,
10461134
"x": 5,
1047-
"y": 41
1135+
"y": 49
10481136
},
10491137
"id": 7,
10501138
"options": {
@@ -1053,9 +1141,7 @@
10531141
"justifyMode": "auto",
10541142
"orientation": "auto",
10551143
"reduceOptions": {
1056-
"calcs": [
1057-
"lastNotNull"
1058-
],
1144+
"calcs": ["lastNotNull"],
10591145
"fields": "",
10601146
"values": false
10611147
},
@@ -1109,7 +1195,7 @@
11091195
"h": 7,
11101196
"w": 5,
11111197
"x": 0,
1112-
"y": 48
1198+
"y": 56
11131199
},
11141200
"id": 2,
11151201
"options": {
@@ -1118,9 +1204,7 @@
11181204
"justifyMode": "auto",
11191205
"orientation": "auto",
11201206
"reduceOptions": {
1121-
"calcs": [
1122-
"lastNotNull"
1123-
],
1207+
"calcs": ["lastNotNull"],
11241208
"fields": "",
11251209
"values": false
11261210
},
@@ -1179,7 +1263,7 @@
11791263
"h": 7,
11801264
"w": 5,
11811265
"x": 5,
1182-
"y": 48
1266+
"y": 56
11831267
},
11841268
"id": 5,
11851269
"options": {
@@ -1188,9 +1272,7 @@
11881272
"justifyMode": "auto",
11891273
"orientation": "auto",
11901274
"reduceOptions": {
1191-
"calcs": [
1192-
"lastNotNull"
1193-
],
1275+
"calcs": ["lastNotNull"],
11941276
"fields": "",
11951277
"values": false
11961278
},
@@ -1241,6 +1323,6 @@
12411323
"timezone": "browser",
12421324
"title": "Aggregator Data",
12431325
"uid": "aggregator",
1244-
"version": 10,
1326+
"version": 6,
12451327
"weekStart": ""
12461328
}

0 commit comments

Comments
 (0)