TRACING-6127: feat: add span_kind filter, show p95 latency instead of avg operation duration (#1044)

* feat: add span_kind filter, show p95 latency instead of avg operation duration

* add Span Kind variable to filter by span kind (defaults to
SPAN_KIND_SERVER to avoid double-counting)
* rename "Duration" to "Latency", and use P95 histogram quantile
  instead of average operation duration
* update error rate unit

Signed-off-by: Andreas Gerstmayr <agerstmayr@redhat.com>

* revert error rate unit

Signed-off-by: Andreas Gerstmayr <agerstmayr@redhat.com>

---------

Signed-off-by: Andreas Gerstmayr <agerstmayr@redhat.com>

Author: andreasgerstmayr

pkg/controllers/uiplugin/apm.go

@@ -40,7 +40,7 @@ func withServiceMetrics(variableMatchers string) dashboard.Option {
 				),
 			),
 		),
-		panelgroup.AddPanel("Duration",
+		panelgroup.AddPanel("Latency",
 			timeseries.Chart(
 				timeseries.WithYAxis(timeseries.YAxis{
 					Format: &common.Format{
@@ -108,7 +108,7 @@ func withOperationMetrics(variableMatchers string) dashboard.Option {
 					},
 					{
 						Name:   "value #3",
-						Header: "Duration",
+						Header: "P95 Latency",
 						Format: &common.Format{
 							Unit:          ptr.To(string(common.MilliSecondsUnit)),
 							DecimalPlaces: 3,
@@ -123,7 +123,7 @@ func withOperationMetrics(variableMatchers string) dashboard.Option {
 			panel.AddQuery(
 				query.PromQL(
 					fmt.Sprintf(`sum(rate({__name__=~"traces_span_metrics_calls(_total)?", %s}[$__rate_interval])) by (span_name) > 0`, variableMatchers),
-					query.SeriesNameFormat("req/s"),
+					query.SeriesNameFormat("Request rate"),
 				),
 			),
 			panel.AddQuery(
@@ -134,16 +134,16 @@ func withOperationMetrics(variableMatchers string) dashboard.Option {
 			),
 			panel.AddQuery(
 				query.PromQL(
-					fmt.Sprintf(`sum(rate({__name__=~"traces_span_metrics_duration(_milliseconds)?_sum", %s}[$__rate_interval])) by (span_name) / sum(rate({__name__=~"traces_span_metrics_duration(_milliseconds)?_count", %s}[$__rate_interval])) by (span_name) > 0`, variableMatchers, variableMatchers),
-					query.SeriesNameFormat("Duration"),
+					fmt.Sprintf(`histogram_quantile(.95, sum(rate({__name__=~"traces_span_metrics_duration(_milliseconds)?_bucket", %s}[$__rate_interval])) by (span_name, le)) > 0`, variableMatchers),
+					query.SeriesNameFormat("P95 Latency"),
 				),
 			),
 		),
 	)
 }
 
 func buildAPMDashboard() (dashboard.Builder, error) {
-	variableMatchers := `namespace="$namespace", service="$collector", service_name="$service"`
+	variableMatchers := `namespace="$namespace", service="$collector", service_name="$service", span_kind=~"${span_kind}"`
 
 	return dashboard.New("apm",
 		dashboard.Name("Application Performance Monitoring (APM)"),
@@ -171,6 +171,20 @@ func buildAPMDashboard() (dashboard.Builder, error) {
 				),
 			),
 		),
+		dashboard.AddVariable("span_kind",
+			listvariable.List(
+				listvariable.DisplayName("Span Kind"),
+				// Filter by SPAN_KIND_SERVER by default to avoid double-counting requests when both the caller and callee are instrumented,
+				// as each side generates its own span (CLIENT and SERVER).
+				listvariable.DefaultValue("SPAN_KIND_SERVER"),
+				listvariable.AllowMultiple(true),
+				listvariable.AllowAllValue(true),
+				listvariable.CustomAllValue(".*"),
+				labelvalues.PrometheusLabelValues("span_kind",
+					labelvalues.Matchers(`{__name__=~"traces_span_metrics_calls(_total)?"}`),
+				),
+			),
+		),
 		withServiceMetrics(variableMatchers),
 		withOperationMetrics(variableMatchers),
 	)