Add example of prometheus exemplars being scraped from a flask app (#245)

Author: aabmass

docs/conf.py

@@ -59,6 +59,8 @@
     "sphinx.ext.githubpages",
     # Support external links to different versions in the Github repo
     "sphinx.ext.extlinks",
+    # Rendered graphviz graphs
+    "sphinx.ext.graphviz",
 ]
 
 intersphinx_mapping = {
@@ -101,9 +103,11 @@
 }
 
 html_context = {
-    "display_github": True, # Integrate GitHub
-    "github_user": "GoogleCloudPlatform", # Username
-    "github_repo": "opentelemetry-operations-python", # Repo name
-    "github_version": "main", # Version
-    "conf_py_path": "/docs/", # Path in the checkout to the docs root
+    "display_github": True,  # Integrate GitHub
+    "github_user": "GoogleCloudPlatform",  # Username
+    "github_repo": "opentelemetry-operations-python",  # Repo name
+    "github_version": "main",  # Version
+    "conf_py_path": "/docs/",  # Path in the checkout to the docs root
 }
+
+graphviz_output_format = "svg"

docs/examples/prometheus_exemplars/README.rst

@@ -0,0 +1,127 @@
+======================================================
+Prometheus Metric Exemplars with OpenTelemetry Tracing
+======================================================
+
+The full code for this example is available `on Github
+<https://github.com/GoogleCloudPlatform/opentelemetry-operations-python/tree/main/docs/examples/prometheus_exemplars>`_.
+
+This end-to-end example shows how to instrument a Flask app with with `Prometheus
+<https://prometheus.io/>`_ metrics linked to OpenTelemetry traces using exemplars. The example
+manually adds exemplars to a Prometheus Histogram which link the metric in Google Cloud managed
+service for Prometheus to Spans in Cloud Trace.
+
+OpenTelemetry Python is configured to send traces to the `OpenTelemetry Collector
+<https://opentelemetry.io/docs/collector/>`_ and the Collector scrapes the python server's
+Prometheus endpoint. The Collector is configured to send metrics to `Google Cloud Managed
+Service for Prometheus <https://cloud.google.com/stackdriver/docs/managed-prometheus>`_ and
+traces to `Google Cloud Trace <https://cloud.google.com/trace/docs/overview>`_.
+
+.. graphviz::
+
+    digraph {
+        rankdir="LR"
+        nodesep=1
+
+        subgraph cluster {
+            server [label="Flask Application"]
+            col [label="OpenTelemetry Collector"]
+        }
+
+        gmp [label="Google Cloud Managed Service for Prometheus" shape="box"]
+        gct [label="Google Cloud Trace" shape="box"]
+
+        server->col [label="OTLP traces"]
+        col->server [label="Scrape Prometheus"]
+        col->gmp [label="metrics"]
+        col->gct [label="traces"]
+    }
+
+To run this example you first need to:
+    * Create a Google Cloud project. You can `create one here <https://console.cloud.google.com/projectcreate>`_.
+    * Enable Cloud Trace API (listed in the Cloud Console as Stackdriver Trace API) in the project `here <https://console.cloud.google.com/apis/library?q=cloud%20trace&filter=visibility:public>`_. If the page says "API Enabled" then you're done! No need to do anything.
+    * Enable Default Application Credentials by creating setting `GOOGLE_APPLICATION_CREDENTIALS <https://cloud.google.com/docs/authentication/getting-started>`_ or by `installing gcloud sdk <https://cloud.google.com/sdk/install>`_ and calling ``gcloud auth application-default login``.
+    * Have docker and docker compose installed on your machine
+
+Attaching Prometheus Exemplars
+------------------------------
+
+Prometheus exemplars can be linked to OpenTelemetry spans by setting the ``span_id`` and
+``trace_id`` attributes (`specification
+<https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/specification/compatibility/prometheus_and_openmetrics.md#exemplars>`_).
+You can get the current span using :func:`opentelemetry.trace.get_current_span`, then format
+its span and trace IDs to hexadecimal strings using :func:`opentelemetry.trace.format_span_id`
+and :func:`opentelemetry.trace.format_trace_id`.
+
+.. literalinclude:: server.py
+    :language: python
+    :dedent:
+    :start-after: [START opentelemetry_prom_exemplars_attach]
+    :end-before: [END opentelemetry_prom_exemplars_attach]
+
+Then make an observation using a `Prometheus Histogram
+<https://prometheus.io/docs/concepts/metric_types/#histogram>`_ as shown below. Google Cloud
+Monitoring can only display exemplars attached to Histograms.
+
+.. literalinclude:: server.py
+    :language: python
+    :dedent:
+    :start-after: [START opentelemetry_prom_exemplars_observe]
+    :end-before: [END opentelemetry_prom_exemplars_observe]
+
+Run
+---
+
+Checkout the example code if you don't already have the repository cloned:
+
+.. code-block:: sh
+
+  git clone https://github.com/GoogleCloudPlatform/opentelemetry-operations-python.git
+  cd docs/examples/prometheus_exemplars
+
+First, set the environment variables needed to provide authentication to the Collector when it
+runs in docker.
+
+.. code-block:: sh
+
+    export USERID=$(id -u)
+    export PROJECT_ID=<your-gcp-project>
+    export GOOGLE_APPLICATION_CREDENTIALS="${HOME}/.config/gcloud/application_default_credentials.json"
+
+Build and start the example containers using ``docker-compose``:
+
+.. code-block:: sh
+
+    docker-compose up --build --abort-on-container-exit
+
+This starts three containers:
+
+#. The Flask server written in ``server.py``. It receives requests and simulates some work by
+   waiting for a random amount of time.
+#. The OpenTelemetry Collector which receives traces from the Flask server by OTLP and scrapes
+   Prometheus metrics from the Flask server's ``/metrics`` endpoint.
+#. A load generator that sends constant requests to the Flask server.
+
+Checking Output
+---------------
+
+While running the example, you can go to `Cloud Monitoring Metrics Explorer page
+<https://console.cloud.google.com/monitoring/metrics-explorer>`_ to see the results. Click on
+the "Metric" dropdown, type ``my_prom_hist``, and select the metric from under "Prometheus
+Target". The full metric name is ``prometheus.googleapis.com/my_prom_hist_seconds/histogram``.
+
+.. image:: select_metric.png
+  :alt: Select the metric
+
+After selecting the metric, you should see something like the image below, with a heatmap
+showing the distribution of request durations in the Python server.
+
+.. image:: heatmap.png
+  :alt: Metrics explorer heatmap
+
+The circles on the heatmap are called "exemplars", which each link to an example span that fell
+within the given bucket on the heatmap. Notice that exemplars are plotted at the time when they
+occurred (x axis) and duration they took (y axis) in the heatmap. Clicking on an exemplar opens
+the Trace details flyout focused on the linked span.
+
+.. image:: trace_details.png
+  :alt: Trace details flyout

docs/examples/prometheus_exemplars/docker-compose.yaml

@@ -0,0 +1,44 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+services:
+  # OpenTelemetry collector. Make sure you set USERID and GOOGLE_APPLICATION_CREDENTIALS
+  # environment variables for your container to authenticate correctly
+  otel-collector:
+    user: ${USERID?set USERID=$(id -u)}
+    image: otel/opentelemetry-collector-contrib:0.77.0
+    environment:
+      - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS?}
+      - PROJECT_ID=${PROJECT_ID?}
+    volumes:
+      - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
+      - ${GOOGLE_APPLICATION_CREDENTIALS}:${GOOGLE_APPLICATION_CREDENTIALS}
+    ports:
+      - 4317:4317 # OTLP gRPC receiver
+
+  # runs server.py
+  python-server:
+    build:
+      context: .
+      dockerfile: server.dockerfile
+    init: true
+    ports:
+      - 6000:6000
+  
+  # load generator that sends requests to the python server
+  hey-loadgen:
+    build:
+      context: .
+      dockerfile: hey.dockerfile
+    command: -c 50 -n 100000000 http://python-server:6000

docs/examples/prometheus_exemplars/heatmap.png

@@ -0,0 +1,16 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+FROM golang:1.20-alpine
+RUN go install github.com/rakyll/hey@latest
+ENTRYPOINT ["hey"]

docs/examples/prometheus_exemplars/hey.dockerfile

@@ -0,0 +1,88 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+
+  # Data sources: metrics
+  prometheus:
+    config:
+      scrape_configs:
+        - job_name: python-server
+          scrape_interval: 5s
+          static_configs:
+            - targets:
+              - python-server:6000
+
+processors:
+  batch:
+
+  memory_limiter:
+    check_interval: 1s
+    limit_percentage: 65
+    spike_limit_percentage: 20
+
+  # Add required GMP attributes if they are missing
+  # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/googlemanagedprometheusexporter#manually-setting-location-cluster-or-namespace
+  resource:
+    attributes:
+    - key: "cluster"
+      value: "example-cluster"
+      action: upsert
+    - key: "namespace"
+      value: "example-apps"
+      action: upsert
+    - key: "location"
+      value: "us-east1"
+      action: upsert
+  
+  transform:
+    # "location", "cluster", "namespace", "job", "instance", and "project_id" are reserved, and
+    # metrics containing these labels will be rejected. Prefix them with exported_ to prevent this.
+    metric_statements:
+    - context: datapoint
+      statements:
+      - set(attributes["exported_location"], attributes["location"])
+      - delete_key(attributes, "location")
+      - set(attributes["exported_cluster"], attributes["cluster"])
+      - delete_key(attributes, "cluster")
+      - set(attributes["exported_namespace"], attributes["namespace"])
+      - delete_key(attributes, "namespace")
+      - set(attributes["exported_job"], attributes["job"])
+      - delete_key(attributes, "job")
+      - set(attributes["exported_instance"], attributes["instance"])
+      - delete_key(attributes, "instance")
+      - set(attributes["exported_project_id"], attributes["project_id"])
+      - delete_key(attributes, "project_id")
+
+exporters:
+  googlecloud:
+    project: ${PROJECT_ID}
+  
+  googlemanagedprometheus:
+    project: ${PROJECT_ID}
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp, prometheus]
+      processors: [memory_limiter, resource, transform, batch]
+      exporters: [googlemanagedprometheus]
+
+    traces:
+      receivers: [otlp]
+      processors: [memory_limiter, resource, batch]
+      exporters: [googlecloud]

docs/examples/prometheus_exemplars/otel-collector-config.yaml

@@ -0,0 +1,42 @@
+backoff==2.2.1
+cachetools==5.2.0
+certifi==2022.12.7
+charset-normalizer==2.1.1
+click==8.1.3
+Deprecated==1.2.13
+Flask==2.2.5
+google-api-core==2.10.2
+google-auth==2.14.1
+google-cloud-monitoring==2.11.3
+google-cloud-trace==1.7.3
+googleapis-common-protos==1.57.0
+grpcio==1.50.0
+grpcio-status==1.50.0
+idna==3.4
+importlib-metadata==6.0.1
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+opentelemetry-api==1.17.0
+opentelemetry-exporter-otlp-proto-grpc==1.17.0
+opentelemetry-instrumentation==0.38b0
+opentelemetry-instrumentation-flask==0.38b0
+opentelemetry-instrumentation-requests==0.38b0
+opentelemetry-instrumentation-wsgi==0.38b0
+opentelemetry-proto==1.17.0
+opentelemetry-sdk==1.17.0
+opentelemetry-semantic-conventions==0.38b0
+opentelemetry-util-http==0.38b0
+prometheus-client==0.16.0
+proto-plus==1.22.1
+protobuf==4.21.9
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+requests==2.28.1
+rsa==4.9
+six==1.16.0
+typing_extensions==4.4.0
+urllib3==1.26.12
+Werkzeug==2.2.3
+wrapt==1.14.1
+zipp==3.15.0

docs/examples/prometheus_exemplars/requirements.txt

@@ -0,0 +1,20 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM python:3.10-alpine
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+COPY server.py .
+ENV FLASK_APP=server.py 
+ENTRYPOINT ["flask", "run", "-h", "0.0.0.0", "-p", "6000"]