Always write enhanced metrics to logs and set DD_ENHANCED_METRICS=true by default

Author: swfunc

datadog_lambda/metric.py

@@ -4,7 +4,6 @@
 # Copyright 2019 Datadog, Inc.
 
 import os
-import sys
 import json
 import time
 import base64
@@ -13,8 +12,7 @@
 import boto3
 from datadog import api
 from datadog.threadstats import ThreadStats
-from datadog_lambda import __version__
-from datadog_lambda.tags import get_enhanced_metrics_tags
+from datadog_lambda.tags import get_enhanced_metrics_tags, tag_dd_lambda_layer
 
 
 ENHANCED_METRICS_NAMESPACE_PREFIX = "aws.lambda.enhanced"
@@ -25,25 +23,6 @@
 lambda_stats.start()
 
 
-def _format_dd_lambda_layer_tag():
-    """
-    Formats the dd_lambda_layer tag, e.g., 'dd_lambda_layer:datadog-python27_1'
-    """
-    runtime = "python{}{}".format(sys.version_info[0], sys.version_info[1])
-    return "dd_lambda_layer:datadog-{}_{}".format(runtime, __version__)
-
-
-def _tag_dd_lambda_layer(tags):
-    """
-    Used by lambda_metric to insert the dd_lambda_layer tag
-    """
-    dd_lambda_layer_tag = _format_dd_lambda_layer_tag()
-    if tags:
-        return tags + [dd_lambda_layer_tag]
-    else:
-        return [dd_lambda_layer_tag]
-
-
 def lambda_metric(metric_name, value, timestamp=None, tags=None):
     """
     Submit a data point to Datadog distribution metrics.
@@ -57,54 +36,79 @@ def lambda_metric(metric_name, value, timestamp=None, tags=None):
     periodically and at the end of the function execution in a
     background thread.
     """
-    tags = _tag_dd_lambda_layer(tags)
+    tags = tag_dd_lambda_layer(tags)
     if os.environ.get("DD_FLUSH_TO_LOG", "").lower() == "true":
-        logger.debug("Sending metric %s to Datadog via log forwarder", metric_name)
-        print(
-            json.dumps(
-                {
-                    "m": metric_name,
-                    "v": value,
-                    "e": timestamp or int(time.time()),
-                    "t": tags,
-                }
-            )
-        )
+        write_metric_point_to_stdout(metric_name, value, timestamp, tags)
     else:
         logger.debug("Sending metric %s to Datadog via lambda layer", metric_name)
         lambda_stats.distribution(metric_name, value, timestamp=timestamp, tags=tags)
 
 
+def write_metric_point_to_stdout(metric_name, value, timestamp=None, tags=[]):
+    """Writes the specified metric point to standard output
+    """
+    logger.debug(
+        "Sending metric %s value %s to Datadog via log forwarder", metric_name, value
+    )
+    print(
+        json.dumps(
+            {
+                "m": metric_name,
+                "v": value,
+                "e": timestamp or int(time.time()),
+                "t": tags,
+            }
+        )
+    )
+
+
 def are_enhanced_metrics_enabled():
     """Check env var to find if enhanced metrics should be submitted
+
+    Returns:
+        boolean for whether enhanced metrics are enabled
     """
-    return os.environ.get("DD_ENHANCED_METRICS", "false").lower() == "true"
+    # DD_ENHANCED_METRICS defaults to true
+    return os.environ.get("DD_ENHANCED_METRICS", "true").lower() == "true"
 
 
-def submit_invocations_metric(lambda_context):
-    """Increment aws.lambda.enhanced.invocations by 1
+def submit_enhanced_metric(metric_name, lambda_context):
+    """Submits the enhanced metric with the given name
+
+    Args:
+        metric_name (str): metric name w/o "aws.lambda.enhanced." prefix i.e. "invocations" or "errors"
+        lambda_context (dict): Lambda context dict passed to the function by AWS
     """
     if not are_enhanced_metrics_enabled():
+        logger.debug(
+            "Not submitting enhanced metric %s because enhanced metrics are disabled"
+        )
         return
 
-    lambda_metric(
-        "{}.invocations".format(ENHANCED_METRICS_NAMESPACE_PREFIX),
+    # Enhanced metrics are always written to logs
+    write_metric_point_to_stdout(
+        "{}.{}".format(ENHANCED_METRICS_NAMESPACE_PREFIX, metric_name),
         1,
         tags=get_enhanced_metrics_tags(lambda_context),
     )
 
 
-def submit_errors_metric(lambda_context):
-    """Increment aws.lambda.enhanced.errors by 1
+def submit_invocations_metric(lambda_context):
+    """Increment aws.lambda.enhanced.invocations by 1, applying runtime, layer, and cold_start tags
+
+    Args:
+        lambda_context (dict): Lambda context dict passed to the function by AWS
     """
-    if not are_enhanced_metrics_enabled():
-        return
+    submit_enhanced_metric("invocations", lambda_context)
 
-    lambda_metric(
-        "{}.errors".format(ENHANCED_METRICS_NAMESPACE_PREFIX),
-        1,
-        tags=get_enhanced_metrics_tags(lambda_context),
-    )
+
+def submit_errors_metric(lambda_context):
+    """Increment aws.lambda.enhanced.errors by 1, applying runtime, layer, and cold_start tags
+
+    Args:
+        lambda_context (dict): Lambda context dict passed to the function by AWS
+    """
+    submit_enhanced_metric("errors", lambda_context)
 
 
 # Set API Key and Host in the module, so they only set once per container

datadog_lambda/tags.py

@@ -1,8 +1,30 @@
+import sys
+
 from platform import python_version_tuple
 
+from datadog_lambda import __version__
 from datadog_lambda.cold_start import get_cold_start_tag
 
 
+def _format_dd_lambda_layer_tag():
+    """
+    Formats the dd_lambda_layer tag, e.g., 'dd_lambda_layer:datadog-python27_1'
+    """
+    runtime = "python{}{}".format(sys.version_info[0], sys.version_info[1])
+    return "dd_lambda_layer:datadog-{}_{}".format(runtime, __version__)
+
+
+def tag_dd_lambda_layer(tags):
+    """
+    Used by lambda_metric to insert the dd_lambda_layer tag
+    """
+    dd_lambda_layer_tag = _format_dd_lambda_layer_tag()
+    if tags:
+        return tags + [dd_lambda_layer_tag]
+    else:
+        return [dd_lambda_layer_tag]
+
+
 def parse_lambda_tags_from_arn(arn):
     """Generate the list of lambda tags based on the data in the arn
     Args:
@@ -42,4 +64,6 @@ def get_enhanced_metrics_tags(lambda_context):
         get_cold_start_tag(),
         "memorysize:{}".format(lambda_context.memory_limit_in_mb),
         get_runtime_tag(),
+        _format_dd_lambda_layer_tag(),
     ]
+

tests/test_metric.py

@@ -1,38 +1,38 @@
 import os
 import unittest
+
 try:
     from unittest.mock import patch, call
 except ImportError:
     from mock import patch, call
 
-from datadog_lambda.metric import (
-    lambda_metric,
-    _format_dd_lambda_layer_tag,
-)
+from datadog_lambda.metric import lambda_metric
+from datadog_lambda.tags import _format_dd_lambda_layer_tag
 
 
 class TestLambdaMetric(unittest.TestCase):
-
     def setUp(self):
-        patcher = patch('datadog_lambda.metric.lambda_stats')
+        patcher = patch("datadog_lambda.metric.lambda_stats")
         self.mock_metric_lambda_stats = patcher.start()
         self.addCleanup(patcher.stop)
 
     def test_lambda_metric_tagged_with_dd_lambda_layer(self):
-        lambda_metric('test', 1)
-        lambda_metric('test', 1, 123, [])
-        lambda_metric('test', 1, tags=['tag1:test'])
+        lambda_metric("test", 1)
+        lambda_metric("test", 1, 123, [])
+        lambda_metric("test", 1, tags=["tag1:test"])
         expected_tag = _format_dd_lambda_layer_tag()
-        self.mock_metric_lambda_stats.distribution.assert_has_calls([
-            call('test', 1, timestamp=None, tags=[expected_tag]),
-            call('test', 1, timestamp=123, tags=[expected_tag]),
-            call('test', 1, timestamp=None, tags=['tag1:test', expected_tag]),
-        ])
+        self.mock_metric_lambda_stats.distribution.assert_has_calls(
+            [
+                call("test", 1, timestamp=None, tags=[expected_tag]),
+                call("test", 1, timestamp=123, tags=[expected_tag]),
+                call("test", 1, timestamp=None, tags=["tag1:test", expected_tag]),
+            ]
+        )
 
     def test_lambda_metric_flush_to_log(self):
-        os.environ["DD_FLUSH_TO_LOG"] = 'True'
+        os.environ["DD_FLUSH_TO_LOG"] = "True"
 
-        lambda_metric('test', 1)
+        lambda_metric("test", 1)
         self.mock_metric_lambda_stats.distribution.assert_not_called()
 
         del os.environ["DD_FLUSH_TO_LOG"]

tests/test_wrapper.py

@@ -62,6 +62,18 @@ def setUp(self):
         self.mock_python_version_tuple.return_value = ("2", "7", "10")
         self.addCleanup(patcher.stop)
 
+        patcher = patch("datadog_lambda.metric.write_metric_point_to_stdout")
+        self.mock_write_metric_point_to_stdout = patcher.start()
+        self.addCleanup(patcher.stop)
+
+        patcher = patch("datadog_lambda.tags._format_dd_lambda_layer_tag")
+        self.mock_format_dd_lambda_layer_tag = patcher.start()
+        # Mock the layer version so we don't have to update tests on every version bump
+        self.mock_format_dd_lambda_layer_tag.return_value = (
+            "dd_lambda_layer:datadog-python27_0.1.0"
+        )
+        self.addCleanup(patcher.stop)
+
     def test_datadog_lambda_wrapper(self):
         @datadog_lambda_wrapper
         def lambda_handler(event, context):
@@ -111,8 +123,6 @@ def lambda_handler(event, context):
         del os.environ["DD_LOGS_INJECTION"]
 
     def test_invocations_metric(self):
-        os.environ["DD_ENHANCED_METRICS"] = "True"
-
         @datadog_lambda_wrapper
         def lambda_handler(event, context):
             lambda_metric("test.metric", 100)
@@ -121,7 +131,7 @@ def lambda_handler(event, context):
 
         lambda_handler(lambda_event, get_mock_context())
 
-        self.mock_lambda_metric.assert_has_calls(
+        self.mock_write_metric_point_to_stdout.assert_has_calls(
             [
                 call(
                     "aws.lambda.enhanced.invocations",
@@ -133,16 +143,13 @@ def lambda_handler(event, context):
                         "cold_start:true",
                         "memorysize:256",
                         "runtime:python2.7",
+                        "dd_lambda_layer:datadog-python27_0.1.0",
                     ],
                 )
             ]
         )
 
-        del os.environ["DD_ENHANCED_METRICS"]
-
     def test_errors_metric(self):
-        os.environ["DD_ENHANCED_METRICS"] = "True"
-
         @datadog_lambda_wrapper
         def lambda_handler(event, context):
             raise RuntimeError()
@@ -152,7 +159,7 @@ def lambda_handler(event, context):
         with self.assertRaises(RuntimeError):
             lambda_handler(lambda_event, get_mock_context())
 
-        self.mock_lambda_metric.assert_has_calls(
+        self.mock_write_metric_point_to_stdout.assert_has_calls(
             [
                 call(
                     "aws.lambda.enhanced.invocations",
@@ -164,6 +171,7 @@ def lambda_handler(event, context):
                         "cold_start:true",
                         "memorysize:256",
                         "runtime:python2.7",
+                        "dd_lambda_layer:datadog-python27_0.1.0",
                     ],
                 ),
                 call(
@@ -176,16 +184,13 @@ def lambda_handler(event, context):
                         "cold_start:true",
                         "memorysize:256",
                         "runtime:python2.7",
+                        "dd_lambda_layer:datadog-python27_0.1.0",
                     ],
                 ),
             ]
         )
 
-        del os.environ["DD_ENHANCED_METRICS"]
-
     def test_enhanced_metrics_cold_start_tag(self):
-        os.environ["DD_ENHANCED_METRICS"] = "True"
-
         @datadog_lambda_wrapper
         def lambda_handler(event, context):
             lambda_metric("test.metric", 100)
@@ -200,7 +205,7 @@ def lambda_handler(event, context):
             lambda_event, get_mock_context(aws_request_id="second-request-id")
         )
 
-        self.mock_lambda_metric.assert_has_calls(
+        self.mock_write_metric_point_to_stdout.assert_has_calls(
             [
                 call(
                     "aws.lambda.enhanced.invocations",
@@ -212,6 +217,7 @@ def lambda_handler(event, context):
                         "cold_start:true",
                         "memorysize:256",
                         "runtime:python2.7",
+                        "dd_lambda_layer:datadog-python27_0.1.0",
                     ],
                 ),
                 call(
@@ -224,14 +230,15 @@ def lambda_handler(event, context):
                         "cold_start:false",
                         "memorysize:256",
                         "runtime:python2.7",
+                        "dd_lambda_layer:datadog-python27_0.1.0",
                     ],
                 ),
             ]
         )
 
-        del os.environ["DD_ENHANCED_METRICS"]
-
     def test_no_enhanced_metrics_without_env_var(self):
+        os.environ["DD_ENHANCED_METRICS"] = "false"
+
         @datadog_lambda_wrapper
         def lambda_handler(event, context):
             raise RuntimeError()
@@ -241,4 +248,6 @@ def lambda_handler(event, context):
         with self.assertRaises(RuntimeError):
             lambda_handler(lambda_event, get_mock_context())
 
-        self.mock_lambda_metric.assert_not_called()
+        self.mock_write_metric_point_to_stdout.assert_not_called()
+
+        del os.environ["DD_ENHANCED_METRICS"]