Merge pull request #14 from apilytics/memory-usage

Send memory usage information together with metrics on Linux

Author: ruohola

.github/workflows/cd.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   release:
     name: "Release"
-    if: ${{ github.ref == 'refs/heads/master' }}
+    if: github.ref == 'refs/heads/master'
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout code"

.github/workflows/ci.yml

@@ -12,7 +12,7 @@ concurrency:
 jobs:
   commits:
     name: "Commits"
-    if: ${{ github.event_name == 'pull_request' }}
+    if: github.event_name == 'pull_request'
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout code"
@@ -93,8 +93,11 @@ jobs:
       fail-fast: true
       matrix:
         python-version: ['3.7', '3.8', '3.9', '3.10']
-
-    runs-on: ubuntu-latest
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    defaults:
+      run:
+        shell: bash
+    runs-on: ${{ matrix.os }}
     steps:
       - name: "Checkout code"
         uses: actions/checkout@v2
@@ -136,7 +139,7 @@ jobs:
           pytest --verbose --cov=. --cov-report=xml .
 
       - name: "Upload coverage"
-        if: matrix.python-version == '3.7'
+        if: matrix.python-version == '3.7' && matrix.os == 'ubuntu-latest'
         uses: codecov/codecov-action@f32b3a3741e1053eb607407145bc9619351dc93b  # v2.1.0
         with:
           files: coverage.xml

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Send current system memory usage and total available memory together with metrics on Linux systems.
+
 ## [1.3.0] - 2022-02-02
 
 ### Added

apilytics/core.py

@@ -1,11 +1,12 @@
 import concurrent.futures
 import json
 import platform
+import re
 import time
 import types
 import urllib.error
 import urllib.request
-from typing import ClassVar, Optional, Type
+from typing import ClassVar, Optional, Tuple, Type
 
 import apilytics
 
@@ -127,6 +128,8 @@ def set_response_info(
         self._response_size = response_size
 
     def _send_metrics(self) -> None:
+        memory_usage, memory_total = _get_used_and_total_memory()
+
         request = urllib.request.Request(
             url="https://www.apilytics.io/api/v1/middleware",
             method="POST",
@@ -158,8 +161,42 @@ def _send_metrics(self) -> None:
                 if self._response_size is not None
                 else {}
             ),
+            **({"memoryUsage": memory_usage} if memory_usage is not None else {}),
+            **({"memoryTotal": memory_total} if memory_total is not None else {}),
         }
         try:
             urllib.request.urlopen(url=request, data=json.dumps(data).encode())
         except urllib.error.URLError:
             pass
+
+
+def _get_used_and_total_memory() -> Tuple[Optional[int], Optional[int]]:
+    """
+    Get information about the used and total system memory.
+
+    Returns:
+        A tuple containing the used and total system memory in bytes.
+        (None, None) if the system is not Linux or if the reading fails.
+        (None, int) if the used memory could not be determined.
+    """
+    used = None
+    total = None
+
+    if platform.system() == "Linux":
+        try:
+            with open("/proc/meminfo") as f:
+                meminfo = f.read()
+        except OSError:
+            pass  # Prepare for everything and anything.
+        else:
+            total_match = re.search(r"MemTotal:\s*(\d+)", meminfo)
+            available_match = re.search(r"MemAvailable:\s*(\d+)", meminfo)
+            if total_match:
+                total = int(total_match.group(1)) * 1024
+                if available_match:
+                    # If MemAvailable exists MemTotal will also exist.
+                    # The reverse is not always true (MemAvailable came in Linux 3.14).
+                    available = int(available_match.group(1)) * 1024
+                    used = total - available
+
+    return used, total

tests/django/test_django.py

@@ -38,13 +38,17 @@ def test_middleware_should_call_apilytics_api(
         "requestSize",
         "responseSize",
         "timeMillis",
+        *(("memoryUsage", "memoryTotal") if platform.system() == "Linux" else ()),
     }
     assert data["path"] == "/"
     assert data["method"] == "GET"
     assert data["statusCode"] == 200
     assert data["requestSize"] == 0
     assert data["responseSize"] > 0
     assert isinstance(data["timeMillis"], int)
+    if platform.system() == "Linux":
+        assert isinstance(data["memoryUsage"], int)
+        assert isinstance(data["memoryTotal"], int)
 
 
 def test_middleware_should_send_query_params(
@@ -123,12 +127,16 @@ def test_middleware_should_work_with_streaming_response(
         "statusCode",
         "requestSize",
         "timeMillis",
+        *(("memoryUsage", "memoryTotal") if platform.system() == "Linux" else ()),
     }
     assert data["path"] == "/streaming"
     assert data["method"] == "GET"
     assert data["statusCode"] == 200
     assert data["requestSize"] == 0
     assert isinstance(data["timeMillis"], int)
+    if platform.system() == "Linux":
+        assert isinstance(data["memoryUsage"], int)
+        assert isinstance(data["memoryTotal"], int)
 
 
 @django.test.override_settings(APILYTICS_API_KEY=None)
@@ -160,10 +168,14 @@ def test_middleware_should_send_data_even_on_errors(
         "statusCode",
         "requestSize",
         "responseSize",
+        *(("memoryUsage", "memoryTotal") if platform.system() == "Linux" else ()),
     }
     assert data["method"] == "GET"
     assert data["path"] == "/error"
     assert data["statusCode"] == 500
     assert data["requestSize"] == 0
     assert data["responseSize"] > 0
     assert isinstance(data["timeMillis"], int)
+    if platform.system() == "Linux":
+        assert isinstance(data["memoryUsage"], int)
+        assert isinstance(data["memoryTotal"], int)

tests/fastapi/test_fastapi.py

@@ -44,6 +44,7 @@ def test_middleware_should_call_apilytics_api(
         "responseSize",
         "userAgent",
         "timeMillis",
+        *(("memoryUsage", "memoryTotal") if platform.system() == "Linux" else ()),
     }
     assert data["path"] == "/"
     assert data["method"] == "GET"
@@ -52,6 +53,9 @@ def test_middleware_should_call_apilytics_api(
     assert data["responseSize"] > 0
     assert data["userAgent"] == "testclient"
     assert isinstance(data["timeMillis"], int)
+    if platform.system() == "Linux":
+        assert isinstance(data["memoryUsage"], int)
+        assert isinstance(data["memoryTotal"], int)
 
 
 def test_middleware_should_send_query_params(
@@ -128,13 +132,17 @@ def test_middleware_should_work_with_streaming_response(
         "requestSize",
         "userAgent",
         "timeMillis",
+        *(("memoryUsage", "memoryTotal") if platform.system() == "Linux" else ()),
     }
     assert data["path"] == "/streaming"
     assert data["method"] == "GET"
     assert data["statusCode"] == 200
     assert data["requestSize"] == 0
     assert data["userAgent"] == "testclient"
     assert isinstance(data["timeMillis"], int)
+    if platform.system() == "Linux":
+        assert isinstance(data["memoryUsage"], int)
+        assert isinstance(data["memoryTotal"], int)
 
 
 @tests.fastapi.conftest.override_middleware(
@@ -166,9 +174,19 @@ def test_middleware_should_send_data_even_on_errors(
 
     __, call_kwargs = mocked_urlopen.call_args
     data = tests.conftest.decode_request_data(call_kwargs["data"])
-    assert data.keys() == {"method", "path", "timeMillis", "userAgent", "requestSize"}
+    assert data.keys() == {
+        "method",
+        "path",
+        "timeMillis",
+        "userAgent",
+        "requestSize",
+        *(("memoryUsage", "memoryTotal") if platform.system() == "Linux" else ()),
+    }
     assert data["method"] == "GET"
     assert data["path"] == "/error"
     assert data["requestSize"] == 0
     assert data["userAgent"] == "testclient"
     assert isinstance(data["timeMillis"], int)
+    if platform.system() == "Linux":
+        assert isinstance(data["memoryUsage"], int)
+        assert isinstance(data["memoryTotal"], int)