Merge branch 'dev' of https://github.com/microsoft/content-processing-solution-accelerator into dev

Author: Prajwal-Microsoft

.github/workflows/test.yml

@@ -33,7 +33,7 @@ jobs:
       - name: Install Backend Dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r src/ContentProcessorAPI/requirements.txt
+          pip install -r src/ContentProcessor/requirements.txt
           pip install pytest-cov
           pip install pytest-asyncio
 
@@ -43,7 +43,7 @@ jobs:
       - name: Check if Backend Test Files Exist
         id: check_backend_tests
         run: |
-          if [ -z "$(find src/ContentProcessorAPI/app/tests -type f -name 'test_*.py')" ]; then
+          if [ -z "$(find src/ContentProcessor/src/tests -type f -name 'test_*.py')" ]; then
             echo "No backend test files found, skipping backend tests."
             echo "skip_backend_tests=true" >> $GITHUB_ENV
           else
@@ -54,8 +54,8 @@ jobs:
       - name: Run Backend Tests with Coverage
         if: env.skip_backend_tests == 'false'
         run: |
-          pytest src/ContentProcessorAPI/app/tests
-          pytest --cov=. --cov-report=term-missing --cov-report=xml
+          cd src/ContentProcessor
+          python -m pytest -vv --cov=. --cov-report=xml --cov-report=term-missing --cov-fail-under=80
 
       - name: Skip Backend Tests
         if: env.skip_backend_tests == 'true'

src/ContentProcessor/pyproject.toml

@@ -28,8 +28,10 @@ dev = [
     "coverage>=7.6.10",
     "pydantic>=2.10.5",
     "pytest>=8.3.4",
+    "pytest-asyncio>=0.25.3",
     "pytest-cov>=6.0.0",
     "pytest-mock>=3.14.0",
+    "mongomock>=2.3.1",
     "ruff>=0.9.1",
 ]
 

src/ContentProcessor/requirements.txt

@@ -0,0 +1,23 @@
+azure-appconfiguration>=1.7.1
+azure-identity>=1.19.0
+azure-storage-blob>=12.24.1
+azure-storage-queue>=12.12.0
+certifi>=2024.12.14
+charset-normalizer>=3.4.1
+openai==1.65.5
+pandas>=2.2.3
+pdf2image>=1.17.0
+poppler-utils>=0.1.0
+pydantic>=2.10.5
+pydantic-settings>=2.7.1
+pymongo>=4.11.2
+python-dotenv>=1.0.1
+tiktoken>=0.9.0
+coverage>=7.6.10
+pydantic>=2.10.5
+pytest>=8.3.4
+pytest-asyncio>=0.25.3
+pytest-cov>=6.0.0
+pytest-mock>=3.14.0
+mongomock>=2.3.1
+ruff>=0.9.1

src/ContentProcessor/src/libs/application/env_config.py

@@ -1,6 +1,7 @@
 from libs.base.application_models import ModelBaseSettings
+from pydantic import Field
 
 
 class EnvConfiguration(ModelBaseSettings):
     # APP_CONFIG_ENDPOINT
-    app_config_endpoint: str
+    app_config_endpoint: str = Field(default="https://example.com")

src/ContentProcessor/src/libs/process_host/handler_process_host.py

@@ -40,11 +40,11 @@ def add_handlers_as_process(
             }
         )
 
-    async def start_handler_processes(self):
+    async def start_handler_processes(self, test_mode: bool = False):
         for handler in self.handlers:
             handler["handler_info"].handler.start()
 
-        while True:
+        while not test_mode:
             for handler in self.handlers:
                 handler["handler_info"].handler.join(timeout=1)
                 if (

src/ContentProcessor/src/main.py

@@ -31,7 +31,7 @@ def _initialize_application(self):
         # Add Azure Credential
         self.application_context.set_credential(DefaultAzureCredential())
 
-    async def run(self):
+    async def run(self, test_mode: bool = False):
         # Get Process lists from the configuration - ex. ["extract", "transform", "evaluate", "save", "custom1", "custom2"....]
         steps = self.application_context.configuration.app_process_steps
 
@@ -53,7 +53,7 @@ async def run(self):
             )
 
         # Start All registered processes
-        await handler_host_manager.start_handler_processes()
+        await handler_host_manager.start_handler_processes(test_mode)
 
 
 async def main():

src/ContentProcessor/src/tests/azure_helper/test_cosmos_mongo.py

@@ -0,0 +1,89 @@
+import pytest
+from libs.azure_helper.comsos_mongo import CosmosMongDBHelper
+import mongomock
+
+
+@pytest.fixture
+def mock_mongo_client(monkeypatch):
+    def mock_mongo_client_init(*args, **kwargs):
+        return mongomock.MongoClient()
+
+    monkeypatch.setattr(
+        "libs.azure_helper.comsos_mongo.MongoClient", mock_mongo_client_init
+    )
+    return mongomock.MongoClient()
+
+
+def test_prepare(mock_mongo_client, monkeypatch):
+    indexes = ["field1", "field2"]
+    helper = CosmosMongDBHelper(
+        "connection_string", "db_name", "container_name", indexes=indexes
+    )
+
+    assert helper.client is not None
+    assert helper.db is not None
+    assert helper.container is not None
+    monkeypatch.setattr(helper.container, "index_information", lambda: indexes)
+    helper._create_indexes(helper.container, indexes)
+    index_info = helper.container.index_information()
+    for index in indexes:
+        assert f"{index}" in index_info
+
+
+def test_insert_document(mock_mongo_client):
+    helper = CosmosMongDBHelper("connection_string", "db_name", "container_name")
+
+    document = {"key": "value"}
+    helper.insert_document(document)
+
+    assert helper.container.find_one(document) is not None
+
+
+def test_find_document(mock_mongo_client):
+    helper = CosmosMongDBHelper("connection_string", "db_name", "container_name")
+
+    query = {"key": "value"}
+    helper.insert_document(query)
+    result = helper.find_document(query)
+
+    assert len(result) == 1
+    assert result[0] == query
+
+
+def test_find_document_with_sort(mock_mongo_client):
+    helper = CosmosMongDBHelper("connection_string", "db_name", "container_name")
+
+    documents = [{"key": "value1", "sort_field": 2}, {"key": "value2", "sort_field": 1}]
+    for doc in documents:
+        helper.insert_document(doc)
+
+    query = {}
+    sort_fields = [("sort_field", 1)]
+    result = helper.find_document(query, sort_fields)
+
+    assert len(result) == 2
+    assert result[0]["key"] == "value2"
+    assert result[1]["key"] == "value1"
+
+
+def test_update_document(mock_mongo_client):
+    helper = CosmosMongDBHelper("connection_string", "db_name", "container_name")
+
+    filter = {"key": "value"}
+    update = {"key": "new_value"}
+    helper.insert_document(filter)
+    helper.update_document(filter, update)
+
+    result = helper.find_document(update)
+    assert len(result) == 1
+    assert result[0]["key"] == "new_value"
+
+
+def test_delete_document(mock_mongo_client):
+    helper = CosmosMongDBHelper("connection_string", "db_name", "container_name")
+
+    helper.insert_document({"Id": "123"})
+    helper.delete_document("123")
+
+    result = helper.find_document({"Id": "123"})
+    assert len(result) == 0

src/ContentProcessor/src/tests/azure_helper/test_storage_blob.py

@@ -0,0 +1,181 @@
+import pytest
+from io import BytesIO
+from libs.azure_helper.storage_blob import StorageBlobHelper
+
+
+@pytest.fixture
+def mock_blob_service_client(mocker):
+    return mocker.patch("libs.azure_helper.storage_blob.BlobServiceClient")
+
+
+@pytest.fixture
+def mock_default_azure_credential(mocker):
+    return mocker.patch("libs.azure_helper.storage_blob.DefaultAzureCredential")
+
+
+@pytest.fixture
+def storage_blob_helper(mock_blob_service_client, mock_default_azure_credential):
+    return StorageBlobHelper(
+        account_url="https://testaccount.blob.core.windows.net",
+        container_name="testcontainer",
+    )
+
+
+def test_get_container_client_with_parent_container(
+    storage_blob_helper, mock_blob_service_client, mocker
+):
+    mock_container_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value = (
+        mock_container_client
+    )
+
+    # Reset call count before the specific action
+    mock_blob_service_client.return_value.get_container_client.reset_mock()
+
+    # Call _get_container_client without passing container_name
+    container_client = storage_blob_helper._get_container_client()
+
+    assert container_client == mock_container_client
+    assert mock_blob_service_client.return_value.get_container_client.call_count == 1
+    mock_blob_service_client.return_value.get_container_client.assert_called_once_with(
+        "testcontainer"
+    )
+
+
+def test_get_container_client_without_container_name(storage_blob_helper):
+    storage_blob_helper.parent_container_name = None
+
+    with pytest.raises(
+        ValueError,
+        match="Container name must be provided either during initialization or as a function argument.",
+    ):
+        storage_blob_helper._get_container_client()
+
+
+def test_upload_file(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+
+    # Mock the open function to simulate reading a file
+    mocker.patch("builtins.open", mocker.mock_open(read_data="test content"))
+
+    storage_blob_helper.upload_file("testcontainer", "testblob", "testfile.txt")
+
+    mock_blob_client.upload_blob.assert_called_once()
+
+
+def test_upload_stream(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+    stream = BytesIO(b"test data")
+
+    storage_blob_helper.upload_stream("testcontainer", "testblob", stream)
+
+    mock_blob_client.upload_blob.assert_called_once_with(stream, overwrite=True)
+
+
+def test_upload_text(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+
+    storage_blob_helper.upload_text("testcontainer", "testblob", "test text")
+
+    mock_blob_client.upload_blob.assert_called_once_with("test text", overwrite=True)
+
+
+def test_download_file(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+    mock_blob_client.download_blob.return_value.readall.return_value = b"test data"
+
+    mock_open = mocker.patch("builtins.open", mocker.mock_open())
+    storage_blob_helper.download_file("testcontainer", "testblob", "downloaded.txt")
+    mock_open.return_value.write.assert_called_once_with(b"test data")
+
+
+def test_download_stream(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+    mock_blob_client.download_blob.return_value.readall.return_value = b"test data"
+
+    stream = storage_blob_helper.download_stream("testcontainer", "testblob")
+
+    assert stream == b"test data"
+
+
+def test_download_text(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+    mock_blob_client.download_blob.return_value.content_as_text.return_value = (
+        "test text"
+    )
+
+    text = storage_blob_helper.download_text("testcontainer", "testblob")
+
+    assert text == "test text"
+
+
+def test_delete_blob(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+
+    storage_blob_helper.delete_blob("testcontainer", "testblob")
+
+    mock_blob_client.delete_blob.assert_called_once()
+
+
+def test_upload_blob_with_str(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+
+    storage_blob_helper.upload_blob("testcontainer", "testblob", "test string data")
+
+    mock_blob_client.upload_blob.assert_called_once_with(
+        "test string data", overwrite=True
+    )
+
+
+def test_upload_blob_with_bytes(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+
+    storage_blob_helper.upload_blob("testcontainer", "testblob", b"test bytes data")
+
+    mock_blob_client.upload_blob.assert_called_once_with(
+        b"test bytes data", overwrite=True
+    )
+
+
+def test_upload_blob_with_io(storage_blob_helper, mock_blob_service_client, mocker):
+    mock_blob_client = mocker.MagicMock()
+    mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = (
+        mock_blob_client
+    )
+    stream = BytesIO(b"test stream data")
+
+    storage_blob_helper.upload_blob("testcontainer", "testblob", stream)
+
+    mock_blob_client.upload_blob.assert_called_once_with(stream, overwrite=True)
+
+
+def test_upload_blob_with_unsupported_type(storage_blob_helper):
+    with pytest.raises(ValueError, match="Unsupported data type for upload"):
+        storage_blob_helper.upload_blob("testcontainer", "testblob", 12345)