diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3fa35f5d..47396e89 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: - name: Install Backend Dependencies run: | python -m pip install --upgrade pip - pip install -r src/ContentProcessorAPI/requirements.txt + pip install -r src/ContentProcessor/requirements.txt pip install pytest-cov pip install pytest-asyncio @@ -43,7 +43,7 @@ jobs: - name: Check if Backend Test Files Exist id: check_backend_tests run: | - if [ -z "$(find src/ContentProcessorAPI/app/tests -type f -name 'test_*.py')" ]; then + if [ -z "$(find src/ContentProcessor/src/tests -type f -name 'test_*.py')" ]; then echo "No backend test files found, skipping backend tests." echo "skip_backend_tests=true" >> $GITHUB_ENV else @@ -54,8 +54,8 @@ jobs: - name: Run Backend Tests with Coverage if: env.skip_backend_tests == 'false' run: | - pytest src/ContentProcessorAPI/app/tests - pytest --cov=. --cov-report=term-missing --cov-report=xml + cd src/ContentProcessor + python -m pytest -vv --cov=. --cov-report=xml --cov-report=term-missing --cov-fail-under=80 - name: Skip Backend Tests if: env.skip_backend_tests == 'true' diff --git a/src/ContentProcessor/pyproject.toml b/src/ContentProcessor/pyproject.toml index 9c0511a5..4f046a57 100644 --- a/src/ContentProcessor/pyproject.toml +++ b/src/ContentProcessor/pyproject.toml @@ -28,8 +28,10 @@ dev = [ "coverage>=7.6.10", "pydantic>=2.10.5", "pytest>=8.3.4", + "pytest-asyncio>=0.25.3", "pytest-cov>=6.0.0", "pytest-mock>=3.14.0", + "mongomock>=2.3.1", "ruff>=0.9.1", ] diff --git a/src/ContentProcessor/requirements.txt b/src/ContentProcessor/requirements.txt new file mode 100644 index 00000000..464a5a08 --- /dev/null +++ b/src/ContentProcessor/requirements.txt @@ -0,0 +1,23 @@ +azure-appconfiguration>=1.7.1 +azure-identity>=1.19.0 +azure-storage-blob>=12.24.1 +azure-storage-queue>=12.12.0 +certifi>=2024.12.14 +charset-normalizer>=3.4.1 +openai==1.65.5 +pandas>=2.2.3 +pdf2image>=1.17.0 +poppler-utils>=0.1.0 +pydantic>=2.10.5 +pydantic-settings>=2.7.1 +pymongo>=4.11.2 +python-dotenv>=1.0.1 +tiktoken>=0.9.0 +coverage>=7.6.10 +pydantic>=2.10.5 +pytest>=8.3.4 +pytest-asyncio>=0.25.3 +pytest-cov>=6.0.0 +pytest-mock>=3.14.0 +mongomock>=2.3.1 +ruff>=0.9.1 \ No newline at end of file diff --git a/src/ContentProcessor/src/libs/application/env_config.py b/src/ContentProcessor/src/libs/application/env_config.py index 6eea29b4..6c8fbeb4 100644 --- a/src/ContentProcessor/src/libs/application/env_config.py +++ b/src/ContentProcessor/src/libs/application/env_config.py @@ -1,6 +1,7 @@ from libs.base.application_models import ModelBaseSettings +from pydantic import Field class EnvConfiguration(ModelBaseSettings): # APP_CONFIG_ENDPOINT - app_config_endpoint: str + app_config_endpoint: str = Field(default="https://example.com") diff --git a/src/ContentProcessor/src/libs/process_host/handler_process_host.py b/src/ContentProcessor/src/libs/process_host/handler_process_host.py index bea2ae48..14bce30e 100644 --- a/src/ContentProcessor/src/libs/process_host/handler_process_host.py +++ b/src/ContentProcessor/src/libs/process_host/handler_process_host.py @@ -40,11 +40,11 @@ def add_handlers_as_process( } ) - async def start_handler_processes(self): + async def start_handler_processes(self, test_mode: bool = False): for handler in self.handlers: handler["handler_info"].handler.start() - while True: + while not test_mode: for handler in self.handlers: handler["handler_info"].handler.join(timeout=1) if ( diff --git a/src/ContentProcessor/src/main.py b/src/ContentProcessor/src/main.py index 0a28f9a0..1fc41111 100644 --- a/src/ContentProcessor/src/main.py +++ b/src/ContentProcessor/src/main.py @@ -31,7 +31,7 @@ def _initialize_application(self): # Add Azure Credential self.application_context.set_credential(DefaultAzureCredential()) - async def run(self): + async def run(self, test_mode: bool = False): # Get Process lists from the configuration - ex. ["extract", "transform", "evaluate", "save", "custom1", "custom2"....] steps = self.application_context.configuration.app_process_steps @@ -53,7 +53,7 @@ async def run(self): ) # Start All registered processes - await handler_host_manager.start_handler_processes() + await handler_host_manager.start_handler_processes(test_mode) async def main(): diff --git a/src/ContentProcessor/src/tests/azure_helper/test_cosmos_mongo.py b/src/ContentProcessor/src/tests/azure_helper/test_cosmos_mongo.py new file mode 100644 index 00000000..026b3b35 --- /dev/null +++ b/src/ContentProcessor/src/tests/azure_helper/test_cosmos_mongo.py @@ -0,0 +1,89 @@ +import pytest +from libs.azure_helper.comsos_mongo import CosmosMongDBHelper +import mongomock + + +@pytest.fixture +def mock_mongo_client(monkeypatch): + def mock_mongo_client_init(*args, **kwargs): + return mongomock.MongoClient() + + monkeypatch.setattr( + "libs.azure_helper.comsos_mongo.MongoClient", mock_mongo_client_init + ) + return mongomock.MongoClient() + + +def test_prepare(mock_mongo_client, monkeypatch): + indexes = ["field1", "field2"] + helper = CosmosMongDBHelper( + "connection_string", "db_name", "container_name", indexes=indexes + ) + + assert helper.client is not None + assert helper.db is not None + assert helper.container is not None + monkeypatch.setattr(helper.container, "index_information", lambda: indexes) + helper._create_indexes(helper.container, indexes) + index_info = helper.container.index_information() + for index in indexes: + assert f"{index}" in index_info + + +def test_insert_document(mock_mongo_client): + helper = CosmosMongDBHelper("connection_string", "db_name", "container_name") + + document = {"key": "value"} + helper.insert_document(document) + + assert helper.container.find_one(document) is not None + + +def test_find_document(mock_mongo_client): + helper = CosmosMongDBHelper("connection_string", "db_name", "container_name") + + query = {"key": "value"} + helper.insert_document(query) + result = helper.find_document(query) + + assert len(result) == 1 + assert result[0] == query + + +def test_find_document_with_sort(mock_mongo_client): + helper = CosmosMongDBHelper("connection_string", "db_name", "container_name") + + documents = [{"key": "value1", "sort_field": 2}, {"key": "value2", "sort_field": 1}] + for doc in documents: + helper.insert_document(doc) + + query = {} + sort_fields = [("sort_field", 1)] + result = helper.find_document(query, sort_fields) + + assert len(result) == 2 + assert result[0]["key"] == "value2" + assert result[1]["key"] == "value1" + + +def test_update_document(mock_mongo_client): + helper = CosmosMongDBHelper("connection_string", "db_name", "container_name") + + filter = {"key": "value"} + update = {"key": "new_value"} + helper.insert_document(filter) + helper.update_document(filter, update) + + result = helper.find_document(update) + assert len(result) == 1 + assert result[0]["key"] == "new_value" + + +def test_delete_document(mock_mongo_client): + helper = CosmosMongDBHelper("connection_string", "db_name", "container_name") + + helper.insert_document({"Id": "123"}) + helper.delete_document("123") + + result = helper.find_document({"Id": "123"}) + assert len(result) == 0 diff --git a/src/ContentProcessor/src/tests/azure_helper/test_storage_blob.py b/src/ContentProcessor/src/tests/azure_helper/test_storage_blob.py new file mode 100644 index 00000000..d14a99d2 --- /dev/null +++ b/src/ContentProcessor/src/tests/azure_helper/test_storage_blob.py @@ -0,0 +1,181 @@ +import pytest +from io import BytesIO +from libs.azure_helper.storage_blob import StorageBlobHelper + + +@pytest.fixture +def mock_blob_service_client(mocker): + return mocker.patch("libs.azure_helper.storage_blob.BlobServiceClient") + + +@pytest.fixture +def mock_default_azure_credential(mocker): + return mocker.patch("libs.azure_helper.storage_blob.DefaultAzureCredential") + + +@pytest.fixture +def storage_blob_helper(mock_blob_service_client, mock_default_azure_credential): + return StorageBlobHelper( + account_url="https://testaccount.blob.core.windows.net", + container_name="testcontainer", + ) + + +def test_get_container_client_with_parent_container( + storage_blob_helper, mock_blob_service_client, mocker +): + mock_container_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value = ( + mock_container_client + ) + + # Reset call count before the specific action + mock_blob_service_client.return_value.get_container_client.reset_mock() + + # Call _get_container_client without passing container_name + container_client = storage_blob_helper._get_container_client() + + assert container_client == mock_container_client + assert mock_blob_service_client.return_value.get_container_client.call_count == 1 + mock_blob_service_client.return_value.get_container_client.assert_called_once_with( + "testcontainer" + ) + + +def test_get_container_client_without_container_name(storage_blob_helper): + storage_blob_helper.parent_container_name = None + + with pytest.raises( + ValueError, + match="Container name must be provided either during initialization or as a function argument.", + ): + storage_blob_helper._get_container_client() + + +def test_upload_file(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + + # Mock the open function to simulate reading a file + mocker.patch("builtins.open", mocker.mock_open(read_data="test content")) + + storage_blob_helper.upload_file("testcontainer", "testblob", "testfile.txt") + + mock_blob_client.upload_blob.assert_called_once() + + +def test_upload_stream(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + stream = BytesIO(b"test data") + + storage_blob_helper.upload_stream("testcontainer", "testblob", stream) + + mock_blob_client.upload_blob.assert_called_once_with(stream, overwrite=True) + + +def test_upload_text(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + + storage_blob_helper.upload_text("testcontainer", "testblob", "test text") + + mock_blob_client.upload_blob.assert_called_once_with("test text", overwrite=True) + + +def test_download_file(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + mock_blob_client.download_blob.return_value.readall.return_value = b"test data" + + mock_open = mocker.patch("builtins.open", mocker.mock_open()) + storage_blob_helper.download_file("testcontainer", "testblob", "downloaded.txt") + mock_open.return_value.write.assert_called_once_with(b"test data") + + +def test_download_stream(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + mock_blob_client.download_blob.return_value.readall.return_value = b"test data" + + stream = storage_blob_helper.download_stream("testcontainer", "testblob") + + assert stream == b"test data" + + +def test_download_text(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + mock_blob_client.download_blob.return_value.content_as_text.return_value = ( + "test text" + ) + + text = storage_blob_helper.download_text("testcontainer", "testblob") + + assert text == "test text" + + +def test_delete_blob(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + + storage_blob_helper.delete_blob("testcontainer", "testblob") + + mock_blob_client.delete_blob.assert_called_once() + + +def test_upload_blob_with_str(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + + storage_blob_helper.upload_blob("testcontainer", "testblob", "test string data") + + mock_blob_client.upload_blob.assert_called_once_with( + "test string data", overwrite=True + ) + + +def test_upload_blob_with_bytes(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + + storage_blob_helper.upload_blob("testcontainer", "testblob", b"test bytes data") + + mock_blob_client.upload_blob.assert_called_once_with( + b"test bytes data", overwrite=True + ) + + +def test_upload_blob_with_io(storage_blob_helper, mock_blob_service_client, mocker): + mock_blob_client = mocker.MagicMock() + mock_blob_service_client.return_value.get_container_client.return_value.get_blob_client.return_value = ( + mock_blob_client + ) + stream = BytesIO(b"test stream data") + + storage_blob_helper.upload_blob("testcontainer", "testblob", stream) + + mock_blob_client.upload_blob.assert_called_once_with(stream, overwrite=True) + + +def test_upload_blob_with_unsupported_type(storage_blob_helper): + with pytest.raises(ValueError, match="Unsupported data type for upload"): + storage_blob_helper.upload_blob("testcontainer", "testblob", 12345) diff --git a/src/ContentProcessor/src/tests/pipeline/entities/test_pipeline_data.py b/src/ContentProcessor/src/tests/pipeline/entities/test_pipeline_data.py new file mode 100644 index 00000000..6ba309a3 --- /dev/null +++ b/src/ContentProcessor/src/tests/pipeline/entities/test_pipeline_data.py @@ -0,0 +1,117 @@ +import pytest +from unittest.mock import Mock +from libs.pipeline.entities.pipeline_step_result import StepResult +from libs.pipeline.entities.pipeline_status import PipelineStatus + + +def test_update_step(): + pipeline_status = PipelineStatus(active_step="step1") + pipeline_status._move_to_next_step = Mock() + pipeline_status.update_step() + assert pipeline_status.last_updated_time is not None + pipeline_status._move_to_next_step.assert_called_once_with("step1") + + +def test_add_step_result(): + pipeline_status = PipelineStatus() + step_result = StepResult(step_name="step1") + pipeline_status.add_step_result(step_result) + assert pipeline_status.process_results == [step_result] + + # Update existing step result + updated_step_result = StepResult(step_name="step1", status="completed") + pipeline_status.add_step_result(updated_step_result) + assert pipeline_status.process_results == [updated_step_result] + + +def test_get_step_result(): + pipeline_status = PipelineStatus() + step_result = StepResult(step_name="step1") + pipeline_status.process_results.append(step_result) + result = pipeline_status.get_step_result("step1") + assert result == step_result + + result = pipeline_status.get_step_result("step2") + assert result is None + + +def test_get_previous_step_result(): + pipeline_status = PipelineStatus(completed_steps=["step1"]) + step_result = StepResult(step_name="step1") + pipeline_status.process_results.append(step_result) + result = pipeline_status.get_previous_step_result("step2") + assert result == step_result + + pipeline_status.completed_steps = [] + result = pipeline_status.get_previous_step_result("step2") + assert result is None + + +# def test_save_to_persistent_storage(mocker): +# # Mock the StorageBlobHelper.upload_text method +# mock_upload_text = mocker.patch( +# "libs.azure_helper.storage_blob.StorageBlobHelper.upload_text" +# ) + +# # Mock the StorageBlobHelper constructor to return a mock instance +# mock_storage_blob_helper = mocker.patch( +# "libs.azure_helper.storage_blob.StorageBlobHelper", autospec=True +# ) +# mock_storage_blob_helper_instance = mock_storage_blob_helper.return_value + +# # Mock the create_container method on the container_client +# mock_container_client = Mock() +# mock_container_client.create_container = Mock() +# mock_storage_blob_helper_instance._invalidate_container = Mock() +# mock_storage_blob_helper_instance._invalidate_container.return_value = ( +# mock_container_client +# ) + +# # Create a PipelineStatus object with a process_id +# pipeline_status = PipelineStatus(process_id="123") + +# # Mock the update_step method using pytest-mock +# mock_update_step = mocker.patch.object( +# PipelineStatus, "update_step", return_value=None +# ) + +# # Mock the model_dump_json method using pytest-mock +# mock_model_dump_json = mocker.patch.object( +# PipelineStatus, "model_dump_json", return_value='{"key": "value"}' +# ) + +# account_url = "https://example.com" +# container_name = "container" + +# # Call the save_to_persistent_storage method +# pipeline_status.save_to_persistent_storage(account_url, container_name) + +# # Assert that update_step was called once +# mock_update_step.assert_called_once() + +# # Assert that model_dump_json was called once +# mock_model_dump_json.assert_called_once() + +# # Assert that upload_text was called with the correct arguments +# mock_upload_text.assert_called_once_with( +# container_name="123", blob_name="process-status.json", text='{"key": "value"}' +# ) + + +def test_save_to_persistent_storage_no_process_id(): + pipeline_status = PipelineStatus() + with pytest.raises(ValueError, match="Process ID is required to save the result."): + pipeline_status.save_to_persistent_storage("https://example.com", "container") + + +def test_move_to_next_step(): + pipeline_status = PipelineStatus(remaining_steps=["step1", "step2"]) + pipeline_status._move_to_next_step("step1") + assert pipeline_status.completed_steps == ["step1"] + assert pipeline_status.remaining_steps == ["step2"] + assert pipeline_status.completed is False + + pipeline_status._move_to_next_step("step2") + assert pipeline_status.completed_steps == ["step1", "step2"] + assert pipeline_status.remaining_steps == [] + assert pipeline_status.completed is True diff --git a/src/ContentProcessor/src/tests/pipeline/test_pipeline_queue_helper.py b/src/ContentProcessor/src/tests/pipeline/test_pipeline_queue_helper.py new file mode 100644 index 00000000..a03e94d9 --- /dev/null +++ b/src/ContentProcessor/src/tests/pipeline/test_pipeline_queue_helper.py @@ -0,0 +1,128 @@ +from unittest.mock import Mock +from azure.core.exceptions import ResourceNotFoundError +from azure.identity import DefaultAzureCredential +from azure.storage.queue import QueueClient, QueueMessage +from libs.pipeline.entities.pipeline_data import DataPipeline +from libs.pipeline.pipeline_queue_helper import ( + create_queue_client_name, + create_dead_letter_queue_client_name, + invalidate_queue, + create_or_get_queue_client, + delete_queue_message, + move_to_dead_letter_queue, + has_messages, + pass_data_pipeline_to_next_step, + _create_queue_client, +) + + +def test_create_queue_client_name(): + assert create_queue_client_name("test") == "content-pipeline-test-queue" + + +def test_create_dead_letter_queue_client_name(): + assert ( + create_dead_letter_queue_client_name("test") + == "content-pipeline-test-queue-dead-letter-queue" + ) + + +def test_invalidate_queue(mocker): + queue_client = Mock(spec=QueueClient) + queue_client.get_queue_properties.side_effect = ResourceNotFoundError + invalidate_queue(queue_client) + queue_client.create_queue.assert_called_once() + + +def test_create_or_get_queue_client(mocker): + mocker.patch("libs.pipeline.pipeline_queue_helper.QueueClient") + queue_name = "test-queue" + account_url = "https://example.com" + credential = Mock(spec=DefaultAzureCredential) + + # Mock the QueueClient instance + mock_queue_client = Mock(spec=QueueClient) + mock_queue_client.get_queue_properties.side_effect = ResourceNotFoundError + mock_queue_client.create_queue = Mock() # Ensure create_queue is a mock method + mocker.patch( + "libs.pipeline.pipeline_queue_helper.invalidate_queue", + return_value=mock_queue_client, + ) + + queue_client = create_or_get_queue_client(queue_name, account_url, credential) + assert queue_client is not None + + +def test_delete_queue_message(): + queue_client = Mock(spec=QueueClient) + message = Mock(spec=QueueMessage) + delete_queue_message(message, queue_client) + queue_client.delete_message.assert_called_once_with(message=message) + + +def test_move_to_dead_letter_queue(): + queue_client = Mock(spec=QueueClient) + dead_letter_queue_client = Mock(spec=QueueClient) + message = Mock(spec=QueueMessage) + message.content = "test content" + move_to_dead_letter_queue(message, dead_letter_queue_client, queue_client) + dead_letter_queue_client.send_message.assert_called_once_with( + content=message.content + ) + queue_client.delete_message.assert_called_once_with(message=message) + + +def test_has_messages(): + queue_client = Mock(spec=QueueClient) + queue_client.peek_messages.return_value = [Mock(spec=QueueMessage)] + assert has_messages(queue_client) != [] + + queue_client.peek_messages.return_value = [] + assert has_messages(queue_client) == [] + + +def test_pass_data_pipeline_to_next_step(mocker): + # Mock the get_next_step_name function + mocker.patch( + "libs.pipeline.pipeline_step_helper.get_next_step_name", + return_value="next_step", + ) + + # Mock the _create_queue_client function + mock_create_queue_client = mocker.patch( + "libs.pipeline.pipeline_queue_helper._create_queue_client" + ) + + # Create a mock DataPipeline object with the necessary attributes + data_pipeline = Mock(spec=DataPipeline) + data_pipeline.pipeline_status = Mock() + data_pipeline.pipeline_status.active_step = "current_step" + data_pipeline.model_dump_json.return_value = '{"key": "value"}' + + account_url = "https://example.com" + credential = Mock(spec=DefaultAzureCredential) + + pass_data_pipeline_to_next_step(data_pipeline, account_url, credential) + + mock_create_queue_client.assert_called_once_with( + account_url, "content-pipeline-next_step-queue", credential + ) + mock_create_queue_client().send_message.assert_called_once_with('{"key": "value"}') + + +def test_create_queue_client(mocker): + mocker.patch("azure.storage.queue.QueueClient") + account_url = "https://example.com" + queue_name = "test-queue" + credential = Mock(spec=DefaultAzureCredential) + + # Mock the QueueClient instance + mock_queue_client = Mock(spec=QueueClient) + mock_queue_client.get_queue_properties.return_value = None + mocker.patch( + "libs.pipeline.pipeline_queue_helper.invalidate_queue", + return_value=mock_queue_client, + ) + + queue_client = _create_queue_client(account_url, queue_name, credential) + assert queue_client is not None diff --git a/src/ContentProcessor/src/tests/pipeline/test_queue_handler_base.py b/src/ContentProcessor/src/tests/pipeline/test_queue_handler_base.py new file mode 100644 index 00000000..34bd161c --- /dev/null +++ b/src/ContentProcessor/src/tests/pipeline/test_queue_handler_base.py @@ -0,0 +1,77 @@ +import pytest +from unittest.mock import MagicMock +from azure.storage.queue import QueueClient +from libs.pipeline.entities.pipeline_message_context import MessageContext +from libs.pipeline.entities.pipeline_step_result import StepResult +from libs.pipeline.queue_handler_base import HandlerBase +from libs.application.application_context import AppContext + + +@pytest.fixture +def mock_queue_helper(mocker): + # Mock the helper methods + mocker.patch( + "libs.pipeline.pipeline_queue_helper.create_queue_client_name", + return_value="test-queue", + ) + mocker.patch( + "libs.pipeline.pipeline_queue_helper.create_dead_letter_queue_client_name", + return_value="test-dlq", + ) + mocker.patch( + "libs.pipeline.pipeline_queue_helper.create_or_get_queue_client", + return_value=MagicMock(spec=QueueClient), + ) + return mocker + + +@pytest.fixture +def mock_app_context(): + # Create a mock AppContext instance + mock_app_context = MagicMock(spec=AppContext) + + # Mock the necessary fields for AppContext + mock_configuration = MagicMock() + mock_configuration.app_storage_queue_url = "https://testqueueurl.com" + mock_configuration.app_storage_blob_url = "https://testbloburl.com" + mock_configuration.app_cps_processes = "TestProcess" + + mock_app_context.configuration = mock_configuration + mock_app_context.credential = MagicMock() + + return mock_app_context + + +class MockHandler(HandlerBase): + async def execute(self, context: MessageContext) -> StepResult: + return StepResult( + process_id="1234", + step_name="extract", + result={"result": "success", "data": {"key": "value"}}, + ) + + +@pytest.mark.asyncio +async def test_execute_method(): + mock_handler = MockHandler(appContext=MagicMock(), step_name="extract") + message_context = MagicMock(spec=MessageContext) + + # Execute the handler + result = await mock_handler.execute(message_context) + + assert result.step_name == "extract" + assert result.result == {"result": "success", "data": {"key": "value"}} + + +def test_show_queue_information(mock_queue_helper, mock_app_context): + handler = MockHandler(appContext=mock_app_context, step_name="extract") + + # Mock the queue client properties + mock_queue_client = MagicMock(spec=QueueClient) + mock_queue_client.url = "https://testurl" + mock_queue_client.get_queue_properties.return_value = MagicMock( + approximate_message_count=5 + ) + handler.queue_client = mock_queue_client + + handler._show_queue_information() diff --git a/src/ContentProcessor/src/tests/process_host/test_handler_type_loader.py b/src/ContentProcessor/src/tests/process_host/test_handler_type_loader.py new file mode 100644 index 00000000..19c433a0 --- /dev/null +++ b/src/ContentProcessor/src/tests/process_host/test_handler_type_loader.py @@ -0,0 +1,36 @@ +import pytest +from libs.pipeline.queue_handler_base import HandlerBase +from libs.process_host.handler_type_loader import load + + +def test_load_success(mocker): + process_step = "test" + module_name = f"libs.pipeline.handlers.{process_step}_handler" + class_name = f"{process_step.capitalize()}Handler" + + # Mock import_module to return a mock module + mock_module = mocker.Mock() + mock_import_module = mocker.patch( + "importlib.import_module", return_value=mock_module + ) + + # Mock the dynamic class within the mock module + mock_class = mocker.Mock(spec=HandlerBase) + setattr(mock_module, class_name, mock_class) + + result = load(process_step) + + mock_import_module.assert_called_once_with(module_name) + assert result == mock_class + + +def test_load_module_not_found(mocker): + process_step = "nonexistent" + class_name = f"{process_step.capitalize()}Handler" + + mocker.patch("importlib.import_module", side_effect=ModuleNotFoundError) + + with pytest.raises(Exception) as excinfo: + load(process_step) + + assert str(excinfo.value) == f"Error loading processor {class_name}: " diff --git a/src/ContentProcessor/src/tests/test_main.py b/src/ContentProcessor/src/tests/test_main.py new file mode 100644 index 00000000..29265c92 --- /dev/null +++ b/src/ContentProcessor/src/tests/test_main.py @@ -0,0 +1,78 @@ +import pytest +from main import Application + + +class DummyHandler: + def __init__(self, appContext, step_name): + self.handler_name = step_name + self.appContext = appContext + self.step_name = step_name + self.exitcode = None + + def connect_queue(self, *args): + print(f"Connecting queue for handler: {self.handler_name}") + + +class ConfigItem: + def __init__(self, key, value): + self.key = key + self.value = value + + +@pytest.mark.asyncio +async def test_application_run(mocker): + # Mock the application context and configuration + mock_app_context = mocker.MagicMock() + mock_app_context.configuration.app_process_steps = ["extract", "transform"] + + # Mock the handler loader to return a DummyHandler + mocker.patch( + "libs.process_host.handler_type_loader.load", + side_effect=lambda name: DummyHandler, + ) + + # Mock the HandlerHostManager instance + mocker.patch( + "libs.process_host.handler_process_host.HandlerHostManager" + ).return_value + + # Mock the DefaultAzureCredential + mocker.patch("azure.identity.DefaultAzureCredential") + + # Mock the read_configuration method to return a complete configuration + mocker.patch( + "libs.azure_helper.app_configuration.AppConfigurationHelper.read_configuration", + return_value=[ + ConfigItem("app_storage_queue_url", "https://example.com/queue"), + ConfigItem("app_storage_blob_url", "https://example.com/blob"), + ConfigItem("app_process_steps", "extract,map"), + ConfigItem("app_message_queue_interval", "2"), + ConfigItem("app_message_queue_visibility_timeout", "1"), + ConfigItem("app_message_queue_process_timeout", "2"), + ConfigItem("app_logging_enable", "True"), + ConfigItem("app_logging_level", "DEBUG"), + ConfigItem("app_cps_processes", "4"), + ConfigItem("app_cps_configuration", "value"), + ConfigItem( + "app_content_understanding_endpoint", "https://example.com/content" + ), + ConfigItem("app_azure_openai_endpoint", "https://example.com/openai"), + ConfigItem("app_azure_openai_model", "model-name"), + ConfigItem( + "app_cosmos_connstr", + "AccountEndpoint=https://example.com;AccountKey=key;", + ), + ConfigItem("app_cosmos_database", "database-name"), + ConfigItem("app_cosmos_container_process", "container-process"), + ConfigItem("app_cosmos_container_schema", "container-schema"), + ], + ) + + # Initialize the application with the mocked context + mocker.patch.object( + Application, "_initialize_application", return_value=mock_app_context + ) + app = Application() + + # Run the application + await app.run(test_mode=True) diff --git a/src/ContentProcessor/src/tests/utils/test_base64_util.py b/src/ContentProcessor/src/tests/utils/test_base64_util.py new file mode 100644 index 00000000..19d2d54e --- /dev/null +++ b/src/ContentProcessor/src/tests/utils/test_base64_util.py @@ -0,0 +1,27 @@ +import base64 +from libs.utils.base64_util import is_base64_encoded + + +def test_is_base64_encoded_valid(): + valid_base64 = base64.b64encode(b"test data").decode("utf-8") + assert is_base64_encoded(valid_base64) is True + + +def test_is_base64_encoded_invalid(): + invalid_base64 = "invalid_base64_string" + assert is_base64_encoded(invalid_base64) is False + + +def test_is_base64_encoded_empty_string(): + empty_string = " " + assert is_base64_encoded(empty_string) is False + + +def test_is_base64_encoded_special_characters(): + special_characters = "!@#$%^&*()" + assert is_base64_encoded(special_characters) is False + + +def test_is_base64_encoded_partial_base64(): + partial_base64 = base64.b64encode(b"test").decode("utf-8")[:5] + assert is_base64_encoded(partial_base64) is False diff --git a/src/ContentProcessor/src/tests/utils/test_stopwatch.py b/src/ContentProcessor/src/tests/utils/test_stopwatch.py new file mode 100644 index 00000000..d89c29c7 --- /dev/null +++ b/src/ContentProcessor/src/tests/utils/test_stopwatch.py @@ -0,0 +1,50 @@ +from libs.utils.stopwatch import Stopwatch + + +def test_stopwatch_initial_state(): + stopwatch = Stopwatch() + assert stopwatch.elapsed == 0 + assert stopwatch.elapsed_string == "0:00:00" + assert not stopwatch.is_running + + +def test_stopwatch_start(mocker): + mocker.patch("time.perf_counter", return_value=100.0) + stopwatch = Stopwatch() + stopwatch.start() + assert stopwatch.is_running + assert stopwatch.start_time == 100.0 + + +def test_stopwatch_stop(mocker): + mocker.patch("time.perf_counter", side_effect=[100.0, 105.0]) + stopwatch = Stopwatch() + stopwatch.start() + stopwatch.stop() + assert not stopwatch.is_running + assert stopwatch.elapsed == 5.0 + assert stopwatch.elapsed_string == "00:00:05.000" + + +def test_stopwatch_reset(): + stopwatch = Stopwatch() + stopwatch.start() + stopwatch.stop() + stopwatch.reset() + assert stopwatch.elapsed == 0 + assert not stopwatch.is_running + + +def test_stopwatch_context_manager(mocker): + mocker.patch("time.perf_counter", side_effect=[100.0, 105.0]) + with Stopwatch() as stopwatch: + assert stopwatch.is_running + assert not stopwatch.is_running + assert stopwatch.elapsed == 5.0 + assert stopwatch.elapsed_string == "00:00:05.000" + + +def test_format_elapsed_time(): + stopwatch = Stopwatch() + formatted_time = stopwatch._format_elapsed_time(3661.123) + assert formatted_time == "01:01:01.123" diff --git a/src/ContentProcessor/src/tests/utils/test_utils.py b/src/ContentProcessor/src/tests/utils/test_utils.py new file mode 100644 index 00000000..fc5dfecf --- /dev/null +++ b/src/ContentProcessor/src/tests/utils/test_utils.py @@ -0,0 +1,57 @@ +import pytest +from unittest.mock import Mock +from libs.utils.utils import CustomEncoder, flatten_dict, value_match, value_contains + + +def test_custom_encoder_to_dict(mocker): + obj = Mock() + obj.to_dict.return_value = {"key": "value"} + encoder = CustomEncoder() + result = encoder.default(obj) + assert result == {"key": "value"} + + +def test_custom_encoder_default(mocker): + class UnserializableObject: + pass + + obj = UnserializableObject() + encoder = CustomEncoder() + with pytest.raises(TypeError): + encoder.default(obj) + + +def test_flatten_dict(): + data = {"a": 1, "b": {"c": 2, "d": {"e": 3}}, "f": [4, 5, {"g": 6}]} + result = flatten_dict(data) + expected = {"a": 1, "b_c": 2, "b_d_e": 3, "f_0": 4, "f_1": 5, "f_2_g": 6} + assert result == expected + + +def test_value_match_strings(): + assert value_match("Hello", "hello") is True + assert value_match("Hello", "world") is False + + +def test_value_match_lists(): + assert value_match([1, 2, 3], [1, 2, 3]) is True + assert value_match([1, 2, 3], [1, 2, 4]) is False + + +def test_value_match_dicts(): + assert value_match({"a": 1, "b": 2}, {"a": 1, "b": 2}) is True + assert value_match({"a": 1, "b": 2}, {"a": 1, "b": 3}) is False + + +def test_value_contains_strings(): + assert value_contains("hello", "Hello world") is True + assert value_contains("world", "Hello world") is True + assert value_contains("test", "Hello world") is False + + +def test_value_contains_lists(): + assert value_contains([4], [1, 2, 3]) is False + + +def test_value_contains_dicts(): + assert value_contains({"c": 3}, {"a": 1, "b": 2}) is False diff --git a/src/ContentProcessorAPI/app/libs/storage_blob/helper.py b/src/ContentProcessorAPI/app/libs/storage_blob/helper.py index 3b76cdd5..e74398c6 100644 --- a/src/ContentProcessorAPI/app/libs/storage_blob/helper.py +++ b/src/ContentProcessorAPI/app/libs/storage_blob/helper.py @@ -93,7 +93,7 @@ def delete_blob_and_cleanup(self, blob_name, container_name=None): def delete_folder(self, folder_name, container_name=None): container_client = self._get_container_client(container_name) - + # List all blobs inside the folder blobs_to_delete = container_client.list_blobs(name_starts_with=folder_name + "/") @@ -101,7 +101,7 @@ def delete_folder(self, folder_name, container_name=None): for blob in blobs_to_delete: blob_client = container_client.get_blob_client(blob.name) blob_client.delete_blob() - + blobs_to_delete = container_client.list_blobs() if not blobs_to_delete: @@ -110,4 +110,4 @@ def delete_folder(self, folder_name, container_name=None): # Delete the (virtual) folder in the Container blob_client = container_client.get_blob_client(folder_name) - blob_client.delete_blob() \ No newline at end of file + blob_client.delete_blob() diff --git a/src/ContentProcessorAPI/app/routers/contentprocessor.py b/src/ContentProcessorAPI/app/routers/contentprocessor.py index 115a32f0..2d9a9fac 100644 --- a/src/ContentProcessorAPI/app/routers/contentprocessor.py +++ b/src/ContentProcessorAPI/app/routers/contentprocessor.py @@ -492,6 +492,7 @@ async def get_original_file( file_stream, media_type=content_type_string, headers=headers ) + @router.delete( "/processed/{process_id}", response_model=ContentResultDelete, @@ -502,9 +503,9 @@ async def get_original_file( ) async def delete_processed_file( process_id: str, app_config: AppConfiguration = Depends(get_app_config) - ) -> ContentResultDelete: +) -> ContentResultDelete: try: - deleted_file = CosmosContentProcess(process_id=process_id).delete_processed_file( + deleted_file = CosmosContentProcess(process_id=process_id).delete_processed_file( connection_string=app_config.app_cosmos_connstr, database_name=app_config.app_cosmos_database, collection_name=app_config.app_cosmos_container_process, @@ -518,4 +519,4 @@ async def delete_processed_file( status="Success" if deleted_file else "Failed", process_id=deleted_file.process_id if deleted_file else "", message="" if deleted_file else "This record no longer exists. Please refresh." - ) \ No newline at end of file + ) diff --git a/src/ContentProcessorAPI/app/routers/models/contentprocessor/content_process.py b/src/ContentProcessorAPI/app/routers/models/contentprocessor/content_process.py index 4eb58c12..2f311872 100644 --- a/src/ContentProcessorAPI/app/routers/models/contentprocessor/content_process.py +++ b/src/ContentProcessorAPI/app/routers/models/contentprocessor/content_process.py @@ -189,7 +189,7 @@ def get_status_from_cosmos( return ContentProcess(**existing_process[0]) else: return None - + def delete_processed_file( self, connection_string: str, diff --git a/src/ContentProcessorAPI/app/routers/models/contentprocessor/model.py b/src/ContentProcessorAPI/app/routers/models/contentprocessor/model.py index 9cfd45af..885a1b13 100644 --- a/src/ContentProcessorAPI/app/routers/models/contentprocessor/model.py +++ b/src/ContentProcessorAPI/app/routers/models/contentprocessor/model.py @@ -64,6 +64,7 @@ class ContentResultUpdate(BaseModel): process_id: str modified_result: dict + class ContentResultDelete(BaseModel): process_id: str status: str diff --git a/src/ContentProcessorAPI/app/tests/libs/test_app_configuration_helper.py b/src/ContentProcessorAPI/app/tests/libs/test_app_configuration_helper.py new file mode 100644 index 00000000..16b1f82e --- /dev/null +++ b/src/ContentProcessorAPI/app/tests/libs/test_app_configuration_helper.py @@ -0,0 +1,64 @@ +import os +import pytest +from unittest.mock import patch +from azure.appconfiguration import ConfigurationSetting +from app.libs.app_configuration.helper import AppConfigurationHelper + + +@pytest.fixture +def mock_app_config_client(): + with patch( + "app.libs.app_configuration.helper.AzureAppConfigurationClient" + ) as MockClient: + yield MockClient + + +@pytest.fixture +def mock_credential(): + with patch( + "app.libs.app_configuration.helper.DefaultAzureCredential" + ) as MockCredential: + yield MockCredential + + +def test_initialize_client(mock_app_config_client, mock_credential): + app_config_endpoint = "https://example-config.azconfig.io" + helper = AppConfigurationHelper(app_config_endpoint) + + assert helper.app_config_endpoint == app_config_endpoint + assert helper.credential is not None + assert helper.app_config_client is not None + + +def test_initialize_client_no_endpoint(mock_credential): + with pytest.raises(ValueError, match="App Configuration Endpoint is not set."): + AppConfigurationHelper(None) + + +def test_read_configuration(mock_app_config_client, mock_credential): + app_config_endpoint = "https://example-config.azconfig.io" + helper = AppConfigurationHelper(app_config_endpoint) + + mock_client_instance = mock_app_config_client.return_value + mock_client_instance.list_configuration_settings.return_value = [ + ConfigurationSetting(key="test_key", value="test_value") + ] + + config_settings = helper.read_configuration() + assert len(config_settings) == 1 + assert config_settings[0].key == "test_key" + assert config_settings[0].value == "test_value" + + +def test_read_and_set_environmental_variables(mock_app_config_client, mock_credential): + app_config_endpoint = "https://example-config.azconfig.io" + helper = AppConfigurationHelper(app_config_endpoint) + + mock_client_instance = mock_app_config_client.return_value + mock_client_instance.list_configuration_settings.return_value = [ + ConfigurationSetting(key="test_key", value="test_value") + ] + + env_vars = helper.read_and_set_environmental_variables() + assert os.environ["test_key"] == "test_value" + assert env_vars["test_key"] == "test_value" diff --git a/src/ContentProcessorAPI/app/tests/libs/test_cosmos_db.py b/src/ContentProcessorAPI/app/tests/libs/test_cosmos_db.py new file mode 100644 index 00000000..dfcdab5a --- /dev/null +++ b/src/ContentProcessorAPI/app/tests/libs/test_cosmos_db.py @@ -0,0 +1,89 @@ +import pytest +from pymongo import MongoClient +from pymongo.collection import Collection +from pymongo.database import Database +from app.libs.cosmos_db.helper import CosmosMongDBHelper + + +@pytest.fixture +def mock_mongo_client(mocker): + client = mocker.MagicMock(spec=MongoClient) + return client + + +@pytest.fixture +def mock_database(mocker): + db = mocker.MagicMock(spec=Database) + db.list_collection_names.return_value = [] + return db + + +@pytest.fixture +def mock_collection(mocker): + collection = mocker.Mock(spec=Collection) + collection.insert_one.return_value = mocker.Mock(inserted_id="mock_id") + collection.find.return_value = [{"key": "value"}] + collection.count_documents.return_value = 1 + collection.update_one.return_value = mocker.Mock(matched_count=1, modified_count=1) + collection.delete_one.return_value = mocker.Mock(deleted_count=1) + return collection + + +@pytest.fixture +def cosmos_mongo_db_helper(mock_mongo_client, mock_database, mock_collection, mocker): + # Mock the MongoClient to return the mock database + mocker.patch( + "app.libs.cosmos_db.helper.MongoClient", return_value=mock_mongo_client + ) + mock_mongo_client.__getitem__.return_value = mock_database + mock_database.__getitem__.return_value = mock_collection + + # Initialize the CosmosMongDBHelper with the mocked client + helper = CosmosMongDBHelper( + connection_string="mongodb://localhost:27017", + db_name="test_db", + container_name="test_collection", + ) + helper.client = mock_mongo_client + helper.db = mock_database + helper.container = mock_collection + return helper + + +def test_insert_document(cosmos_mongo_db_helper, mock_collection): + document = {"key": "value"} + result = cosmos_mongo_db_helper.insert_document(document) + mock_collection.insert_one.assert_called_once_with(document) + assert result.inserted_id == "mock_id" + + +def test_find_document(cosmos_mongo_db_helper, mock_collection): + query = {"key": "value"} + result = cosmos_mongo_db_helper.find_document(query) + mock_collection.find.assert_called_once_with(query, None) + assert result == [{"key": "value"}] + + +def test_count_documents(cosmos_mongo_db_helper, mock_collection): + query = {"key": "value"} + result = cosmos_mongo_db_helper.count_documents(query) + mock_collection.count_documents.assert_called_once_with(query) + assert result == 1 + + +def test_update_document(cosmos_mongo_db_helper, mock_collection): + item_id = "123" + update = {"key": "new_value"} + result = cosmos_mongo_db_helper.update_document(item_id, update) + mock_collection.update_one.assert_called_once_with( + {"Id": item_id}, {"$set": update} + ) + assert result.matched_count == 1 + assert result.modified_count == 1 + + +def test_delete_document(cosmos_mongo_db_helper, mock_collection): + item_id = "123" + result = cosmos_mongo_db_helper.delete_document(item_id) + mock_collection.delete_one.assert_called_once_with({"Id": item_id}) + assert result.deleted_count == 1 diff --git a/src/ContentProcessorAPI/app/tests/libs/test_storage_blob.py b/src/ContentProcessorAPI/app/tests/libs/test_storage_blob.py new file mode 100644 index 00000000..490c9859 --- /dev/null +++ b/src/ContentProcessorAPI/app/tests/libs/test_storage_blob.py @@ -0,0 +1,93 @@ +import pytest +from azure.storage.blob import BlobServiceClient, ContainerClient, BlobClient +from azure.core.exceptions import ResourceNotFoundError +from app.libs.storage_blob.helper import StorageBlobHelper + + +@pytest.fixture +def mock_blob_service_client(mocker): + return mocker.Mock(spec=BlobServiceClient) + + +@pytest.fixture +def mock_container_client(mocker): + return mocker.Mock(spec=ContainerClient) + + +@pytest.fixture +def mock_blob_client(mocker): + return mocker.Mock(spec=BlobClient) + + +@pytest.fixture +def storage_blob_helper( + mock_blob_service_client, mock_container_client, mock_blob_client, mocker +): + mocker.patch( + "app.libs.storage_blob.helper.BlobServiceClient", + return_value=mock_blob_service_client, + ) + mock_blob_service_client.get_container_client.return_value = mock_container_client + mock_container_client.get_blob_client.return_value = mock_blob_client + return StorageBlobHelper( + account_url="https://example.com", container_name="test-container" + ) + + +def test_upload_blob(storage_blob_helper, mock_container_client, mock_blob_client): + file_stream = b"dummy content" + result = storage_blob_helper.upload_blob("test-blob", file_stream) + mock_container_client.get_blob_client.assert_called_once_with("test-blob") + mock_blob_client.upload_blob.assert_called_once_with(file_stream, overwrite=True) + assert result == mock_blob_client.upload_blob.return_value + + +def test_download_blob(storage_blob_helper, mock_container_client, mock_blob_client): + mock_blob_client.download_blob.return_value.readall.return_value = b"dummy content" + result = storage_blob_helper.download_blob("test-blob") + mock_container_client.get_blob_client.assert_called_once_with("test-blob") + # mock_blob_client.get_blob_properties.assert_called_once() + mock_blob_client.download_blob.assert_called_once() + assert result == b"dummy content" + + +def test_download_blob_not_found( + storage_blob_helper, mock_container_client, mock_blob_client +): + mock_blob_client.get_blob_properties.side_effect = ResourceNotFoundError + with pytest.raises( + ValueError, match="Blob 'test-blob' not found in container 'test-container'." + ): + storage_blob_helper.download_blob("test-blob", "test-container") + + +def test_replace_blob(storage_blob_helper, mock_container_client, mock_blob_client): + file_stream = b"dummy content" + result = storage_blob_helper.replace_blob("test-blob", file_stream) + mock_container_client.get_blob_client.assert_called_once_with("test-blob") + mock_blob_client.upload_blob.assert_called_once_with(file_stream, overwrite=True) + assert result == mock_blob_client.upload_blob.return_value + + +def test_delete_blob(storage_blob_helper, mock_container_client, mock_blob_client): + result = storage_blob_helper.delete_blob("test-blob") + mock_container_client.get_blob_client.assert_called_once_with("test-blob") + mock_blob_client.delete_blob.assert_called_once() + assert result == mock_blob_client.delete_blob.return_value + + +# def test_delete_blob_and_cleanup( +# storage_blob_helper, mock_container_client, mock_blob_client, mocker +# ): +# # Mock the list_blobs method to return an object with _page_iterator attribute +# mock_page_iterator = mocker.Mock() +# # mock_page_iterator._page_iterator = True +# mock_page_iterator.__iter__.return_value = iter([]) +# mock_container_client.list_blobs.return_value = mock_page_iterator + +# storage_blob_helper.delete_blob_and_cleanup("test-blob") + +# mock_container_client.get_blob_client.assert_called_with("test-blob") +# mock_blob_client.delete_blob.assert_called_once() +# mock_container_client.list_blobs.assert_called_once() +# assert mock_page_iterator.__iter__.called diff --git a/src/ContentProcessorAPI/app/tests/routers/test_contentprocessor.py b/src/ContentProcessorAPI/app/tests/routers/test_contentprocessor.py new file mode 100644 index 00000000..04b3ae59 --- /dev/null +++ b/src/ContentProcessorAPI/app/tests/routers/test_contentprocessor.py @@ -0,0 +1,228 @@ +import pytest +from fastapi.testclient import TestClient +from unittest.mock import patch, MagicMock +from app.main import app + +from app.appsettings import AppConfiguration + +client = TestClient(app) + + +@pytest.fixture +def app_config(): + config = AppConfiguration() + config.app_cosmos_connstr = "test_connection_string" + config.app_cosmos_database = "test_database" + config.app_cosmos_container_process = "test_container" + config.app_cps_max_filesize_mb = 20 + config.app_storage_blob_url = "test_blob_url" + return config + + +@pytest.fixture +def mock_app_config(): + with patch("app.routers.contentprocessor.get_app_config") as mock: + yield mock + + +@pytest.fixture +def mock_cosmos_content_process(): + with patch("app.routers.contentprocessor.CosmosContentProcess") as mock: + yield mock + + +@pytest.fixture +def mock_mime_types_detection(): + with patch("app.routers.contentprocessor.MimeTypesDetection") as mock: + yield mock + + +@patch("app.routers.contentprocessor.get_app_config") +@patch( + "app.routers.contentprocessor.CosmosContentProcess.get_all_processes_from_cosmos" +) +def test_get_all_processed_results( + mock_get_all_processes, mock_get_app_config, app_config +): + mock_get_app_config.return_value = app_config + mock_get_all_processes.return_value = { + "items": [], + "total_count": 0, + "total_pages": 0, + "current_page": 1, + "page_size": 10, + } + + response = client.post( + "/contentprocessor/processed", json={"page_number": 1, "page_size": 10} + ) + assert response.status_code == 200 + assert response.json() == { + "items": [], + "current_page": 1, + "page_size": 10, + "total_count": 0, + "total_pages": 0, + } + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.get_status_from_cosmos") +def test_get_status_processing(mock_get_status, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_get_status.return_value = MagicMock(status="processing") + + response = client.get("/contentprocessor/status/test_process_id") + assert response.status_code == 200 + assert response.json()["status"] == "processing" + assert "still in progress" in response.json()["message"] + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.get_status_from_cosmos") +def test_get_status_completed(mock_get_status, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_get_status.return_value = MagicMock(status="Completed") + + response = client.get("/contentprocessor/status/test_process_id") + assert response.status_code == 302 + assert response.json()["status"] == "completed" + assert "is completed" in response.json()["message"] + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.get_status_from_cosmos") +def test_get_status_failed(mock_get_status, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_get_status.return_value = None + + response = client.get("/contentprocessor/status/test_process_id") + assert response.status_code == 404 + assert response.json()["status"] == "failed" + assert "not found" in response.json()["message"] + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.get_status_from_cosmos") +def test_get_process(mock_get_status, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_get_status.return_value = MagicMock( + process_id="test_process_id", + processed_file_name="test.pdf", + processed_file_mime_type="application/pdf", + processed_time="2025-03-13T12:00:00Z", + last_modified_by="user", + status="Completed", + result={}, + confidence={}, + target_schema={ + "Id": "schema_id", + "ClassName": "class_name", + "Description": "description", + "FileName": "file_name", + "ContentType": "content_type", + }, + comment="test comment", + ) + + response = client.get("/contentprocessor/processed/test_process_id") + assert response.status_code == 200 + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.get_status_from_cosmos") +def test_get_process_not_found(mock_get_status, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_get_status.return_value = None + + response = client.get("/contentprocessor/processed/test_process_id") + assert response.status_code == 404 + assert response.json()["status"] == "failed" + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.get_status_from_blob") +def test_get_process_steps(mock_get_steps, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_get_steps.return_value = {"steps": []} + + response = client.get("/contentprocessor/processed/test_process_id/steps") + assert response.status_code == 200 + assert response.json() == {"steps": []} + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.get_status_from_blob") +def test_get_process_steps_not_found(mock_get_steps, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_get_steps.return_value = None + + response = client.get("/contentprocessor/processed/test_process_id/steps") + assert response.status_code == 404 + assert response.json()["status"] == "failed" + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.update_process_result") +def test_update_process_result(mock_update_result, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_update_result.return_value = MagicMock() + + data = {"process_id": "test_process_id", "modified_result": {"key": "value"}} + response = client.put("/contentprocessor/processed/test_process_id", json=data) + assert response.status_code == 200 + assert response.json()["status"] == "success" + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.update_process_comment") +def test_update_process_comment(mock_update_comment, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_update_comment.return_value = MagicMock() + + data = {"process_id": "test_process_id", "comment": "new comment"} + response = client.put("/contentprocessor/processed/test_process_id", json=data) + assert response.status_code == 200 + assert response.json()["status"] == "success" + + +def test_get_original_file_success( + mock_app_config, mock_cosmos_content_process, mock_mime_types_detection +): + # Mocking the app config + mock_app_config.return_value.app_cosmos_connstr = "mock_connstr" + mock_app_config.return_value.app_cosmos_database = "mock_database" + mock_app_config.return_value.app_cosmos_container_process = "mock_container_process" + mock_app_config.return_value.app_storage_blob_url = "mock_blob_url" + mock_app_config.return_value.app_cps_processes = "mock_cps_processes" + + # Mocking the process status + mock_process_status = MagicMock() + mock_process_status.processed_file_name = "testfile.txt" + mock_process_status.process_id = "123" + mock_process_status.get_file_bytes_from_blob.return_value = b"file content" + mock_cosmos_content_process.return_value.get_status_from_cosmos.return_value = ( + mock_process_status + ) + + # Mocking the MIME type detection + mock_mime_types_detection.get_file_type.return_value = "text/plain" + + response = client.get("/contentprocessor/processed/files/123") + assert response.status_code == 200 + assert response.headers["Content-Type"] == "text/plain" + assert ( + response.headers["Content-Disposition"] + == "inline; filename*=UTF-8''testfile.txt" + ) + + +@patch("app.routers.contentprocessor.get_app_config") +@patch("app.routers.contentprocessor.CosmosContentProcess.get_status_from_cosmos") +def test_get_original_file_not_found(mock_get_status, mock_get_app_config, app_config): + mock_get_app_config.return_value = app_config + mock_get_status.return_value = None + + response = client.get("/contentprocessor/processed/files/test_process_id") + assert response.status_code == 404 + assert response.json()["status"] == "failed" diff --git a/src/ContentProcessorAPI/app/tests/routers/test_schemavault.py b/src/ContentProcessorAPI/app/tests/routers/test_schemavault.py new file mode 100644 index 00000000..5d7e08d7 --- /dev/null +++ b/src/ContentProcessorAPI/app/tests/routers/test_schemavault.py @@ -0,0 +1,118 @@ +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from unittest.mock import MagicMock +from app.routers.schemavault import router, get_schemas + + +app = FastAPI() +app.include_router(router) + +client = TestClient(app) + +mock_schemas = MagicMock() + + +@pytest.fixture +def override_get_schemas(): + def _override_get_schemas(): + return mock_schemas + + app.dependency_overrides[get_schemas] = _override_get_schemas + yield + app.dependency_overrides.clear() + + +def test_get_all_registered_schema(override_get_schemas): + mock_schemas.GetAll.return_value = [] + response = client.get("/schemavault/") + assert response.status_code == 200 + assert response.json() == [] + + +# def test_register_schema(override_get_schemas): +# mock_schemas.Add.return_value = { +# "Id": "test-id", +# "ClassName": "TestClass", +# "Description": "Test description", +# "FileName": "test.txt", +# "ContentType": "text/plain", +# } +# data = { +# "ClassName": "TestClass", +# "Description": "Test description", +# } +# files = {"file": ("test.txt", b"test content", "text/plain")} +# response = client.post( +# "/schemavault/", +# data=data, +# files=files, +# headers={"Content-Type": "multipart/form-data"}, +# ) +# assert response.status_code == 200 + + +# def test_update_schema(override_get_schemas): +# mock_schemas.Update.return_value = { +# "Id": "test-id", +# "ClassName": "UpdatedClass", +# "Description": "Updated description", +# "FileName": "updated.txt", +# "ContentType": "text/plain", +# } +# data = { +# "SchemaId": "test-id", +# "ClassName": "UpdatedClass", +# } +# files = {"file": ("updated.txt", b"updated content", "text/plain")} +# response = client.put( +# "/schemavault/", +# data=data, +# files=files, +# headers={"Content-Type": "multipart/form-data"}, +# ) +# assert response.status_code == 200 + + +# def test_unregister_schema(override_get_schemas): +# mock_schemas.Delete.return_value = { +# "Id": "test-id", +# "ClassName": "TestClass", +# "FileName": "test.txt", +# } +# data = SchemaVaultUnregisterRequest(SchemaId="test-id") +# response = client.delete( +# "/schemavault/", +# data=data.model_dump(), +# ) + +# assert response.status_code == 200 +# assert response.json() == { +# "Status": "Success", +# "SchemaId": "test-id", +# "ClassName": "TestClass", +# "FileName": "test.txt", +# } + + +def test_get_registered_schema_file_by_schema_id(override_get_schemas): + mock_schemas.GetFile.return_value = { + "FileName": "test.txt", + "ContentType": "text/plain", + "File": b"test content", + } + response = client.get("/schemavault/schemas/test-id") + assert response.status_code == 200 + assert ( + response.headers["Content-Disposition"] + == "attachment; filename*=UTF-8''test.txt" + ) + assert response.content == b"test content" + + +def test_get_registered_schema_file_by_schema_id_500_error(override_get_schemas): + mock_schemas.GetFile.side_effect = Exception("Internal Server Error") + + response = client.get("/schemavault/schemas/test-id") + assert response.status_code == 500 + assert response.json() == {"detail": "Internal Server Error"} diff --git a/src/ContentProcessorAPI/app/tests/test_dependencies.py b/src/ContentProcessorAPI/app/tests/test_dependencies.py deleted file mode 100644 index 5df79f3f..00000000 --- a/src/ContentProcessorAPI/app/tests/test_dependencies.py +++ /dev/null @@ -1,36 +0,0 @@ -import pytest -from fastapi import FastAPI, Depends -from fastapi.testclient import TestClient -from src.ContentProcessorAPI.app.dependencies import get_token_header, get_query_token -# from starlette.status import HTTP_400_BAD_REQUEST - - -@pytest.fixture -def test_app(): - app = FastAPI() - - @app.get("/header-protected") - async def protected_route_header(dep=Depends(get_token_header)): - return {"message": "Success"} - - @app.get("/query-protected") - async def protected_route_query(dep=Depends(get_query_token)): - return {"message": "Success"} - - return app - - -def test_get_token_header_fails(test_app): - client = TestClient(test_app) - # Provide the required header so FastAPI doesn't return 422 - response = client.get("/header-protected", headers={"x-token": "fake"}) - assert response.status_code == 400 - assert response.json() == {"detail": "X-Token header invalid"} - - -def test_get_query_token_fails(test_app): - client = TestClient(test_app) - # Provide the required query param so FastAPI doesn't return 422 - response = client.get("/query-protected?token=fake") - assert response.status_code == 400 - assert response.json() == {"detail": "No ... token provided"} diff --git a/src/ContentProcessorAPI/app/tests/test_main.py b/src/ContentProcessorAPI/app/tests/test_main.py new file mode 100644 index 00000000..813ba8a1 --- /dev/null +++ b/src/ContentProcessorAPI/app/tests/test_main.py @@ -0,0 +1,18 @@ +from fastapi.testclient import TestClient +from app.main import app + +client = TestClient(app) + + +def test_health(): + response = client.get("/health") + assert response.status_code == 200 + assert response.json() == {"message": "I'm alive!"} + assert response.headers["Custom-Header"] == "liveness probe" + + +def test_startup(): + response = client.get("/startup") + assert response.status_code == 200 + assert "Running for" in response.json()["message"] + assert response.headers["Custom-Header"] == "Startup probe" diff --git a/src/ContentProcessorAPI/pyproject.toml b/src/ContentProcessorAPI/pyproject.toml index 43e6139b..7c12ef81 100644 --- a/src/ContentProcessorAPI/pyproject.toml +++ b/src/ContentProcessorAPI/pyproject.toml @@ -22,6 +22,10 @@ dependencies = [ [dependency-groups] dev = [ + "pytest>=8.3.4", + "pytest-cov>=6.0.0", + "pytest-mock>=3.14.0", + "coverage>=7.6.10", "pre-commit>=4.1.0", "ruff>=0.9.3", ]