Merge pull request microsoft#188 from microsoft/PSL-TS-19050

test: Implemented Log execution time per prompt in Report for content processing

Author: Prajwal-Microsoft

tests/e2e-test/pytest.ini

@@ -3,4 +3,5 @@ log_cli = true
 log_cli_level = INFO
 log_file = logs/tests.log
 log_file_level = INFO
-addopts = -p no:warnings
+addopts = -p no:warnings --tb=short
+

tests/e2e-test/readme.MD

@@ -24,7 +24,7 @@ Installing Playwright Pytest from Virtual Environment
 
 Run test cases
 
-- To run test cases from your 'tests' folder : "pytest --html=report.html --self-contained-html"
+- To run test cases from your 'tests/e2e-test' folder : "pytest --html=report.html --self-contained-html"
 
 Create .env file in project root level with web app url and client credentials
 

tests/e2e-test/requirements.txt

@@ -3,4 +3,5 @@ pytest-reporter-html1
 python-dotenv
 pytest-check
 pytest-html
-py
+py
+beautifulsoup4

tests/e2e-test/tests/conftest.py

@@ -1,9 +1,11 @@
 import os
-
+import atexit
+import io
+from bs4 import BeautifulSoup
 import pytest
+import logging
 from config.constants import URL
 from playwright.sync_api import sync_playwright
-from py.xml import html  # type: ignore
 
 
 @pytest.fixture(scope="session")
@@ -25,29 +27,74 @@ def login_logout():
         # perform close the browser
         browser.close()
 
+log_streams = {}
 
 @pytest.hookimpl(tryfirst=True)
-def pytest_html_report_title(report):
-    report.title = "Automation_Content_Processing"
-
+def pytest_runtest_setup(item):
+    # Prepare StringIO for capturing logs
+    stream = io.StringIO()
+    handler = logging.StreamHandler(stream)
+    handler.setLevel(logging.INFO)
 
-# Add a column for descriptions
-def pytest_html_results_table_header(cells):
-    cells.insert(1, html.th("Description"))
+    logger = logging.getLogger()
+    logger.addHandler(handler)
 
+    # Save handler and stream
+    log_streams[item.nodeid] = (handler, stream)
 
-def pytest_html_results_table_row(report, cells):
-    cells.insert(
-        1, html.td(report.description if hasattr(report, "description") else "")
-    )
 
-
-# Add logs and docstring to report
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     outcome = yield
     report = outcome.get_result()
-    report.description = str(item.function.__doc__)
-    os.makedirs("logs", exist_ok=True)
-    extra = getattr(report, "extra", [])
-    report.extra = extra
+
+    handler, stream = log_streams.get(item.nodeid, (None, None))
+
+    if handler and stream:
+        # Make sure logs are flushed
+        handler.flush()
+        log_output = stream.getvalue()
+
+        # Only remove the handler, don't close the stream yet
+        logger = logging.getLogger()
+        logger.removeHandler(handler)
+
+        # Store the log output on the report object for HTML reporting
+        report.description = f"<pre>{log_output.strip()}</pre>"
+
+        # Clean up references
+        log_streams.pop(item.nodeid, None)
+    else:
+        report.description = ""
+
+def pytest_collection_modifyitems(items):
+    for item in items:
+        if hasattr(item, 'callspec'):
+            prompt = item.callspec.params.get("prompt")
+            if prompt:
+                item._nodeid = prompt  # This controls how the test name appears in the report
+
+def rename_duration_column():
+    report_path = os.path.abspath("report.html")  # or your report filename
+    if not os.path.exists(report_path):
+        print("Report file not found, skipping column rename.")
+        return
+
+    with open(report_path, 'r', encoding='utf-8') as f:
+        soup = BeautifulSoup(f, 'html.parser')
+
+    # Find and rename the header
+    headers = soup.select('table#results-table thead th')
+    for th in headers:
+        if th.text.strip() == 'Duration':
+            th.string = 'Execution Time'
+            #print("Renamed 'Duration' to 'Execution Time'")
+            break
+    else:
+        print("'Duration' column not found in report.")
+
+    with open(report_path, 'w', encoding='utf-8') as f:
+        f.write(str(soup))
+
+# Register this function to run after everything is done
+atexit.register(rename_duration_column)

tests/e2e-test/tests/test_contentProcessing_gp_tc.py

@@ -1,41 +1,46 @@
 import logging
-
 import pytest
 from pages.HomePage import HomePage
-
+ 
 logger = logging.getLogger(__name__)
+ 
+# Define step-wise test actions for Golden Path
+golden_path_steps = [
+    ("Validate home page is loaded", lambda home: home.validate_home_page()),
+     ("Select Invoice Schema", lambda home: home.select_schema("Invoice")),
+    ("Upload Invoice documents", lambda home: home.upload_files("Invoice")),
+    ("Refresh page till status is updated to Completed", lambda home: home.refresh()),
+    ("Validate extracted result for Invoice", lambda home: home.validate_invoice_extracted_result()),
+    ("Modify Extracted Data JSON & submit comments", lambda home: home.modify_and_submit_extracted_data()),
+    ("Validate process steps for Invoice", lambda home: home.validate_process_steps()),
+    ("Select Property Loss Damage Claim Form Schema", lambda home: home.select_schema("Property")),
+    ("Upload Property Loss Damage Claim Form documents", lambda home: home.upload_files("Property")),
+    ("Refresh page till status is updated to Completed", lambda home: home.refresh()),
+    ("Validate extracted result for Property Loss Damage Claim Form", lambda home: home.validate_property_extracted_result()),
+    ("Validate process steps for Property Loss Damage Claim Form", lambda home: home.validate_process_steps()),
+    ("Validate Delete files", lambda home: home.delete_files())
+]
+ 
+# Generate readable test step IDs
+golden_path_ids = [f"{i+1:02d}. {desc}" for i, (desc, _) in enumerate(golden_path_steps)]
+
 
+@pytest.mark.parametrize("description, action", golden_path_steps, ids=golden_path_ids)
+def test_content_processing_steps(login_logout, description, action, request):
+    """
+    Executes Golden Path content processing steps with individual log entries.
+    """
+    request.node._nodeid = description
 
-@pytest.mark.testcase_id("TC001")
-def test_ContentProcessing_Golden_path_test(login_logout):
-    """Validate Golden path test case for Content Processing Accelerator"""
     page = login_logout
-    home_page = HomePage(page)
-    logger.info("Step 1: Validate home page is loaded.")
-    home_page.validate_home_page()
-    logger.info("Step 2: Select Invoice Schema.")
-    home_page.select_schema("Invoice")
-    logger.info("Step 3: Upload Invoice documents.")
-    home_page.upload_files("Invoice")
-    logger.info("Step 4: Refresh page till status is updated to Completed.")
-    home_page.refresh()
-    logger.info("Step 5: Validate extracted result for Invoice.")
-    home_page.validate_invoice_extracted_result()
-    logger.info("Step 6: Modify Extracted Data JSON & submit comments.")
-    home_page.modify_and_submit_extracted_data()
-    logger.info("Step 7: Validate process steps for Invoice")
-    home_page.validate_process_steps()
-    logger.info("Step 8: Select Property Loss Damage Claim Form Schema.")
-    home_page.select_schema("Property")
-    logger.info("Step 9: Upload Property Loss Damage Claim Form documents.")
-    home_page.upload_files("Property")
-    logger.info("Step 10: Refresh page till status is updated to Completed.")
-    home_page.refresh()
-    logger.info(
-        "Step 11: Validate extracted result for Property Loss Damage Claim Form."
-    )
-    home_page.validate_property_extracted_result()
-    logger.info("Step 12: Validate process steps for Property Loss Damage Claim Form.")
-    home_page.validate_process_steps()
-    logger.info("Step 13: Validate Delete files.")
-    home_page.delete_files()
+    home = HomePage(page)
+ 
+    logger.info(f"Running test step: {description}")
+    try:
+        action(home)
+    except Exception as e:
+        logger.error(f"Step failed: {description}")
+        raise
+ 
+    # Optionally attach to report
+    request.node._report_sections.append(("call", "log", f"Step passed: {description}"))