test: Implemented Log execution time per prompt in Report

Author: Avijit-Microsoft

infra/deploy_ai_foundry.bicep

@@ -64,7 +64,7 @@ resource applicationInsights 'Microsoft.Insights/components@2020-02-02' = {
     Application_Type: 'web'
     publicNetworkAccessForIngestion: 'Enabled'
     publicNetworkAccessForQuery: 'Enabled'
-    WorkspaceResourceId: logAnalytics.id
+    WorkspaceResourceId: useExisting ? existingLogAnalyticsWorkspaceId : logAnalytics.id
   }
 }
 

tests/e2e-test/pytest.ini

@@ -3,4 +3,4 @@ log_cli = true
 log_cli_level = INFO
 log_file = logs/tests.log
 log_file_level = INFO
-addopts = -p no:warnings
+addopts = -p no:warnings --tb=short

tests/e2e-test/readme.MD

@@ -24,7 +24,7 @@ Installing Playwright Pytest from Virtual Environment
 
 Run test cases
 
-- To run test cases from your 'tests' folder : "pytest --html=report.html --self-contained-html"
+- To run test cases from your 'tests/e2e-test' folder : "pytest --html=report.html --self-contained-html"
 
 Create .env file in project root level with web app url and client credentials
 

tests/e2e-test/requirements.txt

@@ -4,3 +4,4 @@ python-dotenv
 pytest-check
 pytest-html
 py
+beautifulsoup4

tests/e2e-test/tests/conftest.py

@@ -1,11 +1,15 @@
+import atexit
+import io
+import logging
 import os
 
+
+from bs4 import BeautifulSoup
+
 from config.constants import URL
 
 from playwright.sync_api import sync_playwright
 
-from py.xml import html  # type: ignore
-
 import pytest
 
 
@@ -31,22 +35,99 @@ def pytest_html_report_title(report):
 
 
 # Add a column for descriptions
-def pytest_html_results_table_header(cells):
-    cells.insert(1, html.th("Description"))
+# def pytest_html_results_table_header(cells):
+#     cells.insert(1, html.th("Description"))
 
 
-def pytest_html_results_table_row(report, cells):
-    cells.insert(
-        1, html.td(report.description if hasattr(report, "description") else "")
-    )
+# def pytest_html_results_table_row(report, cells):
+#     cells.insert(
+#         1, html.td(report.description if hasattr(report, "description") else "")
+#     )
+
+
+log_streams = {}
+
+
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_setup(item):
+    # Prepare StringIO for capturing logs
+    stream = io.StringIO()
+    handler = logging.StreamHandler(stream)
+    handler.setLevel(logging.INFO)
+
+    logger = logging.getLogger()
+    logger.addHandler(handler)
+
+    # Save handler and stream
+    log_streams[item.nodeid] = (handler, stream)
 
 
-# Add logs and docstring to report
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     outcome = yield
     report = outcome.get_result()
-    report.description = str(item.function.__doc__)
-    os.makedirs("logs", exist_ok=True)
-    extra = getattr(report, "extra", [])
-    report.extra = extra
+
+    handler, stream = log_streams.get(item.nodeid, (None, None))
+
+    if handler and stream:
+        # Make sure logs are flushed
+        handler.flush()
+        log_output = stream.getvalue()
+
+        # Only remove the handler, don't close the stream yet
+        logger = logging.getLogger()
+        logger.removeHandler(handler)
+
+        # Store the log output on the report object for HTML reporting
+        report.description = f"<pre>{log_output.strip()}</pre>"
+
+        # Clean up references
+        log_streams.pop(item.nodeid, None)
+    else:
+        report.description = ""
+
+
+def pytest_collection_modifyitems(items):
+    for item in items:
+        if hasattr(item, 'callspec'):
+            prompt = item.callspec.params.get("prompt")
+            if prompt:
+                item._nodeid = prompt  # This controls how the test name appears in the report
+
+
+def rename_duration_column():
+    report_path = os.path.abspath("report.html")  # or your report filename
+    if not os.path.exists(report_path):
+        print("Report file not found, skipping column rename.")
+        return
+
+    with open(report_path, 'r', encoding='utf-8') as f:
+        soup = BeautifulSoup(f, 'html.parser')
+
+    # Find and rename the header
+    headers = soup.select('table#results-table thead th')
+    for th in headers:
+        if th.text.strip() == 'Duration':
+            th.string = 'Execution Time'
+            # print("Renamed 'Duration' to 'Execution Time'")
+            break
+    else:
+        print("'Duration' column not found in report.")
+
+    with open(report_path, 'w', encoding='utf-8') as f:
+        f.write(str(soup))
+
+
+# Register this function to run after everything is done
+atexit.register(rename_duration_column)
+
+
+# Add logs and docstring to report
+# @pytest.hookimpl(hookwrapper=True)
+# def pytest_runtest_makereport(item, call):
+#     outcome = yield
+#     report = outcome.get_result()
+#     report.description = str(item.function.__doc__)
+#     os.makedirs("logs", exist_ok=True)
+#     extra = getattr(report, "extra", [])
+#     report.extra = extra

tests/e2e-test/tests/test_codegen_gp_tc.py

@@ -5,27 +5,42 @@
 
 import pytest
 
-
 logger = logging.getLogger(__name__)
 
+# Define step-wise test actions for Golden Path
+golden_path_steps = [
+    ("01. Validate home page is loaded", lambda home: home.validate_home_page()),
+    ("02. Validate Upload of other than SQL files", lambda home: home.upload_unsupported_files()),
+    ("03. Validate Upload input files for SQL only", lambda home: home.upload_files()),
+    ("04. Validate translation process for uploaded files", lambda home: _timed_translation(home)),
+    ("05. Check batch history", lambda home: home.validate_batch_history()),
+    ("06. Download all files and return home", lambda home: home.validate_download_files()),
+]
 
-@pytest.mark.testcase_id("TC001")
-def test_CodeGen_Golden_path_test(login_logout):
-    """Validate Golden path test case for Modernize your code Accelerator"""
-    page = login_logout
-    home_page = HomePage(page)
-    logger.info("Step 1: Validate home page is loaded.")
-    home_page.validate_home_page()
-    logger.info("Step 2: Validate Upload of other than SQL files.")
-    home_page.upload_unsupported_files()
-    logger.info("Step 3: Validate Upload input files for SQL only.")
-    home_page.upload_files()
-    logger.info("Step 4: Validate translation process for uploaded files.")
+
+def _timed_translation(home):
     start = time.time()
-    home_page.validate_translate()
+    home.validate_translate()
     end = time.time()
-    print(f"Translation process for uploaded files took {end - start:.2f} seconds")
-    logger.info("Step 5: Check batch history.")
-    home_page.validate_batch_history()
-    logger.info("Step 6: Download all files and return home.")
-    home_page.validate_download_files()
+    logger.info(f"Translation process for uploaded files took {end - start:.2f} seconds")
+
+
+@pytest.mark.parametrize("description, action", golden_path_steps, ids=[desc for desc, _ in golden_path_steps])
+def test_codegen_golden_path(login_logout, description, action, request):
+    """
+    Executes golden path test steps for Modernize Your Code Accelerator with detailed logging.
+    """
+    request.node._nodeid = description  # To improve test output readability
+
+    page = login_logout
+    home = HomePage(page)
+
+    logger.info(f"Running test step: {description}")
+    try:
+        action(home)
+    except Exception:
+        logger.error(f"Step failed: {description}", exc_info=True)
+        raise
+
+    # Optional reporting hook
+    request.node._report_sections.append(("call", "log", f"Step passed: {description}"))