diff --git a/tests/e2e-test/README.md b/tests/e2e-test/README.md index 1957a0d2..69935d96 100644 --- a/tests/e2e-test/README.md +++ b/tests/e2e-test/README.md @@ -20,6 +20,7 @@ This will create a virtual environment directory named microsoft inside your cur Installing Playwright Pytest from Virtual Environment - To install libraries run "pip install -r requirements.txt" +- To install Playwright run "playwright install" Run test cases diff --git a/tests/e2e-test/base/__init__.py b/tests/e2e-test/base/__init__.py index cf50d1cc..daeab9a2 100644 --- a/tests/e2e-test/base/__init__.py +++ b/tests/e2e-test/base/__init__.py @@ -1 +1 @@ -from . import base \ No newline at end of file +"""Initiate base package""" diff --git a/tests/e2e-test/base/base.py b/tests/e2e-test/base/base.py index 759c7701..c34dc3a8 100644 --- a/tests/e2e-test/base/base.py +++ b/tests/e2e-test/base/base.py @@ -1,36 +1,48 @@ -from config.constants import * -import requests import json -from dotenv import load_dotenv -import os + +from config.constants import URL + class BasePage: def __init__(self, page): self.page = page - def scroll_into_view(self,locator): + async def scroll_into_view(self, locator): reference_list = locator - locator.nth(reference_list.count()-1).scroll_into_view_if_needed() + await locator.nth(reference_list.count() - 1).scroll_into_view_if_needed() - def is_visible(self,locator): - locator.is_visible() + async def is_visible(self, locator): + return await locator.is_visible() - def validate_response_status(self, question_api): - load_dotenv() - # The URL of the API endpoint you want to access + async def validate_response_status(self, question_api, expected_status=200): + """Validate API response status for chat endpoint.""" url = f"{URL}/backend/chat" headers = { "Content-Type": "application/json", "Accept": "*/*", } - payload = { - "Question": question_api, # This is your example question, you can modify it as needed - } - # Make the POST request - response = self.page.request.post(url, headers=headers, data=json.dumps(payload), timeout=200000) - # Check the response status code - assert response.status == 200, "Response code is " + str(response.status) + " " + str(response.json()) + payload = {"Question": question_api} + + try: + response = await self.page.context.request.post( + url=url, headers=headers, data=json.dumps(payload), timeout=200_000 + ) + + error_msg = f"Response code is {response.status}" + try: + response_json = await response.json() + error_msg += f" Response: {response_json}" + except Exception: + response_text = await response.text() + error_msg += f" Response text: {response_text}" + + assert response.status == expected_status, error_msg + + await self.page.wait_for_timeout(4000) + return response - \ No newline at end of file + except Exception as e: + print(f"Request failed: {e}") + raise diff --git a/tests/e2e-test/config/constants.py b/tests/e2e-test/config/constants.py index 7d9c8a91..629e36cb 100644 --- a/tests/e2e-test/config/constants.py +++ b/tests/e2e-test/config/constants.py @@ -1,16 +1,19 @@ -from dotenv import load_dotenv import os +from dotenv import load_dotenv + load_dotenv() -URL = os.getenv('url') -if URL.endswith('/'): +URL = os.getenv("url") +if URL.endswith("/"): URL = URL[:-1] # DKM input data chat_question1 = "What are the main factors contributing to the current housing affordability issues?" chat_question2 = "Analyze the two annual reports and compare the positive and negative outcomes YoY. Show the results in a table." -house_10_11_question ="Can you summarize and compare the tables on page 10 and 11?" -handwritten_question1 ="Analyze these forms and create a table with all buyers, sellers, and corresponding purchase prices." -search_1= "Housing Report" -search_2= "Contracts" -contract_details_question = "What liabilities is the buyer responsible for within the contract?" +house_10_11_question = "Can you summarize and compare the tables on page 10 and 11?" +handwritten_question1 = "Analyze these forms and create a table with all buyers, sellers, and corresponding purchase prices." +search_1 = "Housing Report" +search_2 = "Contracts" +contract_details_question = ( + "What liabilities is the buyer responsible for within the contract?" +) diff --git a/tests/e2e-test/pages/__init__.py b/tests/e2e-test/pages/__init__.py index 3363f3e4..8aad2ec4 100644 --- a/tests/e2e-test/pages/__init__.py +++ b/tests/e2e-test/pages/__init__.py @@ -1,2 +1 @@ -from. import loginPage -from. import dkmPage \ No newline at end of file +"""Initiate pages package""" diff --git a/tests/e2e-test/pages/dkmPage.py b/tests/e2e-test/pages/dkmPage.py index c434d19f..7193260d 100644 --- a/tests/e2e-test/pages/dkmPage.py +++ b/tests/e2e-test/pages/dkmPage.py @@ -1,53 +1,54 @@ -from base.base import BasePage -from playwright.sync_api import expect import time + from playwright.sync_api import TimeoutError as PlaywrightTimeoutError +from playwright.sync_api import expect + +from base.base import BasePage + class DkmPage(BasePage): WELCOME_PAGE_TITLE = "(//div[@class='order-5 my-auto pb-3 text-lg font-semibold leading-tight text-white mt-3'])[1]" NEWTOPIC = "//button[normalize-space()='New Topic']" - Suggested_follow_up_questions="body > div:nth-child(3) > div:nth-child(1) > main:nth-child(2) > div:nth-child(1) > div:nth-child(3) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(3) > div:nth-child(1) > div:nth-child(6) > div:nth-child(3) > button:nth-child(2)" + Suggested_follow_up_questions = "body > div:nth-child(3) > div:nth-child(1) > main:nth-child(2) > div:nth-child(1) > div:nth-child(3) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(3) > div:nth-child(1) > div:nth-child(6) > div:nth-child(3) > button:nth-child(2)" SCROLL_DOWN = "//div[10]//div[2]//div[2]//i[1]//img[1]" - ASK_QUESTION ="//textarea[@placeholder='Ask a question or request (ctrl + enter to submit)']" - SEARCH_BOX="//input[@type='search']" - HOUSING_2022 ="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[4]/div[2]/div[2]/span[1]" - HOUSING_2023 ="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[3]/div[2]/div[2]/span[1]" + ASK_QUESTION = ( + "//textarea[@placeholder='Ask a question or request (ctrl + enter to submit)']" + ) + SEARCH_BOX = "//input[@type='search']" + HOUSING_2022 = "//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[4]/div[2]/div[2]/span[1]" + HOUSING_2023 = "//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[3]/div[2]/div[2]/span[1]" CONTRACTS_DETAILS_PAGE = "body > div:nth-child(3) > div:nth-child(1) > main:nth-child(2) > div:nth-child(1) > div:nth-child(2) > div:nth-child(4) > div:nth-child(1) > div:nth-child(1) > div:nth-child(6) > div:nth-child(2) > div:nth-child(2) > div:nth-child(3) > button:nth-child(2)" - DETAILS_PAGE ="body > div:nth-child(3) > div:nth-child(1) > main:nth-child(2) > div:nth-child(1) > div:nth-child(2) > div:nth-child(4) > div:nth-child(1) > div:nth-child(1) > div:nth-child(3) > div:nth-child(2) > div:nth-child(2) > div:nth-child(3) > button:nth-child(2)" - POP_UP_CHAT="//button[@value='Chat Room']" - CLOSE_POP_UP ="//button[@aria-label='close']" - CLLEAR_ALL_POP_UP ="//button[normalize-space()='Clear all']" - HANDWRITTEN_DOC1="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[6]/div[2]/div[2]/span[1]" - HANDWRITTEN_DOC2="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[1]/div[2]/div[2]/span[1]" - HANDWRITTEN_DOC3="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[5]/div[2]/div[2]/span[1]" + DETAILS_PAGE = "body > div:nth-child(3) > div:nth-child(1) > main:nth-child(2) > div:nth-child(1) > div:nth-child(2) > div:nth-child(4) > div:nth-child(1) > div:nth-child(1) > div:nth-child(3) > div:nth-child(2) > div:nth-child(2) > div:nth-child(3) > button:nth-child(2)" + POP_UP_CHAT = "//button[@value='Chat Room']" + CLOSE_POP_UP = "//button[@aria-label='close']" + CLLEAR_ALL_POP_UP = "//button[normalize-space()='Clear all']" + HANDWRITTEN_DOC1 = "//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[6]/div[2]/div[2]/span[1]" + HANDWRITTEN_DOC2 = "//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[1]/div[2]/div[2]/span[1]" + HANDWRITTEN_DOC3 = "//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[5]/div[2]/div[2]/span[1]" SEND_BUTTON = "//button[@aria-label='Send']" POP_UP_CHAT_SEARCH = "(//textarea[@placeholder='Ask a question or request (ctrl + enter to submit)'])[2]" POP_UP_CHAT_SEND = "(//button[@type='submit'])[2]" DOCUMENT_FILTER = "//button[normalize-space()='Accessibility Features']" HEADING_TITLE = "//div[.='Document Knowledge Mining']" - - + def __init__(self, page): self.page = page - - def validate_home_page(self): self.page.wait_for_timeout(5000) - expect(self.page.locator(self.DOCUMENT_FILTER)).to_be_visible() + # expect(self.page.locator(self.DOCUMENT_FILTER)).to_be_visible() expect(self.page.locator(self.HEADING_TITLE)).to_be_visible() self.page.wait_for_timeout(2000) - - def enter_a_question(self,text): + def enter_a_question(self, text): self.page.locator(self.ASK_QUESTION).fill(text) self.page.wait_for_timeout(5000) - def enter_in_search(self,text): + def enter_in_search(self, text): self.page.locator(self.SEARCH_BOX).fill(text) self.page.wait_for_timeout(5000) - def enter_in_popup_search(self,text): + def enter_in_popup_search(self, text): self.page.locator(self.POP_UP_CHAT_SEARCH).fill(text) self.page.wait_for_timeout(5000) self.page.locator(self.POP_UP_CHAT_SEND).click() @@ -67,7 +68,7 @@ def click_on_popup_chat(self): self.page.locator(self.POP_UP_CHAT).click() self.page.wait_for_timeout(5000) - def close_pop_up(self): + def close_pop_up(self): self.page.locator(self.CLOSE_POP_UP).click() self.page.wait_for_timeout(2000) self.page.locator(self.CLLEAR_ALL_POP_UP).click() @@ -78,15 +79,15 @@ def select_handwritten_doc(self): self.page.locator(self.HANDWRITTEN_DOC2).click() self.page.locator(self.HANDWRITTEN_DOC3).click() self.page.wait_for_timeout(2000) - + def click_send_button(self): # Click on send button in question area self.page.locator(self.SEND_BUTTON).click() self.page.wait_for_timeout(5000) - #self.page.wait_for_load_state('networkidle') + # self.page.wait_for_load_state('networkidle') - def wait_until_response_loaded(self,timeout=200000): + def wait_until_response_loaded(self, timeout=200000): start_time = time.time() interval = 0.1 end_time = start_time + timeout / 1000 @@ -97,17 +98,18 @@ def wait_until_response_loaded(self,timeout=200000): return time.sleep(interval) - raise PlaywrightTimeoutError("Response is not generated and it has been timed out.") + raise PlaywrightTimeoutError( + "Response is not generated and it has been timed out." + ) # try: # # Wait for it to appear in the DOM and be visible # locator = self.page.locator(self.ASK_QUESTION) # locator.wait_for(state="enabled", timeout=200000) # adjust timeout as needed # except PlaywrightTimeoutError: # raise Exception("Response is not generated and it has been timed out.") - - - def wait_until_chat_details_response_loaded(self,timeout=200000): - + + def wait_until_chat_details_response_loaded(self, timeout=200000): + start_time = time.time() interval = 0.1 end_time = start_time + timeout / 1000 @@ -118,30 +120,26 @@ def wait_until_chat_details_response_loaded(self,timeout=200000): return time.sleep(interval) - raise PlaywrightTimeoutError("Response is not generated and it has been timed out.") - - + raise PlaywrightTimeoutError( + "Response is not generated and it has been timed out." + ) def click_new_topic(self): self.page.locator(self.NEWTOPIC).click() self.page.wait_for_timeout(2000) - self.page.wait_for_load_state('networkidle') + self.page.wait_for_load_state("networkidle") def get_follow_ques_text(self): - follow_up_question = self.page.locator(self.Suggested_follow_up_questions).text_content() + follow_up_question = self.page.locator( + self.Suggested_follow_up_questions + ).text_content() return follow_up_question - def click_suggested_question(self): + def click_suggested_question(self): self.page.locator(self.Suggested_follow_up_questions).click() self.page.wait_for_timeout(2000) - self.page.wait_for_load_state('networkidle') - - + self.page.wait_for_load_state("networkidle") def click_on_contract_details(self): self.page.locator(self.CONTRACTS_DETAILS_PAGE).click() self.page.wait_for_timeout(12000) - - - - \ No newline at end of file diff --git a/tests/e2e-test/pages/loginPage.py b/tests/e2e-test/pages/loginPage.py index 0ee59f77..0b412556 100644 --- a/tests/e2e-test/pages/loginPage.py +++ b/tests/e2e-test/pages/loginPage.py @@ -13,12 +13,12 @@ class LoginPage(BasePage): def __init__(self, page): self.page = page - def authenticate(self, username,password): + def authenticate(self, username, password): # login with username and password in web url self.page.locator(self.EMAIL_TEXT_BOX).fill(username) self.page.locator(self.NEXT_BUTTON).click() # Wait for the password input field to be available and fill it - self.page.wait_for_load_state('networkidle') + self.page.wait_for_load_state("networkidle") # Enter password self.page.locator(self.PASSWORD_TEXT_BOX).fill(password) # Click on SignIn button @@ -33,4 +33,4 @@ def authenticate(self, username,password): self.page.locator(self.YES_BUTTON).click() self.page.wait_for_timeout(10000) # Wait for the "Articles" button to be available and click it - self.page.wait_for_load_state('networkidle') + self.page.wait_for_load_state("networkidle") diff --git a/tests/e2e-test/tests/conftest.py b/tests/e2e-test/tests/conftest.py index 28771989..2c9e6351 100644 --- a/tests/e2e-test/tests/conftest.py +++ b/tests/e2e-test/tests/conftest.py @@ -1,16 +1,13 @@ -from pathlib import Path -import pytest -from playwright.sync_api import sync_playwright -from config.constants import * -from slugify import slugify -from pages.loginPage import LoginPage -from dotenv import load_dotenv -import os -from py.xml import html # type: ignore +import atexit import io import logging +import os + +import pytest from bs4 import BeautifulSoup -import atexit +from playwright.sync_api import sync_playwright + +from config.constants import URL @pytest.fixture(scope="session") @@ -24,14 +21,8 @@ def login_logout(): # Navigate to the login URL page.goto(URL) # Wait for the login form to appear - page.wait_for_load_state('networkidle') - # login to web url with username and password - #login_page = LoginPage(page) - #load_dotenv() - #login_page.authenticate(os.getenv('user_name'),os.getenv('pass_word')) + page.wait_for_load_state("networkidle") yield page - - # perform close the browser browser.close() @@ -42,9 +33,9 @@ def pytest_html_report_title(report): log_streams = {} + @pytest.hookimpl(tryfirst=True) def pytest_runtest_setup(item): - # Prepare StringIO for capturing logs stream = io.StringIO() handler = logging.StreamHandler(stream) handler.setLevel(logging.INFO) @@ -52,7 +43,6 @@ def pytest_runtest_setup(item): logger = logging.getLogger() logger.addHandler(handler) - # Save handler and stream log_streams[item.nodeid] = (handler, stream) @@ -64,50 +54,45 @@ def pytest_runtest_makereport(item, call): handler, stream = log_streams.get(item.nodeid, (None, None)) if handler and stream: - # Make sure logs are flushed handler.flush() log_output = stream.getvalue() - # Only remove the handler, don't close the stream yet logger = logging.getLogger() logger.removeHandler(handler) - # Store the log output on the report object for HTML reporting report.description = f"
{log_output.strip()}
" - - # Clean up references log_streams.pop(item.nodeid, None) else: report.description = "" + def pytest_collection_modifyitems(items): for item in items: - if hasattr(item, 'callspec'): + if hasattr(item, "callspec"): prompt = item.callspec.params.get("prompt") if prompt: - item._nodeid = prompt # This controls how the test name appears in the report + item._nodeid = prompt + def rename_duration_column(): - report_path = os.path.abspath("report.html") # or your report filename + report_path = os.path.abspath("report.html") if not os.path.exists(report_path): print("Report file not found, skipping column rename.") return - with open(report_path, 'r', encoding='utf-8') as f: - soup = BeautifulSoup(f, 'html.parser') + with open(report_path, "r", encoding="utf-8") as f: + soup = BeautifulSoup(f, "html.parser") - # Find and rename the header - headers = soup.select('table#results-table thead th') + headers = soup.select("table#results-table thead th") for th in headers: - if th.text.strip() == 'Duration': - th.string = 'Execution Time' - #print("Renamed 'Duration' to 'Execution Time'") + if th.text.strip() == "Duration": + th.string = "Execution Time" break else: print("'Duration' column not found in report.") - with open(report_path, 'w', encoding='utf-8') as f: + with open(report_path, "w", encoding="utf-8") as f: f.write(str(soup)) -# Register this function to run after everything is done -atexit.register(rename_duration_column) \ No newline at end of file + +atexit.register(rename_duration_column) diff --git a/tests/e2e-test/tests/test_poc_dkm.py b/tests/e2e-test/tests/test_poc_dkm.py index 64c6fb66..e494a48e 100644 --- a/tests/e2e-test/tests/test_poc_dkm.py +++ b/tests/e2e-test/tests/test_poc_dkm.py @@ -1,11 +1,22 @@ import logging import time + import pytest + +from config.constants import ( + chat_question1, + chat_question2, + house_10_11_question, + search_1, + search_2, + handwritten_question1, + contract_details_question, +) from pages.dkmPage import DkmPage -from config.constants import * logger = logging.getLogger(__name__) + def _store_follow_up_question(dkm): """Helper to store follow-up question text as an attribute on the DkmPage object.""" dkm.follow_up_question = dkm.get_follow_ques_text() @@ -14,55 +25,74 @@ def _store_follow_up_question(dkm): # Define test steps and prompts test_cases = [ ("Validate home page is loaded", lambda dkm: dkm.validate_home_page()), - (f"Ask first chat question: {chat_question1}", lambda dkm: ( - dkm.enter_a_question(chat_question1), - dkm.click_send_button(), - dkm.validate_response_status(chat_question1), - dkm.wait_until_response_loaded() - )), - ("Click on suggested follow-up question", lambda dkm: ( - _store_follow_up_question(dkm), - dkm.click_suggested_question(), - dkm.validate_response_status(dkm.follow_up_question), - dkm.wait_until_response_loaded() - )), + ( + f"Ask first chat question: {chat_question1}", + lambda dkm: ( + dkm.enter_a_question(chat_question1), + dkm.click_send_button(), + dkm.validate_response_status(chat_question1), + dkm.wait_until_response_loaded(), + ), + ), + ( + "Click on suggested follow-up question", + lambda dkm: ( + _store_follow_up_question(dkm), + dkm.click_suggested_question(), + dkm.validate_response_status(dkm.follow_up_question), + dkm.wait_until_response_loaded(), + ), + ), ("Start new topic", lambda dkm: dkm.click_new_topic()), ("Search for 'Housing Report'", lambda dkm: dkm.enter_in_search(search_1)), ("Select housing docs", lambda dkm: dkm.select_housing_checkbox()), - (f"Ask housing chat question: {chat_question2}", lambda dkm: ( - dkm.enter_a_question(chat_question2), - dkm.click_send_button(), - dkm.validate_response_status(chat_question2), - dkm.wait_until_response_loaded() - )), + ( + f"Ask housing chat question: {chat_question2}", + lambda dkm: ( + dkm.enter_a_question(chat_question2), + dkm.click_send_button(), + dkm.validate_response_status(chat_question2), + dkm.wait_until_response_loaded(), + ), + ), ("View details of housing report", lambda dkm: dkm.click_on_details()), - (f"Ask question in housing report popup: {house_10_11_question}", lambda dkm: ( - dkm.click_on_popup_chat(), - dkm.enter_in_popup_search(house_10_11_question), - dkm.validate_response_status(house_10_11_question), - dkm.wait_until_chat_details_response_loaded(), - dkm.close_pop_up() - )), + ( + f"Ask question in housing report popup: {house_10_11_question}", + lambda dkm: ( + dkm.click_on_popup_chat(), + dkm.enter_in_popup_search(house_10_11_question), + dkm.validate_response_status(house_10_11_question), + dkm.wait_until_chat_details_response_loaded(), + dkm.close_pop_up(), + ), + ), ("Search for 'Contracts'", lambda dkm: dkm.enter_in_search(search_2)), ("Select handwritten contract docs", lambda dkm: dkm.select_handwritten_doc()), - (f"Ask question about handwritten contracts: {handwritten_question1}", lambda dkm: ( - dkm.enter_a_question(handwritten_question1), - dkm.click_send_button(), - dkm.validate_response_status(handwritten_question1), - dkm.wait_until_response_loaded() - )), - (f"Ask question in contract details popup: {contract_details_question}", lambda dkm: ( - dkm.click_on_contract_details(), - dkm.click_on_popup_chat(), - dkm.enter_in_popup_search(contract_details_question), - dkm.validate_response_status(contract_details_question), - dkm.wait_until_chat_details_response_loaded(), - dkm.close_pop_up() - )), + ( + f"Ask question about handwritten contracts: {handwritten_question1}", + lambda dkm: ( + dkm.enter_a_question(handwritten_question1), + dkm.click_send_button(), + dkm.validate_response_status(handwritten_question1), + dkm.wait_until_response_loaded(), + ), + ), + ( + f"Ask question in contract details popup: {contract_details_question}", + lambda dkm: ( + dkm.click_on_contract_details(), + dkm.click_on_popup_chat(), + dkm.enter_in_popup_search(contract_details_question), + dkm.validate_response_status(contract_details_question), + dkm.wait_until_chat_details_response_loaded(), + dkm.close_pop_up(), + ), + ), ] # Create custom readable test IDs with step numbers -test_ids = [f"{i+1:02d}. {case[0]}" for i, case in enumerate(test_cases)] +test_ids = [f"{i + 1:02d}. {case[0]}" for i, case in enumerate(test_cases)] + @pytest.mark.parametrize("prompt, action", test_cases, ids=test_ids) def test_dkm_prompt_case(login_logout, prompt, action, request): @@ -86,7 +116,6 @@ def test_dkm_prompt_case(login_logout, prompt, action, request): duration = end - start logger.info(f"Execution Time for '{prompt}': {duration:.2f}s") - # Attach to report - request.node._report_sections.append(( - "call", "log", f"Execution time: {duration:.2f}s" - )) \ No newline at end of file + request.node._report_sections.append( + ("call", "log", f"Execution time: {duration:.2f}s") + )