diff --git a/.github/workflows/test-automation.yml b/.github/workflows/test-automation.yml new file mode 100644 index 00000000..32a23078 --- /dev/null +++ b/.github/workflows/test-automation.yml @@ -0,0 +1,130 @@ +name: Test Automation DKM + +on: + push: + branches: + - main + - dev + paths: + - 'tests/e2e-test/**' + schedule: + - cron: '0 13 * * *' # Runs at 1 PM UTC + workflow_dispatch: + +env: + url: ${{ vars.DKM_URL }} + accelerator_name: "DKM" + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.13' + + - name: Azure CLI Login + uses: azure/login@v2 + with: + creds: '{"clientId":"${{ secrets.AZURE_CLIENT_ID }}","clientSecret":"${{ secrets.AZURE_CLIENT_SECRET }}","subscriptionId":"${{ secrets.AZURE_SUBSCRIPTION_ID }}","tenantId":"${{ secrets.AZURE_TENANT_ID }}"}' + + - name: Start AKS + id: start-aks + uses: azure/cli@v2 + with: + azcliversion: 'latest' + inlineScript: | + az aks install-cli + if [ "$(az aks show --resource-group ${{ vars.DKM_RG }} --name ${{ vars.DKM_AKS_NAME }} --query "powerState.code" -o tsv)" = "Running" ]; then echo "AKS is running"; else az aks start --resource-group ${{ vars.DKM_RG }} --name ${{ vars.DKM_AKS_NAME }}; fi + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r tests/e2e-test/requirements.txt + + - name: Ensure browsers are installed + run: python -m playwright install --with-deps chromium + + - name: Run tests(1) + id: test1 + run: | + xvfb-run pytest --headed --html=report/report.html --self-contained-html + working-directory: tests/e2e-test + continue-on-error: true + + - name: Sleep for 30 seconds + if: ${{ steps.test1.outcome == 'failure' }} + run: sleep 30s + shell: bash + + - name: Run tests(2) + id: test2 + if: ${{ steps.test1.outcome == 'failure' }} + run: | + xvfb-run pytest --headed --html=report/report.html --self-contained-html + working-directory: tests/e2e-test + continue-on-error: true + + - name: Sleep for 60 seconds + if: ${{ steps.test2.outcome == 'failure' }} + run: sleep 60s + shell: bash + + - name: Run tests(3) + id: test3 + if: ${{ steps.test2.outcome == 'failure' }} + run: | + xvfb-run pytest --headed --html=report/report.html --self-contained-html + working-directory: tests/e2e-test + + - name: Upload test report + id: upload_report + uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: test-report + path: tests/e2e-test/report/* + + - name: Send Notification + if: always() + run: | + RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + REPORT_URL=${{ steps.upload_report.outputs.artifact-url }} + IS_SUCCESS=${{ steps.test1.outcome == 'success' || steps.test2.outcome == 'success' || steps.test3.outcome == 'success' }} + # Construct the email body + if [ "$IS_SUCCESS" = "true" ]; then + EMAIL_BODY=$(cat <Dear Team,

We would like to inform you that the ${{ env.accelerator_name }} Test Automation process has completed successfully.

Run URL: ${RUN_URL}

Test Report: ${REPORT_URL}

Best regards,
Your Automation Team

", + "subject": "${{ env.accelerator_name }} Test Automation - Success" + } + EOF + ) + else + EMAIL_BODY=$(cat <Dear Team,

We would like to inform you that the ${{ env.accelerator_name }} Test Automation process has encountered an issue and has failed to complete successfully.

Run URL: ${RUN_URL}

Test Report: ${REPORT_URL}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

", + "subject": "${{ env.accelerator_name }} Test Automation - Failure" + } + EOF + ) + fi + + # Send the notification + curl -X POST "${{ secrets.EMAILNOTIFICATION_LOGICAPP_URL_TA }}" \ + -H "Content-Type: application/json" \ + -d "$EMAIL_BODY" || echo "Failed to send notification" + + - name: Stop AKS + if: always() + uses: azure/cli@v2 + with: + azcliversion: 'latest' + inlineScript: | + az aks install-cli + if [ "$(az aks show --resource-group ${{ vars.DKM_RG }} --name ${{ vars.DKM_AKS_NAME }} --query "powerState.code" -o tsv)" = "Running" ]; then az aks stop --resource-group ${{ vars.DKM_RG }} --name ${{ vars.DKM_AKS_NAME }}; else echo "AKS is already stopped"; fi + az logout \ No newline at end of file diff --git a/tests/e2e-test/.gitignore b/tests/e2e-test/.gitignore new file mode 100644 index 00000000..6f792d69 --- /dev/null +++ b/tests/e2e-test/.gitignore @@ -0,0 +1,167 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/latest/usage/project/#working-with-version-control +.pdm.toml +.pdm-python +.pdm-build/ + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +microsoft/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +.idea/ +archive/ +report/ +screenshots/ +report.html diff --git a/tests/e2e-test/README.md b/tests/e2e-test/README.md new file mode 100644 index 00000000..ea844696 --- /dev/null +++ b/tests/e2e-test/README.md @@ -0,0 +1,34 @@ +# Test Automation for Document Knowledge Mining Accelerator + +Write end-to-end tests for your web apps with [Playwright](https://github.com/microsoft/playwright-python) and [pytest](https://docs.pytest.org/en/stable/). + +- Support for **all modern browsers** including Chromium, WebKit and Firefox. +- Support for **headless and headed** execution. +- **Built-in fixtures** that provide browser primitives to test functions. + +Pre-Requisites: + +- Install Visual Studio Code: Download and Install Visual Studio Code(VSCode). +- Install NodeJS: Download and Install Node JS + +Create and Activate Python Virtual Environment + +- From your directory open and run cmd : "python -m venv microsoft" +This will create a virtual environment directory named microsoft inside your current directory +- To enable virtual environment, copy location for "microsoft\Scripts\activate.bat" and run from cmd + +Installing Playwright Pytest from Virtual Environment + +- To install libraries run "pip install -r requirements.txt" + +Run test cases + +- To run test cases from your 'tests' folder : "pytest --html=report.html --self-contained-html" + +Create .env file in project root level with web app url and client credentials + +- create a .env file in project root level and the application url. please refer 'sample_dotenv_file.txt' file. + +## Documentation + +See on [playwright.dev](https://playwright.dev/python/docs/test-runners) for examples and more detailed information. diff --git a/tests/e2e-test/base/__init__.py b/tests/e2e-test/base/__init__.py new file mode 100644 index 00000000..cf50d1cc --- /dev/null +++ b/tests/e2e-test/base/__init__.py @@ -0,0 +1 @@ +from . import base \ No newline at end of file diff --git a/tests/e2e-test/base/base.py b/tests/e2e-test/base/base.py new file mode 100644 index 00000000..759c7701 --- /dev/null +++ b/tests/e2e-test/base/base.py @@ -0,0 +1,36 @@ +from config.constants import * +import requests +import json +from dotenv import load_dotenv +import os + +class BasePage: + def __init__(self, page): + self.page = page + + def scroll_into_view(self,locator): + reference_list = locator + locator.nth(reference_list.count()-1).scroll_into_view_if_needed() + + def is_visible(self,locator): + locator.is_visible() + + def validate_response_status(self, question_api): + load_dotenv() + # The URL of the API endpoint you want to access + url = f"{URL}/backend/chat" + + headers = { + "Content-Type": "application/json", + "Accept": "*/*", + } + payload = { + "Question": question_api, # This is your example question, you can modify it as needed + } + # Make the POST request + response = self.page.request.post(url, headers=headers, data=json.dumps(payload), timeout=200000) + + # Check the response status code + assert response.status == 200, "Response code is " + str(response.status) + " " + str(response.json()) + + \ No newline at end of file diff --git a/tests/e2e-test/config/constants.py b/tests/e2e-test/config/constants.py new file mode 100644 index 00000000..7d9c8a91 --- /dev/null +++ b/tests/e2e-test/config/constants.py @@ -0,0 +1,16 @@ +from dotenv import load_dotenv +import os + +load_dotenv() +URL = os.getenv('url') +if URL.endswith('/'): + URL = URL[:-1] + +# DKM input data +chat_question1 = "What are the main factors contributing to the current housing affordability issues?" +chat_question2 = "Analyze the two annual reports and compare the positive and negative outcomes YoY. Show the results in a table." +house_10_11_question ="Can you summarize and compare the tables on page 10 and 11?" +handwritten_question1 ="Analyze these forms and create a table with all buyers, sellers, and corresponding purchase prices." +search_1= "Housing Report" +search_2= "Contracts" +contract_details_question = "What liabilities is the buyer responsible for within the contract?" diff --git a/tests/e2e-test/pages/__init__.py b/tests/e2e-test/pages/__init__.py new file mode 100644 index 00000000..3363f3e4 --- /dev/null +++ b/tests/e2e-test/pages/__init__.py @@ -0,0 +1,2 @@ +from. import loginPage +from. import dkmPage \ No newline at end of file diff --git a/tests/e2e-test/pages/dkmPage.py b/tests/e2e-test/pages/dkmPage.py new file mode 100644 index 00000000..c434d19f --- /dev/null +++ b/tests/e2e-test/pages/dkmPage.py @@ -0,0 +1,147 @@ +from base.base import BasePage +from playwright.sync_api import expect +import time +from playwright.sync_api import TimeoutError as PlaywrightTimeoutError + +class DkmPage(BasePage): + WELCOME_PAGE_TITLE = "(//div[@class='order-5 my-auto pb-3 text-lg font-semibold leading-tight text-white mt-3'])[1]" + NEWTOPIC = "//button[normalize-space()='New Topic']" + Suggested_follow_up_questions="body > div:nth-child(3) > div:nth-child(1) > main:nth-child(2) > div:nth-child(1) > div:nth-child(3) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1) > div:nth-child(3) > div:nth-child(1) > div:nth-child(6) > div:nth-child(3) > button:nth-child(2)" + SCROLL_DOWN = "//div[10]//div[2]//div[2]//i[1]//img[1]" + ASK_QUESTION ="//textarea[@placeholder='Ask a question or request (ctrl + enter to submit)']" + SEARCH_BOX="//input[@type='search']" + HOUSING_2022 ="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[4]/div[2]/div[2]/span[1]" + HOUSING_2023 ="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[3]/div[2]/div[2]/span[1]" + CONTRACTS_DETAILS_PAGE = "body > div:nth-child(3) > div:nth-child(1) > main:nth-child(2) > div:nth-child(1) > div:nth-child(2) > div:nth-child(4) > div:nth-child(1) > div:nth-child(1) > div:nth-child(6) > div:nth-child(2) > div:nth-child(2) > div:nth-child(3) > button:nth-child(2)" + DETAILS_PAGE ="body > div:nth-child(3) > div:nth-child(1) > main:nth-child(2) > div:nth-child(1) > div:nth-child(2) > div:nth-child(4) > div:nth-child(1) > div:nth-child(1) > div:nth-child(3) > div:nth-child(2) > div:nth-child(2) > div:nth-child(3) > button:nth-child(2)" + POP_UP_CHAT="//button[@value='Chat Room']" + CLOSE_POP_UP ="//button[@aria-label='close']" + CLLEAR_ALL_POP_UP ="//button[normalize-space()='Clear all']" + HANDWRITTEN_DOC1="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[6]/div[2]/div[2]/span[1]" + HANDWRITTEN_DOC2="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[1]/div[2]/div[2]/span[1]" + HANDWRITTEN_DOC3="//body[1]/div[2]/div[1]/main[1]/div[1]/div[2]/div[4]/div[1]/div[1]/div[5]/div[2]/div[2]/span[1]" + SEND_BUTTON = "//button[@aria-label='Send']" + POP_UP_CHAT_SEARCH = "(//textarea[@placeholder='Ask a question or request (ctrl + enter to submit)'])[2]" + POP_UP_CHAT_SEND = "(//button[@type='submit'])[2]" + DOCUMENT_FILTER = "//button[normalize-space()='Accessibility Features']" + HEADING_TITLE = "//div[.='Document Knowledge Mining']" + + + def __init__(self, page): + self.page = page + + + + def validate_home_page(self): + self.page.wait_for_timeout(5000) + expect(self.page.locator(self.DOCUMENT_FILTER)).to_be_visible() + expect(self.page.locator(self.HEADING_TITLE)).to_be_visible() + self.page.wait_for_timeout(2000) + + + def enter_a_question(self,text): + self.page.locator(self.ASK_QUESTION).fill(text) + self.page.wait_for_timeout(5000) + + def enter_in_search(self,text): + self.page.locator(self.SEARCH_BOX).fill(text) + self.page.wait_for_timeout(5000) + + def enter_in_popup_search(self,text): + self.page.locator(self.POP_UP_CHAT_SEARCH).fill(text) + self.page.wait_for_timeout(5000) + self.page.locator(self.POP_UP_CHAT_SEND).click() + # self.page.wait_for_load_state('networkidle') + + def select_housing_checkbox(self): + self.page.locator(self.HOUSING_2022).click() + self.page.locator(self.HOUSING_2023).click() + self.page.wait_for_timeout(5000) + + def click_on_details(self): + self.page.wait_for_timeout(5000) + self.page.locator(self.DETAILS_PAGE).click() + self.page.wait_for_timeout(13000) + + def click_on_popup_chat(self): + self.page.locator(self.POP_UP_CHAT).click() + self.page.wait_for_timeout(5000) + + def close_pop_up(self): + self.page.locator(self.CLOSE_POP_UP).click() + self.page.wait_for_timeout(2000) + self.page.locator(self.CLLEAR_ALL_POP_UP).click() + self.page.wait_for_timeout(2000) + + def select_handwritten_doc(self): + self.page.locator(self.HANDWRITTEN_DOC1).click() + self.page.locator(self.HANDWRITTEN_DOC2).click() + self.page.locator(self.HANDWRITTEN_DOC3).click() + self.page.wait_for_timeout(2000) + + def click_send_button(self): + # Click on send button in question area + self.page.locator(self.SEND_BUTTON).click() + self.page.wait_for_timeout(5000) + + #self.page.wait_for_load_state('networkidle') + + def wait_until_response_loaded(self,timeout=200000): + start_time = time.time() + interval = 0.1 + end_time = start_time + timeout / 1000 + locator = self.page.locator(self.ASK_QUESTION) + + while time.time() < end_time: + if locator.is_enabled(): + return + time.sleep(interval) + + raise PlaywrightTimeoutError("Response is not generated and it has been timed out.") + # try: + # # Wait for it to appear in the DOM and be visible + # locator = self.page.locator(self.ASK_QUESTION) + # locator.wait_for(state="enabled", timeout=200000) # adjust timeout as needed + # except PlaywrightTimeoutError: + # raise Exception("Response is not generated and it has been timed out.") + + + def wait_until_chat_details_response_loaded(self,timeout=200000): + + start_time = time.time() + interval = 0.1 + end_time = start_time + timeout / 1000 + locator = self.page.locator(self.POP_UP_CHAT_SEARCH) + + while time.time() < end_time: + if locator.is_enabled(): + return + time.sleep(interval) + + raise PlaywrightTimeoutError("Response is not generated and it has been timed out.") + + + + def click_new_topic(self): + self.page.locator(self.NEWTOPIC).click() + self.page.wait_for_timeout(2000) + self.page.wait_for_load_state('networkidle') + + def get_follow_ques_text(self): + follow_up_question = self.page.locator(self.Suggested_follow_up_questions).text_content() + return follow_up_question + + def click_suggested_question(self): + self.page.locator(self.Suggested_follow_up_questions).click() + self.page.wait_for_timeout(2000) + self.page.wait_for_load_state('networkidle') + + + + def click_on_contract_details(self): + self.page.locator(self.CONTRACTS_DETAILS_PAGE).click() + self.page.wait_for_timeout(12000) + + + + \ No newline at end of file diff --git a/tests/e2e-test/pages/loginPage.py b/tests/e2e-test/pages/loginPage.py new file mode 100644 index 00000000..0ee59f77 --- /dev/null +++ b/tests/e2e-test/pages/loginPage.py @@ -0,0 +1,36 @@ +from base.base import BasePage + + +class LoginPage(BasePage): + + EMAIL_TEXT_BOX = "//input[@type='email']" + NEXT_BUTTON = "//input[@type='submit']" + PASSWORD_TEXT_BOX = "//input[@type='password']" + SIGNIN_BUTTON = "//input[@id='idSIButton9']" + YES_BUTTON = "//input[@id='idSIButton9']" + PERMISSION_ACCEPT_BUTTON = "//input[@type='submit']" + + def __init__(self, page): + self.page = page + + def authenticate(self, username,password): + # login with username and password in web url + self.page.locator(self.EMAIL_TEXT_BOX).fill(username) + self.page.locator(self.NEXT_BUTTON).click() + # Wait for the password input field to be available and fill it + self.page.wait_for_load_state('networkidle') + # Enter password + self.page.locator(self.PASSWORD_TEXT_BOX).fill(password) + # Click on SignIn button + self.page.locator(self.SIGNIN_BUTTON).click() + # Wait for 5 seconds to ensure the login process completes + self.page.wait_for_timeout(20000) # Wait for 20 seconds + if self.page.locator(self.PERMISSION_ACCEPT_BUTTON).is_visible(): + self.page.locator(self.PERMISSION_ACCEPT_BUTTON).click() + self.page.wait_for_timeout(10000) + else: + # Click on YES button + self.page.locator(self.YES_BUTTON).click() + self.page.wait_for_timeout(10000) + # Wait for the "Articles" button to be available and click it + self.page.wait_for_load_state('networkidle') diff --git a/tests/e2e-test/pytest.ini b/tests/e2e-test/pytest.ini new file mode 100644 index 00000000..76eb64fc --- /dev/null +++ b/tests/e2e-test/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +log_cli = true +log_cli_level = INFO +log_file = logs/tests.log +log_file_level = INFO +addopts = -p no:warnings diff --git a/tests/e2e-test/requirements.txt b/tests/e2e-test/requirements.txt new file mode 100644 index 00000000..1b0ac0d7 --- /dev/null +++ b/tests/e2e-test/requirements.txt @@ -0,0 +1,6 @@ +pytest-playwright +pytest-reporter-html1 +python-dotenv +pytest-check +pytest-html +py diff --git a/tests/e2e-test/sample_dotenv_file.txt b/tests/e2e-test/sample_dotenv_file.txt new file mode 100644 index 00000000..bee18d2f --- /dev/null +++ b/tests/e2e-test/sample_dotenv_file.txt @@ -0,0 +1 @@ +url = 'web app url' \ No newline at end of file diff --git a/tests/e2e-test/tests/__init__.py b/tests/e2e-test/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e-test/tests/conftest.py b/tests/e2e-test/tests/conftest.py new file mode 100644 index 00000000..5823badc --- /dev/null +++ b/tests/e2e-test/tests/conftest.py @@ -0,0 +1,86 @@ +from pathlib import Path +import pytest +from playwright.sync_api import sync_playwright +from config.constants import * +from slugify import slugify +from pages.loginPage import LoginPage +from dotenv import load_dotenv +import os +from py.xml import html # type: ignore +import io +import logging + + +@pytest.fixture(scope="session") +def login_logout(): + # perform login and browser close once in a session + with sync_playwright() as p: + browser = p.chromium.launch(headless=False, args=["--start-maximized"]) + context = browser.new_context(no_viewport=True) + context.set_default_timeout(120000) + page = context.new_page() + # Navigate to the login URL + page.goto(URL) + # Wait for the login form to appear + page.wait_for_load_state('networkidle') + # login to web url with username and password + #login_page = LoginPage(page) + #load_dotenv() + #login_page.authenticate(os.getenv('user_name'),os.getenv('pass_word')) + yield page + + # perform close the browser + browser.close() + + +@pytest.hookimpl(tryfirst=True) +def pytest_html_report_title(report): + report.title = "Test Automation DKM" + + +log_streams = {} + +@pytest.hookimpl(tryfirst=True) +def pytest_runtest_setup(item): + # Prepare StringIO for capturing logs + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setLevel(logging.INFO) + + logger = logging.getLogger() + logger.addHandler(handler) + + # Save handler and stream + log_streams[item.nodeid] = (handler, stream) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + + handler, stream = log_streams.get(item.nodeid, (None, None)) + + if handler and stream: + # Make sure logs are flushed + handler.flush() + log_output = stream.getvalue() + + # Only remove the handler, don't close the stream yet + logger = logging.getLogger() + logger.removeHandler(handler) + + # Store the log output on the report object for HTML reporting + report.description = f"
{log_output.strip()}
" + + # Clean up references + log_streams.pop(item.nodeid, None) + else: + report.description = "" + +def pytest_collection_modifyitems(items): + for item in items: + if hasattr(item, 'callspec'): + prompt = item.callspec.params.get("prompt") + if prompt: + item._nodeid = prompt # This controls how the test name appears in the report \ No newline at end of file diff --git a/tests/e2e-test/tests/test_poc_dkm.py b/tests/e2e-test/tests/test_poc_dkm.py new file mode 100644 index 00000000..64c6fb66 --- /dev/null +++ b/tests/e2e-test/tests/test_poc_dkm.py @@ -0,0 +1,92 @@ +import logging +import time +import pytest +from pages.dkmPage import DkmPage +from config.constants import * + +logger = logging.getLogger(__name__) + +def _store_follow_up_question(dkm): + """Helper to store follow-up question text as an attribute on the DkmPage object.""" + dkm.follow_up_question = dkm.get_follow_ques_text() + + +# Define test steps and prompts +test_cases = [ + ("Validate home page is loaded", lambda dkm: dkm.validate_home_page()), + (f"Ask first chat question: {chat_question1}", lambda dkm: ( + dkm.enter_a_question(chat_question1), + dkm.click_send_button(), + dkm.validate_response_status(chat_question1), + dkm.wait_until_response_loaded() + )), + ("Click on suggested follow-up question", lambda dkm: ( + _store_follow_up_question(dkm), + dkm.click_suggested_question(), + dkm.validate_response_status(dkm.follow_up_question), + dkm.wait_until_response_loaded() + )), + ("Start new topic", lambda dkm: dkm.click_new_topic()), + ("Search for 'Housing Report'", lambda dkm: dkm.enter_in_search(search_1)), + ("Select housing docs", lambda dkm: dkm.select_housing_checkbox()), + (f"Ask housing chat question: {chat_question2}", lambda dkm: ( + dkm.enter_a_question(chat_question2), + dkm.click_send_button(), + dkm.validate_response_status(chat_question2), + dkm.wait_until_response_loaded() + )), + ("View details of housing report", lambda dkm: dkm.click_on_details()), + (f"Ask question in housing report popup: {house_10_11_question}", lambda dkm: ( + dkm.click_on_popup_chat(), + dkm.enter_in_popup_search(house_10_11_question), + dkm.validate_response_status(house_10_11_question), + dkm.wait_until_chat_details_response_loaded(), + dkm.close_pop_up() + )), + ("Search for 'Contracts'", lambda dkm: dkm.enter_in_search(search_2)), + ("Select handwritten contract docs", lambda dkm: dkm.select_handwritten_doc()), + (f"Ask question about handwritten contracts: {handwritten_question1}", lambda dkm: ( + dkm.enter_a_question(handwritten_question1), + dkm.click_send_button(), + dkm.validate_response_status(handwritten_question1), + dkm.wait_until_response_loaded() + )), + (f"Ask question in contract details popup: {contract_details_question}", lambda dkm: ( + dkm.click_on_contract_details(), + dkm.click_on_popup_chat(), + dkm.enter_in_popup_search(contract_details_question), + dkm.validate_response_status(contract_details_question), + dkm.wait_until_chat_details_response_loaded(), + dkm.close_pop_up() + )), +] + +# Create custom readable test IDs with step numbers +test_ids = [f"{i+1:02d}. {case[0]}" for i, case in enumerate(test_cases)] + +@pytest.mark.parametrize("prompt, action", test_cases, ids=test_ids) +def test_dkm_prompt_case(login_logout, prompt, action, request): + """ + Executes each DKM user interaction step as an independent test case, + logs execution time, and attaches it to the test report. + """ + page = login_logout + dkm_page = DkmPage(page) + logger.info(f"Running test step: {prompt}") + + start = time.time() + if isinstance(action, tuple): + for step in action: + if callable(step): + step() + else: + action(dkm_page) + end = time.time() + + duration = end - start + logger.info(f"Execution Time for '{prompt}': {duration:.2f}s") + + # Attach to report + request.node._report_sections.append(( + "call", "log", f"Execution time: {duration:.2f}s" + )) \ No newline at end of file