From 33147d60f69403ee785f71abd2fdc636f4528692 Mon Sep 17 00:00:00 2001 From: Shreyas-Microsoft Date: Thu, 3 Apr 2025 11:56:13 +0530 Subject: [PATCH 01/47] Vertically align landing page --- src/frontend/src/pages/landingPage.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/frontend/src/pages/landingPage.css b/src/frontend/src/pages/landingPage.css index 37a3e815..690c185c 100644 --- a/src/frontend/src/pages/landingPage.css +++ b/src/frontend/src/pages/landingPage.css @@ -1,3 +1,7 @@ +main { + padding-top: 8rem !important; +} + .main-content { transition: margin-right 0.3s ease-in-out; /* Smooth transition */ margin-right: 0px; /* Default margin */ From f360316fd866415ec58b23bd7816729a562a9a0b Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Thu, 3 Apr 2025 12:39:18 +0530 Subject: [PATCH 02/47] added github files --- .github/CODEOWNERS | 5 + .github/ISSUE_TEMPLATE/bug_report.md | 45 +++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 32 +++++++ .github/ISSUE_TEMPLATE/subtask.md | 22 +++++ .github/PULL_REQUEST_TEMPLATE.md | 39 ++++++++ .github/dependabot.yml | 38 ++++++++ .github/workflows/Create-Release.yml | 65 +++++++++++++ .github/workflows/pr-title-checker.yml | 22 +++++ .github/workflows/pylint.yml | 33 +++++++ .github/workflows/stale-bot.yml | 82 +++++++++++++++++ .github/workflows/test.yml | 106 ++++++++++++++++++++++ 11 files changed, 489 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/subtask.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/Create-Release.yml create mode 100644 .github/workflows/pr-title-checker.yml create mode 100644 .github/workflows/pylint.yml create mode 100644 .github/workflows/stale-bot.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..9fead0fe --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# Lines starting with '#' are comments. +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in the repo. +* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft @aniaroramsft @brittneek @Vinay-Microsoft diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..882ebd79 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,45 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +# Describe the bug +A clear and concise description of what the bug is. + +# Expected behavior +A clear and concise description of what you expected to happen. + +# How does this bug make you feel? +_Share a gif from [giphy](https://giphy.com/) to tells us how you'd feel_ + +--- + +# Debugging information + +## Steps to reproduce +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +## Screenshots +If applicable, add screenshots to help explain your problem. + +## Logs + +If applicable, add logs to help the engineer debug the problem. + +--- + +# Tasks + +_To be filled in by the engineer picking up the issue_ + +- [ ] Task 1 +- [ ] Task 2 +- [ ] ... diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..3496fc82 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +# Motivation + +A clear and concise description of why this feature would be useful and the value it would bring. +Explain any alternatives considered and why they are not sufficient. + +# How would you feel if this feature request was implemented? + +_Share a gif from [giphy](https://giphy.com/) to tells us how you'd feel. Format: ![alt_text](https://media.giphy.com/media/xxx/giphy.gif)_ + +# Requirements + +A list of requirements to consider this feature delivered +- Requirement 1 +- Requirement 2 +- ... + +# Tasks + +_To be filled in by the engineer picking up the issue_ + +- [ ] Task 1 +- [ ] Task 2 +- [ ] ... diff --git a/.github/ISSUE_TEMPLATE/subtask.md b/.github/ISSUE_TEMPLATE/subtask.md new file mode 100644 index 00000000..9f86c843 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/subtask.md @@ -0,0 +1,22 @@ +--- +name: Sub task +about: A sub task +title: '' +labels: subtask +assignees: '' + +--- + +Required by + +# Description + +A clear and concise description of what this subtask is. + +# Tasks + +_To be filled in by the engineer picking up the subtask + +- [ ] Task 1 +- [ ] Task 2 +- [ ] ... diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..34a53da4 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,39 @@ +## Purpose + +* ... + +## Does this introduce a breaking change? + + +- [ ] Yes +- [ ] No + + + +## Golden Path Validation +- [ ] I have tested the primary workflows (the "golden path") to ensure they function correctly without errors. + +## Deployment Validation +- [ ] I have validated the deployment process successfully and all services are running as expected with this change. + +## What to Check +Verify that the following are valid +* ... + +## Other Information + + + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..508a62b8 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,38 @@ +version: 2 +updates: + # GitHub Actions dependencies + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + commit-message: + prefix: "build" + target-branch: "dependabotchanges" + open-pull-requests-limit: 100 + + - package-ecosystem: "pip" + directory: "/src/backend" + schedule: + interval: "monthly" + commit-message: + prefix: "build" + target-branch: "dependabotchanges" + open-pull-requests-limit: 100 + + - package-ecosystem: "pip" + directory: "/src/frontend" + schedule: + interval: "monthly" + commit-message: + prefix: "build" + target-branch: "dependabotchanges" + open-pull-requests-limit: 100 + + - package-ecosystem: "npm" + directory: "/src/frontend" + schedule: + interval: "monthly" + commit-message: + prefix: "build" + target-branch: "dependabotchanges" + open-pull-requests-limit: 100 diff --git a/.github/workflows/Create-Release.yml b/.github/workflows/Create-Release.yml new file mode 100644 index 00000000..0d51134d --- /dev/null +++ b/.github/workflows/Create-Release.yml @@ -0,0 +1,65 @@ +on: + push: + branches: + - main + +permissions: + contents: write + pull-requests: write + +name: Create-Release + +jobs: + create-release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.workflow_run.head_sha }} + + - uses: codfish/semantic-release-action@v3 + id: semantic + with: + tag-format: 'v${version}' + additional-packages: | + ['conventional-changelog-conventionalcommits@7'] + plugins: | + [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits" + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { type: 'feat', section: 'Features', hidden: false }, + { type: 'fix', section: 'Bug Fixes', hidden: false }, + { type: 'perf', section: 'Performance Improvements', hidden: false }, + { type: 'revert', section: 'Reverts', hidden: false }, + { type: 'docs', section: 'Other Updates', hidden: false }, + { type: 'style', section: 'Other Updates', hidden: false }, + { type: 'chore', section: 'Other Updates', hidden: false }, + { type: 'refactor', section: 'Other Updates', hidden: false }, + { type: 'test', section: 'Other Updates', hidden: false }, + { type: 'build', section: 'Other Updates', hidden: false }, + { type: 'ci', section: 'Other Updates', hidden: false } + ] + } + } + ], + '@semantic-release/github' + ] + env: + GITHUB_TOKEN: ${{ secrets.TOKEN }} + - run: echo ${{ steps.semantic.outputs.release-version }} + + - run: echo "$OUTPUTS" + env: + OUTPUTS: ${{ toJson(steps.semantic.outputs) }} + \ No newline at end of file diff --git a/.github/workflows/pr-title-checker.yml b/.github/workflows/pr-title-checker.yml new file mode 100644 index 00000000..b7e70e56 --- /dev/null +++ b/.github/workflows/pr-title-checker.yml @@ -0,0 +1,22 @@ +name: "PR Title Checker" + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + merge_group: + +permissions: + pull-requests: read + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + if: ${{ github.event_name != 'merge_group' }} + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pylint.yml b/.github/workflows/pylint.yml new file mode 100644 index 00000000..d784267d --- /dev/null +++ b/.github/workflows/pylint.yml @@ -0,0 +1,33 @@ +name: PyLint + +on: [push] + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + + steps: + # Step 1: Checkout code + - name: Checkout code + uses: actions/checkout@v4 + + # Step 2: Set up Python environment + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r src/backend/requirements.txt + + # Step 3: Run all code quality checks + - name: Pylint + run: | + echo "Running Pylint..." + python -m flake8 --config=.flake8 --verbose . + \ No newline at end of file diff --git a/.github/workflows/stale-bot.yml b/.github/workflows/stale-bot.yml new file mode 100644 index 00000000..c9157580 --- /dev/null +++ b/.github/workflows/stale-bot.yml @@ -0,0 +1,82 @@ +name: "Manage Stale Issues, PRs & Unmerged Branches" +on: + schedule: + - cron: '30 1 * * *' # Runs daily at 1:30 AM UTC + workflow_dispatch: # Allows manual triggering +permissions: + contents: write + issues: write + pull-requests: write +jobs: + stale: + runs-on: ubuntu-latest + steps: + - name: Mark Stale Issues and PRs + uses: actions/stale@v9 + with: + stale-issue-message: "This issue is stale because it has been open 180 days with no activity. Remove stale label or comment, or it will be closed in 30 days." + stale-pr-message: "This PR is stale because it has been open 180 days with no activity. Please update or it will be closed in 30 days." + days-before-stale: 180 + days-before-close: 30 + exempt-issue-labels: "keep" + exempt-pr-labels: "keep" + cleanup-branches: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch full history for accurate branch checks + - name: Fetch All Branches + run: git fetch --all --prune + - name: List Merged Branches With No Activity in Last 3 Months + run: | + + echo "Branch Name,Last Commit Date,Committer,Committed In Branch,Action" > merged_branches_report.csv + + for branch in $(git for-each-ref --format '%(refname:short) %(committerdate:unix)' refs/remotes/origin | awk -v date=$(date -d '3 months ago' +%s) '$2 < date {print $1}'); do + if [[ "$branch" != "origin/main" && "$branch" != "origin/dev" ]]; then + branch_name=${branch#origin/} + # Ensure the branch exists locally before getting last commit date + git fetch origin "$branch_name" || echo "Could not fetch branch: $branch_name" + last_commit_date=$(git log -1 --format=%ci "origin/$branch_name" || echo "Unknown") + committer_name=$(git log -1 --format=%cn "origin/$branch_name" || echo "Unknown") + committed_in_branch=$(git branch -r --contains "origin/$branch_name" | tr -d ' ' | paste -sd "," -) + echo "$branch_name,$last_commit_date,$committer_name,$committed_in_branch,Delete" >> merged_branches_report.csv + fi + done + - name: List PR Approved and Merged Branches Older Than 30 Days + run: | + + for branch in $(gh api repos/${{ github.repository }}/pulls --jq '.[] | select(.merged_at != null and (.base.ref == "main" or .base.ref == "dev")) | select(.merged_at | fromdateiso8601 < (now - 2592000)) | .head.ref'); do + # Ensure the branch exists locally before getting last commit date + git fetch origin "$branch" || echo "Could not fetch branch: $branch" + last_commit_date=$(git log -1 --format=%ci origin/$branch || echo "Unknown") + committer_name=$(git log -1 --format=%cn origin/$branch || echo "Unknown") + committed_in_branch=$(git branch -r --contains "origin/$branch" | tr -d ' ' | paste -sd "," -) + echo "$branch,$last_commit_date,$committer_name,$committed_in_branch,Delete" >> merged_branches_report.csv + done + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: List Open PR Branches With No Activity in Last 3 Months + run: | + + for branch in $(gh api repos/${{ github.repository }}/pulls --state open --jq '.[] | select(.base.ref == "main" or .base.ref == "dev") | .head.ref'); do + # Ensure the branch exists locally before getting last commit date + git fetch origin "$branch" || echo "Could not fetch branch: $branch" + last_commit_date=$(git log -1 --format=%ci origin/$branch || echo "Unknown") + committer_name=$(git log -1 --format=%cn origin/$branch || echo "Unknown") + if [[ $(date -d "$last_commit_date" +%s) -lt $(date -d '3 months ago' +%s) ]]; then + # If no commit in the last 3 months, mark for deletion + committed_in_branch=$(git branch -r --contains "origin/$branch" | tr -d ' ' | paste -sd "," -) + echo "$branch,$last_commit_date,$committer_name,$committed_in_branch,Delete" >> merged_branches_report.csv + fi + done + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Upload CSV Report of Inactive Branches + uses: actions/upload-artifact@v4 + with: + name: merged-branches-report + path: merged_branches_report.csv + retention-days: 30 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..3f245b24 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,106 @@ +name: Test Workflow with Coverage - Code-Gen + +on: + push: + branches: + - main + - dev + - demo + pull_request: + types: + - opened + - ready_for_review + - reopened + - synchronize + branches: + - main + - dev + - demo + +jobs: +# frontend_tests: +# runs-on: ubuntu-latest + +# steps: +# - name: Checkout code +# uses: actions/checkout@v3 + +# - name: Set up Node.js +# uses: actions/setup-node@v3 +# with: +# node-version: '20' + +# - name: Check if Frontend Test Files Exist +# id: check_frontend_tests +# run: | +# if [ -z "$(find src/tests/frontend -type f -name '*.test.js' -o -name '*.test.ts' -o -name '*.test.tsx')" ]; then +# echo "No frontend test files found, skipping frontend tests." +# echo "skip_frontend_tests=true" >> $GITHUB_ENV +# else +# echo "Frontend test files found, running tests." +# echo "skip_frontend_tests=false" >> $GITHUB_ENV +# fi + +# - name: Install Frontend Dependencies +# if: env.skip_frontend_tests == 'false' +# run: | +# cd src/frontend +# npm install + +# - name: Run Frontend Tests with Coverage +# if: env.skip_frontend_tests == 'false' +# run: | +# cd src/tests/frontend +# npm run test -- --coverage + +# - name: Skip Frontend Tests +# if: env.skip_frontend_tests == 'true' +# run: | +# echo "Skipping frontend tests because no test files were found." + + backend_tests: + runs-on: ubuntu-latest + + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Backend Dependencies + run: | + python -m pip install --upgrade pip + pip install -r src/backend/requirements.txt + pip install -r src/frontend/requirements.txt + pip install pytest-cov + pip install pytest-asyncio + - name: Set PYTHONPATH + run: echo "PYTHONPATH=$PWD/src/backend" >> $GITHUB_ENV + + - name: Check if Backend Test Files Exist + id: check_backend_tests + run: | + if [ -z "$(find src/tests/backend -type f -name '*_test.py')" ]; then + echo "No backend test files found, skipping backend tests." + echo "skip_backend_tests=true" >> $GITHUB_ENV + else + echo "Backend test files found, running tests." + echo "skip_backend_tests=false" >> $GITHUB_ENV + fi + + - name: Run Backend Tests with Coverage + if: env.skip_backend_tests == 'false' + run: | + cd src/tests/backend + pytest --cov=. --cov-report=term-missing --cov-report=xml + + + + - name: Skip Backend Tests + if: env.skip_backend_tests == 'true' + run: | + echo "Skipping backend tests because no test files were found." From 04da7af831ae4e1e8604c519f24dc134f66c979f Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Thu, 3 Apr 2025 13:29:53 +0530 Subject: [PATCH 03/47] edit 3 --- .flake8 | 5 +++++ .github/CODEOWNERS | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..93f63e5d --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 88 +extend-ignore = E501 +exclude = .venv, frontend +ignore = E203, W503, G004, G200 \ No newline at end of file diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9fead0fe..92ebe267 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,4 @@ # Each line is a file pattern followed by one or more owners. # These owners will be the default owners for everything in the repo. -* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft @aniaroramsft @brittneek @Vinay-Microsoft +* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft @aniaroramsft @marktayl1 @Vinay-Microsoft From 4974cc864b4dc8c9976a2f7c6b2c5239ef84356b Mon Sep 17 00:00:00 2001 From: "Priyanka Singhal (Persistent Systems Inc)" Date: Thu, 3 Apr 2025 13:34:12 +0530 Subject: [PATCH 04/47] Build Docker image and push to container registry --- .github/workflows/build-docker-images.yml | 43 +++++++++++++ .github/workflows/build-docker.yml | 78 +++++++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 .github/workflows/build-docker-images.yml create mode 100644 .github/workflows/build-docker.yml diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml new file mode 100644 index 00000000..43018075 --- /dev/null +++ b/.github/workflows/build-docker-images.yml @@ -0,0 +1,43 @@ +name: Build Docker and Optional Push + +on: + push: + branches: + - main + - dev + - demo + - hotfix + pull_request: + branches: + - main + - dev + - demo + - hotfix + types: + - opened + - ready_for_review + - reopened + - synchronize + merge_group: + workflow_dispatch: + +jobs: + docker-build: + strategy: + matrix: + include: + - app_name: cmsabackend + dockerfile: docker/Backend.Dockerfile + password_secret: DOCKER_PASSWORD + - app_name: cmsafrontend + dockerfile: docker/Frontend.Dockerfile + password_secret: DOCKER_PASSWORD + uses: ./.github/workflows/build-docker.yml + with: + registry: cmsacontainerreg.azurecr.io + username: cmsacontainerreg + password_secret: ${{ matrix.password_secret }} + app_name: ${{ matrix.app_name }} + dockerfile: ${{ matrix.dockerfile }} + push: ${{ github.event_name == 'push' || github.base_ref == 'main' || github.base_ref == 'dev' || github.base_ref == 'demo' || github.base_ref == 'hotfix' }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 00000000..03deeb78 --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,78 @@ +name: Reusable Docker build and push workflow + +on: + workflow_call: + inputs: + registry: + required: true + type: string + username: + required: true + type: string + password_secret: + required: true + type: string + app_name: + required: true + type: string + dockerfile: + required: true + type: string + push: + required: true + type: boolean + secrets: + DOCKER_PASSWORD: + required: true + +jobs: + docker-build: + runs-on: ubuntu-latest + steps: + + - name: Checkout + uses: actions/checkout@v4 + + - name: Docker Login + if: ${{ inputs.push }} + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ inputs.username }} + password: ${{ secrets[inputs.password_secret] }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Get current date + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Determine Tag Name Based on Branch + id: determine_tag + run: | + if [[ "${{ github.base_ref }}" == "main" ]]; then + echo "tagname=latest" >> $GITHUB_OUTPUT + elif [[ "${{ github.base_ref }}" == "dev" ]]; then + echo "tagname=dev" >> $GITHUB_OUTPUT + elif [[ "${{ github.base_ref }}" == "demo" ]]; then + echo "tagname=demo" >> $GITHUB_OUTPUT + elif [[ "${{ github.base_ref }}" == "hotfix" ]]; then + echo "tagname=hotfix" >> $GITHUB_OUTPUT + elif [[ "${{ github.base_ref }}" == "dependabotchanges" ]]; then + echo "tagname=dependabotchanges" >> $GITHUB_OUTPUT + else + echo "tagname=default" >> $GITHUB_OUTPUT + fi + + + - name: Build Docker Image and optionally push + uses: docker/build-push-action@v6 + with: + context: . + file: ${{ inputs.dockerfile }} + push: ${{ inputs.push }} + cache-from: type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name}}:${{ steps.determine_tag.outputs.tagname }} + tags: | + ${{ inputs.registry }}/${{ inputs.app_name}}:${{ steps.determine_tag.outputs.tagname }} + ${{ inputs.registry }}/${{ inputs.app_name}}:${{ steps.determine_tag.outputs.tagname }}_${{ steps.date.outputs.date }}_${{ github.run_number }} \ No newline at end of file From c6a5b8dec00e416a48272ada68fec0263868c9ac Mon Sep 17 00:00:00 2001 From: "Priyanka Singhal (Persistent Systems Inc)" Date: Thu, 3 Apr 2025 14:44:56 +0530 Subject: [PATCH 05/47] skip docker login on push --- .github/workflows/build-docker-images.yml | 2 +- .github/workflows/build-docker.yml | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index 43018075..7519d620 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -39,5 +39,5 @@ jobs: password_secret: ${{ matrix.password_secret }} app_name: ${{ matrix.app_name }} dockerfile: ${{ matrix.dockerfile }} - push: ${{ github.event_name == 'push' || github.base_ref == 'main' || github.base_ref == 'dev' || github.base_ref == 'demo' || github.base_ref == 'hotfix' }} + push: ${{ github.ref_name == 'main' || github.ref_name == 'dev' || github.ref_name == 'demo' || github.ref_name == 'hotfix' }} secrets: inherit \ No newline at end of file diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 03deeb78..70c2d232 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -32,6 +32,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 + + - name: push debug + run: echo ${{ inputs.push }} - name: Docker Login if: ${{ inputs.push }} @@ -51,15 +54,15 @@ jobs: - name: Determine Tag Name Based on Branch id: determine_tag run: | - if [[ "${{ github.base_ref }}" == "main" ]]; then + if [[ "${{ github.ref_name }}" == "main" ]]; then echo "tagname=latest" >> $GITHUB_OUTPUT - elif [[ "${{ github.base_ref }}" == "dev" ]]; then + elif [[ "${{ github.ref_name }}" == "dev" ]]; then echo "tagname=dev" >> $GITHUB_OUTPUT - elif [[ "${{ github.base_ref }}" == "demo" ]]; then + elif [[ "${{ github.ref_name }}" == "demo" ]]; then echo "tagname=demo" >> $GITHUB_OUTPUT - elif [[ "${{ github.base_ref }}" == "hotfix" ]]; then + elif [[ "${{ github.ref_name }}" == "hotfix" ]]; then echo "tagname=hotfix" >> $GITHUB_OUTPUT - elif [[ "${{ github.base_ref }}" == "dependabotchanges" ]]; then + elif [[ "${{ github.ref_name }}" == "dependabotchanges" ]]; then echo "tagname=dependabotchanges" >> $GITHUB_OUTPUT else echo "tagname=default" >> $GITHUB_OUTPUT From 02ae0538d9996ee495f2eb055084bed6dc576099 Mon Sep 17 00:00:00 2001 From: "Priyanka Singhal (Persistent Systems Inc)" Date: Thu, 3 Apr 2025 14:48:40 +0530 Subject: [PATCH 06/47] removed debug step --- .github/workflows/build-docker.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 70c2d232..d253f320 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -32,9 +32,6 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - - name: push debug - run: echo ${{ inputs.push }} - name: Docker Login if: ${{ inputs.push }} @@ -62,8 +59,6 @@ jobs: echo "tagname=demo" >> $GITHUB_OUTPUT elif [[ "${{ github.ref_name }}" == "hotfix" ]]; then echo "tagname=hotfix" >> $GITHUB_OUTPUT - elif [[ "${{ github.ref_name }}" == "dependabotchanges" ]]; then - echo "tagname=dependabotchanges" >> $GITHUB_OUTPUT else echo "tagname=default" >> $GITHUB_OUTPUT fi From b14199ad206ae424f9b01d9d2c0645dbdca0f791 Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Fri, 4 Apr 2025 13:10:10 +0530 Subject: [PATCH 07/47] feat: added one click deployment github action pipeline --- .github/workflows/deploy.yml | 259 +++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 00000000..16c5f286 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,259 @@ +name: Validate Deployment + +on: + push: + branches: + - main + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v3 + + - name: Setup Azure CLI + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az --version # Verify installation + + - name: Login to Azure + run: | + az login --service-principal -u ${{ secrets.AZURE_CLIENT_ID }} -p ${{ secrets.AZURE_CLIENT_SECRET }} --tenant ${{ secrets.AZURE_TENANT_ID }} + + - name: Install Bicep CLI + run: az bicep install + + - name: Generate Resource Group Name + id: generate_rg_name + run: | + echo "Generating a unique resource group name..." + TIMESTAMP=$(date +%Y%m%d%H%M%S) + COMMON_PART="ci-mycsa" + UNIQUE_RG_NAME="${COMMON_PART}${TIMESTAMP}" + echo "RESOURCE_GROUP_NAME=${UNIQUE_RG_NAME}" >> $GITHUB_ENV + echo "Generated Resource_GROUP_PREFIX: ${UNIQUE_RG_NAME}" + + + - name: Check and Create Resource Group + id: check_create_rg + run: | + set -e + echo "Checking if resource group exists..." + rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) + if [ "$rg_exists" = "false" ]; then + echo "Resource group does not exist. Creating..." + az group create --name ${{ env.RESOURCE_GROUP_NAME }} --location northcentralus || { echo "Error creating resource group"; exit 1; } + else + echo "Resource group already exists." + fi + + + - name: Deploy Bicep Template + id: deploy + run: | + set -e + az deployment group create \ + --resource-group ${{ env.RESOURCE_GROUP_NAME }} \ + --template-file infra/main.bicep \ + --parameters ResourcePrefix=codegen AiLocation=northcentralus + + + - name: Send Notification on Failure + if: failure() + run: | + RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + + # Construct the email body + EMAIL_BODY=$(cat <Dear Team,

We would like to inform you that the Modernize-your-code-solution-accelerator Automation process has encountered an issue and has failed to complete successfully.

Build URL: ${RUN_URL}
${OUTPUT}

Please investigate the matter at your earliest convenience.

Best regards,
Your Automation Team

" + } + EOF + ) + + # Send the notification + curl -X POST "${{ secrets.LOGIC_APP_URL }}" \ + -H "Content-Type: application/json" \ + -d "$EMAIL_BODY" || echo "Failed to send notification" + + + - name: Get Log Analytics Workspace from Resource Group + id: get_log_analytics_workspace + run: | + + set -e + echo "Fetching Log Analytics workspace from resource group ${{ env.RESOURCE_GROUP_NAME }}..." + + # Run the az monitor log-analytics workspace list command to get the workspace name + log_analytics_workspace_name=$(az monitor log-analytics workspace list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[0].name" -o tsv) + + if [ -z "$log_analytics_workspace_name" ]; then + echo "No Log Analytics workspace found in resource group ${{ env.RESOURCE_GROUP_NAME }}." + exit 1 + else + echo "LOG_ANALYTICS_WORKSPACE_NAME=${log_analytics_workspace_name}" >> $GITHUB_ENV + echo "Log Analytics workspace name: ${log_analytics_workspace_name}" + fi + + + - name: List KeyVaults and Store in Array + id: list_keyvaults + run: | + + set -e + echo "Listing all KeyVaults in the resource group ${RESOURCE_GROUP_NAME}..." + + # Get the list of KeyVaults in the specified resource group + keyvaults=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --query "[?type=='Microsoft.KeyVault/vaults'].name" -o tsv) + + if [ -z "$keyvaults" ]; then + echo "No KeyVaults found in resource group ${RESOURCE_GROUP_NAME}." + echo "KEYVAULTS=[]" >> $GITHUB_ENV # If no KeyVaults found, set an empty array + else + echo "KeyVaults found: $keyvaults" + + # Format the list into an array with proper formatting (no trailing comma) + keyvault_array="[" + first=true + for kv in $keyvaults; do + if [ "$first" = true ]; then + keyvault_array="$keyvault_array\"$kv\"" + first=false + else + keyvault_array="$keyvault_array,\"$kv\"" + fi + done + keyvault_array="$keyvault_array]" + + # Output the formatted array and save it to the environment variable + echo "KEYVAULTS=$keyvault_array" >> $GITHUB_ENV + fi + + - name: Purge log analytics workspace + id: log_analytics_workspace + run: | + + set -e + # Purge Log Analytics Workspace + echo "Purging the Log Analytics Workspace..." + if ! az monitor log-analytics workspace delete --force --resource-group ${{ env.RESOURCE_GROUP_NAME }} --workspace-name ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }} --yes --verbose; then + echo "Failed to purge Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}" + else + echo "Purged the Log Analytics workspace: ${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}" + fi + + echo "Log analytics workspace resource purging completed successfully" + + + - name: Delete Bicep Deployment + if: success() + run: | + set -e + echo "Checking if resource group exists..." + rg_exists=$(az group exists --name ${{ env.RESOURCE_GROUP_NAME }}) + if [ "$rg_exists" = "true" ]; then + echo "Resource group exist. Cleaning..." + az group delete \ + --name ${{ env.RESOURCE_GROUP_NAME }} \ + --yes \ + --no-wait + echo "Resource group deleted... ${{ env.RESOURCE_GROUP_NAME }}" + else + echo "Resource group does not exists." + fi + + + - name: Wait for resource deletion to complete + run: | + + # List of keyvaults + KEYVAULTS="${{ env.KEYVAULTS }}" + + # Remove the surrounding square brackets, if they exist + stripped_keyvaults=$(echo "$KEYVAULTS" | sed 's/\[\|\]//g') + + # Convert the comma-separated string into an array + IFS=',' read -r -a resources_to_check <<< "$stripped_keyvaults" + + # Append new resources to the array + resources_to_check+=("${{ env.LOG_ANALYTICS_WORKSPACE_NAME }}") + + echo "List of resources to check: ${resources_to_check[@]}" + + # Maximum number of retries + max_retries=3 + + # Retry intervals in seconds (30, 60, 120) + retry_intervals=(30 60 120) + + # Retry mechanism to check resources + retries=0 + while true; do + resource_found=false + + # Get the list of resources in YAML format again on each retry + resource_list=$(az resource list --resource-group ${{ env.RESOURCE_GROUP_NAME }} --output yaml) + + # Iterate through the resources to check + for resource in "${resources_to_check[@]}"; do + echo "Checking resource: $resource" + if echo "$resource_list" | grep -q "name: $resource"; then + echo "Resource '$resource' exists in the resource group." + resource_found=true + else + echo "Resource '$resource' does not exist in the resource group." + fi + done + + # If any resource exists, retry + if [ "$resource_found" = true ]; then + retries=$((retries + 1)) + if [ "$retries" -gt "$max_retries" ]; then + echo "Maximum retry attempts reached. Exiting." + break + else + # Wait for the appropriate interval for the current retry + echo "Waiting for ${retry_intervals[$retries-1]} seconds before retrying..." + sleep ${retry_intervals[$retries-1]} + fi + else + echo "No resources found. Exiting." + break + fi + done + + + - name: Purging the Resources + if: success() + run: | + + set -e + # List of keyvaults + KEYVAULTS="${{ env.KEYVAULTS }}" + + # Remove the surrounding square brackets, if they exist + stripped_keyvaults=$(echo "$KEYVAULTS" | sed 's/\[\|\]//g') + + # Convert the comma-separated string into an array + IFS=',' read -r -a keyvault_array <<< "$stripped_keyvaults" + + echo "Using KeyVaults Array..." + for keyvault_name in "${keyvault_array[@]}"; do + echo "Processing KeyVault: $keyvault_name" + # Check if the KeyVault is soft-deleted + deleted_vaults=$(az keyvault list-deleted --query "[?name=='$keyvault_name']" -o json --subscription ${{ secrets.AZURE_SUBSCRIPTION_ID }}) + + # If the KeyVault is found in the soft-deleted state, purge it + if [ "$(echo "$deleted_vaults" | jq length)" -gt 0 ]; then + echo "KeyVault '$keyvault_name' is soft-deleted. Proceeding to purge..." + # Purge the KeyVault + if az keyvault purge --name "$keyvault_name" --no-wait; then + echo "Successfully purged KeyVault '$keyvault_name'." + else + echo "Failed to purge KeyVault '$keyvault_name'." + fi + else + echo "KeyVault '$keyvault_name' is not soft-deleted. No action taken." + fi + done From e182cffa46730e8c90947c4c366f7dee8fb44a3d Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Mon, 7 Apr 2025 12:47:10 +0530 Subject: [PATCH 08/47] pylint issues fixed --- .flake8 | 2 +- src/backend/api/api_routes.py | 30 +++++++------ src/backend/api/auth/auth_utils.py | 12 ++--- src/backend/api/auth/sample_user.py | 2 +- src/backend/api/status_updates.py | 2 + src/backend/app.py | 14 +++--- src/backend/common/config/config.py | 1 + src/backend/common/database/cosmosdb.py | 16 +++---- src/backend/common/database/database_base.py | 42 +++++++++-------- .../common/database/database_factory.py | 1 - src/backend/common/logger/app_logger.py | 3 +- src/backend/common/models/api.py | 20 ++++----- src/backend/common/services/batch_service.py | 45 ++++++++++--------- src/backend/common/storage/blob_azure.py | 15 +++---- src/backend/common/storage/blob_base.py | 34 +++++++------- src/backend/common/storage/blob_factory.py | 1 - src/backend/sql_agents/__init__.py | 5 ++- src/backend/sql_agents/agent_config.py | 1 - src/backend/sql_agents/fixer/agent.py | 10 ++--- src/backend/sql_agents/fixer/response.py | 4 +- .../sql_agents/helpers/selection_function.py | 8 ++-- .../helpers/termination_function.py | 8 ++-- src/backend/sql_agents/helpers/utils.py | 2 +- src/backend/sql_agents/migrator/agent.py | 8 ++-- src/backend/sql_agents/migrator/response.py | 10 ++--- src/backend/sql_agents/picker/agent.py | 11 +++-- src/backend/sql_agents/picker/response.py | 4 +- .../sql_agents/semantic_verifier/agent.py | 11 +++-- .../sql_agents/semantic_verifier/response.py | 4 +- .../sql_agents/syntax_checker/agent.py | 11 +++-- .../sql_agents/syntax_checker/plug_ins.py | 6 +-- .../sql_agents/syntax_checker/response.py | 4 +- src/backend/sql_agents_start.py | 33 +++++++------- .../backend/common/config/config_test.py | 2 +- .../backend/common/database/cosmosdb_test.py | 14 +++--- .../common/database/database_base_test.py | 8 ++-- .../common/database/database_factory_test.py | 10 ++++- .../backend/common/storage/blob_azure_test.py | 30 +++++++++++-- .../backend/common/storage/blob_base_test.py | 9 ++-- .../common/storage/blob_factory_test.py | 38 ++++++++++++---- 40 files changed, 268 insertions(+), 223 deletions(-) diff --git a/.flake8 b/.flake8 index 93f63e5d..0df06ab8 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ max-line-length = 88 extend-ignore = E501 exclude = .venv, frontend -ignore = E203, W503, G004, G200 \ No newline at end of file +ignore = E203, W503, G004, G200,B008,ANN,D100,D101,D102,D103,D104,D105,D106,D107 \ No newline at end of file diff --git a/src/backend/api/api_routes.py b/src/backend/api/api_routes.py index 8a3d5a8d..d234b6c1 100644 --- a/src/backend/api/api_routes.py +++ b/src/backend/api/api_routes.py @@ -1,4 +1,4 @@ -"""FastAPI API routes for file processing and conversion""" +"""FastAPI API routes for file processing and conversion.""" import asyncio import io @@ -6,8 +6,10 @@ from api.auth.auth_utils import get_authenticated_user from api.status_updates import app_connection_manager, close_connection + from common.logger.app_logger import AppLogger from common.services.batch_service import BatchService + from fastapi import ( APIRouter, File, @@ -24,13 +26,14 @@ logger = AppLogger("APIRoutes") # start processing the batch -from sql_agents_start import process_batch_async +from sql_agents_start import process_batch_async # noqa: E402 @router.post("/start-processing") async def start_processing(request: Request): """ - Start processing files for a given batch + Start processing files for a given batch. + --- tags: - File Processing @@ -50,6 +53,7 @@ async def start_processing(request: Request): responses: 200: description: Processing initiated successfully + content: application/json: schema: @@ -61,6 +65,7 @@ async def start_processing(request: Request): type: string 400: description: Invalid processing request + 500: description: Internal server error """ @@ -89,7 +94,7 @@ async def start_processing(request: Request): ) async def download_files(batch_id: str): """ - Download files as ZIP + Download files as ZIP. --- tags: @@ -118,7 +123,6 @@ async def download_files(batch_id: str): type: string example: Batch not found """ - # call batch_service get_batch_for_zip to get all files for batch_id batch_service = BatchService() await batch_service.initialize_database() @@ -172,7 +176,7 @@ async def batch_status_updates( websocket: WebSocket, batch_id: str ): # , request: Request): """ - WebSocket endpoint for real-time batch status updates + Web-Socket endpoint for real-time batch status updates. --- tags: @@ -248,7 +252,7 @@ async def batch_status_updates( @router.get("/batch-story/{batch_id}") async def get_batch_status(request: Request, batch_id: str): """ - Retrieve batch history and file statuses + Retrieve batch history and file statuses. --- tags: @@ -371,9 +375,7 @@ async def get_batch_status(request: Request, batch_id: str): @router.get("/batch-summary/{batch_id}") async def get_batch_summary(request: Request, batch_id: str): - """ - Retrieve batch summary for a given batch ID. - """ + """Retrieve batch summary for a given batch ID.""" try: batch_service = BatchService() await batch_service.initialize_database() @@ -404,7 +406,7 @@ async def upload_file( request: Request, file: UploadFile = File(...), batch_id: str = Form(...) ): """ - Upload file for conversion + Upload file for conversion. --- tags: @@ -634,7 +636,7 @@ async def get_file_details(request: Request, file_id: str): @router.delete("/delete-batch/{batch_id}") async def delete_batch_details(request: Request, batch_id: str): """ - delete batch history using batch_id + Delete batch history using batch_id. --- tags: @@ -689,7 +691,7 @@ async def delete_batch_details(request: Request, batch_id: str): @router.delete("/delete-file/{file_id}") async def delete_file_details(request: Request, file_id: str): """ - delete file history using batch_id + Delete file history using batch_id. --- tags: @@ -747,7 +749,7 @@ async def delete_file_details(request: Request, file_id: str): @router.delete("/delete_all") async def delete_all_details(request: Request): """ - delete all the history of batches, files and logs + Delete all the history of batches, files and logs. --- tags: diff --git a/src/backend/api/auth/auth_utils.py b/src/backend/api/auth/auth_utils.py index c186b2cf..da6a6b23 100644 --- a/src/backend/api/auth/auth_utils.py +++ b/src/backend/api/auth/auth_utils.py @@ -1,10 +1,12 @@ -from fastapi import Request, HTTPException -import logging import base64 import json +import logging from typing import Dict + from api.auth.sample_user import sample_user +from fastapi import HTTPException, Request + logger = logging.getLogger(__name__) @@ -26,19 +28,19 @@ def __init__(self, user_details: Dict): def get_tenant_id(client_principal_b64: str) -> str: - """Extract tenant ID from base64 encoded client principal""" + """Extract tenant ID from base64 encoded client principal.""" try: decoded_bytes = base64.b64decode(client_principal_b64) decoded_string = decoded_bytes.decode("utf-8") user_info = json.loads(decoded_string) return user_info.get("tid", "") - except Exception as ex: + except Exception : logger.exception("Error decoding client principal") return "" def get_authenticated_user(request: Request) -> UserDetails: - """Get authenticated user details from request headers""" + """Get authenticated user details from request headers.""" user_object = {} headers = dict(request.headers) # Check if we're in production with real headers diff --git a/src/backend/api/auth/sample_user.py b/src/backend/api/auth/sample_user.py index e15ef56e..64bb2bee 100644 --- a/src/backend/api/auth/sample_user.py +++ b/src/backend/api/auth/sample_user.py @@ -5,4 +5,4 @@ "x-ms-client-principal-idp": "aad", "x-ms-token-aad-id-token": "dev.token", "x-ms-client-principal": "your_base_64_encoded_token" -} \ No newline at end of file +} diff --git a/src/backend/api/status_updates.py b/src/backend/api/status_updates.py index 67f932b4..7bf9f09f 100644 --- a/src/backend/api/status_updates.py +++ b/src/backend/api/status_updates.py @@ -1,5 +1,6 @@ """ Holds collection of websocket connections. + from clients registering for status updates. These socket references are used to send updates to registered clients from the backend processing code. @@ -11,6 +12,7 @@ from typing import Dict from common.models.api import FileProcessUpdate, FileProcessUpdateJSONEncoder + from fastapi import WebSocket logger = logging.getLogger(__name__) diff --git a/src/backend/app.py b/src/backend/app.py index b7b2173c..95d08302 100644 --- a/src/backend/app.py +++ b/src/backend/app.py @@ -1,12 +1,14 @@ -import uvicorn - -# Import our route modules +"""Create and configure the FastAPI application.""" from api.api_routes import router as backend_router + from common.logger.app_logger import AppLogger + from dotenv import load_dotenv + from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware +import uvicorn # from agent_services.agents_routes import router as agents_router # Load environment variables @@ -17,9 +19,7 @@ def create_app() -> FastAPI: - """ - Factory function to create and configure the FastAPI application - """ + """Create and return the FastAPI application instance.""" app = FastAPI(title="Code Gen Accelerator", version="1.0.0") # Configure CORS @@ -37,7 +37,7 @@ def create_app() -> FastAPI: @app.get("/health") async def health_check(): - """Health check endpoint""" + """Health check endpoint.""" return {"status": "healthy"} return app diff --git a/src/backend/common/config/config.py b/src/backend/common/config/config.py index 9d5d1ad8..3b774d60 100644 --- a/src/backend/common/config/config.py +++ b/src/backend/common/config/config.py @@ -1,6 +1,7 @@ import os from azure.identity.aio import ClientSecretCredential, DefaultAzureCredential + from dotenv import load_dotenv load_dotenv() diff --git a/src/backend/common/database/cosmosdb.py b/src/backend/common/database/cosmosdb.py index 8444a81a..a9e17e2c 100644 --- a/src/backend/common/database/cosmosdb.py +++ b/src/backend/common/database/cosmosdb.py @@ -1,5 +1,4 @@ from datetime import datetime, timezone -from enum import Enum from typing import Dict, List, Optional from uuid import UUID, uuid4 @@ -7,9 +6,9 @@ from azure.cosmos.aio import CosmosClient from azure.cosmos.aio._database import DatabaseProxy from azure.cosmos.exceptions import ( - CosmosResourceExistsError, - CosmosResourceNotFoundError, + CosmosResourceExistsError ) + from common.database.database_base import DatabaseBase from common.logger.app_logger import AppLogger from common.models.api import ( @@ -20,6 +19,7 @@ LogType, ProcessStatus, ) + from semantic_kernel.contents import AuthorRole @@ -208,7 +208,7 @@ async def get_batch_files(self, batch_id: str) -> List[Dict]: raise async def get_batch_from_id(self, batch_id: str) -> Dict: - """Retrieve a batch from the database using the batch ID""" + """Retrieve a batch from the database using the batch ID.""" try: query = "SELECT * FROM c WHERE c.batch_id = @batch_id" params = [{"name": "@batch_id", "value": batch_id}] @@ -225,7 +225,7 @@ async def get_batch_from_id(self, batch_id: str) -> Dict: raise async def get_user_batches(self, user_id: str) -> Dict: - """Retrieve all batches for a given user""" + """Retrieve all batches for a given user.""" try: query = "SELECT * FROM c WHERE c.user_id = @user_id" params = [{"name": "@user_id", "value": user_id}] @@ -242,7 +242,7 @@ async def get_user_batches(self, user_id: str) -> Dict: raise async def get_file_logs(self, file_id: str) -> List[Dict]: - """Retrieve all logs for a given file""" + """Retrieve all logs for a given file.""" try: query = ( "SELECT * FROM c WHERE c.file_id = @file_id ORDER BY c.timestamp DESC" @@ -322,7 +322,7 @@ async def add_file_log( agent_type: AgentType, author_role: AuthorRole, ) -> None: - """Log a file status update""" + """Log a file status update.""" try: log_id = uuid4() log_entry = FileLog( @@ -343,7 +343,7 @@ async def add_file_log( async def update_batch_entry( self, batch_id: str, user_id: str, status: ProcessStatus, file_count: int ): - """Update batch status""" + """Update batch status.""" try: batch = await self.get_batch(user_id, batch_id) if not batch: diff --git a/src/backend/common/database/database_base.py b/src/backend/common/database/database_base.py index a54f3c33..961426b5 100644 --- a/src/backend/common/database/database_base.py +++ b/src/backend/common/database/database_base.py @@ -1,67 +1,65 @@ import uuid from abc import ABC, abstractmethod -from datetime import datetime -from enum import Enum from typing import Dict, List, Optional -from common.logger.app_logger import AppLogger -from common.models.api import AgentType, BatchRecord, FileRecord, LogType, ProcessStatus +from common.models.api import AgentType, BatchRecord, FileRecord, LogType + from semantic_kernel.contents import AuthorRole class DatabaseBase(ABC): - """Abstract base class for database operations""" + """Abstract base class for database operations.""" @abstractmethod async def initialize_cosmos(self) -> None: - """Initialize the cosmosdb client and create container if needed""" + """Initialize the cosmosdb client and create container if needed.""" pass @abstractmethod async def create_batch(self, user_id: str, batch_id: uuid.UUID) -> BatchRecord: - """Create a new conversion batch""" + """Create a new conversion batch.""" pass @abstractmethod async def get_file_logs(self, file_id: str) -> Dict: - """Retrieve all logs for a file""" + """Retrieve all logs for a file.""" pass @abstractmethod async def get_batch_from_id(self, batch_id: str) -> Dict: - """Retrieve all logs for a file""" + """Retrieve all logs for a file.""" pass @abstractmethod async def get_batch_files(self, batch_id: str) -> List[Dict]: - """Retrieve all files for a batch""" + """Retrieve all files for a batch.""" pass @abstractmethod async def delete_file_logs(self, file_id: str) -> None: - """Delete all logs for a file""" + """Delete all logs for a file.""" pass @abstractmethod async def get_user_batches(self, user_id: str) -> Dict: - """Retrieve all batches for a user""" + """Retrieve all batches for a user.""" pass @abstractmethod async def add_file( self, batch_id: uuid.UUID, file_id: uuid.UUID, file_name: str, storage_path: str ) -> FileRecord: - """Add a file entry to the database""" + """Add a file entry to the database.""" pass @abstractmethod async def get_batch(self, user_id: str, batch_id: str) -> Optional[Dict]: - """Retrieve a batch and its associated files""" + """Retrieve a batch and its associated files.""" pass @abstractmethod async def get_file(self, file_id: str) -> Optional[Dict]: - """Retrieve a file entry along with its logs""" + """Retrieve a file entry along with its logs.""" pass @abstractmethod @@ -74,12 +72,12 @@ async def add_file_log( agent_type: AgentType, author_role: AuthorRole, ) -> None: - """Log a file status update""" + """Log a file status update.""" pass @abstractmethod async def update_file(self, file_record: FileRecord) -> None: - """update file record""" + """Update file record.""" pass @abstractmethod @@ -88,25 +86,25 @@ async def update_batch(self, batch_record: BatchRecord) -> BatchRecord: @abstractmethod async def delete_all(self, user_id: str) -> None: - """Delete all batches, files, and logs for a user""" + """Delete all batches, files, and logs for a user.""" pass @abstractmethod async def delete_batch(self, user_id: str, batch_id: str) -> None: - """Delete a batch along with its files and logs""" + """Delete a batch along with its files and logs.""" pass @abstractmethod async def delete_file(self, user_id: str, batch_id: str, file_id: str) -> None: - """Delete a file and its logs, and update batch file count""" + """Delete a file and its logs, and update batch file count.""" pass @abstractmethod async def get_batch_history(self, user_id: str, batch_id: str) -> List[Dict]: - """Retrieve all logs for a batch""" + """Retrieve all logs for a batch.""" pass @abstractmethod async def close(self) -> None: - """Close database connection""" + """Close database connection.""" pass diff --git a/src/backend/common/database/database_factory.py b/src/backend/common/database/database_factory.py index 1306a520..ee92677f 100644 --- a/src/backend/common/database/database_factory.py +++ b/src/backend/common/database/database_factory.py @@ -1,6 +1,5 @@ from typing import Optional -from azure.cosmos.aio import CosmosClient from common.config.config import Config from common.database.cosmosdb import CosmosDBClient from common.database.database_base import DatabaseBase diff --git a/src/backend/common/logger/app_logger.py b/src/backend/common/logger/app_logger.py index 5642ea7f..b9aed467 100644 --- a/src/backend/common/logger/app_logger.py +++ b/src/backend/common/logger/app_logger.py @@ -1,7 +1,6 @@ +import json import logging -from datetime import datetime from typing import Any -import json class LogLevel: diff --git a/src/backend/common/models/api.py b/src/backend/common/models/api.py index 15c9525a..7bf280a7 100644 --- a/src/backend/common/models/api.py +++ b/src/backend/common/models/api.py @@ -1,9 +1,9 @@ from __future__ import annotations import json +import logging from datetime import datetime from enum import Enum -import logging from typing import Dict, List from uuid import UUID @@ -125,7 +125,7 @@ def __init__( @staticmethod def fromdb(data: Dict) -> FileLog: - """Convert str to UUID after fetching from the database""" + """Convert str to UUID after fetching from the database.""" return FileLog( log_id=UUID(data["log_id"]), # Convert str → UUID file_id=UUID(data["file_id"]), # Convert str → UUID @@ -142,7 +142,7 @@ def fromdb(data: Dict) -> FileLog: ) def dict(self) -> Dict: - """Convert UUID to str before inserting into the database""" + """Convert UUID to str before inserting into the database.""" return { "id": str(self.log_id), # Convert UUID → str "log_id": str(self.log_id), # Convert UUID → str @@ -185,7 +185,7 @@ def __init__( @staticmethod def fromdb(data: Dict) -> FileRecord: - """Convert str to UUID after fetching from the database""" + """Convert str to UUID after fetching from the database.""" return FileRecord( file_id=UUID(data["file_id"]), # Convert str → UUID batch_id=UUID(data["batch_id"]), # Convert str → UUID @@ -203,7 +203,7 @@ def fromdb(data: Dict) -> FileRecord: ) def dict(self) -> Dict: - """Convert UUID to str before inserting into the database""" + """Convert UUID to str before inserting into the database.""" return { "id": str(self.file_id), "file_id": str(self.file_id), # Convert UUID → str @@ -221,7 +221,7 @@ def dict(self) -> Dict: class FileProcessUpdate: - "websocket payload for file process updates" + """websocket payload for file process updates.""" def __init__( self, @@ -259,9 +259,7 @@ def dict(self) -> Dict: class FileProcessUpdateJSONEncoder(json.JSONEncoder): - """ - Custom JSON encoder for serializing FileProcessUpdate, ProcessStatus, and FileResult objects. - """ + """Custom JSON encoder for serializing FileProcessUpdate, ProcessStatus, and FileResult objects.""" def default(self, obj): # Check if the object is an instance of FileProcessUpdate, ProcessStatus, or FileResult @@ -294,7 +292,7 @@ def __init__( self.status = status def dict(self) -> Dict: - """Convert UUID to str before inserting into the database""" + """Convert UUID to str before inserting into the database.""" return { "batch_id": str(self.batch_id), # Convert UUID → str for DB "user_id": self.user_id, @@ -355,7 +353,7 @@ def fromdb(data: Dict) -> BatchRecord: ) def dict(self) -> Dict: - """Convert UUID to str before inserting into the database""" + """Convert UUID to str before inserting into the database.""" return { "id": str(self.batch_id), "batch_id": str(self.batch_id), # Convert UUID → str for DB diff --git a/src/backend/common/services/batch_service.py b/src/backend/common/services/batch_service.py index bbfecc13..0d5a6096 100644 --- a/src/backend/common/services/batch_service.py +++ b/src/backend/common/services/batch_service.py @@ -14,7 +14,9 @@ ProcessStatus, ) from common.storage.blob_factory import BlobStorageFactory + from fastapi import HTTPException, UploadFile + from semantic_kernel.contents import AuthorRole @@ -29,7 +31,7 @@ async def initialize_database(self): self.database = await DatabaseFactory.get_database() async def get_batch(self, batch_id: UUID, user_id: str) -> Optional[Dict]: - """Retrieve batch details including files""" + """Retrieve batch details including files.""" batch = await self.database.get_batch(user_id, batch_id) if not batch: return None @@ -38,7 +40,7 @@ async def get_batch(self, batch_id: UUID, user_id: str) -> Optional[Dict]: return {"batch": batch, "files": files} async def get_file(self, file_id: str) -> Optional[Dict]: - """Retrieve file details""" + """Retrieve file details.""" file = await self.database.get_file(file_id) if not file: return None @@ -46,7 +48,7 @@ async def get_file(self, file_id: str) -> Optional[Dict]: return {"file": file} async def get_file_report(self, file_id: str) -> Optional[Dict]: - """Retrieve file logs""" + """Retrieve file logs.""" file = await self.database.get_file(file_id) file_record = FileRecord.fromdb(file) batch = await self.database.get_batch_from_id(str(file_record.batch_id)) @@ -59,7 +61,7 @@ async def get_file_report(self, file_id: str) -> Optional[Dict]: storage = await BlobStorageFactory.get_storage() if file_record.translated_path not in ["", None]: translated_content = await storage.get_file(file_record.translated_path) - except (FileNotFoundError, IOError) as e: + except IOError as e: self.logger.error(f"Error downloading file content: {str(e)}") return { @@ -71,20 +73,19 @@ async def get_file_report(self, file_id: str) -> Optional[Dict]: } async def get_file_translated(self, file: dict): - """Retrieve file logs""" - + """Retrieve file logs.""" translated_content = "" try: storage = await BlobStorageFactory.get_storage() if file["translated_path"] not in ["", None]: translated_content = await storage.get_file(file["translated_path"]) - except (FileNotFoundError, IOError) as e: + except IOError as e: self.logger.error(f"Error downloading file content: {str(e)}") return translated_content async def get_batch_for_zip(self, batch_id: str) -> List[Tuple[str, str]]: - """Retrieve batch details including files in a single zip archive""" + """Retrieve batch details including files in a single zip archive.""" files = [] try: files_meta = await self.database.get_batch_files(batch_id) @@ -108,7 +109,7 @@ async def get_batch_for_zip(self, batch_id: str) -> List[Tuple[str, str]]: raise # Re-raise for caller handling async def get_batch_summary(self, batch_id: str, user_id: str) -> Optional[Dict]: - """Retrieve file logs""" + """Retrieve file logs.""" try: try: batch = await self.database.get_batch(user_id, batch_id) @@ -148,7 +149,7 @@ async def get_batch_summary(self, batch_id: str, user_id: str) -> Optional[Dict] raise # Re-raise for caller handling async def delete_batch(self, batch_id: UUID, user_id: str): - """Delete a batch along with its files and logs""" + """Delete a batch along with its files and logs.""" batch = await self.database.get_batch(user_id, batch_id) if batch: await self.database.delete_batch(user_id, batch_id) @@ -157,7 +158,7 @@ async def delete_batch(self, batch_id: UUID, user_id: str): return {"message": "Batch deleted successfully", "batch_id": str(batch_id)} async def delete_file(self, file_id: UUID, user_id: str): - """Delete a file and its logs, and update batch file count""" + """Delete a file and its logs, and update batch file count.""" try: # Ensure storage is available storage = await BlobStorageFactory.get_storage() @@ -208,11 +209,11 @@ async def delete_file(self, file_id: UUID, user_id: str): raise RuntimeError("File deletion failed") from e async def delete_all(self, user_id: str): - """Delete all batches, files, and logs for a user""" + """Delete all batches, files, and logs for a user.""" return await self.database.delete_all(user_id) async def get_all_batches(self, user_id: str): - """Retrieve all batches for a user""" + """Retrieve all batches for a user.""" return await self.database.get_user_batches(user_id) def is_valid_uuid(self, value: str) -> bool: @@ -235,7 +236,7 @@ def generate_file_path( return file_path async def upload_file_to_batch(self, batch_id: str, user_id: str, file: UploadFile): - """Upload a file, create entries in the database, and log the process""" + """Upload a file, create entries in the database, and log the process.""" try: # Ensure storage is available storage = await BlobStorageFactory.get_storage() @@ -362,7 +363,7 @@ async def update_file( error_count: int, syntax_count: int, ): - """Update file entry in the database""" + """Update file entry in the database.""" file = await self.database.get_file(file_id) if not file: raise HTTPException(status_code=404, detail="File not found") @@ -376,7 +377,7 @@ async def update_file( return file_record async def update_file_record(self, file_record: FileRecord): - """Update file entry in the database""" + """Update file entry in the database.""" await self.database.update_file(file_record) async def create_file_log( @@ -388,7 +389,7 @@ async def create_file_log( agent_type: AgentType, author_role: AuthorRole, ): - """Create a new file log entry in the database""" + """Create a new file log entry in the database.""" await self.database.add_file_log( UUID(file_id), description, @@ -399,7 +400,7 @@ async def create_file_log( ) async def update_batch(self, batch_id: str, status: ProcessStatus): - """Update batch status to completed""" + """Update batch status to completed.""" batch = await self.database.get_batch_from_id(batch_id) if not batch: raise HTTPException(status_code=404, detail="Batch not found") @@ -409,7 +410,7 @@ async def update_batch(self, batch_id: str, status: ProcessStatus): await self.database.update_batch(batch_record) async def create_candidate(self, file_id: str, candidate: str): - """Create a new candidate entry in the database and upload the candita file to storage""" + """Create a new candidate entry in the database and upload the candita file to storage.""" # Ensure storage is available storage = await BlobStorageFactory.get_storage() if not storage: @@ -462,7 +463,7 @@ async def batch_files_final_update(self, batch_id: str): # file didn't completed successfully file_record.status = ProcessStatus.COMPLETED - if(file_record.translated_path == None or file_record.translated_path == ""): + if (file_record.translated_path is None or file_record.translated_path == ""): file_record.file_result = FileResult.ERROR error_count, syntax_count = await self.get_file_counts( @@ -519,11 +520,11 @@ async def get_file_counts(self, file_id: str): return error_count, syntax_count async def get_batch_from_id(self, batch_id: str): - """Retrieve a batch record from the database""" + """Retrieve a batch record from the database.""" return await self.database.get_batch_from_id(batch_id) async def delete_all_from_storage_cosmos(self, user_id: str): - """Delete a all files from storage, remove its database entry, logs""" + """Delete a all files from storage, remove its database entry, logs.""" try: # Ensure storage is available storage = await BlobStorageFactory.get_storage() diff --git a/src/backend/common/storage/blob_azure.py b/src/backend/common/storage/blob_azure.py index 839c07cd..097cfd76 100644 --- a/src/backend/common/storage/blob_azure.py +++ b/src/backend/common/storage/blob_azure.py @@ -1,9 +1,8 @@ from typing import Any, BinaryIO, Dict, Optional -from azure.core.exceptions import ResourceExistsError from azure.identity import DefaultAzureCredential from azure.storage.blob import BlobServiceClient -from common.config.config import Config + from common.logger.app_logger import AppLogger from common.storage.blob_base import BlobStorageBase @@ -42,7 +41,7 @@ async def upload_file( content_type: Optional[str] = None, metadata: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: - """Upload a file to Azure Blob Storage""" + """Upload a file to Azure Blob Storage.""" try: blob_client = self.container_client.get_blob_client(blob_path) @@ -51,7 +50,7 @@ async def upload_file( raise try: # Upload the file - upload_results = blob_client.upload_blob( + upload_results = blob_client.upload_blob( # noqa: F841 file_content, content_type=content_type, metadata=metadata, @@ -78,7 +77,7 @@ async def upload_file( raise async def get_file(self, blob_path: str) -> BinaryIO: - """Download a file from Azure Blob Storage""" + """Download a file from Azure Blob Storage.""" try: blob_client = self.container_client.get_blob_client(blob_path) download_stream = blob_client.download_blob() @@ -95,7 +94,7 @@ async def get_file(self, blob_path: str) -> BinaryIO: raise async def delete_file(self, blob_path: str) -> bool: - """Delete a file from Azure Blob Storage""" + """Delete a file from Azure Blob Storage.""" try: blob_client = self.container_client.get_blob_client(blob_path) blob_client.delete_blob() @@ -108,7 +107,7 @@ async def delete_file(self, blob_path: str) -> bool: return False async def list_files(self, prefix: Optional[str] = None) -> list[Dict[str, Any]]: - """List files in Azure Blob Storage""" + """List files in Azure Blob Storage.""" try: blobs = [] async for blob in self.container_client.list_blobs(name_starts_with=prefix): @@ -128,7 +127,7 @@ async def list_files(self, prefix: Optional[str] = None) -> list[Dict[str, Any]] raise async def close(self) -> None: - """Close blob storage connections""" + """Close blob storage connections.""" if self.service_client: self.service_client.close() self.logger.info("Closed blob storage connection") diff --git a/src/backend/common/storage/blob_base.py b/src/backend/common/storage/blob_base.py index af7b0c94..44955840 100644 --- a/src/backend/common/storage/blob_base.py +++ b/src/backend/common/storage/blob_base.py @@ -1,27 +1,27 @@ from abc import ABC, abstractmethod -from typing import BinaryIO, Optional, Dict, Any +from typing import Any, BinaryIO, Dict, Optional -class BlobStorageBase(ABC): - """Abstract base class for blob storage operations""" +class BlobStorageBase(ABC): + """Abstract base class for blob storage operations.""" @abstractmethod async def upload_file( - self, + self, file_content: BinaryIO, blob_path: str, content_type: Optional[str] = None, metadata: Optional[Dict[str, str]] = None ) -> Dict[str, Any]: """ - Upload a file to blob storage - + Upload a file to blob storage. + Args: file_content: The file content to upload blob_path: The path where to store the blob content_type: Optional content type of the file metadata: Optional metadata to store with the blob - + Returns: Dict containing upload details (url, size, etc.) """ @@ -30,11 +30,11 @@ async def upload_file( @abstractmethod async def get_file(self, blob_path: str) -> BinaryIO: """ - Retrieve a file from blob storage - + Retrieve a file from blob storage. + Args: blob_path: Path to the blob - + Returns: File content as a binary stream """ @@ -43,11 +43,11 @@ async def get_file(self, blob_path: str) -> BinaryIO: @abstractmethod async def delete_file(self, blob_path: str) -> bool: """ - Delete a file from blob storage - + Delete a file from blob storage. + Args: blob_path: Path to the blob to delete - + Returns: True if deletion was successful """ @@ -56,12 +56,12 @@ async def delete_file(self, blob_path: str) -> bool: @abstractmethod async def list_files(self, prefix: Optional[str] = None) -> list[Dict[str, Any]]: """ - List files in blob storage - + List files in blob storage. + Args: prefix: Optional prefix to filter blobs - + Returns: List of blob details """ - pass \ No newline at end of file + pass diff --git a/src/backend/common/storage/blob_factory.py b/src/backend/common/storage/blob_factory.py index 9e47fd8e..fc855635 100644 --- a/src/backend/common/storage/blob_factory.py +++ b/src/backend/common/storage/blob_factory.py @@ -38,7 +38,6 @@ async def close_storage() -> None: async def main(): storage = await BlobStorageFactory.get_storage() # Use the storage instance... - files = await storage.list_files() blob = await storage.get_file("q1_informix.sql") print(blob) await BlobStorageFactory.close_storage() diff --git a/src/backend/sql_agents/__init__.py b/src/backend/sql_agents/__init__.py index 06480628..4251f942 100644 --- a/src/backend/sql_agents/__init__.py +++ b/src/backend/sql_agents/__init__.py @@ -1,6 +1,7 @@ -"""This module initializes the agents and helpers for the""" +"""This module initializes the agents and helpers for the.""" from common.models.api import AgentType + from sql_agents.fixer.agent import setup_fixer_agent from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion from sql_agents.helpers.utils import get_prompt @@ -10,7 +11,7 @@ from sql_agents.syntax_checker.agent import setup_syntax_checker_agent # Import the configuration function -from .agent_config import AgentsConfigDialect, create_config +from .agent_config import create_config __all__ = [ "setup_migrator_agent", diff --git a/src/backend/sql_agents/agent_config.py b/src/backend/sql_agents/agent_config.py index d8152354..8f46372c 100644 --- a/src/backend/sql_agents/agent_config.py +++ b/src/backend/sql_agents/agent_config.py @@ -1,6 +1,5 @@ """Configuration for the agents module.""" -import json import os from enum import Enum diff --git a/src/backend/sql_agents/fixer/agent.py b/src/backend/sql_agents/fixer/agent.py index 2ace3bcc..033b51c9 100644 --- a/src/backend/sql_agents/fixer/agent.py +++ b/src/backend/sql_agents/fixer/agent.py @@ -3,18 +3,18 @@ import logging from common.models.api import AgentType -from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion -from sql_agents.helpers.utils import get_prompt + from semantic_kernel.agents import ChatCompletionAgent from semantic_kernel.kernel import KernelArguments -from semantic_kernel.prompt_template import PromptTemplateConfig + from sql_agents.agent_config import AgentModelDeployment, AgentsConfigDialect from sql_agents.fixer.response import FixerResponse +from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion +from sql_agents.helpers.utils import get_prompt logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) - logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -22,7 +22,7 @@ def setup_fixer_agent( name: AgentType, config: AgentsConfigDialect, deployment_name: AgentModelDeployment ) -> ChatCompletionAgent: - """Setup the fixer agent.""" + """Set up the fixer agent.""" _deployment_name = deployment_name.value _name = name.value kernel = create_kernel_with_chat_completion(_name, _deployment_name) diff --git a/src/backend/sql_agents/fixer/response.py b/src/backend/sql_agents/fixer/response.py index 39bf521d..4cb7b2a0 100644 --- a/src/backend/sql_agents/fixer/response.py +++ b/src/backend/sql_agents/fixer/response.py @@ -2,9 +2,7 @@ class FixerResponse(BaseModel): - """ - Model for the response of the fixer - """ + """Model for the response of the fixer.""" thought: str fixed_query: str diff --git a/src/backend/sql_agents/helpers/selection_function.py b/src/backend/sql_agents/helpers/selection_function.py index 4e3c045c..35481a45 100644 --- a/src/backend/sql_agents/helpers/selection_function.py +++ b/src/backend/sql_agents/helpers/selection_function.py @@ -1,4 +1,4 @@ -"""selection_function.py""" +"""selection_function.py.""" from semantic_kernel.functions import KernelFunctionFromPrompt @@ -6,7 +6,7 @@ def setup_selection_function( name, migrator_name, picker_name, syntax_checker_name, fixer_name ): - """Setup the selection function.""" + """Set up the selection function.""" selection_function = KernelFunctionFromPrompt( function_name=name, prompt=f""" @@ -19,12 +19,12 @@ def setup_selection_function( - {picker_name.value} - {syntax_checker_name.value} - {fixer_name.value} - + Follow these instructions to determine the next participant: 1. After user input, it is always {migrator_name.value}'s turn. 2. After {migrator_name.value}, it is always {picker_name.value}'s turn. 3. After {picker_name.value}, it is always {syntax_checker_name.value}'s turn. - + The next two steps are repeated until the migration is complete: 4. After {syntax_checker_name.value}, it is {fixer_name.value}'s turn. 5. After {fixer_name.value}, it is {syntax_checker_name.value}'s turn. diff --git a/src/backend/sql_agents/helpers/termination_function.py b/src/backend/sql_agents/helpers/termination_function.py index 443fd2d8..5b97ae25 100644 --- a/src/backend/sql_agents/helpers/termination_function.py +++ b/src/backend/sql_agents/helpers/termination_function.py @@ -1,19 +1,19 @@ -""" Helper function to set up the termination function for the semantic kernel. """ +"""Helper function to set up the termination function for the semantic kernel.""" from semantic_kernel.functions import KernelFunctionFromPrompt def setup_termination_function(name, termination_keyword): - """Setup the termination function for the semantic kernel.""" + """Set up the termination function for the semantic kernel.""" termination_function = KernelFunctionFromPrompt( function_name=name, prompt=f""" Examine the response and determine whether the query migration is complete. If so, respond with a single word without explanation: {termination_keyword}. - + INPUT: - Your input will be a JSON structure that contains a "syntax_errors" key. - + RULES: - If "syntax_errors" is an empty list, migration is complete. - If "syntax_errors" is not empty, migration is not complete. diff --git a/src/backend/sql_agents/helpers/utils.py b/src/backend/sql_agents/helpers/utils.py index 28e1a744..d2000ab9 100644 --- a/src/backend/sql_agents/helpers/utils.py +++ b/src/backend/sql_agents/helpers/utils.py @@ -14,7 +14,7 @@ def get_prompt(agent_type: str) -> str: def is_text(content): - """Check if the content is text and not empty""" + """Check if the content is text and not empty.""" if isinstance(content, str): if len(content) == 0: return False diff --git a/src/backend/sql_agents/migrator/agent.py b/src/backend/sql_agents/migrator/agent.py index b881006d..390dfa1d 100644 --- a/src/backend/sql_agents/migrator/agent.py +++ b/src/backend/sql_agents/migrator/agent.py @@ -3,11 +3,13 @@ import logging from common.models.api import AgentType -from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion -from sql_agents.helpers.utils import get_prompt + from semantic_kernel.agents import ChatCompletionAgent from semantic_kernel.functions import KernelArguments + from sql_agents.agent_config import AgentModelDeployment, AgentsConfigDialect +from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion +from sql_agents.helpers.utils import get_prompt from sql_agents.migrator.response import MigratorResponse logger = logging.getLogger(__name__) @@ -17,7 +19,7 @@ def setup_migrator_agent( name: AgentType, config: AgentsConfigDialect, deployment_name: AgentModelDeployment ) -> ChatCompletionAgent: - """Setup the migrator agent.""" + """Set up the migrator agent.""" _deployment_name = deployment_name.value _name = name.value NUM_CANDIDATES = 3 diff --git a/src/backend/sql_agents/migrator/response.py b/src/backend/sql_agents/migrator/response.py index da8124d0..fa74f827 100644 --- a/src/backend/sql_agents/migrator/response.py +++ b/src/backend/sql_agents/migrator/response.py @@ -2,21 +2,17 @@ class MigratorCandidate(BaseModel): - """ - Model for a single candidate for migration - """ + """Model for a single candidate for migration.""" plan: str candidate_query: str class MigratorResponse(BaseModel): - """ - Model for the response of the migrator - """ + """Model for the response of the migrator.""" input_summary: str candidates: list[MigratorCandidate] input_error: str | None = None summary: str | None = None - rai_error: str | None = None \ No newline at end of file + rai_error: str | None = None diff --git a/src/backend/sql_agents/picker/agent.py b/src/backend/sql_agents/picker/agent.py index c724c130..867c7903 100644 --- a/src/backend/sql_agents/picker/agent.py +++ b/src/backend/sql_agents/picker/agent.py @@ -1,15 +1,18 @@ -"""Picker agent setup.""" +"""Set up the Picker agent.""" import logging from common.models.api import AgentType -from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion -from sql_agents.helpers.utils import get_prompt + from semantic_kernel.agents import ChatCompletionAgent from semantic_kernel.kernel import KernelArguments + from sql_agents.agent_config import AgentModelDeployment, AgentsConfigDialect +from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion +from sql_agents.helpers.utils import get_prompt from sql_agents.picker.response import PickerResponse + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -19,7 +22,7 @@ def setup_picker_agent( name: AgentType, config: AgentsConfigDialect, deployment_name: AgentModelDeployment ) -> ChatCompletionAgent: - """Setup the picker agent.""" + """Set up the picker agent.""" _deployment_name = deployment_name.value _name = name.value kernel = create_kernel_with_chat_completion(_name, _deployment_name) diff --git a/src/backend/sql_agents/picker/response.py b/src/backend/sql_agents/picker/response.py index eaad7c86..33a3804b 100644 --- a/src/backend/sql_agents/picker/response.py +++ b/src/backend/sql_agents/picker/response.py @@ -7,9 +7,7 @@ class PickerCandidateSummary(BaseModel): class PickerResponse(BaseModel): - """ - The response of the picker agent. - """ + """The response of the picker agent.""" source_summary: str candidate_summaries: list[PickerCandidateSummary] diff --git a/src/backend/sql_agents/semantic_verifier/agent.py b/src/backend/sql_agents/semantic_verifier/agent.py index ab60adaa..c20dc3dc 100644 --- a/src/backend/sql_agents/semantic_verifier/agent.py +++ b/src/backend/sql_agents/semantic_verifier/agent.py @@ -1,15 +1,18 @@ -"""This module contains the setup for the semantic verifier agent.""" +"""Set up the semantic verifier agent.""" import logging from common.models.api import AgentType -from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion -from sql_agents.helpers.utils import get_prompt + from semantic_kernel.agents import ChatCompletionAgent from semantic_kernel.kernel import KernelArguments + from sql_agents.agent_config import AgentModelDeployment, AgentsConfigDialect +from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion +from sql_agents.helpers.utils import get_prompt from sql_agents.semantic_verifier.response import SemanticVerifierResponse + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -21,7 +24,7 @@ def setup_semantic_verifier_agent( source_query: str, target_query: str, ) -> ChatCompletionAgent: - """Setup the semantic verifier agent.""" + """Set up the semantic verifier agent.""" _deployment_name = deployment_name.value _name = name.value kernel = create_kernel_with_chat_completion(_name, _deployment_name) diff --git a/src/backend/sql_agents/semantic_verifier/response.py b/src/backend/sql_agents/semantic_verifier/response.py index 0c3f5ddc..ab771a40 100644 --- a/src/backend/sql_agents/semantic_verifier/response.py +++ b/src/backend/sql_agents/semantic_verifier/response.py @@ -2,9 +2,7 @@ class SemanticVerifierResponse(BaseModel): - """ - Response model for the semantic verifier agent - """ + """Response model for the semantic verifier agent.""" analysis: str judgement: str diff --git a/src/backend/sql_agents/syntax_checker/agent.py b/src/backend/sql_agents/syntax_checker/agent.py index 0c709935..9ee89ec4 100644 --- a/src/backend/sql_agents/syntax_checker/agent.py +++ b/src/backend/sql_agents/syntax_checker/agent.py @@ -1,17 +1,20 @@ -"""This module contains the syntax checker agent.""" +"""Set up the syntax checker agent.""" import logging from common.models.api import AgentType -from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion -from sql_agents.helpers.utils import get_prompt + from semantic_kernel.agents import ChatCompletionAgent from semantic_kernel.connectors.ai import FunctionChoiceBehavior from semantic_kernel.kernel import KernelArguments + from sql_agents.agent_config import AgentModelDeployment, AgentsConfigDialect +from sql_agents.helpers.sk_utils import create_kernel_with_chat_completion +from sql_agents.helpers.utils import get_prompt from sql_agents.syntax_checker.plug_ins import SyntaxCheckerPlugin from sql_agents.syntax_checker.response import SyntaxCheckerResponse + logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) @@ -19,7 +22,7 @@ def setup_syntax_checker_agent( name: AgentType, config: AgentsConfigDialect, deployment_name: AgentModelDeployment ) -> ChatCompletionAgent: - """Setup the syntax checker agent.""" + """Set up the syntax checker agent.""" _deployment_name = deployment_name.value _name = name.value kernel = create_kernel_with_chat_completion(_name, _deployment_name) diff --git a/src/backend/sql_agents/syntax_checker/plug_ins.py b/src/backend/sql_agents/syntax_checker/plug_ins.py index f5f27032..ca689987 100644 --- a/src/backend/sql_agents/syntax_checker/plug_ins.py +++ b/src/backend/sql_agents/syntax_checker/plug_ins.py @@ -27,7 +27,7 @@ def check_syntax( ) -> Annotated[ str, """ - Returns a json list of errors in the format of + Returns a json list of errors in the format of. [ { "Line": , @@ -39,13 +39,11 @@ def check_syntax( """, ]: """Check the TSQL syntax using tsqlParser.""" - print(f"Called syntaxCheckerPlugin with: {candidate_sql}") return self._call_tsqlparser(candidate_sql) def _call_tsqlparser(self, param): - """Select the executable based on the operating system""" - + """Select the executable based on the operating system.""" print("cwd =" + os.getcwd()) print(f"Calling tsqlParser with: {param}") if platform.system() == "Windows": diff --git a/src/backend/sql_agents/syntax_checker/response.py b/src/backend/sql_agents/syntax_checker/response.py index 14fd3a43..79070183 100644 --- a/src/backend/sql_agents/syntax_checker/response.py +++ b/src/backend/sql_agents/syntax_checker/response.py @@ -10,9 +10,7 @@ class SyntaxErrorInt(BaseModel): class SyntaxCheckerResponse(BaseModel): - """ - Response model for the syntax checker agent - """ + """Response model for the syntax checker agent.""" thought: str syntax_errors: List[SyntaxErrorInt] diff --git a/src/backend/sql_agents_start.py b/src/backend/sql_agents_start.py index a9d3796a..5636756d 100644 --- a/src/backend/sql_agents_start.py +++ b/src/backend/sql_agents_start.py @@ -1,15 +1,10 @@ -""" -This script demonstrates how to use the backend agents to migrate a query from one SQL dialect to another. -""" +"""This script demonstrates how to use the backend agents to migrate a query from one SQL dialect to another.""" -import asyncio import json import logging -import os -import sys -from pathlib import Path -from api.status_updates import close_connection, send_status_update +from api.status_updates import send_status_update + from common.models.api import ( AgentType, FileProcessUpdate, @@ -20,10 +15,9 @@ ) from common.services.batch_service import BatchService from common.storage.blob_factory import BlobStorageFactory + from fastapi import HTTPException -from sql_agents.helpers.selection_function import setup_selection_function -from sql_agents.helpers.termination_function import setup_termination_function -from sql_agents.helpers.utils import is_text + from semantic_kernel.agents import AgentGroupChat from semantic_kernel.agents.strategies import ( KernelFunctionSelectionStrategy, @@ -36,6 +30,7 @@ ChatMessageContent, ) from semantic_kernel.exceptions.service_exceptions import ServiceResponseException + from sql_agents import ( create_kernel_with_chat_completion, setup_fixer_agent, @@ -46,6 +41,9 @@ ) from sql_agents.agent_config import AgentModelDeployment, create_config from sql_agents.fixer.response import FixerResponse +from sql_agents.helpers.selection_function import setup_selection_function +from sql_agents.helpers.termination_function import setup_termination_function +from sql_agents.helpers.utils import is_text from sql_agents.migrator.response import MigratorResponse from sql_agents.picker.response import PickerResponse from sql_agents.semantic_verifier.response import SemanticVerifierResponse @@ -78,8 +76,11 @@ def extract_query(content): - """Extract the query from a chat that contains the following template: - # "migrated_query": 'SELECT TOP 10 * FROM mytable'""" + """ + Extract the query from a chat that contains the following template:. + + # "migrated_query": 'SELECT TOP 10 * FROM mytable' + """ if "migrated_query" in content: sub_str = content.split("migrated_query")[1] return sub_str.split(":")[1].strip().strip('"') @@ -136,7 +137,7 @@ async def configure_agents(): async def convert( source_script, file: FileRecord, batch_service: BatchService, agent_config ) -> str: - """setup agents, selection and termination.""" + """Set up agents, selection and termination.""" logger.info("Migrating query: %s\n", source_script) history_reducer = ChatHistoryTruncationReducer( @@ -432,7 +433,7 @@ async def invoke_semantic_verifier( async def process_batch_async(batch_id: str): - """Run main script with dummy Cosmos data""" + """Run main script with dummy Cosmos data.""" logger.info("Processing batch: %s", batch_id) storage = await BlobStorageFactory.get_storage() batch_service = BatchService() @@ -542,7 +543,7 @@ async def process_batch_async(batch_id: str): async def process_error( ex: Exception, file_record: FileRecord, batch_service: BatchService ): - """insert data base write to file record stating invalid file and send ws notification""" + """Insert data base write to file record stating invalid file and send ws notification.""" await batch_service.create_file_log( str(file_record.file_id), "Error processing file {}".format(ex), diff --git a/src/tests/backend/common/config/config_test.py b/src/tests/backend/common/config/config_test.py index 16f52ea9..87531bbc 100644 --- a/src/tests/backend/common/config/config_test.py +++ b/src/tests/backend/common/config/config_test.py @@ -22,7 +22,7 @@ class TestConfigInitialization(unittest.TestCase): clear=True, ) def test_config_initialization(self): - """Test if all attributes are correctly assigned from environment variables""" + """Test if all attributes are correctly assigned from environment variables.""" config = Config() # Ensure every attribute is accessed diff --git a/src/tests/backend/common/database/cosmosdb_test.py b/src/tests/backend/common/database/cosmosdb_test.py index 44521e18..7ef364a6 100644 --- a/src/tests/backend/common/database/cosmosdb_test.py +++ b/src/tests/backend/common/database/cosmosdb_test.py @@ -1,19 +1,15 @@ import asyncio +import enum import uuid from datetime import datetime -import enum -import pytest + from azure.cosmos import PartitionKey, exceptions from common.database.cosmosdb import CosmosDBClient -from common.models.api import ( - BatchRecord, - FileRecord, - ProcessStatus, - FileLog, - LogType, -) from common.logger.app_logger import AppLogger +from common.models.api import ProcessStatus + +import pytest # --- Enums for Testing --- diff --git a/src/tests/backend/common/database/database_base_test.py b/src/tests/backend/common/database/database_base_test.py index 6000d86d..0e9d1fec 100644 --- a/src/tests/backend/common/database/database_base_test.py +++ b/src/tests/backend/common/database/database_base_test.py @@ -1,12 +1,11 @@ -import asyncio import uuid -import pytest -from datetime import datetime from enum import Enum # Import the abstract base class and related models/enums. from common.database.database_base import DatabaseBase -from common.models.api import BatchRecord, FileRecord, ProcessStatus +from common.models.api import ProcessStatus + +import pytest DatabaseBase.__abstractmethods__ = set() @@ -63,6 +62,7 @@ def close(self): def get_dummy_status(): """ Try to use a specific ProcessStatus value (e.g. PROCESSING). + If that member is not available, just return the first member in the enum. """ try: diff --git a/src/tests/backend/common/database/database_factory_test.py b/src/tests/backend/common/database/database_factory_test.py index b597e56a..bdf99d35 100644 --- a/src/tests/backend/common/database/database_factory_test.py +++ b/src/tests/backend/common/database/database_factory_test.py @@ -1,7 +1,8 @@ -import pytest from common.config.config import Config from common.database.database_factory import DatabaseFactory +import pytest + class DummyConfig: cosmosdb_endpoint = "dummy_endpoint" @@ -20,6 +21,7 @@ def __init__(self, endpoint, credential, database_name, batch_container, file_co self.file_container = file_container self.log_container = log_container + def dummy_config_init(self): self.cosmosdb_endpoint = DummyConfig.cosmosdb_endpoint self.cosmosdb_database = DummyConfig.cosmosdb_database @@ -29,19 +31,23 @@ def dummy_config_init(self): # Provide a dummy method for credentials. self.get_azure_credentials = lambda: "dummy_credential" + @pytest.fixture(autouse=True) def patch_config(monkeypatch): # Patch the __init__ of Config so that an instance will have the required attributes. monkeypatch.setattr(Config, "__init__", dummy_config_init) + @pytest.fixture(autouse=True) def patch_cosmosdb_client(monkeypatch): # Patch CosmosDBClient in the module under test to use our dummy client. monkeypatch.setattr("common.database.database_factory.CosmosDBClient", DummyCosmosDBClient) + def test_get_database(): """ - Test that DatabaseFactory.get_database() correctly returns an instance of the + Test that DatabaseFactory.get_database() correctly returns an instance of the. + dummy CosmosDB client with the expected configuration values. """ # When get_database() is called, it creates a new Config() instance. diff --git a/src/tests/backend/common/storage/blob_azure_test.py b/src/tests/backend/common/storage/blob_azure_test.py index 2abb8c8e..900bf0e9 100644 --- a/src/tests/backend/common/storage/blob_azure_test.py +++ b/src/tests/backend/common/storage/blob_azure_test.py @@ -1,17 +1,20 @@ # blob_azure_test.py -import asyncio from datetime import datetime -import pytest from unittest.mock import AsyncMock, MagicMock, patch # Import the class under test -from common.storage.blob_azure import AzureBlobStorage from azure.core.exceptions import ResourceExistsError +from common.storage.blob_azure import AzureBlobStorage + + +import pytest + class DummyBlob: """A dummy blob item returned by list_blobs.""" + def __init__(self, name, size, creation_time, content_type, metadata): self.name = name self.size = size @@ -19,8 +22,10 @@ def __init__(self, name, size, creation_time, content_type, metadata): self.content_settings = MagicMock(content_type=content_type) self.metadata = metadata + class DummyAsyncIterator: """A dummy async iterator that yields the given items.""" + def __init__(self, items): self.items = items self.index = 0 @@ -35,18 +40,22 @@ async def __anext__(self): self.index += 1 return item + class DummyDownloadStream: """A dummy download stream whose content_as_bytes method returns a fixed byte string.""" + async def content_as_bytes(self): return b"file content" # --- Fixtures --- + @pytest.fixture def dummy_storage(): # Create an instance with dummy connection string and container name. return AzureBlobStorage("dummy_connection_string", "dummy_container") + @pytest.fixture def dummy_container_client(): container = MagicMock() @@ -55,12 +64,14 @@ def dummy_container_client(): container.get_blob_client = MagicMock() return container + @pytest.fixture def dummy_service_client(dummy_container_client): service = MagicMock() service.get_container_client.return_value = dummy_container_client return service + @pytest.fixture def dummy_blob_client(): blob_client = MagicMock() @@ -73,6 +84,7 @@ def dummy_blob_client(): # --- Tests for AzureBlobStorage methods --- + @pytest.mark.asyncio async def test_initialize_creates_container(dummy_storage, dummy_service_client, dummy_container_client): with patch("common.storage.blob_azure.BlobServiceClient.from_connection_string", return_value=dummy_service_client) as mock_from_conn: @@ -83,6 +95,7 @@ async def test_initialize_creates_container(dummy_storage, dummy_service_client, dummy_service_client.get_container_client.assert_called_once_with("dummy_container") dummy_container_client.create_container.assert_awaited_once() + @pytest.mark.asyncio async def test_initialize_container_already_exists(dummy_storage, dummy_service_client, dummy_container_client): with patch("common.storage.blob_azure.BlobServiceClient.from_connection_string", return_value=dummy_service_client): @@ -93,6 +106,7 @@ async def test_initialize_container_already_exists(dummy_storage, dummy_service_ dummy_container_client.create_container.assert_awaited_once() mock_debug.assert_called_with("Container dummy_container already exists") + @pytest.mark.asyncio async def test_initialize_failure(dummy_storage): # Simulate failure during initialization. @@ -102,6 +116,7 @@ async def test_initialize_failure(dummy_storage): await dummy_storage.initialize() mock_error.assert_called() + @pytest.mark.asyncio async def test_upload_file_success(dummy_storage, dummy_blob_client): # Patch get_blob_client to return our dummy blob client. @@ -127,6 +142,7 @@ async def test_upload_file_success(dummy_storage, dummy_blob_client): assert result["url"] == dummy_blob_client.url assert result["etag"] == "dummy_etag" + @pytest.mark.asyncio async def test_upload_file_error(dummy_storage, dummy_blob_client): dummy_storage.container_client = MagicMock() @@ -135,6 +151,7 @@ async def test_upload_file_error(dummy_storage, dummy_blob_client): with pytest.raises(Exception, match="Upload failed"): await dummy_storage.upload_file(b"data", "blob.txt", "text/plain", {}) + @pytest.mark.asyncio async def test_get_file_success(dummy_storage, dummy_blob_client): dummy_storage.container_client = MagicMock() @@ -146,6 +163,7 @@ async def test_get_file_success(dummy_storage, dummy_blob_client): dummy_blob_client.download_blob.assert_awaited() assert result == b"file content" + @pytest.mark.asyncio async def test_get_file_error(dummy_storage, dummy_blob_client): dummy_storage.container_client = MagicMock() @@ -154,6 +172,7 @@ async def test_get_file_error(dummy_storage, dummy_blob_client): with pytest.raises(Exception, match="Download error"): await dummy_storage.get_file("nonexistent.txt") + @pytest.mark.asyncio async def test_delete_file_success(dummy_storage, dummy_blob_client): dummy_storage.container_client = MagicMock() @@ -164,6 +183,7 @@ async def test_delete_file_success(dummy_storage, dummy_blob_client): dummy_blob_client.delete_blob.assert_awaited() assert result is True + @pytest.mark.asyncio async def test_delete_file_error(dummy_storage, dummy_blob_client): dummy_storage.container_client = MagicMock() @@ -172,6 +192,7 @@ async def test_delete_file_error(dummy_storage, dummy_blob_client): result = await dummy_storage.delete_file("blob.txt") assert result is False + @pytest.mark.asyncio async def test_list_files_success(dummy_storage): dummy_storage.container_client = MagicMock() @@ -185,10 +206,12 @@ async def test_list_files_success(dummy_storage): names = {item["name"] for item in result} assert names == {"file1.txt", "file2.txt"} + @pytest.mark.asyncio async def test_list_files_failure(dummy_storage): dummy_storage.container_client = MagicMock() # Define list_blobs to return an invalid object (simulate error) + async def invalid_list_blobs(*args, **kwargs): # Return a plain string (which does not implement __aiter__) return "invalid" @@ -196,6 +219,7 @@ async def invalid_list_blobs(*args, **kwargs): with pytest.raises(Exception): await dummy_storage.list_files("") + @pytest.mark.asyncio async def test_close(dummy_storage): dummy_storage.service_client = MagicMock() diff --git a/src/tests/backend/common/storage/blob_base_test.py b/src/tests/backend/common/storage/blob_base_test.py index b4b0361e..561007ed 100644 --- a/src/tests/backend/common/storage/blob_base_test.py +++ b/src/tests/backend/common/storage/blob_base_test.py @@ -1,14 +1,13 @@ -import pytest -import asyncio -import uuid from datetime import datetime -from typing import BinaryIO, Dict, Any +from typing import Any, BinaryIO, Dict # Import the abstract base class from the production code. from common.storage.blob_base import BlobStorageBase - +import pytest # Create a dummy concrete subclass of BlobStorageBase that calls the parent's abstract methods. + + class DummyBlobStorage(BlobStorageBase): async def initialize(self) -> None: # Call the parent (which is just a pass) diff --git a/src/tests/backend/common/storage/blob_factory_test.py b/src/tests/backend/common/storage/blob_factory_test.py index e19af495..47e344ff 100644 --- a/src/tests/backend/common/storage/blob_factory_test.py +++ b/src/tests/backend/common/storage/blob_factory_test.py @@ -1,10 +1,7 @@ -# blob_factory_test.py import asyncio -import json import os import sys -import pytest -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock # Adjust sys.path so that the project root is found. sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) @@ -22,21 +19,26 @@ sys.modules["azure.monitor.events.extension"] = MagicMock() # --- Import the module under test --- -from common.storage.blob_factory import BlobStorageFactory -from common.storage.blob_base import BlobStorageBase -from common.storage.blob_azure import AzureBlobStorage +from common.storage.blob_base import BlobStorageBase # noqa: E402 +from common.storage.blob_factory import BlobStorageFactory # noqa: E402 + +import pytest # noqa: E402 # --- Dummy configuration for testing --- + + class DummyConfig: azure_blob_connection_string = "dummy_connection_string" azure_blob_container_name = "dummy_container" # --- Fixture to patch Config in our tests --- + + @pytest.fixture(autouse=True) def patch_config(monkeypatch): # Import the real Config from your project. from common.config.config import Config - + def dummy_init(self): self.azure_blob_connection_string = DummyConfig.azure_blob_connection_string self.azure_blob_container_name = DummyConfig.azure_blob_container_name @@ -81,6 +83,8 @@ async def close(self): self.initialized = False # --- Fixture to patch AzureBlobStorage --- + + @pytest.fixture(autouse=True) def patch_azure_blob_storage(monkeypatch): monkeypatch.setattr("common.storage.blob_factory.AzureBlobStorage", DummyAzureBlobStorage) @@ -88,6 +92,7 @@ def patch_azure_blob_storage(monkeypatch): # -------------------- Tests for BlobStorageFactory -------------------- + @pytest.mark.asyncio async def test_get_storage_success(): """Test that get_storage returns an initialized DummyAzureBlobStorage instance and is a singleton.""" @@ -99,13 +104,16 @@ async def test_get_storage_success(): storage2 = await BlobStorageFactory.get_storage() assert storage is storage2 + @pytest.mark.asyncio async def test_get_storage_missing_config(monkeypatch): """ Test that get_storage raises a ValueError when configuration is missing. + We simulate missing connection string and container name. """ from common.config.config import Config + def dummy_init_missing(self): self.azure_blob_connection_string = "" self.azure_blob_container_name = "" @@ -113,6 +121,7 @@ def dummy_init_missing(self): with pytest.raises(ValueError, match="Azure Blob Storage configuration is missing"): await BlobStorageFactory.get_storage() + @pytest.mark.asyncio async def test_close_storage_success(): """Test that close_storage calls close() on the storage instance and resets the singleton.""" @@ -125,6 +134,7 @@ async def test_close_storage_success(): # -------------------- File Upload Tests -------------------- + @pytest.mark.asyncio async def test_upload_file_success(): """Test that upload_file successfully uploads a file and returns metadata.""" @@ -139,6 +149,7 @@ async def test_upload_file_success(): assert result["size"] == len(file_content) assert blob_path in storage.files + @pytest.mark.asyncio async def test_upload_file_error(monkeypatch): """Test that an exception during file upload is propagated.""" @@ -150,6 +161,7 @@ async def test_upload_file_error(monkeypatch): # -------------------- File Retrieval Tests -------------------- + @pytest.mark.asyncio async def test_get_file_success(): """Test that get_file retrieves the correct file content.""" @@ -161,6 +173,7 @@ async def test_get_file_success(): result = await storage.get_file(blob_path) assert result == file_content + @pytest.mark.asyncio async def test_get_file_not_found(): """Test that get_file raises FileNotFoundError when file does not exist.""" @@ -171,6 +184,7 @@ async def test_get_file_not_found(): # -------------------- File Deletion Tests -------------------- + @pytest.mark.asyncio async def test_delete_file_success(): """Test that delete_file removes an existing file.""" @@ -181,6 +195,7 @@ async def test_delete_file_success(): await storage.delete_file(blob_path) assert blob_path not in storage.files + @pytest.mark.asyncio async def test_delete_file_nonexistent(): """Test that deleting a non-existent file does not raise an error.""" @@ -192,6 +207,7 @@ async def test_delete_file_nonexistent(): # -------------------- File Listing Tests -------------------- + @pytest.mark.asyncio async def test_list_files_with_prefix(): """Test that list_files returns files that match the given prefix.""" @@ -205,6 +221,7 @@ async def test_list_files_with_prefix(): result = await storage.list_files("folder/") assert set(result) == {"folder/a.txt", "folder/b.txt"} + @pytest.mark.asyncio async def test_list_files_no_files(): """Test that list_files returns an empty list when no files match the prefix.""" @@ -216,6 +233,7 @@ async def test_list_files_no_files(): # -------------------- Additional Basic Tests -------------------- + @pytest.mark.asyncio async def test_dummy_azure_blob_storage_initialize(): """Test that initializing DummyAzureBlobStorage sets the initialized flag.""" @@ -224,6 +242,7 @@ async def test_dummy_azure_blob_storage_initialize(): await storage.initialize() assert storage.initialized is True + @pytest.mark.asyncio async def test_dummy_azure_blob_storage_upload_and_retrieve(): """Test that a file uploaded to DummyAzureBlobStorage can be retrieved.""" @@ -238,6 +257,7 @@ async def test_dummy_azure_blob_storage_upload_and_retrieve(): retrieved = await storage.get_file(blob_path) assert retrieved == content + @pytest.mark.asyncio async def test_dummy_azure_blob_storage_close(): """Test that close() sets initialized to False.""" @@ -248,6 +268,7 @@ async def test_dummy_azure_blob_storage_close(): # -------------------- Test for BlobStorageFactory Singleton Usage -------------------- + def test_common_usage_of_blob_factory(): """Test that manually setting the singleton in BlobStorageFactory works as expected.""" # Create a dummy storage instance. @@ -257,6 +278,7 @@ def test_common_usage_of_blob_factory(): storage = asyncio.run(BlobStorageFactory.get_storage()) assert storage is dummy_storage + if __name__ == "__main__": # Run tests when this file is executed directly. asyncio.run(pytest.main()) From 550beb0589cf57de13fb4d5e36e7cafa671d0a38 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Mon, 7 Apr 2025 12:54:43 +0530 Subject: [PATCH 09/47] edit 2 --- src/tests/backend/common/storage/blob_azure_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tests/backend/common/storage/blob_azure_test.py b/src/tests/backend/common/storage/blob_azure_test.py index 900bf0e9..2f743020 100644 --- a/src/tests/backend/common/storage/blob_azure_test.py +++ b/src/tests/backend/common/storage/blob_azure_test.py @@ -216,7 +216,7 @@ async def invalid_list_blobs(*args, **kwargs): # Return a plain string (which does not implement __aiter__) return "invalid" dummy_storage.container_client.list_blobs = invalid_list_blobs - with pytest.raises(Exception): + with pytest.raises(Exception): # noqa B017 await dummy_storage.list_files("") From ff3d973b96ff7f19aeeb636315018a3c060434c4 Mon Sep 17 00:00:00 2001 From: "Vishal Shinde (Persistent Systems Inc)" Date: Mon, 7 Apr 2025 19:10:19 +0530 Subject: [PATCH 10/47] resolved bug:15313 --- src/frontend/src/api/utils.tsx | 2 +- src/frontend/src/pages/modernizationPage.tsx | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/frontend/src/api/utils.tsx b/src/frontend/src/api/utils.tsx index 6c625a8c..f66517c2 100644 --- a/src/frontend/src/api/utils.tsx +++ b/src/frontend/src/api/utils.tsx @@ -75,7 +75,7 @@ export const useStyles = makeStyles({ }, selectedCard: { border: "var(--NeutralStroke2.Rest)", - backgroundColor: "rgb(221, 217, 217)", + backgroundColor: "#EBEBEB", }, mainContent: { flex: 1, diff --git a/src/frontend/src/pages/modernizationPage.tsx b/src/frontend/src/pages/modernizationPage.tsx index c5a3ab82..082d44ff 100644 --- a/src/frontend/src/pages/modernizationPage.tsx +++ b/src/frontend/src/pages/modernizationPage.tsx @@ -489,7 +489,7 @@ const ModernizationPage = () => { // State for the loading component const [showLoading, setShowLoading] = useState(true); const [loadingError, setLoadingError] = useState(null); - + const [selectedFilebg, setSelectedFile] = useState(null); const [selectedFileId, setSelectedFileId] = React.useState("") const [fileId, setFileId] = React.useState(""); const [expandedSections, setExpandedSections] = React.useState([]) @@ -1239,6 +1239,10 @@ const ModernizationPage = () => { navigate("/"); }; + const handleClick = (file: string) => { + setSelectedFile(file === selectedFilebg ? null : file); + }; + return (
@@ -1296,6 +1300,10 @@ const ModernizationPage = () => { // Don't allow selecting queued files if (file.status === "ready_to_process") return; setSelectedFileId(file.id); + handleClick(file.id); + }} + style={{ + backgroundColor: selectedFilebg === file.id ? "#EBEBEB" : "var(--NeutralBackground1-Rest)", }} > {isSummary ? ( From 8672990720e2bd4ed673c601ac8c857277c03296 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Wed, 9 Apr 2025 11:42:44 +0530 Subject: [PATCH 11/47] added api_test.py --- src/tests/backend/common/models/api_test.py | 117 ++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 src/tests/backend/common/models/api_test.py diff --git a/src/tests/backend/common/models/api_test.py b/src/tests/backend/common/models/api_test.py new file mode 100644 index 00000000..c06e45fb --- /dev/null +++ b/src/tests/backend/common/models/api_test.py @@ -0,0 +1,117 @@ +import pytest +from uuid import uuid4 +from datetime import datetime +from backend.common.models.api import ( + FileLog, FileRecord, FileProcessUpdate, FileProcessUpdateJSONEncoder, + QueueBatch, BatchRecord, + LogType, AgentType, AuthorRole, ProcessStatus, FileResult, TranslateType +) + +@pytest.fixture +def common_datetime(): + return datetime.now() + +@pytest.fixture +def uuid_pair(): + return str(uuid4()), str(uuid4()) + +def test_filelog_fromdb_and_dict(uuid_pair, common_datetime): + log_id, file_id = uuid_pair + data = { + "log_id": log_id, + "file_id": file_id, + "description": "test log", + "last_candidate": "some_candidate", + "log_type": "SUCCESS", + "agent_type": "migrator", + "author_role": "user", + "timestamp": common_datetime.isoformat(), + } + log = FileLog.fromdb(data) + assert log.log_id.hex == log_id.replace("-", "") + assert log.dict()["log_type"] == "info" + + assert log.dict()["author_role"] == "user" + +def test_filerecord_fromdb_and_dict(uuid_pair, common_datetime): + file_id, batch_id = uuid_pair + data = { + "file_id": file_id, + "batch_id": batch_id, + "original_name": "file.sql", + "blob_path": "/blob/file.sql", + "translated_path": "/translated/file.sql", + "status": "in_progress", + "file_result": "warning", + "error_count": 2, + "syntax_count": 5, + "created_at": common_datetime.isoformat(), + "updated_at": common_datetime.isoformat(), + } + record = FileRecord.fromdb(data) + assert record.file_id.hex == file_id.replace("-", "") + assert record.dict()["status"] == "ready_to_process" + assert record.dict()["file_result"] == "warning" + +def test_fileprocessupdate_dict(uuid_pair): + file_id, batch_id = uuid_pair + update = FileProcessUpdate( + file_id=file_id, + batch_id=batch_id, + process_status=ProcessStatus.COMPLETED, + file_result=FileResult.SUCCESS, + agent_type=AgentType.FIXER, + agent_message="Translation done", + ) + result = update.dict() + assert result["process_status"] == "completed" + assert result["file_result"] == "success" + assert result["agent_type"] == "fixer" + assert result["agent_message"] == "Translation done" + +def test_fileprocessupdate_json_encoder(uuid_pair): + file_id, batch_id = uuid_pair + update = FileProcessUpdate( + file_id=file_id, + batch_id=batch_id, + process_status=ProcessStatus.FAILED, + file_result=FileResult.ERROR, + agent_type=AgentType.HUMAN, + agent_message="Something failed", + ) + json_string = FileProcessUpdateJSONEncoder().encode(update) + assert "failed" in json_string + assert "human" in json_string + +def test_queuebatch_dict(uuid_pair, common_datetime): + batch_id, _ = uuid_pair + batch = QueueBatch( + batch_id=batch_id, + user_id="user123", + translate_from="en", + translate_to="tsql", + created_at=common_datetime, + updated_at=common_datetime, + status=ProcessStatus.IN_PROGRESS, + ) + result = batch.dict() + assert result["status"] == "in_process" + assert result["user_id"] == "user123" + +def test_batchrecord_fromdb_and_dict(uuid_pair, common_datetime): + batch_id, _ = uuid_pair + data = { + "batch_id": batch_id, + "user_id": "user123", + "file_count": 3, + "created_at": common_datetime.isoformat(), + "updated_at": common_datetime.isoformat(), + "status": "completed", + "from_language": "Informix", + "to_language": "T-SQL" + } + record = BatchRecord.fromdb(data) + assert record.status == ProcessStatus.COMPLETED + assert record.from_language == TranslateType.INFORMIX + assert record.to_language == TranslateType.TSQL + assert record.dict()["status"] == "completed" From 4ffa2afea5273045219f69f4f9c0ffb1e9799166 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Wed, 9 Apr 2025 14:07:25 +0530 Subject: [PATCH 12/47] added conftest.py --- src/tests/conftest.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/tests/conftest.py diff --git a/src/tests/conftest.py b/src/tests/conftest.py new file mode 100644 index 00000000..cad4e268 --- /dev/null +++ b/src/tests/conftest.py @@ -0,0 +1,12 @@ +import os +import sys + +# Determine the project root relative to this conftest.py file. +# This file is at: /src/tests/conftest.py +# We want to add: /src/backend to sys.path. +current_dir = os.path.dirname(os.path.abspath(__file__)) +project_root = os.path.abspath(os.path.join(current_dir, "..")) # Goes from tests to src +backend_path = os.path.join(project_root, "backend") +sys.path.insert(0, backend_path) + +print("Adjusted sys.path:", sys.path) From 7e0937cb355be699123967988691c5ab2e650f40 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Wed, 9 Apr 2025 19:29:58 +0530 Subject: [PATCH 13/47] solved pylint issue of api_test.py --- src/tests/backend/common/models/api_test.py | 20 ++- src/tests/backend/common/services/__init__.py | 0 .../common/services/batch_service_test.py | 147 ++++++++++++++++++ 3 files changed, 160 insertions(+), 7 deletions(-) create mode 100644 src/tests/backend/common/services/__init__.py create mode 100644 src/tests/backend/common/services/batch_service_test.py diff --git a/src/tests/backend/common/models/api_test.py b/src/tests/backend/common/models/api_test.py index c06e45fb..b338efc0 100644 --- a/src/tests/backend/common/models/api_test.py +++ b/src/tests/backend/common/models/api_test.py @@ -1,20 +1,21 @@ -import pytest -from uuid import uuid4 from datetime import datetime -from backend.common.models.api import ( - FileLog, FileRecord, FileProcessUpdate, FileProcessUpdateJSONEncoder, - QueueBatch, BatchRecord, - LogType, AgentType, AuthorRole, ProcessStatus, FileResult, TranslateType -) +from uuid import uuid4 + +from backend.common.models.api import AgentType, BatchRecord, FileLog, FileProcessUpdate, FileProcessUpdateJSONEncoder, FileRecord, FileResult, ProcessStatus, QueueBatch, TranslateType + +import pytest + @pytest.fixture def common_datetime(): return datetime.now() + @pytest.fixture def uuid_pair(): return str(uuid4()), str(uuid4()) + def test_filelog_fromdb_and_dict(uuid_pair, common_datetime): log_id, file_id = uuid_pair data = { @@ -33,6 +34,7 @@ def test_filelog_fromdb_and_dict(uuid_pair, common_datetime): assert log.dict()["author_role"] == "user" + def test_filerecord_fromdb_and_dict(uuid_pair, common_datetime): file_id, batch_id = uuid_pair data = { @@ -53,6 +55,7 @@ def test_filerecord_fromdb_and_dict(uuid_pair, common_datetime): assert record.dict()["status"] == "ready_to_process" assert record.dict()["file_result"] == "warning" + def test_fileprocessupdate_dict(uuid_pair): file_id, batch_id = uuid_pair update = FileProcessUpdate( @@ -69,6 +72,7 @@ def test_fileprocessupdate_dict(uuid_pair): assert result["agent_type"] == "fixer" assert result["agent_message"] == "Translation done" + def test_fileprocessupdate_json_encoder(uuid_pair): file_id, batch_id = uuid_pair update = FileProcessUpdate( @@ -83,6 +87,7 @@ def test_fileprocessupdate_json_encoder(uuid_pair): assert "failed" in json_string assert "human" in json_string + def test_queuebatch_dict(uuid_pair, common_datetime): batch_id, _ = uuid_pair batch = QueueBatch( @@ -98,6 +103,7 @@ def test_queuebatch_dict(uuid_pair, common_datetime): assert result["status"] == "in_process" assert result["user_id"] == "user123" + def test_batchrecord_fromdb_and_dict(uuid_pair, common_datetime): batch_id, _ = uuid_pair data = { diff --git a/src/tests/backend/common/services/__init__.py b/src/tests/backend/common/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/tests/backend/common/services/batch_service_test.py b/src/tests/backend/common/services/batch_service_test.py new file mode 100644 index 00000000..e5efa561 --- /dev/null +++ b/src/tests/backend/common/services/batch_service_test.py @@ -0,0 +1,147 @@ +import pytest +from unittest.mock import AsyncMock, MagicMock, patch +from uuid import uuid4 +from datetime import datetime +from fastapi import UploadFile, HTTPException + +from common.services.batch_service import BatchService +from common.models.api import ( + FileRecord, + BatchRecord, + FileResult, + LogType, + AgentType, + ProcessStatus, +) + +# ---------- Helpers ---------- +def make_file_record(**overrides): + return FileRecord( + file_id=overrides.get("file_id", "file1"), + batch_id=overrides.get("batch_id", "batch123"), + original_name=overrides.get("original_name", "file.txt"), + blob_path=overrides.get("blob_path", "blob/path/file.txt"), + translated_path=overrides.get("translated_path", "translated/file.txt"), + status=overrides.get("status", ProcessStatus.READY_TO_PROCESS), + error_count=overrides.get("error_count", 0), + syntax_count=overrides.get("syntax_count", 0), + created_at=overrides.get("created_at", datetime.utcnow()), + updated_at=overrides.get("updated_at", datetime.utcnow()) + ) + +def make_batch_record(**overrides): + return BatchRecord( + batch_id=overrides.get("batch_id", "batch123"), + user_id=overrides.get("user_id", "user1"), + file_count=overrides.get("file_count", 1), + created_at=overrides.get("created_at", datetime.utcnow().isoformat()), + updated_at=overrides.get("updated_at", datetime.utcnow().isoformat()), + status=overrides.get("status", ProcessStatus.READY_TO_PROCESS), + ) + +# ---------- Fixtures ---------- +@pytest.fixture +def batch_service(): + service = BatchService() + service.logger = MagicMock() + service.database = AsyncMock() + return service + +# ---------- Tests ---------- +@pytest.mark.asyncio +async def test_get_batch_success(batch_service): + batch_id = uuid4() + user_id = "test_user" + batch_service.database.get_batch.return_value = {"batch_id": str(batch_id)} + batch_service.database.get_batch_files.return_value = [{"file_id": "f1"}] + + result = await batch_service.get_batch(batch_id, user_id) + assert result["batch"]["batch_id"] == str(batch_id) + assert result["files"] == [{"file_id": "f1"}] + +@pytest.mark.asyncio +async def test_get_file_not_found(batch_service): + batch_service.database.get_file.return_value = None + result = await batch_service.get_file("missing_file_id") + assert result is None + +def test_is_valid_uuid_valid(batch_service): + assert batch_service.is_valid_uuid(str(uuid4())) is True + +def test_generate_file_path(batch_service): + path = batch_service.generate_file_path("batch1", "user1", "file1", "file@.txt") + assert path == "user1/batch1/file1/file_.txt" + +@pytest.mark.asyncio +@patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) +async def test_get_file_report_success(mock_storage, batch_service): + file_id = "file1" + file_record = make_file_record(file_id=file_id) + batch_record = make_batch_record(batch_id=file_record.batch_id) + + batch_service.database.get_file.return_value = file_record.dict() + batch_service.database.get_batch_from_id.return_value = batch_record.dict() + batch_service.database.get_file_logs.return_value = [{"log_type": "INFO"}] + + with patch("common.models.api.FileRecord.fromdb", return_value=file_record), \ + patch("common.models.api.BatchRecord.fromdb", return_value=batch_record), \ + patch.object(mock_storage, "get_file", new=AsyncMock(return_value="translated content")): + + result = await batch_service.get_file_report(file_id) + assert result["translated_content"] == "translated content" + +@pytest.mark.asyncio +@patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) +async def test_upload_file_to_batch_creates_batch(mock_storage, batch_service): + batch_id = str(uuid4()) + user_id = "test_user" + filename = "doc.txt" + file_mock = MagicMock(spec=UploadFile) + file_mock.filename = filename + file_mock.content_type = "text/plain" + file_mock.read = AsyncMock(return_value=b"content") + + # Simulate batch creation + batch_service.database.get_batch.return_value = None + batch_service.database.create_batch.return_value = {"batch_id": batch_id} + batch_service.database.get_batch_files.return_value = [{"file_id": "f1"}] + batch_service.database.get_file.return_value = {"file_id": "new_id"} + + mock_storage.upload_file.return_value = None + + file_record = make_file_record(file_id="new_id", batch_id=batch_id) + + with patch("common.models.api.FileRecord.fromdb", return_value=file_record), \ + patch("uuid.uuid4", return_value=uuid4()): + + result = await batch_service.upload_file_to_batch(batch_id, user_id, file_mock) + assert "file" in result + assert "batch" in result + +@pytest.mark.asyncio +@patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) +async def test_delete_batch_and_files_batch_not_found(mock_storage, batch_service): + batch_service.database.get_batch.return_value = None + result = await batch_service.delete_batch_and_files("batch123", "user1") + assert result["message"] == "Batch not found" + +@pytest.mark.asyncio +async def test_update_file_not_found(batch_service): + batch_service.database.get_file.return_value = None + with pytest.raises(HTTPException) as exc_info: + await batch_service.update_file("file123", ProcessStatus.COMPLETED, FileResult.SUCCESS, 1, 2) + assert exc_info.value.status_code == 404 + +@pytest.mark.asyncio +async def test_batch_files_final_update_with_error_log(batch_service): + file_id = str(uuid4()) + file_record = make_file_record(file_id=file_id, translated_path=None, status=ProcessStatus.IN_PROGRESS) + + batch_service.database.get_batch_files.return_value = [file_record.dict()] + batch_service.get_file_counts = AsyncMock(return_value=(1, 0)) + batch_service.update_file_record = AsyncMock() + batch_service.create_file_log = AsyncMock() + + with patch("common.models.api.FileRecord.fromdb", return_value=file_record): + await batch_service.batch_files_final_update("batch1") + batch_service.update_file_record.assert_awaited() From 98e122fd8446c433d1fad56caad23ddb0696106e Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Fri, 11 Apr 2025 14:19:46 +0530 Subject: [PATCH 14/47] feat: added unit test cases for cosmosdb_test.py file --- .../backend/common/database/cosmosdb_test.py | 1091 +++++++++-------- 1 file changed, 599 insertions(+), 492 deletions(-) diff --git a/src/tests/backend/common/database/cosmosdb_test.py b/src/tests/backend/common/database/cosmosdb_test.py index 7ef364a6..a6e7f1ed 100644 --- a/src/tests/backend/common/database/cosmosdb_test.py +++ b/src/tests/backend/common/database/cosmosdb_test.py @@ -1,618 +1,725 @@ +import pytest import asyncio -import enum -import uuid -from datetime import datetime - -from azure.cosmos import PartitionKey, exceptions - -from common.database.cosmosdb import CosmosDBClient +import os +import sys +from unittest import mock + +from unittest.mock import AsyncMock, patch +from uuid import uuid4 +from datetime import datetime, timezone +from azure.cosmos.exceptions import CosmosResourceExistsError + +# Add backend directory to sys.path +sys.path.insert( + 0, + os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..", "backend")), +) + +from common.models.api import ( + AgentType, + BatchRecord, + FileLog, + LogType, + ProcessStatus, + FileRecord, + AuthorRole, +) from common.logger.app_logger import AppLogger -from common.models.api import ProcessStatus - -import pytest - +from common.database.cosmosdb import ( + CosmosDBClient, +) +from azure.cosmos.aio import CosmosClient -# --- Enums for Testing --- -class DummyProcessStatus(enum.Enum): - READY_TO_PROCESS = "READY" - PROCESSING = "PROCESSING" +# Mocked data for the test +endpoint = "https://fake.cosmosdb.azure.com" +credential = "fake_credential" +database_name = "test_database" +batch_container = "batch_container" +file_container = "file_container" +log_container = "log_container" -class DummyLogType(enum.Enum): - INFO = "INFO" - ERROR = "ERROR" - - -@pytest.fixture(autouse=True) -def patch_enums(monkeypatch): - monkeypatch.setattr("common.models.api.ProcessStatus", DummyProcessStatus) - monkeypatch.setattr("common.models.api.LogType", DummyLogType) - - -# --- implementations to simulate Cosmos DB behavior --- -async def async_query_generator(items): - for item in items: - yield item - - -async def async_query_error_generator(*args, **kwargs): - raise Exception("Error in query") - if False: - yield - - -class DummyContainerClient: - def __init__(self, container_name): - self.container_name = container_name - self.created_items = [] - self.deleted_items = [] - self._query_items_func = None - - async def create_item(self, body): - self.created_items.append(body) - - async def replace_item(self, item, body): - return body - - async def delete_item(self, item, partition_key=None): - self.deleted_items.append((item, partition_key)) - - async def delete_items(self, key): - self.deleted_items.append(key) - - async def query_items(self, query, parameters): - if self._query_items_func: - async for item in self._query_items_func(query, parameters): - yield item - else: - if False: - yield - - def set_query_items(self, func): - self._query_items_func = func - - -class DummyDatabase: - def __init__(self, database_name): - self.database_name = database_name - self.containers = {} - - async def create_container(self, id, partition_key): - if id in self.containers: - raise exceptions.CosmosResourceExistsError(404, "Container exists") - container = DummyContainerClient(id) - self.containers[id] = container - return container - - def get_container_client(self, container_name): - return self.containers.get(container_name, DummyContainerClient(container_name)) - - -class DummyCosmosClient: - def __init__(self, url, credential): - self.url = url - self.credential = credential - self._database = DummyDatabase("dummy_db") - self.closed = False - - def get_database_client(self, database_name): - return self._database - - def close(self): - self.closed = True - - -class FakeCosmosDBClient(CosmosDBClient): - async def _async_init( - self, - endpoint: str, - credential: any, - database_name: str, - batch_container: str, - file_container: str, - log_container: str, - ): - self.endpoint = endpoint - self.credential = credential - self.database_name = database_name - self.batch_container_name = batch_container - self.file_container_name = file_container - self.log_container_name = log_container - self.logger = AppLogger("CosmosDB") - self.client = DummyCosmosClient(endpoint, credential) - db = self.client.get_database_client(database_name) - self.batch_container = await db.create_container( - batch_container, PartitionKey(path="/batch_id") - ) - self.file_container = await db.create_container( - file_container, PartitionKey(path="/file_id") - ) - self.log_container = await db.create_container( - log_container, PartitionKey(path="/log_id") - ) - - @classmethod - async def create( - cls, - endpoint, - credential, - database_name, - batch_container, - file_container, - log_container, - ): - instance = cls.__new__(cls) - await instance._async_init( - endpoint, - credential, - database_name, - batch_container, - file_container, - log_container, - ) - return instance - - # Minimal implementations for abstract methods not under test. - async def delete_file_logs(self, file_id: str) -> None: - await self.log_container.delete_items(file_id) - - async def log_batch_status( - self, batch_id: str, status: ProcessStatus, processed_files: int - ) -> None: - return - - -# --- Fixture --- @pytest.fixture -def cosmosdb_client(event_loop): - client = event_loop.run_until_complete( - FakeCosmosDBClient.create( - endpoint="dummy_endpoint", - credential="dummy_credential", - database_name="dummy_db", - batch_container="batch", - file_container="file", - log_container="log", - ) +def cosmos_db_client(): + return CosmosDBClient( + endpoint=endpoint, + credential=credential, + database_name=database_name, + batch_container=batch_container, + file_container=file_container, + log_container=log_container, ) - return client -# --- Test Cases --- @pytest.mark.asyncio -async def test_initialization_success(cosmosdb_client): - assert cosmosdb_client.client is not None - assert cosmosdb_client.batch_container is not None - assert cosmosdb_client.file_container is not None - assert cosmosdb_client.log_container is not None +async def test_initialize_cosmos(cosmos_db_client, mocker): + # Mocking CosmosClient and its methods + mock_client = mocker.patch.object(CosmosClient, 'get_database_client', return_value=mock.MagicMock()) + mock_database = mock_client.return_value + + # Use AsyncMock for asynchronous methods + mock_batch_container = mock.MagicMock() + mock_file_container = mock.MagicMock() + mock_log_container = mock.MagicMock() + + # Use AsyncMock to mock asynchronous container creation + mock_database.create_container = AsyncMock(side_effect=[ + mock_batch_container, + mock_file_container, + mock_log_container + ]) + + # Call the initialize_cosmos method + await cosmos_db_client.initialize_cosmos() + + # Assert that the containers were created or fetched successfully + mock_database.create_container.assert_any_call(id=batch_container, partition_key=mock.ANY) + mock_database.create_container.assert_any_call(id=file_container, partition_key=mock.ANY) + mock_database.create_container.assert_any_call(id=log_container, partition_key=mock.ANY) + + # Check the client and containers were set + assert cosmos_db_client.client is not None + assert cosmos_db_client.batch_container == mock_batch_container + assert cosmos_db_client.file_container == mock_file_container + assert cosmos_db_client.log_container == mock_log_container @pytest.mark.asyncio -async def test_init_error(monkeypatch): - async def fake_async_init(*args, **kwargs): - raise Exception("client error") +async def test_initialize_cosmos_with_error(cosmos_db_client, mocker): + # Mocking CosmosClient and its methods + mock_client = mocker.patch.object(CosmosClient, 'get_database_client', return_value=mock.MagicMock()) + mock_database = mock_client.return_value + + # Simulate a general exception during container creation + mock_database.create_container = AsyncMock(side_effect=Exception("Failed to create container")) - monkeypatch.setattr(FakeCosmosDBClient, "_async_init", fake_async_init) + # Call the initialize_cosmos method and expect it to raise an error with pytest.raises(Exception) as exc_info: - await FakeCosmosDBClient.create("dummy", "dummy", "dummy", "a", "b", "c") - assert "client error" in str(exc_info.value) + await cosmos_db_client.initialize_cosmos() + + # Assert that the exception message matches the expected message + assert str(exc_info.value) == "Failed to create container" @pytest.mark.asyncio -async def test_get_or_create_container_existing(monkeypatch, cosmosdb_client): - db = DummyDatabase("dummy_db") - existing = DummyContainerClient("existing") - db.containers["existing"] = existing +async def test_initialize_cosmos_container_exists_error(cosmos_db_client, mocker): + # Mocking CosmosClient and its methods + mock_client = mocker.patch.object(CosmosClient, 'get_database_client', return_value=mock.MagicMock()) + mock_database = mock_client.return_value + + # Simulating CosmosResourceExistsError for container creation + mock_database.create_container = AsyncMock(side_effect=CosmosResourceExistsError) - async def fake_create_container(id, partition_key): - raise exceptions.CosmosResourceExistsError(404, "Container exists") + # Use AsyncMock for asynchronous methods + mock_batch_container = mock.MagicMock() + mock_file_container = mock.MagicMock() + mock_log_container = mock.MagicMock() - monkeypatch.setattr(db, "create_container", fake_create_container) - monkeypatch.setattr(db, "get_container_client", lambda name: existing) + # Use AsyncMock to mock asynchronous container creation + mock_database.create_container = AsyncMock(side_effect=[ + mock_batch_container, + mock_file_container, + mock_log_container + ]) - # Directly call _get_or_create_container on a new instance. - instance = FakeCosmosDBClient.__new__(FakeCosmosDBClient) - instance.logger = AppLogger("CosmosDB") - result = await instance._get_or_create_container(db, "existing", "/id") - assert result is existing + # Call the initialize_cosmos method + await cosmos_db_client.initialize_cosmos() + + # Assert that the container creation method was called with the correct arguments + mock_database.create_container.assert_any_call(id='batch_container', partition_key=mock.ANY) + mock_database.create_container.assert_any_call(id='file_container', partition_key=mock.ANY) + mock_database.create_container.assert_any_call(id='log_container', partition_key=mock.ANY) + + # Check that existing containers are returned (mocked containers) + assert cosmos_db_client.batch_container == mock_batch_container + assert cosmos_db_client.file_container == mock_file_container + assert cosmos_db_client.log_container == mock_log_container @pytest.mark.asyncio -async def test_create_batch_success(monkeypatch, cosmosdb_client): - called = False +async def test_create_batch_new(cosmos_db_client, mocker): + user_id = "user_1" + batch_id = uuid4() - async def fake_create_item(body): - nonlocal called - called = True + # Mock container creation + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) - monkeypatch.setattr( - cosmosdb_client.batch_container, "create_item", fake_create_item - ) - bid = uuid.uuid4() - batch = await cosmosdb_client.create_batch("user1", bid) - assert batch.batch_id == bid - assert batch.user_id == "user1" - assert called + # Mock the method to return the batch + mock_batch_container.create_item = AsyncMock(return_value=None) + # Call the method + batch = await cosmos_db_client.create_batch(user_id, batch_id) + + # Assert that the batch is created + assert batch.batch_id == batch_id + assert batch.user_id == user_id + assert batch.status == ProcessStatus.READY_TO_PROCESS + + mock_batch_container.create_item.assert_called_once_with(body=batch.dict()) @pytest.mark.asyncio -async def test_create_batch_error(monkeypatch, cosmosdb_client): - async def fake_create_item(body): - raise Exception("Batch creation error") +async def test_create_batch_exists(cosmos_db_client, mocker): + user_id = "user_1" + batch_id = uuid4() - monkeypatch.setattr( - cosmosdb_client.batch_container, "create_item", fake_create_item - ) - with pytest.raises(Exception) as exc_info: - await cosmosdb_client.create_batch("user1", uuid.uuid4()) - assert "Batch creation error" in str(exc_info.value) + # Mock container creation and get_batch + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mock_batch_container.create_item = AsyncMock(side_effect=CosmosResourceExistsError) + # Mock the get_batch method + mock_get_batch = AsyncMock(return_value=BatchRecord( + batch_id=batch_id, + user_id=user_id, + file_count=0, + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + status=ProcessStatus.READY_TO_PROCESS + )) + mocker.patch.object(cosmos_db_client, 'get_batch', mock_get_batch) -@pytest.mark.asyncio -async def test_add_file_success(monkeypatch, cosmosdb_client): - called = False + # Call the method + batch = await cosmos_db_client.create_batch(user_id, batch_id) - async def fake_create_item(body): - nonlocal called - called = True + # Assert that batch was fetched (not created) due to already existing + assert batch.batch_id == batch_id + assert batch.user_id == user_id + assert batch.status == ProcessStatus.READY_TO_PROCESS - monkeypatch.setattr(cosmosdb_client.file_container, "create_item", fake_create_item) - bid = uuid.uuid4() - fid = uuid.uuid4() - fs = await cosmosdb_client.add_file(bid, fid, "test.txt", "path/to/blob") - assert fs.file_id == fid - assert fs.original_name == "test.txt" - assert fs.blob_path == "path/to/blob" - assert called + mock_get_batch.assert_called_once_with(user_id, str(batch_id)) @pytest.mark.asyncio -async def test_add_file_error(monkeypatch, cosmosdb_client): - async def fake_create_item(body): - raise Exception("Add file error") - - monkeypatch.setattr( - cosmosdb_client.file_container, - "create_item", - lambda *args, **kwargs: fake_create_item(*args, **kwargs), - ) - with pytest.raises(Exception) as exc_info: - await cosmosdb_client.add_file( - uuid.uuid4(), uuid.uuid4(), "test.txt", "path/to/blob" - ) - assert "Add file error" in str(exc_info.value) +async def test_add_file(cosmos_db_client, mocker): + batch_id = uuid4() + file_id = uuid4() + file_name = "file.txt" + storage_path = "/path/to/storage" + # Mock file container creation + mock_file_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) -@pytest.mark.asyncio -async def test_get_batch_success(monkeypatch, cosmosdb_client): - batch_item = { - "id": "batch1", - "user_id": "user1", - "created_at": datetime.utcnow().isoformat(), - } - file_item = {"file_id": "file1", "batch_id": "batch1"} + # Mock the create_item method + mock_file_container.create_item = AsyncMock(return_value=None) - async def fake_query_items_batch(*args, **kwargs): - for item in [batch_item]: - yield item + # Call the method + file_record = await cosmos_db_client.add_file(batch_id, file_id, file_name, storage_path) - async def fake_query_items_files(*args, **kwargs): - for item in [file_item]: - yield item + # Assert that the file record is created + assert file_record.file_id == file_id + assert file_record.batch_id == batch_id + assert file_record.original_name == file_name + assert file_record.blob_path == storage_path + assert file_record.status == ProcessStatus.READY_TO_PROCESS - cosmosdb_client.batch_container.set_query_items(fake_query_items_batch) - cosmosdb_client.file_container.set_query_items(fake_query_items_files) - result = await cosmosdb_client.get_batch("user1", "batch1") - assert result is not None - assert result.get("id") == "batch1" + mock_file_container.create_item.assert_called_once_with(body=file_record.dict()) @pytest.mark.asyncio -async def test_get_batch_not_found(monkeypatch, cosmosdb_client): - async def fake_query_items(*args, **kwargs): - if False: - yield +async def test_update_file(cosmos_db_client, mocker): + file_id = uuid4() + file_record = FileRecord( + file_id=file_id, + batch_id=uuid4(), + original_name="file.txt", + blob_path="/path/to/storage", + translated_path="", + status=ProcessStatus.READY_TO_PROCESS, + error_count=0, + syntax_count=0, + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc) + ) + + # Mock file container replace_item method + mock_file_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + mock_file_container.replace_item = AsyncMock(return_value=None) - cosmosdb_client.batch_container.set_query_items(fake_query_items) - result = await cosmosdb_client.get_batch("user1", "nonexistent") - assert result is None + # Call the method + updated_file_record = await cosmos_db_client.update_file(file_record) + + # Assert that the file record is updated + assert updated_file_record.file_id == file_id + + mock_file_container.replace_item.assert_called_once_with(item=str(file_id), body=file_record.dict()) @pytest.mark.asyncio -async def test_get_batch_error(monkeypatch, cosmosdb_client): - async def fake_query_items(*args, **kwargs): - raise Exception("Query batch error") - if False: - yield - - monkeypatch.setattr( - cosmosdb_client.batch_container, - "query_items", - lambda *args, **kwargs: fake_query_items(*args, **kwargs), +async def test_update_batch(cosmos_db_client, mocker): + batch_record = BatchRecord( + batch_id=uuid4(), + user_id="user_1", + file_count=0, + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + status=ProcessStatus.READY_TO_PROCESS ) - with pytest.raises(Exception) as exc_info: - await cosmosdb_client.get_batch("user1", "batch1") - assert "Query batch error" in str(exc_info.value) + # Mock batch container replace_item method + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mock_batch_container.replace_item = AsyncMock(return_value=None) -@pytest.mark.asyncio -async def test_get_file_success(monkeypatch, cosmosdb_client): - file_item = {"file_id": "file1", "original_name": "test.txt"} + # Call the method + updated_batch_record = await cosmos_db_client.update_batch(batch_record) - async def fake_query_items(*args, **kwargs): - for item in [file_item]: - yield item + # Assert that the batch record is updated + assert updated_batch_record.batch_id == batch_record.batch_id - cosmosdb_client.file_container.set_query_items(fake_query_items) - result = await cosmosdb_client.get_file("file1") - assert result == file_item + mock_batch_container.replace_item.assert_called_once_with(item=str(batch_record.batch_id), body=batch_record.dict()) @pytest.mark.asyncio -async def test_get_file_error(monkeypatch, cosmosdb_client): - async def fake_query_items(*args, **kwargs): - raise Exception("Query file error") - if False: - yield - - monkeypatch.setattr( - cosmosdb_client.file_container, - "query_items", - lambda *args, **kwargs: fake_query_items(*args, **kwargs), +async def test_get_batch(cosmos_db_client, mocker): + user_id = "user_1" + batch_id = str(uuid4()) + + # Mock batch container query_items method + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, "batch_container", mock_batch_container) + + # Simulate the query result + expected_batch = { + "batch_id": batch_id, + "user_id": user_id, + "file_count": 0, + "status": ProcessStatus.READY_TO_PROCESS, + } + + # We define the async generator function that will yield the expected batch + async def mock_query_items(query, parameters): + yield expected_batch + + # Assign the async generator to query_items mock + mock_batch_container.query_items.side_effect = mock_query_items + # Call the method + batch = await cosmos_db_client.get_batch(user_id, batch_id) + + # Assert the batch is returned correctly + assert batch["batch_id"] == batch_id + assert batch["user_id"] == user_id + + mock_batch_container.query_items.assert_called_once_with( + query="SELECT * FROM c WHERE c.batch_id = @batch_id and c.user_id = @user_id", + parameters=[ + {"name": "@batch_id", "value": batch_id}, + {"name": "@user_id", "value": user_id}, + ], ) - with pytest.raises(Exception) as exc_info: - await cosmosdb_client.get_file("file1") - assert "Query file error" in str(exc_info.value) @pytest.mark.asyncio -async def test_get_batch_files_success(monkeypatch, cosmosdb_client): - file_item = {"file_id": "file1", "batch_id": "batch1"} - - async def fake_query_items(*args, **kwargs): - for item in [file_item]: - yield item +async def test_get_file(cosmos_db_client, mocker): + file_id = str(uuid4()) + + # Mock file container query_items method + mock_file_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + + # Simulate the query result + expected_file = { + "file_id": file_id, + "status": ProcessStatus.READY_TO_PROCESS, + "original_name": "file.txt", + "blob_path": "/path/to/file" + } - cosmosdb_client.file_container.set_query_items(fake_query_items) - files = await cosmosdb_client.get_batch_files("user1", "batch1") - assert files == [file_item] + # We define the async generator function that will yield the expected batch + async def mock_query_items(query, parameters): + yield expected_file + # Assign the async generator to query_items mock + mock_file_container.query_items.side_effect = mock_query_items -@pytest.mark.asyncio -async def test_get_user_batches_success(monkeypatch, cosmosdb_client): - batch_item1 = {"id": "batch1", "user_id": "user1"} - batch_item2 = {"id": "batch2", "user_id": "user1"} + # Call the method + file = await cosmos_db_client.get_file(file_id) - async def fake_query_items(*args, **kwargs): - for item in [batch_item1, batch_item2]: - yield item + # Assert the file is returned correctly + assert file["file_id"] == file_id + assert file["status"] == ProcessStatus.READY_TO_PROCESS - cosmosdb_client.batch_container.set_query_items(fake_query_items) - result = await cosmosdb_client.get_user_batches("user1") - assert result == [batch_item1, batch_item2] + mock_file_container.query_items.assert_called_once() @pytest.mark.asyncio -async def test_get_user_batches_error(monkeypatch, cosmosdb_client): - async def fake_query_items(*args, **kwargs): - raise Exception("User batches error") - if False: - yield - - monkeypatch.setattr( - cosmosdb_client.batch_container, - "query_items", - lambda *args, **kwargs: fake_query_items(*args, **kwargs), - ) - with pytest.raises(Exception) as exc_info: - await cosmosdb_client.get_user_batches("user1") - assert "User batches error" in str(exc_info.value) +async def test_get_batch_files(cosmos_db_client, mocker): + batch_id = str(uuid4()) + + # Mock file container query_items method + mock_file_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + + # Simulate the query result for multiple files + expected_files = [ + { + "file_id": str(uuid4()), + "status": ProcessStatus.READY_TO_PROCESS, + "original_name": "file1.txt", + "blob_path": "/path/to/file1" + }, + { + "file_id": str(uuid4()), + "status": ProcessStatus.IN_PROGRESS, + "original_name": "file2.txt", + "blob_path": "/path/to/file2" + } + ] + + # Define the async generator function to yield the expected files + async def mock_query_items(query, parameters): + for file in expected_files: + yield file + + # Set the side_effect of query_items to simulate async iteration + mock_file_container.query_items.side_effect = mock_query_items + + # Call the method + files = await cosmos_db_client.get_batch_files(batch_id) + + # Assert the files list contains the correct files + assert len(files) == len(expected_files) + assert files[0]["file_id"] == expected_files[0]["file_id"] + assert files[1]["file_id"] == expected_files[1]["file_id"] + + mock_file_container.query_items.assert_called_once() @pytest.mark.asyncio -async def test_get_file_logs_success(monkeypatch, cosmosdb_client): - log_item = { - "file_id": "file1", - "description": "log", - "timestamp": datetime.utcnow().isoformat(), +async def test_get_batch_from_id(cosmos_db_client, mocker): + batch_id = str(uuid4()) + + # Mock batch container query_items method + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + + # Simulate the query result + expected_batch = { + "batch_id": batch_id, + "status": ProcessStatus.READY_TO_PROCESS, + "user_id": "user_123", } - async def fake_query_items(*args, **kwargs): - for item in [log_item]: - yield item + # Define the async generator function that will yield the expected batch + async def mock_query_items(query, parameters): + yield expected_batch - cosmosdb_client.log_container.set_query_items(fake_query_items) - result = await cosmosdb_client.get_file_logs("file1") - assert result == [log_item] + # Assign the async generator to query_items mock + mock_batch_container.query_items.side_effect = mock_query_items + # Call the method + batch = await cosmos_db_client.get_batch_from_id(batch_id) -@pytest.mark.asyncio -async def test_get_file_logs_error(monkeypatch, cosmosdb_client): - async def fake_query_items(*args, **kwargs): - raise Exception("Log query error") - if False: - yield - - monkeypatch.setattr( - cosmosdb_client.log_container, - "query_items", - lambda *args, **kwargs: fake_query_items(*args, **kwargs), - ) - with pytest.raises(Exception) as exc_info: - await cosmosdb_client.get_file_logs("file1") - assert "Log query error" in str(exc_info.value) + # Assert the batch is returned correctly + assert batch["batch_id"] == batch_id + assert batch["status"] == ProcessStatus.READY_TO_PROCESS + + mock_batch_container.query_items.assert_called_once() @pytest.mark.asyncio -async def test_delete_all_success(monkeypatch, cosmosdb_client): - async def fake_delete_items(key): - return +async def test_get_user_batches(cosmos_db_client, mocker): + user_id = "user_123" - monkeypatch.setattr( - cosmosdb_client.batch_container, "delete_items", fake_delete_items - ) - monkeypatch.setattr( - cosmosdb_client.file_container, "delete_items", fake_delete_items - ) - monkeypatch.setattr( - cosmosdb_client.log_container, "delete_items", fake_delete_items - ) - await cosmosdb_client.delete_all("user1") + # Mock batch container query_items method + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + # Simulate the query result + expected_batches = [ + {"batch_id": str(uuid4()), "status": ProcessStatus.READY_TO_PROCESS, "user_id": user_id}, + {"batch_id": str(uuid4()), "status": ProcessStatus.IN_PROGRESS, "user_id": user_id} + ] -@pytest.mark.asyncio -async def test_delete_all_error(monkeypatch, cosmosdb_client): - async def fake_delete_items(key): - raise Exception("Delete all error") + # Define the async generator function that will yield the expected batches + async def mock_query_items(query, parameters): + for batch in expected_batches: + yield batch - monkeypatch.setattr( - cosmosdb_client.batch_container, "delete_items", fake_delete_items - ) - with pytest.raises(Exception) as exc_info: - await cosmosdb_client.delete_all("user1") - assert "Delete all error" in str(exc_info.value) + # Assign the async generator to query_items mock + mock_batch_container.query_items.side_effect = mock_query_items + + # Call the method + batches = await cosmos_db_client.get_user_batches(user_id) + + # Assert the batches are returned correctly + assert len(batches) == 2 + assert batches[0]["status"] == ProcessStatus.READY_TO_PROCESS + assert batches[1]["status"] == ProcessStatus.IN_PROGRESS + + mock_batch_container.query_items.assert_called_once() @pytest.mark.asyncio -async def test_delete_logs_success(monkeypatch, cosmosdb_client): - async def fake_delete_items(key): - return +async def test_get_file_logs(cosmos_db_client, mocker): + file_id = str(uuid4()) + + # Mock log container query_items method + mock_log_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'log_container', mock_log_container) + + # Simulate the query result with new log structure + expected_logs = [ + { + "log_id": str(uuid4()), + "file_id": file_id, + "description": "Log entry 1", + "last_candidate": "candidate_1", + "log_type": LogType.INFO, + "agent_type": AgentType.FIXER, + "author_role": AuthorRole.ASSISTANT, + "timestamp": datetime(2025, 4, 7, 12, 0, 0) + }, + { + "log_id": str(uuid4()), + "file_id": file_id, + "description": "Log entry 2", + "last_candidate": "candidate_2", + "log_type": LogType.ERROR, + "agent_type": AgentType.HUMAN, + "author_role": AuthorRole.USER, + "timestamp": datetime(2025, 4, 7, 12, 5, 0) + } + ] + + # Define the async generator function that will yield the expected logs + async def mock_query_items(query, parameters): + for log in expected_logs: + yield log + + # Assign the async generator to query_items mock + mock_log_container.query_items.side_effect = mock_query_items + + # Call the method + logs = await cosmos_db_client.get_file_logs(file_id) + + # Assert the logs are returned correctly + assert len(logs) == 2 + assert logs[0]["description"] == "Log entry 1" + assert logs[1]["description"] == "Log entry 2" + assert logs[0]["log_type"] == LogType.INFO + assert logs[1]["log_type"] == LogType.ERROR + assert logs[0]["timestamp"] == datetime(2025, 4, 7, 12, 0, 0) + assert logs[1]["timestamp"] == datetime(2025, 4, 7, 12, 5, 0) + + mock_log_container.query_items.assert_called_once() - monkeypatch.setattr( - cosmosdb_client.log_container, "delete_items", fake_delete_items - ) - await cosmosdb_client.delete_logs("file1") + +@pytest.mark.asyncio +async def test_delete_all(cosmos_db_client, mocker): + user_id = str(uuid4()) + + # Mock containers with AsyncMock + mock_batch_container = AsyncMock() + mock_file_container = AsyncMock() + mock_log_container = AsyncMock() + + # Patching the containers with mock objects + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + mocker.patch.object(cosmos_db_client, 'log_container', mock_log_container) + + # Mock the delete_item method for all containers + mock_batch_container.delete_item = AsyncMock(return_value=None) + mock_file_container.delete_item = AsyncMock(return_value=None) + mock_log_container.delete_item = AsyncMock(return_value=None) + + # Call the delete_all method + await cosmos_db_client.delete_all(user_id) + + mock_batch_container.delete_item.assert_called_once() + mock_file_container.delete_item.assert_called_once() + mock_log_container.delete_item.assert_called_once() @pytest.mark.asyncio -async def test_delete_batch_success(monkeypatch, cosmosdb_client): - delete_calls = [] +async def test_delete_logs(cosmos_db_client, mocker): + file_id = str(uuid4()) - async def fake_delete_items(key): - delete_calls.append(key) + # Mock the log container with AsyncMock + mock_log_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'log_container', mock_log_container) - async def fake_delete_item(item, partition_key): - delete_calls.append((item, partition_key)) + # Simulate the query result for logs + log_ids = [str(uuid4()), str(uuid4())] - monkeypatch.setattr( - cosmosdb_client.file_container, "delete_items", fake_delete_items - ) - monkeypatch.setattr( - cosmosdb_client.log_container, "delete_items", fake_delete_items - ) - monkeypatch.setattr( - cosmosdb_client.batch_container, "delete_item", fake_delete_item - ) - await cosmosdb_client.delete_batch("user1", "batch1") - assert len(delete_calls) == 3 + # Define the async generator function to simulate query result + async def mock_query_items(query, parameters): + for log_id in log_ids: + yield {"id": log_id} + + # Assign the async generator to query_items mock + mock_log_container.query_items.side_effect = mock_query_items + + # Mock delete_item method for log_container + mock_log_container.delete_item = AsyncMock(return_value=None) + + # Call the delete_logs method + await cosmos_db_client.delete_logs(file_id) + + # Assert delete_item is called for each log id + for log_id in log_ids: + mock_log_container.delete_item.assert_any_call(log_id, partition_key=log_id) + + mock_log_container.query_items.assert_called_once() @pytest.mark.asyncio -async def test_delete_file_success(monkeypatch, cosmosdb_client): - calls = [] +async def test_delete_batch(cosmos_db_client, mocker): + user_id = str(uuid4()) + batch_id = str(uuid4()) - async def fake_delete_items(key): - calls.append(("log_delete", key)) + # Mock the batch container with AsyncMock + mock_batch_container = AsyncMock() + mocker.patch.object(cosmos_db_client, "batch_container", mock_batch_container) - async def fake_delete_item(file_id): - calls.append(("file_delete", file_id)) + # Call the delete_batch method + await cosmos_db_client.delete_batch(user_id, batch_id) - monkeypatch.setattr( - cosmosdb_client.log_container, "delete_items", fake_delete_items - ) - monkeypatch.setattr(cosmosdb_client.file_container, "delete_item", fake_delete_item) - await cosmosdb_client.delete_file("user1", "batch1", "file1") - assert ("log_delete", "file1") in calls - assert ("file_delete", "file1") in calls + mock_batch_container.delete_item.assert_called_once() @pytest.mark.asyncio -async def test_log_file_status_success(monkeypatch, cosmosdb_client): - called = False +async def test_delete_file(cosmos_db_client, mocker): + user_id = str(uuid4()) + file_id = str(uuid4()) - async def fake_create_item(body): - nonlocal called - called = True + # Mock containers with AsyncMock + mock_file_container = AsyncMock() + mock_log_container = AsyncMock() - monkeypatch.setattr(cosmosdb_client.log_container, "create_item", fake_create_item) - await cosmosdb_client.log_file_status( - "file1", DummyProcessStatus.READY_TO_PROCESS, "desc", DummyLogType.INFO - ) - assert called + # Patching the containers with mock objects + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + mocker.patch.object(cosmos_db_client, 'log_container', mock_log_container) + + # Mock the delete_logs method (since it's called in delete_file) + mocker.patch.object(cosmos_db_client, 'delete_logs', return_value=None) + + # Call the delete_file method + await cosmos_db_client.delete_file(user_id, file_id) + + cosmos_db_client.delete_logs.assert_called_once_with(file_id) + + mock_file_container.delete_item.assert_called_once_with(file_id, partition_key=file_id) @pytest.mark.asyncio -async def test_log_file_status_error(monkeypatch, cosmosdb_client): - async def fake_create_item(body): - raise Exception("Log error") - - monkeypatch.setattr( - cosmosdb_client.log_container, - "create_item", - lambda *args, **kwargs: fake_create_item(*args, **kwargs), +async def test_add_file_log(cosmos_db_client, mocker): + file_id = uuid4() + description = "File processing started" + last_candidate = "candidate_123" + log_type = LogType.INFO + agent_type = AgentType.MIGRATOR + author_role = AuthorRole.ASSISTANT + + # Mock log container create_item method + mock_log_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'log_container', mock_log_container) + + # Mock the create_item method + mock_log_container.create_item = AsyncMock(return_value=None) + + # Call the method + await cosmos_db_client.add_file_log( + file_id, description, last_candidate, log_type, agent_type, author_role ) - with pytest.raises(Exception) as exc_info: - await cosmosdb_client.log_file_status( - "file1", DummyProcessStatus.READY_TO_PROCESS, "desc", DummyLogType.INFO - ) - assert "Log error" in str(exc_info.value) + + mock_log_container.create_item.assert_called_once() @pytest.mark.asyncio -async def test_update_batch_entry_success(monkeypatch, cosmosdb_client): - dummy_batch = { - "id": "batch1", - "user_id": "user1", - "status": DummyProcessStatus.READY_TO_PROCESS, - "updated_at": datetime.utcnow().isoformat(), +async def test_update_batch_entry(cosmos_db_client, mocker): + batch_id = "batch_123" + user_id = "user_123" + status = ProcessStatus.IN_PROGRESS + file_count = 5 + + # Mock batch container replace_item method + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + + # Mock the get_batch method + mocker.patch.object(cosmos_db_client, 'get_batch', return_value={ + "batch_id": batch_id, + "status": ProcessStatus.READY_TO_PROCESS.value, + "user_id": user_id, "file_count": 0, - } + "updated_at": "2025-04-07T00:00:00Z" + }) - async def fake_get_batch(user_id, batch_id): - return dummy_batch.copy() + # Mock the replace_item method + mock_batch_container.replace_item = AsyncMock(return_value=None) - monkeypatch.setattr(cosmosdb_client, "get_batch", fake_get_batch) - updated_body = None + # Call the method + updated_batch = await cosmos_db_client.update_batch_entry(batch_id, user_id, status, file_count) - async def fake_replace_item(item, body): - nonlocal updated_body - updated_body = body - return body + # Assert that replace_item was called with the correct arguments + mock_batch_container.replace_item.assert_called_once_with(item=batch_id, body={ + "batch_id": batch_id, + "status": status.value, + "user_id": user_id, + "file_count": file_count, + "updated_at": updated_batch["updated_at"] + }) - monkeypatch.setattr( - cosmosdb_client.batch_container, "replace_item", fake_replace_item - ) - new_status = DummyProcessStatus.PROCESSING - file_count = 5 - result = await cosmosdb_client.update_batch_entry( - "batch1", "user1", new_status, file_count - ) - assert result["file_count"] == file_count - assert result["status"] == new_status.value - assert updated_body is not None + # Assert the returned batch matches expected values + assert updated_batch["batch_id"] == batch_id + assert updated_batch["status"] == status.value + assert updated_batch["file_count"] == file_count @pytest.mark.asyncio -async def test_update_batch_entry_not_found(monkeypatch, cosmosdb_client): - monkeypatch.setattr( - cosmosdb_client, "get_batch", lambda u, b: asyncio.sleep(0, result=None) - ) - with pytest.raises(ValueError, match="Batch not found"): - await cosmosdb_client.update_batch_entry( - "nonexistent", "user1", DummyProcessStatus.READY_TO_PROCESS, 0 - ) +async def test_close(cosmos_db_client, mocker): + # Mock the client and logger + mock_client = mock.MagicMock() + mock_logger = mock.MagicMock() + cosmos_db_client.client = mock_client + cosmos_db_client.logger = mock_logger + # Call the method + await cosmos_db_client.close() -@pytest.mark.asyncio -async def test_close(monkeypatch, cosmosdb_client): - closed = False + # Assert that the client was closed + mock_client.close.assert_called_once() - def fake_close(): - nonlocal closed - closed = True + # Assert that logger's info method was called + mock_logger.info.assert_called_once_with("Closed Cosmos DB connection") - monkeypatch.setattr(cosmosdb_client.client, "close", fake_close) - await cosmosdb_client.close() - assert closed + +@pytest.mark.asyncio +async def test_get_batch_history(cosmos_db_client, mocker): + user_id = "user_123" + limit = 5 + offset = 0 + sort_order = "DESC" + + # Mock batch container query_items method + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + + # Simulate the query result for batches + expected_batches = [ + {"batch_id": "batch_1", "status": ProcessStatus.IN_PROGRESS.value, "user_id": user_id, "file_count": 5}, + {"batch_id": "batch_2", "status": ProcessStatus.COMPLETED.value, "user_id": user_id, "file_count": 3}, + ] + + # Define the async generator function to simulate query result + async def mock_query_items(query, parameters): + for batch in expected_batches: + yield batch + + # Assign the async generator to query_items mock + mock_batch_container.query_items.side_effect = mock_query_items + + # Call the method + batches = await cosmos_db_client.get_batch_history(user_id, limit, sort_order, offset) + + # Assert the returned batches are correct + assert len(batches) == len(expected_batches) + assert batches[0]["batch_id"] == expected_batches[0]["batch_id"] + + mock_batch_container.query_items.assert_called_once() From 3e090de7890b9a75a33ed50b47124d1fad47f90a Mon Sep 17 00:00:00 2001 From: "Vishal Shinde (Persistent Systems Inc)" Date: Fri, 11 Apr 2025 17:18:36 +0530 Subject: [PATCH 15/47] feat: added unit test cases for config.py, database_base.py, database_factory.py and blob_azure.py file --- src/backend/common/database/database_base.py | 70 ++-- .../common/database/database_factory.py | 36 +- .../backend/common/config/config_test.py | 132 ++++--- .../common/database/database_base_test.py | 140 +++---- .../common/database/database_factory_test.py | 110 +++--- .../backend/common/storage/blob_azure_test.py | 357 +++++++++--------- 6 files changed, 423 insertions(+), 422 deletions(-) diff --git a/src/backend/common/database/database_base.py b/src/backend/common/database/database_base.py index 961426b5..8e22b7c4 100644 --- a/src/backend/common/database/database_base.py +++ b/src/backend/common/database/database_base.py @@ -12,55 +12,55 @@ class DatabaseBase(ABC): @abstractmethod async def initialize_cosmos(self) -> None: - """Initialize the cosmosdb client and create container if needed.""" - pass + """Initialize the cosmosdb client and create container if needed""" + pass # pragma: no cover @abstractmethod async def create_batch(self, user_id: str, batch_id: uuid.UUID) -> BatchRecord: - """Create a new conversion batch.""" - pass + """Create a new conversion batch""" + pass # pragma: no cover @abstractmethod async def get_file_logs(self, file_id: str) -> Dict: - """Retrieve all logs for a file.""" - pass + """Retrieve all logs for a file""" + pass # pragma: no cover @abstractmethod async def get_batch_from_id(self, batch_id: str) -> Dict: - """Retrieve all logs for a file.""" - pass + """Retrieve all logs for a file""" + pass # pragma: no cover @abstractmethod async def get_batch_files(self, batch_id: str) -> List[Dict]: - """Retrieve all files for a batch.""" - pass + """Retrieve all files for a batch""" + pass # pragma: no cover @abstractmethod async def delete_file_logs(self, file_id: str) -> None: - """Delete all logs for a file.""" - pass + """Delete all logs for a file""" + pass # pragma: no cover @abstractmethod async def get_user_batches(self, user_id: str) -> Dict: - """Retrieve all batches for a user.""" - pass + """Retrieve all batches for a user""" + pass # pragma: no cover @abstractmethod async def add_file( self, batch_id: uuid.UUID, file_id: uuid.UUID, file_name: str, storage_path: str ) -> FileRecord: - """Add a file entry to the database.""" - pass + """Add a file entry to the database""" + pass # pragma: no cover @abstractmethod async def get_batch(self, user_id: str, batch_id: str) -> Optional[Dict]: - """Retrieve a batch and its associated files.""" - pass + """Retrieve a batch and its associated files""" + pass # pragma: no cover @abstractmethod async def get_file(self, file_id: str) -> Optional[Dict]: - """Retrieve a file entry along with its logs.""" - pass + """Retrieve a file entry along with its logs""" + pass # pragma: no cover @abstractmethod async def add_file_log( @@ -72,39 +72,39 @@ async def add_file_log( agent_type: AgentType, author_role: AuthorRole, ) -> None: - """Log a file status update.""" - pass + """Log a file status update""" + pass # pragma: no cover @abstractmethod async def update_file(self, file_record: FileRecord) -> None: - """Update file record.""" - pass + """update file record""" + pass # pragma: no cover @abstractmethod async def update_batch(self, batch_record: BatchRecord) -> BatchRecord: - pass + pass # pragma: no cover @abstractmethod async def delete_all(self, user_id: str) -> None: - """Delete all batches, files, and logs for a user.""" - pass + """Delete all batches, files, and logs for a user""" + pass # pragma: no cover @abstractmethod async def delete_batch(self, user_id: str, batch_id: str) -> None: - """Delete a batch along with its files and logs.""" - pass + """Delete a batch along with its files and logs""" + pass # pragma: no cover @abstractmethod async def delete_file(self, user_id: str, batch_id: str, file_id: str) -> None: - """Delete a file and its logs, and update batch file count.""" - pass + """Delete a file and its logs, and update batch file count""" + pass # pragma: no cover @abstractmethod async def get_batch_history(self, user_id: str, batch_id: str) -> List[Dict]: - """Retrieve all logs for a batch.""" - pass + """Retrieve all logs for a batch""" + pass # pragma: no cover @abstractmethod async def close(self) -> None: - """Close database connection.""" - pass + """Close database connection""" + pass # pragma: no cover diff --git a/src/backend/common/database/database_factory.py b/src/backend/common/database/database_factory.py index ee92677f..c2f7de9d 100644 --- a/src/backend/common/database/database_factory.py +++ b/src/backend/common/database/database_factory.py @@ -1,3 +1,4 @@ +import asyncio from typing import Optional from common.config.config import Config @@ -33,25 +34,20 @@ async def get_database(): # Note that you have to assign yourself data plane access to Cosmos in script for this to work locally. See # https://learn.microsoft.com/en-us/azure/cosmos-db/table/security/how-to-grant-data-plane-role-based-access?tabs=built-in-definition%2Ccsharp&pivots=azure-interface-cli # Note that your principal id is your entra object id for your user account. -if __name__ == "__main__": - # Example usage - import asyncio - - async def main(): - database = await DatabaseFactory.get_database() - # Use the database instance... - await database.initialize_cosmos() - await database.create_batch("mark1", "123e4567-e89b-12d3-a456-426614174000") - await database.add_file( - "123e4567-e89b-12d3-a456-426614174000", - "123e4567-e89b-12d3-a456-426614174001", - "q1_informix.sql", - "https://cmsamarktaylstor.blob.core.windows.net/cmsablob", - ) - tstbatch = await database.get_batch( - "mark1", "123e4567-e89b-12d3-a456-426614174000" - ) - print(tstbatch) - await database.close() +async def main(): + database = await DatabaseFactory.get_database() + await database.initialize_cosmos() + await database.create_batch("mark1", "123e4567-e89b-12d3-a456-426614174000") + await database.add_file( + "123e4567-e89b-12d3-a456-426614174000", + "123e4567-e89b-12d3-a456-426614174001", + "q1_informix.sql", + "https://cmsamarktaylstor.blob.core.windows.net/cmsablob", + ) + tstbatch = await database.get_batch("mark1", "123e4567-e89b-12d3-a456-426614174000") + print(tstbatch) + await database.close() + +if __name__ == "__main__": asyncio.run(main()) diff --git a/src/tests/backend/common/config/config_test.py b/src/tests/backend/common/config/config_test.py index 87531bbc..dc4306d8 100644 --- a/src/tests/backend/common/config/config_test.py +++ b/src/tests/backend/common/config/config_test.py @@ -1,62 +1,70 @@ -import unittest -from unittest.mock import patch - -# from config import Config -from common.config.config import Config - - -class TestConfigInitialization(unittest.TestCase): - @patch.dict( - "os.environ", - { - "AZURE_TENANT_ID": "test-tenant-id", - "AZURE_CLIENT_ID": "test-client-id", - "AZURE_CLIENT_SECRET": "test-client-secret", - "COSMOSDB_DATABASE": "test-database", - "COSMOSDB_BATCH_CONTAINER": "test-batch-container", - "COSMOSDB_FILE_CONTAINER": "test-file-container", - "COSMOSDB_LOG_CONTAINER": "test-log-container", - "AZURE_BLOB_CONTAINER_NAME": "test-blob-container-name", - "AZURE_BLOB_ACCOUNT_NAME": "test-blob-account-name", - }, - clear=True, - ) - def test_config_initialization(self): - """Test if all attributes are correctly assigned from environment variables.""" - config = Config() - - # Ensure every attribute is accessed - self.assertEqual(config.azure_tenant_id, "test-tenant-id") - self.assertEqual(config.azure_client_id, "test-client-id") - self.assertEqual(config.azure_client_secret, "test-client-secret") - - self.assertEqual(config.cosmosdb_endpoint, "test-cosmosdb-endpoint") - self.assertEqual(config.cosmosdb_database, "test-database") - self.assertEqual(config.cosmosdb_batch_container, "test-batch-container") - self.assertEqual(config.cosmosdb_file_container, "test-file-container") - self.assertEqual(config.cosmosdb_log_container, "test-log-container") - - self.assertEqual(config.azure_blob_container_name, "test-blob-container-name") - self.assertEqual(config.azure_blob_account_name, "test-blob-account-name") - - @patch.dict( - "os.environ", - { - "COSMOSDB_ENDPOINT": "test-cosmosdb-endpoint", - "COSMOSDB_DATABASE": "test-database", - "COSMOSDB_BATCH_CONTAINER": "test-batch-container", - "COSMOSDB_FILE_CONTAINER": "test-file-container", - "COSMOSDB_LOG_CONTAINER": "test-log-container", - }, - ) - def test_cosmosdb_config_initialization(self): - config = Config() - self.assertEqual(config.cosmosdb_endpoint, "test-cosmosdb-endpoint") - self.assertEqual(config.cosmosdb_database, "test-database") - self.assertEqual(config.cosmosdb_batch_container, "test-batch-container") - self.assertEqual(config.cosmosdb_file_container, "test-file-container") - self.assertEqual(config.cosmosdb_log_container, "test-log-container") - - -if __name__ == "__main__": - unittest.main() +import os +import sys +import pytest + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..', 'backend'))) + +@pytest.fixture(autouse=True) +def clear_env(monkeypatch): + # Clear environment variables that might affect tests. + keys = [ + "AZURE_TENANT_ID", + "AZURE_CLIENT_ID", + "AZURE_CLIENT_SECRET", + "COSMOSDB_ENDPOINT", + "COSMOSDB_DATABASE", + "COSMOSDB_BATCH_CONTAINER", + "COSMOSDB_FILE_CONTAINER", + "COSMOSDB_LOG_CONTAINER", + "AZURE_BLOB_CONTAINER_NAME", + "AZURE_BLOB_ACCOUNT_NAME", + ] + for key in keys: + monkeypatch.delenv(key, raising=False) + +def test_config_initialization(monkeypatch): + # Set the full configuration environment variables. + monkeypatch.setenv("AZURE_TENANT_ID", "test-tenant-id") + monkeypatch.setenv("AZURE_CLIENT_ID", "test-client-id") + monkeypatch.setenv("AZURE_CLIENT_SECRET", "test-client-secret") + monkeypatch.setenv("COSMOSDB_ENDPOINT", "test-cosmosdb-endpoint") + monkeypatch.setenv("COSMOSDB_DATABASE", "test-database") + monkeypatch.setenv("COSMOSDB_BATCH_CONTAINER", "test-batch-container") + monkeypatch.setenv("COSMOSDB_FILE_CONTAINER", "test-file-container") + monkeypatch.setenv("COSMOSDB_LOG_CONTAINER", "test-log-container") + monkeypatch.setenv("AZURE_BLOB_CONTAINER_NAME", "test-blob-container-name") + monkeypatch.setenv("AZURE_BLOB_ACCOUNT_NAME", "test-blob-account-name") + + # Local import to avoid triggering circular imports during module collection. + from common.config.config import Config + config = Config() + + assert config.azure_tenant_id == "test-tenant-id" + assert config.azure_client_id == "test-client-id" + assert config.azure_client_secret == "test-client-secret" + assert config.cosmosdb_endpoint == "test-cosmosdb-endpoint" + assert config.cosmosdb_database == "test-database" + assert config.cosmosdb_batch_container == "test-batch-container" + assert config.cosmosdb_file_container == "test-file-container" + assert config.cosmosdb_log_container == "test-log-container" + assert config.azure_blob_container_name == "test-blob-container-name" + assert config.azure_blob_account_name == "test-blob-account-name" + +def test_cosmosdb_config_initialization(monkeypatch): + # Set only cosmosdb-related environment variables. + monkeypatch.setenv("COSMOSDB_ENDPOINT", "test-cosmosdb-endpoint") + monkeypatch.setenv("COSMOSDB_DATABASE", "test-database") + monkeypatch.setenv("COSMOSDB_BATCH_CONTAINER", "test-batch-container") + monkeypatch.setenv("COSMOSDB_FILE_CONTAINER", "test-file-container") + monkeypatch.setenv("COSMOSDB_LOG_CONTAINER", "test-log-container") + + from common.config.config import Config + config = Config() + + assert config.cosmosdb_endpoint == "test-cosmosdb-endpoint" + assert config.cosmosdb_database == "test-database" + assert config.cosmosdb_batch_container == "test-batch-container" + assert config.cosmosdb_file_container == "test-file-container" + assert config.cosmosdb_log_container == "test-log-container" + + \ No newline at end of file diff --git a/src/tests/backend/common/database/database_base_test.py b/src/tests/backend/common/database/database_base_test.py index 0e9d1fec..8dd57d81 100644 --- a/src/tests/backend/common/database/database_base_test.py +++ b/src/tests/backend/common/database/database_base_test.py @@ -1,68 +1,70 @@ +import asyncio import uuid +import pytest +import os +import sys +from datetime import datetime from enum import Enum - -# Import the abstract base class and related models/enums. + +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../backend'))) + from common.database.database_base import DatabaseBase -from common.models.api import ProcessStatus - -import pytest - +from common.models.api import BatchRecord, FileRecord, ProcessStatus + +# Allow instantiation of the abstract base class by clearing its abstract methods. DatabaseBase.__abstractmethods__ = set() - - + @pytest.fixture def db_instance(): - # Instantiate the DatabaseBase directly. + # Create a concrete implementation of DatabaseBase using async methods. class ConcreteDatabase(DatabaseBase): - def create_batch(self, user_id, batch_id): + async def create_batch(self, user_id, batch_id): pass - - def get_file_logs(self, file_id): + + async def get_file_logs(self, file_id): pass - - def get_batch_files(self, user_id, batch_id): + + async def get_batch_files(self, user_id, batch_id): pass - - def delete_file_logs(self, file_id): + + async def delete_file_logs(self, file_id): pass - - def get_user_batches(self, user_id): + + async def get_user_batches(self, user_id): pass - - def add_file(self, batch_id, file_id, file_name, file_path): + + async def add_file(self, batch_id, file_id, file_name, file_path): pass - - def get_batch(self, user_id, batch_id): + + async def get_batch(self, user_id, batch_id): pass - - def get_file(self, file_id): + + async def get_file(self, file_id): pass - - def log_file_status(self, file_id, status, description, log_type): + + async def log_file_status(self, file_id, status, description, log_type): pass - - def log_batch_status(self, batch_id, status, file_count): + + async def log_batch_status(self, batch_id, status, file_count): pass - - def delete_all(self, user_id): + + async def delete_all(self, user_id): pass - - def delete_batch(self, user_id, batch_id): + + async def delete_batch(self, user_id, batch_id): pass - - def delete_file(self, user_id, batch_id, file_id): + + async def delete_file(self, user_id, batch_id, file_id): pass - - def close(self): + + async def close(self): pass - + return ConcreteDatabase() - - + def get_dummy_status(): """ Try to use a specific ProcessStatus value (e.g. PROCESSING). - If that member is not available, just return the first member in the enum. """ try: @@ -71,97 +73,79 @@ def get_dummy_status(): members = list(ProcessStatus) if members: return members[0] - # If the enum is empty, create a dummy one + # If the enum is empty, create a dummy one. DummyStatus = Enum("DummyStatus", {"DUMMY": "dummy"}) return DummyStatus.DUMMY - - + @pytest.mark.asyncio async def test_create_batch(db_instance): result = await db_instance.create_batch("user1", uuid.uuid4()) - # Since the method is abstract (and implemented as pass), result is None. + # Since the method is implemented as pass, result is None. assert result is None - - + @pytest.mark.asyncio async def test_get_file_logs(db_instance): result = await db_instance.get_file_logs("file1") assert result is None - - + @pytest.mark.asyncio async def test_get_batch_files(db_instance): result = await db_instance.get_batch_files("user1", "batch1") assert result is None - - + @pytest.mark.asyncio async def test_delete_file_logs(db_instance): result = await db_instance.delete_file_logs("file1") assert result is None - - + @pytest.mark.asyncio async def test_get_user_batches(db_instance): result = await db_instance.get_user_batches("user1") assert result is None - - + @pytest.mark.asyncio async def test_add_file(db_instance): - result = await db_instance.add_file( - uuid.uuid4(), uuid.uuid4(), "test.txt", "/dummy/path" - ) + result = await db_instance.add_file(uuid.uuid4(), uuid.uuid4(), "test.txt", "/dummy/path") assert result is None - - + @pytest.mark.asyncio async def test_get_batch(db_instance): result = await db_instance.get_batch("user1", "batch1") assert result is None - - + @pytest.mark.asyncio async def test_get_file(db_instance): result = await db_instance.get_file("file1") assert result is None - - + @pytest.mark.asyncio async def test_log_file_status(db_instance): - # Use an existing member for file status—here we use COMPLETED. - result = await db_instance.log_file_status( - "file1", ProcessStatus.COMPLETED, "desc", "log_type" - ) + # Using ProcessStatus.COMPLETED as an example. + result = await db_instance.log_file_status("file1", ProcessStatus.COMPLETED, "desc", "log_type") assert result is None - - + @pytest.mark.asyncio async def test_log_batch_status(db_instance): dummy_status = get_dummy_status() result = await db_instance.log_batch_status("batch1", dummy_status, 5) assert result is None - - + @pytest.mark.asyncio async def test_delete_all(db_instance): result = await db_instance.delete_all("user1") assert result is None - - + @pytest.mark.asyncio async def test_delete_batch(db_instance): result = await db_instance.delete_batch("user1", "batch1") assert result is None - - + @pytest.mark.asyncio async def test_delete_file(db_instance): result = await db_instance.delete_file("user1", "batch1", "file1") assert result is None - - + @pytest.mark.asyncio async def test_close(db_instance): result = await db_instance.close() - assert result is None + assert result is None \ No newline at end of file diff --git a/src/tests/backend/common/database/database_factory_test.py b/src/tests/backend/common/database/database_factory_test.py index bdf99d35..67ad35ab 100644 --- a/src/tests/backend/common/database/database_factory_test.py +++ b/src/tests/backend/common/database/database_factory_test.py @@ -1,63 +1,79 @@ -from common.config.config import Config -from common.database.database_factory import DatabaseFactory - +import os +import sys import pytest +import asyncio +from unittest.mock import AsyncMock, patch +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..', 'backend'))) -class DummyConfig: - cosmosdb_endpoint = "dummy_endpoint" - cosmosdb_database = "dummy_database" - cosmosdb_batch_container = "dummy_batch" - cosmosdb_file_container = "dummy_file" - cosmosdb_log_container = "dummy_log" +@pytest.fixture(autouse=True) +def patch_config(monkeypatch): + """Patch Config class to use dummy values.""" + from common.config.config import Config + def dummy_init(self): + """Mocked __init__ method for Config to set dummy values.""" + self.cosmosdb_endpoint = "dummy_endpoint" + self.cosmosdb_database = "dummy_database" + self.cosmosdb_batch_container = "dummy_batch" + self.cosmosdb_file_container = "dummy_file" + self.cosmosdb_log_container = "dummy_log" + self.get_azure_credentials = lambda: "dummy_credential" -class DummyCosmosDBClient: - def __init__(self, endpoint, credential, database_name, batch_container, file_container, log_container): - self.endpoint = endpoint - self.credential = credential - self.database_name = database_name - self.batch_container = batch_container - self.file_container = file_container - self.log_container = log_container + monkeypatch.setattr(Config, "__init__", dummy_init) # Replace the init method +@pytest.fixture(autouse=True) +def patch_cosmosdb_client(monkeypatch): + """Patch CosmosDBClient to use a dummy implementation.""" + from common.database.database_factory import CosmosDBClient -def dummy_config_init(self): - self.cosmosdb_endpoint = DummyConfig.cosmosdb_endpoint - self.cosmosdb_database = DummyConfig.cosmosdb_database - self.cosmosdb_batch_container = DummyConfig.cosmosdb_batch_container - self.cosmosdb_file_container = DummyConfig.cosmosdb_file_container - self.cosmosdb_log_container = DummyConfig.cosmosdb_log_container - # Provide a dummy method for credentials. - self.get_azure_credentials = lambda: "dummy_credential" + class DummyCosmosDBClient: + def __init__(self, endpoint, credential, database_name, batch_container, file_container, log_container): + self.endpoint = endpoint + self.credential = credential + self.database_name = database_name + self.batch_container = batch_container + self.file_container = file_container + self.log_container = log_container + async def initialize_cosmos(self): + pass -@pytest.fixture(autouse=True) -def patch_config(monkeypatch): - # Patch the __init__ of Config so that an instance will have the required attributes. - monkeypatch.setattr(Config, "__init__", dummy_config_init) + async def create_batch(self, *args, **kwargs): + pass + async def add_file(self, *args, **kwargs): + pass -@pytest.fixture(autouse=True) -def patch_cosmosdb_client(monkeypatch): - # Patch CosmosDBClient in the module under test to use our dummy client. - monkeypatch.setattr("common.database.database_factory.CosmosDBClient", DummyCosmosDBClient) + async def get_batch(self, *args, **kwargs): + return "mock_batch" + async def close(self): + pass -def test_get_database(): - """ - Test that DatabaseFactory.get_database() correctly returns an instance of the. + monkeypatch.setattr("common.database.database_factory.CosmosDBClient", DummyCosmosDBClient) - dummy CosmosDB client with the expected configuration values. - """ - # When get_database() is called, it creates a new Config() instance. - db_instance = DatabaseFactory.get_database() +@pytest.mark.asyncio +async def test_get_database(): + """Test database retrieval using the factory.""" + from common.database.database_factory import DatabaseFactory - # Verify that the returned instance is our dummy client with the expected attributes. - assert isinstance(db_instance, DummyCosmosDBClient) - assert db_instance.endpoint == DummyConfig.cosmosdb_endpoint + db_instance = await DatabaseFactory.get_database() + + assert db_instance.endpoint == "dummy_endpoint" assert db_instance.credential == "dummy_credential" - assert db_instance.database_name == DummyConfig.cosmosdb_database - assert db_instance.batch_container == DummyConfig.cosmosdb_batch_container - assert db_instance.file_container == DummyConfig.cosmosdb_file_container - assert db_instance.log_container == DummyConfig.cosmosdb_log_container + assert db_instance.database_name == "dummy_database" + assert db_instance.batch_container == "dummy_batch" + assert db_instance.file_container == "dummy_file" + assert db_instance.log_container == "dummy_log" + +@pytest.mark.asyncio +async def test_main_function(): + """Test the main function in database factory.""" + with patch("common.database.database_factory.DatabaseFactory.get_database", new_callable=AsyncMock, return_value=AsyncMock()) as mock_get_database, patch("builtins.print") as mock_print: + + from common.database.database_factory import main + await main() + + mock_get_database.assert_called_once() + mock_print.assert_called() # Ensures print is executed diff --git a/src/tests/backend/common/storage/blob_azure_test.py b/src/tests/backend/common/storage/blob_azure_test.py index 2f743020..573cd085 100644 --- a/src/tests/backend/common/storage/blob_azure_test.py +++ b/src/tests/backend/common/storage/blob_azure_test.py @@ -1,228 +1,225 @@ -# blob_azure_test.py - -from datetime import datetime -from unittest.mock import AsyncMock, MagicMock, patch - -# Import the class under test -from azure.core.exceptions import ResourceExistsError - -from common.storage.blob_azure import AzureBlobStorage - - +import json +import os +import sys import pytest +from unittest.mock import AsyncMock, patch, MagicMock +from io import BytesIO +# Add backend directory to sys.path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..', 'backend'))) -class DummyBlob: - """A dummy blob item returned by list_blobs.""" - - def __init__(self, name, size, creation_time, content_type, metadata): - self.name = name - self.size = size - self.creation_time = creation_time - self.content_settings = MagicMock(content_type=content_type) - self.metadata = metadata - - -class DummyAsyncIterator: - """A dummy async iterator that yields the given items.""" - - def __init__(self, items): - self.items = items - self.index = 0 - - def __aiter__(self): - return self - - async def __anext__(self): - if self.index >= len(self.items): - raise StopAsyncIteration - item = self.items[self.index] - self.index += 1 - return item - - -class DummyDownloadStream: - """A dummy download stream whose content_as_bytes method returns a fixed byte string.""" +from common.storage.blob_azure import AzureBlobStorage - async def content_as_bytes(self): - return b"file content" -# --- Fixtures --- +@pytest.fixture +def mock_blob_service(): + """Fixture to mock Azure Blob Storage service client""" + with patch("common.storage.blob_azure.BlobServiceClient") as mock_service: + mock_service_instance = MagicMock() + mock_container_client = MagicMock() + mock_blob_client = MagicMock() + # Set up mock methods + mock_service.return_value = mock_service_instance + mock_service_instance.get_container_client.return_value = mock_container_client + mock_container_client.get_blob_client.return_value = mock_blob_client -@pytest.fixture -def dummy_storage(): - # Create an instance with dummy connection string and container name. - return AzureBlobStorage("dummy_connection_string", "dummy_container") + yield mock_service_instance, mock_container_client, mock_blob_client @pytest.fixture -def dummy_container_client(): - container = MagicMock() - container.create_container = AsyncMock() - container.list_blobs = MagicMock() # Will be overridden per test. - container.get_blob_client = MagicMock() - return container +def blob_storage(mock_blob_service): + """Fixture to initialize AzureBlobStorage with mocked dependencies""" + service_client, container_client, blob_client = mock_blob_service + return AzureBlobStorage(account_name="test_account", container_name="test_container") -@pytest.fixture -def dummy_service_client(dummy_container_client): - service = MagicMock() - service.get_container_client.return_value = dummy_container_client - return service +@pytest.mark.asyncio +async def test_upload_file(blob_storage, mock_blob_service): + """Test uploading a file""" + _, _, mock_blob_client = mock_blob_service + mock_blob_client.upload_blob.return_value = MagicMock() + mock_blob_client.get_blob_properties.return_value = MagicMock( + size=1024, + content_settings=MagicMock(content_type="text/plain"), + creation_time="2024-03-15T12:00:00Z", + etag="dummy_etag", + ) + + file_content = BytesIO(b"dummy data") + + result = await blob_storage.upload_file(file_content, "test_blob.txt", "text/plain") + + assert result["path"] == "test_blob.txt" + assert result["size"] == 1024 + assert result["content_type"] == "text/plain" + assert result["created_at"] == "2024-03-15T12:00:00Z" + assert result["etag"] == "dummy_etag" + assert "url" in result -@pytest.fixture -def dummy_blob_client(): - blob_client = MagicMock() - blob_client.upload_blob = AsyncMock() - blob_client.get_blob_properties = AsyncMock() - blob_client.download_blob = AsyncMock() - blob_client.delete_blob = AsyncMock() - blob_client.url = "https://dummy.blob.core.windows.net/dummy_container/dummy_blob" - return blob_client +@pytest.mark.asyncio +async def test_upload_file_exception(blob_storage, mock_blob_service): + """Test upload_file when an exception occurs""" + _, _, mock_blob_client = mock_blob_service + mock_blob_client.upload_blob.side_effect = Exception("Upload failed") -# --- Tests for AzureBlobStorage methods --- + with pytest.raises(Exception, match="Upload failed"): + await blob_storage.upload_file(BytesIO(b"dummy data"), "test_blob.txt") @pytest.mark.asyncio -async def test_initialize_creates_container(dummy_storage, dummy_service_client, dummy_container_client): - with patch("common.storage.blob_azure.BlobServiceClient.from_connection_string", return_value=dummy_service_client) as mock_from_conn: - # Simulate normal container creation. - dummy_container_client.create_container = AsyncMock() - await dummy_storage.initialize() - mock_from_conn.assert_called_once_with("dummy_connection_string") - dummy_service_client.get_container_client.assert_called_once_with("dummy_container") - dummy_container_client.create_container.assert_awaited_once() +async def test_get_file(blob_storage, mock_blob_service): + """Test downloading a file""" + _, _, mock_blob_client = mock_blob_service + mock_blob_client.download_blob.return_value.readall.return_value = b"dummy data" + result = await blob_storage.get_file("test_blob.txt") -@pytest.mark.asyncio -async def test_initialize_container_already_exists(dummy_storage, dummy_service_client, dummy_container_client): - with patch("common.storage.blob_azure.BlobServiceClient.from_connection_string", return_value=dummy_service_client): - # Simulate container already existing. - dummy_container_client.create_container = AsyncMock(side_effect=ResourceExistsError("Container exists")) - with patch.object(dummy_storage.logger, "debug") as mock_debug: - await dummy_storage.initialize() - dummy_container_client.create_container.assert_awaited_once() - mock_debug.assert_called_with("Container dummy_container already exists") + assert result == "dummy data" @pytest.mark.asyncio -async def test_initialize_failure(dummy_storage): - # Simulate failure during initialization. - with patch("common.storage.blob_azure.BlobServiceClient.from_connection_string", side_effect=Exception("Init error")): - with patch.object(dummy_storage.logger, "error") as mock_error: - with pytest.raises(Exception, match="Init error"): - await dummy_storage.initialize() - mock_error.assert_called() +async def test_get_file_exception(blob_storage, mock_blob_service): + """Test get_file when an exception occurs""" + _, _, mock_blob_client = mock_blob_service + mock_blob_client.download_blob.side_effect = Exception("Download failed") + + with pytest.raises(Exception, match="Download failed"): + await blob_storage.get_file("test_blob.txt") @pytest.mark.asyncio -async def test_upload_file_success(dummy_storage, dummy_blob_client): - # Patch get_blob_client to return our dummy blob client. - dummy_storage.container_client = MagicMock() - dummy_storage.container_client.get_blob_client.return_value = dummy_blob_client - - # Create a dummy properties object. - dummy_properties = MagicMock() - dummy_properties.size = 1024 - dummy_properties.content_settings = MagicMock(content_type="text/plain") - dummy_properties.creation_time = datetime(2023, 1, 1) - dummy_properties.etag = "dummy_etag" - dummy_blob_client.get_blob_properties = AsyncMock(return_value=dummy_properties) - - file_content = b"Hello, world!" - result = await dummy_storage.upload_file(file_content, "dummy_blob.txt", "text/plain", {"key": "value"}) - dummy_storage.container_client.get_blob_client.assert_called_once_with("dummy_blob.txt") - dummy_blob_client.upload_blob.assert_awaited_with(file_content, content_type="text/plain", metadata={"key": "value"}, overwrite=True) - dummy_blob_client.get_blob_properties.assert_awaited() - assert result["path"] == "dummy_blob.txt" - assert result["size"] == 1024 - assert result["content_type"] == "text/plain" - assert result["url"] == dummy_blob_client.url - assert result["etag"] == "dummy_etag" +async def test_delete_file(blob_storage, mock_blob_service): + """Test deleting a file""" + _, _, mock_blob_client = mock_blob_service + mock_blob_client.delete_blob.return_value = None + result = await blob_storage.delete_file("test_blob.txt") -@pytest.mark.asyncio -async def test_upload_file_error(dummy_storage, dummy_blob_client): - dummy_storage.container_client = MagicMock() - dummy_storage.container_client.get_blob_client.return_value = dummy_blob_client - dummy_blob_client.upload_blob = AsyncMock(side_effect=Exception("Upload failed")) - with pytest.raises(Exception, match="Upload failed"): - await dummy_storage.upload_file(b"data", "blob.txt", "text/plain", {}) + assert result is True @pytest.mark.asyncio -async def test_get_file_success(dummy_storage, dummy_blob_client): - dummy_storage.container_client = MagicMock() - dummy_storage.container_client.get_blob_client.return_value = dummy_blob_client - # Make download_blob return a DummyDownloadStream (not wrapped in extra coroutine) - dummy_blob_client.download_blob = AsyncMock(return_value=DummyDownloadStream()) - result = await dummy_storage.get_file("blob.txt") - dummy_storage.container_client.get_blob_client.assert_called_once_with("blob.txt") - dummy_blob_client.download_blob.assert_awaited() - assert result == b"file content" +async def test_delete_file_exception(blob_storage, mock_blob_service): + """Test delete_file when an exception occurs""" + _, _, mock_blob_client = mock_blob_service + mock_blob_client.delete_blob.side_effect = Exception("Delete failed") + result = await blob_storage.delete_file("test_blob.txt") -@pytest.mark.asyncio -async def test_get_file_error(dummy_storage, dummy_blob_client): - dummy_storage.container_client = MagicMock() - dummy_storage.container_client.get_blob_client.return_value = dummy_blob_client - dummy_blob_client.download_blob = AsyncMock(side_effect=Exception("Download error")) - with pytest.raises(Exception, match="Download error"): - await dummy_storage.get_file("nonexistent.txt") + assert result is False @pytest.mark.asyncio -async def test_delete_file_success(dummy_storage, dummy_blob_client): - dummy_storage.container_client = MagicMock() - dummy_storage.container_client.get_blob_client.return_value = dummy_blob_client - dummy_blob_client.delete_blob = AsyncMock() - result = await dummy_storage.delete_file("blob.txt") - dummy_storage.container_client.get_blob_client.assert_called_once_with("blob.txt") - dummy_blob_client.delete_blob.assert_awaited() - assert result is True +async def test_list_files(blob_storage, mock_blob_service): + """Test listing files in a container""" + _, mock_container_client, _ = mock_blob_service + + class AsyncIterator: + """Helper class to create an async iterator""" + def __init__(self, items): + self._items = items + + def __aiter__(self): + self._iter = iter(self._items) + return self + + async def __anext__(self): + try: + return next(self._iter) + except StopIteration: + raise StopAsyncIteration + + mock_blobs = [ + MagicMock(name="file1.txt"), + MagicMock(name="file2.txt"), + ] + + # Explicitly set attributes to avoid MagicMock issues + mock_blobs[0].name = "file1.txt" + mock_blobs[0].size = 123 + mock_blobs[0].creation_time = "2024-03-15T12:00:00Z" + mock_blobs[0].content_settings = MagicMock(content_type="text/plain") + mock_blobs[0].metadata = {} + + mock_blobs[1].name = "file2.txt" + mock_blobs[1].size = 456 + mock_blobs[1].creation_time = "2024-03-16T12:00:00Z" + mock_blobs[1].content_settings = MagicMock(content_type="application/json") + mock_blobs[1].metadata = {} + + mock_container_client.list_blobs = MagicMock(return_value=AsyncIterator(mock_blobs)) + + result = await blob_storage.list_files() + assert len(result) == 2 + assert result[0]["name"] == "file1.txt" + assert result[0]["size"] == 123 + assert result[0]["created_at"] == "2024-03-15T12:00:00Z" + assert result[0]["content_type"] == "text/plain" + assert result[0]["metadata"] == {} -@pytest.mark.asyncio -async def test_delete_file_error(dummy_storage, dummy_blob_client): - dummy_storage.container_client = MagicMock() - dummy_storage.container_client.get_blob_client.return_value = dummy_blob_client - dummy_blob_client.delete_blob = AsyncMock(side_effect=Exception("Delete error")) - result = await dummy_storage.delete_file("blob.txt") - assert result is False + assert result[1]["name"] == "file2.txt" + assert result[1]["size"] == 456 + assert result[1]["created_at"] == "2024-03-16T12:00:00Z" + assert result[1]["content_type"] == "application/json" + assert result[1]["metadata"] == {} @pytest.mark.asyncio -async def test_list_files_success(dummy_storage): - dummy_storage.container_client = MagicMock() - # Create two dummy blobs. - blob1 = DummyBlob("file1.txt", 100, datetime(2023, 1, 1), "text/plain", {"a": "1"}) - blob2 = DummyBlob("file2.txt", 200, datetime(2023, 1, 2), "text/plain", {"b": "2"}) - async_iterator = DummyAsyncIterator([blob1, blob2]) - dummy_storage.container_client.list_blobs.return_value = async_iterator - result = await dummy_storage.list_files("file") - assert len(result) == 2 - names = {item["name"] for item in result} - assert names == {"file1.txt", "file2.txt"} +async def test_list_files_exception(blob_storage, mock_blob_service): + """Test list_files when an exception occurs""" + _, mock_container_client, _ = mock_blob_service + mock_container_client.list_blobs.side_effect = Exception("List failed") + + with pytest.raises(Exception, match="List failed"): + await blob_storage.list_files() @pytest.mark.asyncio -async def test_list_files_failure(dummy_storage): - dummy_storage.container_client = MagicMock() - # Define list_blobs to return an invalid object (simulate error) +async def test_close(blob_storage, mock_blob_service): + """Test closing the storage client""" + service_client, _, _ = mock_blob_service - async def invalid_list_blobs(*args, **kwargs): - # Return a plain string (which does not implement __aiter__) - return "invalid" - dummy_storage.container_client.list_blobs = invalid_list_blobs - with pytest.raises(Exception): # noqa B017 - await dummy_storage.list_files("") + await blob_storage.close() + service_client.close.assert_called_once() @pytest.mark.asyncio -async def test_close(dummy_storage): - dummy_storage.service_client = MagicMock() - dummy_storage.service_client.close = AsyncMock() - await dummy_storage.close() - dummy_storage.service_client.close.assert_awaited() +async def test_blob_storage_init_exception(): + """Test that an exception during initialization logs the error message""" + with patch("common.storage.blob_azure.BlobServiceClient") as mock_service, \ + patch("logging.getLogger") as mock_logger: # Patch logging globally + + # Mock logger instance + mock_logger_instance = MagicMock() + mock_logger.return_value = mock_logger_instance + + # Simulate an exception when creating BlobServiceClient + mock_service.side_effect = Exception("Connection failed") + + # Try to initialize AzureBlobStorage + try: + AzureBlobStorage(account_name="test_account", container_name="test_container") + except Exception: + pass # Prevent test failure due to the exception + + # Construct the expected JSON log format + expected_error_log = json.dumps({ + "message": "Failed to initialize Azure Blob Storage", + "context": { + "error": "Connection failed", + "account_name": "test_account" + } + }) + + expected_debug_log = json.dumps({ + "message": "Container test_container already exists" + }) + + # Assert that error logging happened with the expected JSON string + mock_logger_instance.error.assert_called_once_with(expected_error_log) + + # Assert that debug log is written for container existence + mock_logger_instance.debug.assert_called_once_with(expected_debug_log) \ No newline at end of file From 667d88017b9f43fca6d172a992c0fb5aa0761e85 Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Tue, 15 Apr 2025 12:03:50 +0530 Subject: [PATCH 16/47] fix: removed duplicate import --- src/backend/api/api_routes.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/backend/api/api_routes.py b/src/backend/api/api_routes.py index d3c561e6..ff264f02 100644 --- a/src/backend/api/api_routes.py +++ b/src/backend/api/api_routes.py @@ -25,9 +25,6 @@ router = APIRouter() logger = AppLogger("APIRoutes") -# start processing the batch -from sql_agents_start import process_batch_async # noqa: E402 - # start processing the batch @router.post("/start-processing") async def start_processing(request: Request): From e09573d9a917f9dcb829ffb8cd4c0e475727d095 Mon Sep 17 00:00:00 2001 From: Shreyas-Microsoft Date: Tue, 15 Apr 2025 14:05:19 +0530 Subject: [PATCH 17/47] Consistent agent naming --- src/backend/api/api_routes.py | 2 +- src/frontend/src/api/utils.tsx | 36 +++++++++++++++----- src/frontend/src/pages/modernizationPage.tsx | 10 +++--- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/backend/api/api_routes.py b/src/backend/api/api_routes.py index d3c561e6..88986a19 100644 --- a/src/backend/api/api_routes.py +++ b/src/backend/api/api_routes.py @@ -26,7 +26,7 @@ logger = AppLogger("APIRoutes") # start processing the batch -from sql_agents_start import process_batch_async # noqa: E402 +# from sql_agents_start import process_batch_async # noqa: E402 # start processing the batch @router.post("/start-processing") diff --git a/src/frontend/src/api/utils.tsx b/src/frontend/src/api/utils.tsx index f66517c2..95ce5459 100644 --- a/src/frontend/src/api/utils.tsx +++ b/src/frontend/src/api/utils.tsx @@ -294,15 +294,35 @@ export const determineFileStatus = (file) => { return "error"; }; // Function to format agent type strings -export const formatAgent = (str = "Agents") => { - if (!str) return "Agents"; - return str +export const formatAgent = (str = "Agent") => { + if (!str) return "agent"; + + const cleaned = str .replace(/[^a-zA-Z\s]/g, " ") // Remove non-alphabetic characters - .replace(/\s+/g, " ") // Replace multiple spaces with a single space - .trim() // Remove leading/trailing spaces - .split(" ") // Split words - .map(word => word.charAt(0).toUpperCase() + word.slice(1).toLowerCase()) // Capitalize first letter - .join(" ") || "Agents"; // Ensure default "Agent" if empty + .replace(/\s+/g, " ") // Collapse multiple spaces + .trim() + .replace(/\bAgents\b/i, "Agent"); // Singularize "Agents" if it's the only word + + const words = cleaned + .split(" ") + .filter(Boolean) + .map(w => w.toLowerCase()); + + const hasAgent = words.includes("agent"); + + // Capitalize all except "agent" (unless it's the only word) + const result = words.map((word, index) => { + if (word === "agent") { + return words.length === 1 ? "Agent" : "agent"; // Capitalize if it's the only word + } + return word.charAt(0).toUpperCase() + word.slice(1); + }); + + if (!hasAgent) { + result.push("agent"); + } + + return result.join(" "); }; // Function to handle rate limit errors and ensure descriptions end with a dot diff --git a/src/frontend/src/pages/modernizationPage.tsx b/src/frontend/src/pages/modernizationPage.tsx index 082d44ff..685842c2 100644 --- a/src/frontend/src/pages/modernizationPage.tsx +++ b/src/frontend/src/pages/modernizationPage.tsx @@ -425,11 +425,11 @@ enum ProcessingStage { } enum Agents { - Verifier = "Semantic Verifier", - Checker = "Syntax Checker", - Picker = "Picker", - Migrator = "Migrator", - Agents = "Agents" + Verifier = "Semantic Verifier agent", + Checker = "Syntax Checker agent", + Picker = "Picker agent", + Migrator = "Migrator agent", + Agents = "Agent" } From 1525f73303800af834753fe720104d7d6f062efc Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Tue, 15 Apr 2025 14:12:16 +0530 Subject: [PATCH 18/47] fix: pylint fixes --- src/backend/api/api_routes.py | 11 +++++++---- src/backend/common/config/config.py | 1 - src/backend/common/database/database_base.py | 3 +-- src/backend/sql_agents/agents/agent_base.py | 1 + src/backend/sql_agents/convert_script.py | 6 ++++-- src/backend/sql_agents/process_batch.py | 16 +++++++++++----- 6 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/backend/api/api_routes.py b/src/backend/api/api_routes.py index ff264f02..a233bd26 100644 --- a/src/backend/api/api_routes.py +++ b/src/backend/api/api_routes.py @@ -4,6 +4,12 @@ import io import zipfile +from api.auth.auth_utils import get_authenticated_user +from api.status_updates import app_connection_manager, close_connection + +from common.logger.app_logger import AppLogger +from common.services.batch_service import BatchService + from fastapi import ( APIRouter, File, @@ -16,15 +22,12 @@ ) from fastapi.responses import Response -from api.auth.auth_utils import get_authenticated_user -from api.status_updates import app_connection_manager, close_connection -from common.logger.app_logger import AppLogger -from common.services.batch_service import BatchService from sql_agents.process_batch import process_batch_async router = APIRouter() logger = AppLogger("APIRoutes") + # start processing the batch @router.post("/start-processing") async def start_processing(request: Request): diff --git a/src/backend/common/config/config.py b/src/backend/common/config/config.py index ac2791fa..24eb2fe8 100644 --- a/src/backend/common/config/config.py +++ b/src/backend/common/config/config.py @@ -17,7 +17,6 @@ from azure.identity.aio import ClientSecretCredential, DefaultAzureCredential - class Config: """Configuration class for the application.""" diff --git a/src/backend/common/database/database_base.py b/src/backend/common/database/database_base.py index 2440a5e3..156d6bdf 100644 --- a/src/backend/common/database/database_base.py +++ b/src/backend/common/database/database_base.py @@ -4,11 +4,10 @@ from abc import ABC, abstractmethod from typing import Dict, List, Optional -from common.models.api import AgentType, BatchRecord, FileRecord, LogType +from common.models.api import BatchRecord, FileRecord, LogType from semantic_kernel.contents import AuthorRole -from common.models.api import BatchRecord, FileRecord, LogType from sql_agents.helpers.models import AgentType diff --git a/src/backend/sql_agents/agents/agent_base.py b/src/backend/sql_agents/agents/agent_base.py index 656f74b7..34bb9e81 100644 --- a/src/backend/sql_agents/agents/agent_base.py +++ b/src/backend/sql_agents/agents/agent_base.py @@ -8,6 +8,7 @@ ResponseFormatJsonSchema, ResponseFormatJsonSchemaType, ) + from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent from semantic_kernel.functions import KernelArguments diff --git a/src/backend/sql_agents/convert_script.py b/src/backend/sql_agents/convert_script.py index d7fabfe1..ccc82de3 100644 --- a/src/backend/sql_agents/convert_script.py +++ b/src/backend/sql_agents/convert_script.py @@ -7,9 +7,8 @@ import json import logging -from semantic_kernel.contents import AuthorRole, ChatMessageContent - from api.status_updates import send_status_update + from common.models.api import ( FileProcessUpdate, FileRecord, @@ -18,6 +17,9 @@ ProcessStatus, ) from common.services.batch_service import BatchService + +from semantic_kernel.contents import AuthorRole, ChatMessageContent + from sql_agents.agents.fixer.response import FixerResponse from sql_agents.agents.migrator.response import MigratorResponse from sql_agents.agents.picker.response import PickerResponse diff --git a/src/backend/sql_agents/process_batch.py b/src/backend/sql_agents/process_batch.py index a142b2e5..9f4918a0 100644 --- a/src/backend/sql_agents/process_batch.py +++ b/src/backend/sql_agents/process_batch.py @@ -6,13 +6,10 @@ import logging +from api.status_updates import send_status_update + from azure.identity.aio import DefaultAzureCredential -from fastapi import HTTPException -from semantic_kernel.agents import AzureAIAgent # pylint: disable=E0611 -from semantic_kernel.contents import AuthorRole -from semantic_kernel.exceptions.service_exceptions import ServiceResponseException -from api.status_updates import send_status_update from common.models.api import ( FileProcessUpdate, FileRecord, @@ -22,6 +19,15 @@ ) from common.services.batch_service import BatchService from common.storage.blob_factory import BlobStorageFactory + +from fastapi import HTTPException + + +from semantic_kernel.agents import AzureAIAgent # pylint: disable=E0611 +from semantic_kernel.contents import AuthorRole +from semantic_kernel.exceptions.service_exceptions import ServiceResponseException + + from sql_agents.agents.agent_config import AgentBaseConfig from sql_agents.convert_script import convert_script from sql_agents.helpers.agents_manager import SqlAgents From 5f7c2524a98c0237337489948f9ad74a867e5f7a Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Tue, 15 Apr 2025 15:00:05 +0530 Subject: [PATCH 19/47] fix: pylint fix --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 0df06ab8..6401ce97 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ max-line-length = 88 extend-ignore = E501 exclude = .venv, frontend -ignore = E203, W503, G004, G200,B008,ANN,D100,D101,D102,D103,D104,D105,D106,D107 \ No newline at end of file +ignore = E203, W503, G004, G200,B008,ANN,D \ No newline at end of file From 6f1032ade700fff4427e4ab3a0492a3eed1cae85 Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Tue, 15 Apr 2025 15:12:18 +0530 Subject: [PATCH 20/47] fix: pylint fixes --- .flake8 | 2 +- src/backend/sql_agents/helpers/agents_manager.py | 5 +++-- src/backend/sql_agents/helpers/comms_manager.py | 3 +-- src/backend/sql_agents/process_batch.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.flake8 b/.flake8 index 6401ce97..51f3adb3 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ max-line-length = 88 extend-ignore = E501 exclude = .venv, frontend -ignore = E203, W503, G004, G200,B008,ANN,D \ No newline at end of file +ignore = E203, W503, G004, G200,B008,ANN,D100,D101,D102,D103,D104,D105,D106,D107,D205,D400,D401,D200 \ No newline at end of file diff --git a/src/backend/sql_agents/helpers/agents_manager.py b/src/backend/sql_agents/helpers/agents_manager.py index a3244ff3..8767b796 100644 --- a/src/backend/sql_agents/helpers/agents_manager.py +++ b/src/backend/sql_agents/helpers/agents_manager.py @@ -33,7 +33,8 @@ def __init__(self): @classmethod async def create(cls, config: AgentBaseConfig): """Create the SQL agents for migration. - Required as init cannot be async""" + Required as init cannot be async + """ self = cls() # Create an instance try: self.agent_config = config @@ -71,7 +72,7 @@ def idx_agents(self): } async def delete_agents(self): - """cleans up the agents from Azure Foundry""" + """Cleans up the agents from Azure Foundry""" try: for agent in self.agents: await self.agent_config.ai_project_client.agents.delete_agent(agent.id) diff --git a/src/backend/sql_agents/helpers/comms_manager.py b/src/backend/sql_agents/helpers/comms_manager.py index 2a006296..d465ef07 100644 --- a/src/backend/sql_agents/helpers/comms_manager.py +++ b/src/backend/sql_agents/helpers/comms_manager.py @@ -20,8 +20,7 @@ class SelectionStrategy(SequentialSelectionStrategy): # Select the next agent that should take the next turn in the chat async def select_agent(self, agents, history): - """ "Check which agent should take the next turn in the chat.""" - + """Check which agent should take the next turn in the chat.""" match history[-1].name: case AgentType.MIGRATOR.value: # The Migrator should go first diff --git a/src/backend/sql_agents/process_batch.py b/src/backend/sql_agents/process_batch.py index 9f4918a0..132c574f 100644 --- a/src/backend/sql_agents/process_batch.py +++ b/src/backend/sql_agents/process_batch.py @@ -42,7 +42,7 @@ async def process_batch_async( batch_id: str, convert_from: str = "informix", convert_to: str = "tsql" ): - """central batch processing function to process each file in the batch""" + """Central batch processing function to process each file in the batch""" logger.info("Processing batch: %s", batch_id) storage = await BlobStorageFactory.get_storage() batch_service = BatchService() @@ -160,7 +160,7 @@ async def process_batch_async( async def process_error( ex: Exception, file_record: FileRecord, batch_service: BatchService ): - """insert data base write to file record stating invalid file and send ws notification""" + """Insert data base write to file record stating invalid file and send ws notification""" await batch_service.create_file_log( file_id=str(file_record.file_id), description=f"Error processing file {ex}", From 13c7916570aba0e0547b0a34bb924f734b6a8573 Mon Sep 17 00:00:00 2001 From: "Vishal Shinde (Persistent Systems Inc)" Date: Tue, 15 Apr 2025 18:28:03 +0530 Subject: [PATCH 21/47] resolved bug:15142 --- src/backend/api/api_routes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/api/api_routes.py b/src/backend/api/api_routes.py index a233bd26..998fda93 100644 --- a/src/backend/api/api_routes.py +++ b/src/backend/api/api_routes.py @@ -2,6 +2,7 @@ import asyncio import io +from typing import Optional import zipfile from api.auth.auth_utils import get_authenticated_user @@ -800,7 +801,7 @@ async def delete_all_details(request: Request): @router.get("/batch-history") -async def list_batch_history(request: Request, offset: int = 0, limit: int = 25): +async def list_batch_history(request: Request, offset: int = 0,limit: Optional[int] = None): """ Retrieve batch processing history for the authenticated user. From 397ac230781f0a40fac9f5224ad54d11810af0e8 Mon Sep 17 00:00:00 2001 From: "Vishal Shinde (Persistent Systems Inc)" Date: Tue, 15 Apr 2025 18:34:13 +0530 Subject: [PATCH 22/47] pylint issue fix --- src/backend/api/api_routes.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/backend/api/api_routes.py b/src/backend/api/api_routes.py index 998fda93..35265fd8 100644 --- a/src/backend/api/api_routes.py +++ b/src/backend/api/api_routes.py @@ -2,8 +2,9 @@ import asyncio import io -from typing import Optional import zipfile +from typing import Optional + from api.auth.auth_utils import get_authenticated_user from api.status_updates import app_connection_manager, close_connection @@ -801,7 +802,7 @@ async def delete_all_details(request: Request): @router.get("/batch-history") -async def list_batch_history(request: Request, offset: int = 0,limit: Optional[int] = None): +async def list_batch_history(request: Request, offset: int = 0, limit: Optional[int] = None): """ Retrieve batch processing history for the authenticated user. From fdc64812459d86a38b388bce9d2a618dcaef3b74 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Tue, 15 Apr 2025 20:08:34 +0530 Subject: [PATCH 23/47] updated batch_service_test.py --- .../common/services/batch_service_test.py | 865 +++++++++++++++--- 1 file changed, 761 insertions(+), 104 deletions(-) diff --git a/src/tests/backend/common/services/batch_service_test.py b/src/tests/backend/common/services/batch_service_test.py index e5efa561..dc97dfff 100644 --- a/src/tests/backend/common/services/batch_service_test.py +++ b/src/tests/backend/common/services/batch_service_test.py @@ -1,147 +1,804 @@ -import pytest +from io import BytesIO from unittest.mock import AsyncMock, MagicMock, patch from uuid import uuid4 -from datetime import datetime -from fastapi import UploadFile, HTTPException +from common.models.api import AgentType, AuthorRole, BatchRecord, FileResult, LogType, ProcessStatus from common.services.batch_service import BatchService -from common.models.api import ( - FileRecord, - BatchRecord, - FileResult, - LogType, - AgentType, - ProcessStatus, -) - -# ---------- Helpers ---------- -def make_file_record(**overrides): - return FileRecord( - file_id=overrides.get("file_id", "file1"), - batch_id=overrides.get("batch_id", "batch123"), - original_name=overrides.get("original_name", "file.txt"), - blob_path=overrides.get("blob_path", "blob/path/file.txt"), - translated_path=overrides.get("translated_path", "translated/file.txt"), - status=overrides.get("status", ProcessStatus.READY_TO_PROCESS), - error_count=overrides.get("error_count", 0), - syntax_count=overrides.get("syntax_count", 0), - created_at=overrides.get("created_at", datetime.utcnow()), - updated_at=overrides.get("updated_at", datetime.utcnow()) - ) -def make_batch_record(**overrides): - return BatchRecord( - batch_id=overrides.get("batch_id", "batch123"), - user_id=overrides.get("user_id", "user1"), - file_count=overrides.get("file_count", 1), - created_at=overrides.get("created_at", datetime.utcnow().isoformat()), - updated_at=overrides.get("updated_at", datetime.utcnow().isoformat()), - status=overrides.get("status", ProcessStatus.READY_TO_PROCESS), - ) +from fastapi import HTTPException, UploadFile + +import pytest + +import pytest_asyncio + -# ---------- Fixtures ---------- @pytest.fixture +def mock_service(mocker): + service = BatchService() + service.logger = mocker.Mock() + service.database = MagicMock() + + return service + + +@pytest_asyncio.fixture +async def service(): + svc = BatchService() + svc.logger = MagicMock() + return svc + + def batch_service(): + service = BatchService() # Correct constructor + service.database = MagicMock() # Inject mock database + return service + + +@pytest.mark.asyncio +@patch("common.services.batch_service.DatabaseFactory.get_database", new_callable=AsyncMock) +async def test_initialize_database(mock_get_db, service): + mock_db = AsyncMock() + mock_get_db.return_value = mock_db + await service.initialize_database() + assert service.database == mock_db + + +@pytest.mark.asyncio +async def test_get_batch_found(service): + service.database = AsyncMock() + batch_id = uuid4() + user_id = "user123" + service.database.get_batch.return_value = {"id": str(batch_id)} + service.database.get_batch_files.return_value = [{"file_id": "f1"}] + result = await service.get_batch(batch_id, user_id) + assert result["batch"] == {"id": str(batch_id)} + assert result["files"] == [{"file_id": "f1"}] + + +@pytest.mark.asyncio +async def test_get_batch_not_found(service): + service.database = AsyncMock() + batch_id = uuid4() + user_id = "user123" + service.database.get_batch.return_value = None + result = await service.get_batch(batch_id, user_id) + assert result is None + + +@pytest.mark.asyncio +async def test_get_file_found(service): + service.database = AsyncMock() + service.database.get_file.return_value = {"file_id": "file123"} + result = await service.get_file("file123") + assert result == {"file": {"file_id": "file123"}} + + +@pytest.mark.asyncio +async def test_get_file_not_found(service): + service.database = AsyncMock() + service.database.get_file.return_value = None + result = await service.get_file("notfound") + assert result is None + + +@pytest.mark.asyncio +@patch("common.services.batch_service.BlobStorageFactory.get_storage", new_callable=AsyncMock) +@patch("common.models.api.FileRecord.fromdb") +@patch("common.models.api.BatchRecord.fromdb") +async def test_get_file_report_success(mock_batch_fromdb, mock_file_fromdb, mock_get_storage, service): + service.database = AsyncMock() + file_id = "file123" + mock_file = {"batch_id": uuid4(), "translated_path": "some/path"} + mock_batch = {"batch_id": "batch123"} + mock_logs = [{"log": "log1"}] + mock_translated = "translated content" + service.database.get_file.return_value = mock_file + service.database.get_batch_from_id.return_value = mock_batch + service.database.get_file_logs.return_value = mock_logs + mock_file_fromdb.return_value = MagicMock(dict=lambda: mock_file, batch_id=mock_file["batch_id"], translated_path="some/path") + mock_batch_fromdb.return_value = MagicMock(dict=lambda: mock_batch) + mock_storage = AsyncMock() + mock_storage.get_file.return_value = mock_translated + mock_get_storage.return_value = mock_storage + result = await service.get_file_report(file_id) + assert result["file"] == mock_file + assert result["batch"] == mock_batch + assert result["logs"] == mock_logs + assert result["translated_content"] == mock_translated + + +@pytest.mark.asyncio +@patch("common.services.batch_service.BlobStorageFactory.get_storage", new_callable=AsyncMock) +async def test_get_file_translated_success(mock_get_storage, service): + file = {"translated_path": "some/path"} + mock_storage = AsyncMock() + mock_storage.get_file.return_value = "translated" + mock_get_storage.return_value = mock_storage + result = await service.get_file_translated(file) + assert result == "translated" + + +@pytest.mark.asyncio +@patch("common.services.batch_service.BlobStorageFactory.get_storage", new_callable=AsyncMock) +async def test_get_file_translated_error(mock_get_storage, service): + file = {"translated_path": "some/path"} + mock_storage = AsyncMock() + mock_storage.get_file.side_effect = IOError("Failed to download") + mock_get_storage.return_value = mock_storage + result = await service.get_file_translated(file) + assert result == "" + + +@pytest.mark.asyncio +async def test_get_batch_for_zip(service): + service.database = AsyncMock() + service.get_file_translated = AsyncMock(return_value="file-content") + service.database.get_batch_files.return_value = [ + {"original_name": "doc1.txt", "translated_path": "path1"}, + {"original_name": "doc2.txt", "translated_path": "path2"}, + ] + result = await service.get_batch_for_zip("batch1") + assert len(result) == 2 + assert result[0][0] == "rslt_doc1.txt" + assert result[0][1] == "file-content" + + +@pytest.mark.asyncio +@patch("common.models.api.BatchRecord.fromdb") +async def test_get_batch_summary_success(mock_batch_fromdb, service): + service.database = AsyncMock() + mock_batch = {"batch_id": "batch1"} + mock_batch_record = MagicMock(dict=lambda: {"batch_id": "batch1"}) + mock_batch_fromdb.return_value = mock_batch_record + service.database.get_batch.return_value = mock_batch + service.database.get_batch_files.return_value = [ + {"file_id": "file1", "translated_path": "path1"}, + {"file_id": "file2", "translated_path": None}, + ] + service.database.get_file_logs.return_value = ["log1"] + service.get_file_translated = AsyncMock(return_value="translated") + result = await service.get_batch_summary("batch1", "user1") + assert "files" in result + assert "batch" in result + assert result["files"][0]["logs"] == ["log1"] + assert result["files"][0]["translated_content"] == "translated" + + +@pytest.mark.asyncio +async def test_batch_zip_with_no_files(service): + service.database = AsyncMock() + service.database.get_batch_files.return_value = [] + service.get_file_translated = AsyncMock() + result = await service.get_batch_for_zip("batch_empty") + assert result == [] + + +def test_is_valid_uuid(): + service = BatchService() + valid = str(uuid4()) + invalid = "not-a-uuid" + assert service.is_valid_uuid(valid) + assert not service.is_valid_uuid(invalid) + + +def test_generate_file_path(): + service = BatchService() + path = service.generate_file_path("batch1", "user1", "file1", "test@file.pdf") + assert path == "user1/batch1/file1/test_file.pdf" + + +@pytest.mark.asyncio +async def test_delete_batch_existing(): service = BatchService() - service.logger = MagicMock() service.database = AsyncMock() - return service + batch_id = uuid4() + service.database.get_batch.return_value = {"id": str(batch_id)} + service.database.delete_batch.return_value = None + result = await service.delete_batch(batch_id, "user1") + assert result["message"] == "Batch deleted successfully" + assert result["batch_id"] == str(batch_id) + -# ---------- Tests ---------- @pytest.mark.asyncio -async def test_get_batch_success(batch_service): +async def test_delete_file_success(): + service = BatchService() + service.database = AsyncMock() + file_id = uuid4() + batch_id = uuid4() + mock_file = MagicMock() + mock_file.batch_id = batch_id + mock_file.blob_path = "some/path/file.pdf" + mock_file.translated_path = "some/path/file_translated.pdf" + with patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage: + mock_storage.return_value.delete_file.return_value = True + service.database.get_file.return_value = mock_file + service.database.get_batch.return_value = {"id": str(batch_id)} + service.database.get_batch_files.return_value = [1, 2] + with patch("common.models.api.FileRecord.fromdb", return_value=mock_file), \ + patch("common.models.api.BatchRecord.fromdb") as mock_batch_record: + mock_record = MagicMock() + mock_record.file_count = 1 + service.database.update_batch.return_value = None + mock_batch_record.return_value = mock_record + result = await service.delete_file(file_id, "user1") + assert result["message"] == "File deleted successfully" + assert result["file_id"] == str(file_id) + + +@pytest.mark.asyncio +async def test_upload_file_to_batch_dict_batch(): + service = BatchService() + service.database = AsyncMock() + file = UploadFile(filename="hello@file.txt", file=BytesIO(b"test content")) + batch_id = str(uuid4()) + file_id = str(uuid4()) + with patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage, \ + patch("uuid.uuid4", return_value=file_id), \ + patch("common.models.api.FileRecord.fromdb", return_value={"blob_path": "path"}): + + mock_storage.return_value.upload_file.return_value = None + service.database.get_batch.side_effect = [None, {"file_count": 0}] + service.database.create_batch.return_value = {} + service.database.get_batch_files.return_value = ["file1", "file2"] + service.database.get_file.return_value = {"filename": file.filename} + service.database.update_batch_entry.return_value = {"batch_id": batch_id, "file_count": 2} + result = await service.upload_file_to_batch(batch_id, "user1", file) + assert "batch" in result + assert "file" in result + + +@pytest.mark.asyncio +async def test_upload_file_to_batch_invalid_storage(): + service = BatchService() + service.database = AsyncMock() + file = UploadFile(filename="file.txt", file=BytesIO(b"data")) + with patch("common.storage.blob_factory.BlobStorageFactory.get_storage", return_value=None): + with pytest.raises(RuntimeError) as exc_info: + await service.upload_file_to_batch(str(uuid4()), "user1", file) + # Check outer exception message + assert str(exc_info.value) == "File upload failed" + + # Check original cause of the exception + assert isinstance(exc_info.value.__cause__, RuntimeError) + assert str(exc_info.value.__cause__) == "Storage service not initialized" + + +@pytest.mark.asyncio +async def test_delete_batch_success(mock_service): batch_id = uuid4() user_id = "test_user" - batch_service.database.get_batch.return_value = {"batch_id": str(batch_id)} - batch_service.database.get_batch_files.return_value = [{"file_id": "f1"}] + mock_service.database.get_batch = AsyncMock(return_value={"id": str(batch_id)}) + mock_service.database.delete_batch = AsyncMock() + result = await mock_service.delete_batch(batch_id, user_id) + assert result["message"] == "Batch deleted successfully" + assert result["batch_id"] == str(batch_id) + + +def test_is_valid_uuid_valid(mock_service): + assert mock_service.is_valid_uuid(str(uuid4())) is True + + +def test_is_valid_uuid_invalid(mock_service): + assert mock_service.is_valid_uuid("not-a-uuid") is False + + +def test_generate_file_path_only_filename(): + service = BatchService() + path = service.generate_file_path(None, None, None, "weird@name!.txt") + assert path.endswith("weird_name_.txt") + + +def test_is_valid_uuid_empty_string(): + service = BatchService() + assert not service.is_valid_uuid("") + + +def test_is_valid_uuid_partial_uuid(): + service = BatchService() + assert not service.is_valid_uuid("1234abcd") - result = await batch_service.get_batch(batch_id, user_id) - assert result["batch"]["batch_id"] == str(batch_id) - assert result["files"] == [{"file_id": "f1"}] @pytest.mark.asyncio -async def test_get_file_not_found(batch_service): - batch_service.database.get_file.return_value = None - result = await batch_service.get_file("missing_file_id") +async def test_delete_file_file_not_found(): + service = BatchService() + service.database = AsyncMock() + file_id = str(uuid4()) + + service.database.get_file.return_value = None + result = await service.delete_file(file_id, "user1") assert result is None -def test_is_valid_uuid_valid(batch_service): - assert batch_service.is_valid_uuid(str(uuid4())) is True -def test_generate_file_path(batch_service): - path = batch_service.generate_file_path("batch1", "user1", "file1", "file@.txt") - assert path == "user1/batch1/file1/file_.txt" +@pytest.mark.asyncio +async def test_upload_file_to_batch_storage_upload_fails(): + service = BatchService() + service.database = AsyncMock() + file = UploadFile(filename="test.txt", file=BytesIO(b"abc")) + file_id = str(uuid4()) + + with patch("common.storage.blob_factory.BlobStorageFactory.get_storage") as mock_get_storage, \ + patch("uuid.uuid4", return_value=file_id): + mock_storage = AsyncMock() + mock_storage.upload_file.side_effect = RuntimeError("upload failed") + mock_get_storage.return_value = mock_storage + + service.database.get_batch.side_effect = [None, {"file_count": 0}] + service.database.create_batch.return_value = {} + service.database.get_batch_files.return_value = [] + service.database.update_batch_entry.return_value = {} + + with pytest.raises(RuntimeError, match="File upload failed"): + await service.upload_file_to_batch("batch123", "user1", file) + + @pytest.mark.asyncio + async def test_update_file_counts_success(service): + service.database = AsyncMock() + file_id = str(uuid4()) + mock_file = {"file_id": file_id} + mock_logs = [ + {"log_type": LogType.ERROR.value}, + {"log_type": LogType.WARNING.value}, + {"log_type": LogType.WARNING.value}, + ] + service.database.get_file.return_value = mock_file + service.database.get_file_logs.return_value = mock_logs + with patch("common.models.api.FileRecord.fromdb", return_value=MagicMock()) as mock_file_record: + await service.update_file_counts(file_id) + mock_file_record.assert_called_once() + service.database.update_file.assert_called_once() + + @pytest.mark.asyncio + async def test_update_file_counts_no_logs(service): + service.database = AsyncMock() + file_id = str(uuid4()) + mock_file = {"file_id": file_id} + service.database.get_file.return_value = mock_file + service.database.get_file_logs.return_value = [] + with patch("common.models.api.FileRecord.fromdb", return_value=MagicMock()) as mock_file_record: + await service.update_file_counts(file_id) + mock_file_record.assert_called_once() + service.database.update_file.assert_called_once() + + @pytest.mark.asyncio + async def test_get_file_counts_success(service): + service.database = AsyncMock() + file_id = str(uuid4()) + mock_logs = [ + {"log_type": LogType.ERROR.value}, + {"log_type": LogType.WARNING.value}, + {"log_type": LogType.WARNING.value}, + ] + service.database.get_file_logs.return_value = mock_logs + error_count, syntax_count = await service.get_file_counts(file_id) + assert error_count == 1 + assert syntax_count == 2 + + @pytest.mark.asyncio + async def test_get_file_counts_no_logs(service): + service.database = AsyncMock() + file_id = str(uuid4()) + service.database.get_file_logs.return_value = [] + error_count, syntax_count = await service.get_file_counts(file_id) + assert error_count == 0 + assert syntax_count == 0 + + @pytest.mark.asyncio + async def test_get_batch_history_success(service): + service.database = AsyncMock() + user_id = "user123" + mock_history = [{"batch_id": "batch1"}, {"batch_id": "batch2"}] + service.database.get_batch_history.return_value = mock_history + result = await service.get_batch_history(user_id, limit=10, offset=0) + assert result == mock_history + + @pytest.mark.asyncio + async def test_get_batch_history_no_history(service): + service.database = AsyncMock() + user_id = "user123" + service.database.get_batch_history.return_value = [] + result = await service.get_batch_history(user_id, limit=10, offset=0) + assert result == [] + + @pytest.mark.asyncio + @patch("common.services.batch_service.DatabaseFactory.get_database", new_callable=AsyncMock) + async def test_initialize_database_success(mock_get_database, service): + # Arrange + mock_database = AsyncMock() + mock_get_database.return_value = mock_database + + # Act + await service.initialize_database() + + # Assert + assert service.database == mock_database + mock_get_database.assert_called_once() + + @pytest.mark.asyncio + @patch("common.services.batch_service.DatabaseFactory.get_database", new_callable=AsyncMock) + async def test_initialize_database_failure(mock_get_database, service): + # Arrange + mock_get_database.side_effect = RuntimeError("Database initialization failed") + + # Act & Assert + with pytest.raises(RuntimeError, match="Database initialization failed"): + await service.initialize_database() + mock_get_database.assert_called_once() + + @pytest.mark.asyncio + @patch("common.services.batch_service.DatabaseFactory.get_database", new_callable=AsyncMock) + async def test_initialize_database_success(mock_get_database, service): + # Arrange + mock_database = AsyncMock() + mock_get_database.return_value = mock_database + + # Act + await service.initialize_database() + + # Assert + assert service.database == mock_database + mock_get_database.assert_called_once() + + @pytest.mark.asyncio + @patch("common.services.batch_service.DatabaseFactory.get_database", new_callable=AsyncMock) + async def test_initialize_database_failure(mock_get_database, service): + # Arrange + mock_get_database.side_effect = RuntimeError("Database initialization failed") + + # Act & Assert + with pytest.raises(RuntimeError, match="Database initialization failed"): + await service.initialize_database() + mock_get_database.assert_called_once() + @pytest.mark.asyncio -@patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) -async def test_get_file_report_success(mock_storage, batch_service): - file_id = "file1" - file_record = make_file_record(file_id=file_id) - batch_record = make_batch_record(batch_id=file_record.batch_id) +async def test_update_file_success(): + service = BatchService() + service.database = AsyncMock() + file_id = str(uuid4()) + mock_file = {"file_id": file_id} + mock_record = MagicMock() + mock_record.error_count = 0 + mock_record.syntax_count = 0 - batch_service.database.get_file.return_value = file_record.dict() - batch_service.database.get_batch_from_id.return_value = batch_record.dict() - batch_service.database.get_file_logs.return_value = [{"log_type": "INFO"}] + service.database.get_file.return_value = mock_file + with patch("common.models.api.FileRecord.fromdb", return_value=mock_record): + await service.update_file(file_id, ProcessStatus.COMPLETED, FileResult.SUCCESS, 1, 2) + assert mock_record.error_count == 1 + assert mock_record.syntax_count == 2 + service.database.update_file.assert_called_once() - with patch("common.models.api.FileRecord.fromdb", return_value=file_record), \ - patch("common.models.api.BatchRecord.fromdb", return_value=batch_record), \ - patch.object(mock_storage, "get_file", new=AsyncMock(return_value="translated content")): - result = await batch_service.get_file_report(file_id) - assert result["translated_content"] == "translated content" +@pytest.mark.asyncio +async def test_update_file_record(): + service = BatchService() + service.database = AsyncMock() + mock_file_record = MagicMock() + await service.update_file_record(mock_file_record) + service.database.update_file.assert_called_once_with(mock_file_record) + @pytest.mark.asyncio -@patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) -async def test_upload_file_to_batch_creates_batch(mock_storage, batch_service): +async def test_create_file_log(): + service = BatchService() + service.database = AsyncMock() + file_id = str(uuid4()) + await service.create_file_log( + file_id=file_id, + description="test log", + last_candidate="candidate", + log_type=LogType.SUCCESS, + agent_type=AgentType.HUMAN, + author_role=AuthorRole.USER + ) + service.database.add_file_log.assert_called_once() + + +@pytest.mark.asyncio +async def test_update_batch_success(): + service = BatchService() + service.database = AsyncMock() batch_id = str(uuid4()) - user_id = "test_user" - filename = "doc.txt" - file_mock = MagicMock(spec=UploadFile) - file_mock.filename = filename - file_mock.content_type = "text/plain" - file_mock.read = AsyncMock(return_value=b"content") + mock_batch = {"batch_id": batch_id} + mock_batch_record = MagicMock() + service.database.get_batch_from_id.return_value = mock_batch + with patch("common.models.api.BatchRecord.fromdb", return_value=mock_batch_record): + await service.update_batch(batch_id, ProcessStatus.COMPLETED) + service.database.update_batch.assert_called_once_with(mock_batch_record) - # Simulate batch creation - batch_service.database.get_batch.return_value = None - batch_service.database.create_batch.return_value = {"batch_id": batch_id} - batch_service.database.get_batch_files.return_value = [{"file_id": "f1"}] - batch_service.database.get_file.return_value = {"file_id": "new_id"} - mock_storage.upload_file.return_value = None +@pytest.mark.asyncio +async def test_delete_batch_and_files_success(): + service = BatchService() + service.database = AsyncMock() + batch_id = str(uuid4()) + user_id = "user" + mock_file = MagicMock() + mock_file.file_id = uuid4() + mock_file.blob_path = "blob/file" + mock_file.translated_path = "blob/translated" + service.database.get_batch.return_value = {"batch_id": batch_id} + service.database.get_batch_files.return_value = [mock_file] + + with patch("common.models.api.FileRecord.fromdb", return_value=mock_file), \ + patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage: + mock_storage.return_value.delete_file.return_value = True + result = await service.delete_batch_and_files(batch_id, user_id) + assert result["message"] == "Files deleted successfully" - file_record = make_file_record(file_id="new_id", batch_id=batch_id) - with patch("common.models.api.FileRecord.fromdb", return_value=file_record), \ - patch("uuid.uuid4", return_value=uuid4()): +@pytest.mark.asyncio +async def test_batch_files_final_update(): + service = BatchService() + service.database = AsyncMock() + file_id = str(uuid4()) + file = { + "file_id": file_id, + "translated_path": "", + "status": "IN_PROGRESS" + } + service.database.get_batch_files.return_value = [file] + with patch("common.models.api.FileRecord.fromdb", return_value=MagicMock(file_id=file_id, translated_path="", status=None)), \ + patch.object(service, "get_file_counts", return_value=(1, 1)), \ + patch.object(service, "create_file_log", new_callable=AsyncMock), \ + patch.object(service, "update_file_record", new_callable=AsyncMock): + await service.batch_files_final_update("batch1") - result = await batch_service.upload_file_to_batch(batch_id, user_id, file_mock) - assert "file" in result + +@pytest.mark.asyncio +async def test_delete_all_from_storage_cosmos_success(): + service = BatchService() + service.database = AsyncMock() + user_id = "user123" + file_id = str(uuid4()) + batch_id = str(uuid4()) + mock_file = { + "translated_path": "translated/path" + } + + service.get_all_batches = AsyncMock(return_value=[{"batch_id": batch_id}]) + service.database.get_file.return_value = mock_file + service.database.list_files = AsyncMock(return_value=[{"name": f"user/{batch_id}/{file_id}/file.txt"}]) + + with patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage: + mock_storage.return_value.list_files.return_value = [{"name": f"user/{batch_id}/{file_id}/file.txt"}] + mock_storage.return_value.delete_file.return_value = True + result = await service.delete_all_from_storage_cosmos(user_id) + assert result["message"] == "All user data deleted successfully" + + +@pytest.mark.asyncio +async def test_create_candidate_success(): + service = BatchService() + service.database = AsyncMock() + file_id = str(uuid4()) + batch_id = str(uuid4()) + user_id = "user123" + mock_file = {"batch_id": batch_id, "original_name": "doc.txt"} + mock_batch = {"user_id": user_id} + + with patch("common.models.api.FileRecord.fromdb", return_value=MagicMock(original_name="doc.txt", batch_id=batch_id)), \ + patch("common.models.api.BatchRecord.fromdb", return_value=MagicMock(user_id=user_id)), \ + patch.object(service, "get_file_counts", return_value=(0, 1)), \ + patch.object(service, "update_file_record", new_callable=AsyncMock), \ + patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage: + + mock_storage.return_value.upload_file.return_value = None + service.database.get_file.return_value = mock_file + service.database.get_batch_from_id.return_value = mock_batch + await service.create_candidate(file_id, "Some content") + + +@pytest.mark.asyncio +async def test_batch_files_final_update_success_path(): + service = BatchService() + service.database = AsyncMock() + file_id = str(uuid4()) + file = { + "file_id": file_id, + "translated_path": "some/path", + "status": "IN_PROGRESS" + } + + mock_file_record = MagicMock(translated_path="some/path", file_id=file_id) + service.database.get_batch_files.return_value = [file] + + with patch("common.models.api.FileRecord.fromdb", return_value=mock_file_record), \ + patch.object(service, "update_file_record", new_callable=AsyncMock): + await service.batch_files_final_update("batch123") + + +@pytest.mark.asyncio +async def test_get_file_counts_logs_none(): + service = BatchService() + service.database = AsyncMock() + service.database.get_file_logs.return_value = None + error_count, syntax_count = await service.get_file_counts("file_id") + assert error_count == 0 + assert syntax_count == 0 + + +@pytest.mark.asyncio +async def test_create_candidate_upload_error(): + service = BatchService() + service.database = AsyncMock() + file_id = str(uuid4()) + mock_file = {"batch_id": str(uuid4()), "original_name": "doc.txt"} + mock_batch = {"user_id": "user1"} + + with patch("common.models.api.FileRecord.fromdb", return_value=MagicMock(original_name="doc.txt", batch_id=mock_file["batch_id"])), \ + patch("common.models.api.BatchRecord.fromdb", return_value=MagicMock(user_id="user1")), \ + patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage, \ + patch.object(service, "get_file_counts", return_value=(1, 1)), \ + patch.object(service, "update_file_record", new_callable=AsyncMock): + + mock_storage.return_value.upload_file.side_effect = Exception("Upload fail") + service.database.get_file.return_value = mock_file + service.database.get_batch_from_id.return_value = mock_batch + + await service.create_candidate(file_id, "candidate content") + + +@pytest.mark.asyncio +async def test_get_batch_history_failure(): + service = BatchService() + service.logger = MagicMock() + service.database = AsyncMock() + + service.database.get_batch_history.side_effect = RuntimeError("DB failure") + + with pytest.raises(RuntimeError, match="Error retrieving batch history"): + await service.get_batch_history("user1", limit=5, offset=0) + + +@pytest.mark.asyncio +async def test_delete_file_logs_exception(): + service = BatchService() + service.database = AsyncMock() + file_id = str(uuid4()) + batch_id = str(uuid4()) + mock_file = MagicMock() + mock_file.batch_id = batch_id + mock_file.blob_path = "blob" + mock_file.translated_path = "translated" + with patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage: + mock_storage.return_value.delete_file.return_value = True + service.database.get_file.return_value = mock_file + service.database.get_batch.return_value = {"id": str(batch_id)} + service.database.get_batch_files.return_value = [1, 2] + + with patch("common.models.api.FileRecord.fromdb", return_value=mock_file), \ + patch("common.models.api.BatchRecord.fromdb") as mock_batch_record: + mock_record = MagicMock() + mock_record.file_count = 2 + mock_batch_record.return_value = mock_record + service.database.update_batch.side_effect = Exception("Update failed") + + result = await service.delete_file(file_id, "user1") + assert result["message"] == "File deleted successfully" + + +@pytest.mark.asyncio +async def test_upload_file_to_batch_batchrecord(): + service = BatchService() + service.database = AsyncMock() + file = UploadFile(filename="test.txt", file=BytesIO(b"test content")) + batch_id = str(uuid4()) + file_id = str(uuid4()) + + # Create a mock BatchRecord instance + mock_batch_record = MagicMock(spec=BatchRecord) + mock_batch_record.file_count = 0 + mock_batch_record.updated_at = None + + with patch("uuid.uuid4", return_value=file_id), \ + patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage, \ + patch("common.models.api.FileRecord.fromdb", return_value={"blob_path": "blob/path"}), \ + patch("common.models.api.BatchRecord.fromdb", return_value=mock_batch_record): + + mock_storage.return_value.upload_file.return_value = None + # This will trigger the BatchRecord path + service.database.get_batch.side_effect = [mock_batch_record] + service.database.get_batch_files.return_value = ["file1", "file2"] + service.database.get_file.return_value = {"file_id": file_id} + service.database.update_batch_entry.return_value = mock_batch_record + + result = await service.upload_file_to_batch(batch_id, "user1", file) assert "batch" in result + assert "file" in result + @pytest.mark.asyncio -@patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) -async def test_delete_batch_and_files_batch_not_found(mock_storage, batch_service): - batch_service.database.get_batch.return_value = None - result = await batch_service.delete_batch_and_files("batch123", "user1") - assert result["message"] == "Batch not found" +async def test_upload_file_to_batch_unknown_type(): + service = BatchService() + service.database = AsyncMock() + file = UploadFile(filename="file.txt", file=BytesIO(b"data")) + file_id = str(uuid4()) + + with patch("uuid.uuid4", return_value=file_id), \ + patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage, \ + patch("common.models.api.FileRecord.fromdb", return_value={"blob_path": "path"}): + + mock_storage.return_value.upload_file.return_value = None + service.database.get_batch.side_effect = [object()] # Unknown type + service.database.get_batch_files.return_value = [] + service.database.get_file.return_value = {"file_id": file_id} + + with pytest.raises(RuntimeError, match="File upload failed"): + await service.upload_file_to_batch("batch123", "user1", file) + @pytest.mark.asyncio -async def test_update_file_not_found(batch_service): - batch_service.database.get_file.return_value = None - with pytest.raises(HTTPException) as exc_info: - await batch_service.update_file("file123", ProcessStatus.COMPLETED, FileResult.SUCCESS, 1, 2) - assert exc_info.value.status_code == 404 +@patch("common.services.batch_service.BlobStorageFactory.get_storage", new_callable=AsyncMock) +@patch("common.models.api.FileRecord.fromdb") +@patch("common.models.api.BatchRecord.fromdb") +async def test_get_file_report_ioerror(mock_batch_fromdb, mock_file_fromdb, mock_get_storage): + service = BatchService() + service.database = AsyncMock() + file_id = "file123" + mock_file = {"batch_id": uuid4(), "translated_path": "some/path"} + mock_batch = {"batch_id": "batch123"} + mock_logs = [{"log": "log1"}] + + mock_file_fromdb.return_value = MagicMock(dict=lambda: mock_file, batch_id=mock_file["batch_id"], translated_path="some/path") + mock_batch_fromdb.return_value = MagicMock(dict=lambda: mock_batch) + service.database.get_file.return_value = mock_file + service.database.get_batch_from_id.return_value = mock_batch + service.database.get_file_logs.return_value = mock_logs + + mock_storage = AsyncMock() + mock_storage.get_file.side_effect = IOError("Boom") + mock_get_storage.return_value = mock_storage + + result = await service.get_file_report(file_id) + assert result["translated_content"] == "" + @pytest.mark.asyncio -async def test_batch_files_final_update_with_error_log(batch_service): +@patch("common.models.api.BatchRecord.fromdb") +async def test_get_batch_summary_log_exception(mock_batch_fromdb): + service = BatchService() + service.database = AsyncMock() + mock_batch = {"batch_id": "batch1"} + mock_batch_record = MagicMock(dict=lambda: {"batch_id": "batch1"}) + mock_batch_fromdb.return_value = mock_batch_record + + service.database.get_batch.return_value = mock_batch + service.database.get_batch_files.return_value = [{"file_id": "file1", "translated_path": None}] + service.database.get_file_logs.side_effect = Exception("DB log fail") + + result = await service.get_batch_summary("batch1", "user1") + assert result["files"][0]["logs"] == [] + + +@pytest.mark.asyncio +async def test_update_file_not_found(): + service = BatchService() + service.database = AsyncMock() + service.database.get_file.return_value = None + with pytest.raises(HTTPException) as exc: + await service.update_file("invalid_id", ProcessStatus.COMPLETED, FileResult.SUCCESS, 0, 0) + assert exc.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_create_candidate_success_flow(): + service = BatchService() + service.database = AsyncMock() file_id = str(uuid4()) - file_record = make_file_record(file_id=file_id, translated_path=None, status=ProcessStatus.IN_PROGRESS) + batch_id = str(uuid4()) + user_id = "user1" + + mock_file = {"batch_id": batch_id, "original_name": "test.txt"} + mock_batch = {"user_id": user_id} + + with patch("common.models.api.FileRecord.fromdb", return_value=MagicMock(original_name="test.txt", batch_id=batch_id)), \ + patch("common.models.api.BatchRecord.fromdb", return_value=MagicMock(user_id=user_id)), \ + patch("common.storage.blob_factory.BlobStorageFactory.get_storage", new_callable=AsyncMock) as mock_storage, \ + patch.object(service, "get_file_counts", return_value=(0, 0)), \ + patch.object(service, "update_file_record", new_callable=AsyncMock): - batch_service.database.get_batch_files.return_value = [file_record.dict()] - batch_service.get_file_counts = AsyncMock(return_value=(1, 0)) - batch_service.update_file_record = AsyncMock() - batch_service.create_file_log = AsyncMock() + service.database.get_file.return_value = mock_file + service.database.get_batch_from_id.return_value = mock_batch + mock_storage.return_value.upload_file.return_value = None - with patch("common.models.api.FileRecord.fromdb", return_value=file_record): - await batch_service.batch_files_final_update("batch1") - batch_service.update_file_record.assert_awaited() + await service.create_candidate(file_id, "candidate content") From ab2e4f02c4434aa557efd4eeb3ea2bfd40de424c Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Thu, 17 Apr 2025 10:32:58 +0530 Subject: [PATCH 24/47] feat: scheduled one click deployment for twice a day --- .github/workflows/deploy.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 16c5f286..cd08d6f7 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -4,6 +4,8 @@ on: push: branches: - main + schedule: + - cron: '0 5,17 * * *' # Runs at 5:00 AM and 5:00 PM GMT jobs: deploy: From 0533dc87a19d93937880d621fe14849bf3ef0e6a Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Thu, 17 Apr 2025 16:19:15 +0530 Subject: [PATCH 25/47] fix: disable purge protection of key vault --- infra/deploy_keyvault.bicep | 2 -- infra/main.json | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/infra/deploy_keyvault.bicep b/infra/deploy_keyvault.bicep index 5222a9f8..ac8102d0 100644 --- a/infra/deploy_keyvault.bicep +++ b/infra/deploy_keyvault.bicep @@ -35,9 +35,7 @@ resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { enabledForDeployment: true enabledForDiskEncryption: true enabledForTemplateDeployment: true - enableSoftDelete: false enableRbacAuthorization: true - enablePurgeProtection: true publicNetworkAccess: 'enabled' sku: { family: 'A' diff --git a/infra/main.json b/infra/main.json index 0a14b568..db20a178 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.34.44.8038", - "templateHash": "13937422806437579370" + "templateHash": "15962798193197746525" } }, "parameters": { @@ -451,7 +451,7 @@ "_generator": { "name": "bicep", "version": "0.34.44.8038", - "templateHash": "10664495342911727649" + "templateHash": "1179876312013038352" } }, "parameters": { @@ -506,7 +506,6 @@ "enabledForTemplateDeployment": true, "enableSoftDelete": false, "enableRbacAuthorization": true, - "enablePurgeProtection": true, "publicNetworkAccess": "enabled", "sku": { "family": "A", From 52e55adc635eee0d98660792bcc362c0acfc549b Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Mon, 21 Apr 2025 16:30:33 +0530 Subject: [PATCH 26/47] feat: updated main.json file --- infra/main.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/infra/main.json b/infra/main.json index db20a178..a077f1a8 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.34.44.8038", - "templateHash": "15962798193197746525" + "templateHash": "2966583538132786271" } }, "parameters": { @@ -451,7 +451,7 @@ "_generator": { "name": "bicep", "version": "0.34.44.8038", - "templateHash": "1179876312013038352" + "templateHash": "7479964703030361933" } }, "parameters": { @@ -504,7 +504,6 @@ "enabledForDeployment": true, "enabledForDiskEncryption": true, "enabledForTemplateDeployment": true, - "enableSoftDelete": false, "enableRbacAuthorization": true, "publicNetworkAccess": "enabled", "sku": { From 1bfbef0678aadcff7fa347abf9c51c0dcf3e481a Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Tue, 22 Apr 2025 11:19:41 +0530 Subject: [PATCH 27/47] added app_test file --- src/tests/backend/app_test.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/tests/backend/app_test.py diff --git a/src/tests/backend/app_test.py b/src/tests/backend/app_test.py new file mode 100644 index 00000000..610e36c3 --- /dev/null +++ b/src/tests/backend/app_test.py @@ -0,0 +1,33 @@ +from backend.app import create_app + +from fastapi import FastAPI + +from httpx import ASGITransport +from httpx import AsyncClient + +import pytest + + +@pytest.fixture +def app() -> FastAPI: + """Fixture to create a test app instance.""" + return create_app() + + +@pytest.mark.asyncio +async def test_health_check(app: FastAPI): + """Test the /health endpoint returns a healthy status.""" + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as ac: + response = await ac.get("/health") + assert response.status_code == 200 + assert response.json() == {"status": "healthy"} + + +@pytest.mark.asyncio +async def test_backend_routes_exist(app: FastAPI): + """Ensure /api routes are available (smoke test).""" + # Check available routes include /api prefix from backend_router + routes = [route.path for route in app.router.routes] + backend_routes = [r for r in routes if r.startswith("/api")] + assert backend_routes, "No backend routes found under /api prefix" From dfe155edd9dc1ce87ff960ae54852077a6ff66b4 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Tue, 22 Apr 2025 12:23:40 +0530 Subject: [PATCH 28/47] editing test workflow --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f245b24..66a60b3e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,6 +6,7 @@ on: - main - dev - demo + - psl-backend-unit-test pull_request: types: - opened From ad088a2c5ad76b45b0ce69e8edf939bed56c3c74 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Tue, 22 Apr 2025 12:28:16 +0530 Subject: [PATCH 29/47] updated batch-service-test --- .../common/services/batch_service_test.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/tests/backend/common/services/batch_service_test.py b/src/tests/backend/common/services/batch_service_test.py index dc97dfff..21fd3a67 100644 --- a/src/tests/backend/common/services/batch_service_test.py +++ b/src/tests/backend/common/services/batch_service_test.py @@ -263,25 +263,6 @@ async def test_upload_file_to_batch_invalid_storage(): assert str(exc_info.value.__cause__) == "Storage service not initialized" -@pytest.mark.asyncio -async def test_delete_batch_success(mock_service): - batch_id = uuid4() - user_id = "test_user" - mock_service.database.get_batch = AsyncMock(return_value={"id": str(batch_id)}) - mock_service.database.delete_batch = AsyncMock() - result = await mock_service.delete_batch(batch_id, user_id) - assert result["message"] == "Batch deleted successfully" - assert result["batch_id"] == str(batch_id) - - -def test_is_valid_uuid_valid(mock_service): - assert mock_service.is_valid_uuid(str(uuid4())) is True - - -def test_is_valid_uuid_invalid(mock_service): - assert mock_service.is_valid_uuid("not-a-uuid") is False - - def test_generate_file_path_only_filename(): service = BatchService() path = service.generate_file_path(None, None, None, "weird@name!.txt") From e8a1de724ee49aa7704f3ce48bc3a46ae741a3ca Mon Sep 17 00:00:00 2001 From: Roopan P M Date: Tue, 22 Apr 2025 16:14:08 +0530 Subject: [PATCH 30/47] main json updated --- infra/main.json | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/infra/main.json b/infra/main.json index e0b0e54a..716e8a79 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.34.44.8038", - "templateHash": "8953208502938265930" + "templateHash": "6290258568261172226" } }, "parameters": { @@ -57,7 +57,7 @@ "uniqueId": "[toLower(uniqueString(subscription().id, parameters('Prefix'), resourceGroup().location))]", "UniquePrefix": "[format('cm{0}', padLeft(take(variables('uniqueId'), 12), 12, '0'))]", "ResourcePrefix": "[take(format('cm{0}{1}', parameters('Prefix'), variables('UniquePrefix')), 15)]", - "imageVersion": "rc1", + "imageVersion": "fnd01", "location": "[resourceGroup().location]", "dblocation": "[resourceGroup().location]", "cosmosdbDatabase": "cmsadb", @@ -349,6 +349,19 @@ "[extensionResourceId(format('/subscriptions/{0}/resourceGroups/{1}', subscription().subscriptionId, resourceGroup().name), 'Microsoft.Resources/deployments', 'deploy_ai_foundry')]" ] }, + { + "type": "Microsoft.Authorization/roleAssignments", + "apiVersion": "2022-04-01", + "scope": "[format('Microsoft.MachineLearningServices/workspaces/{0}', format('{0}-prj', variables('ResourcePrefix')))]", + "name": "[guid(toLower(format('{0}Bck-ca', variables('ResourcePrefix'))), resourceId('Microsoft.MachineLearningServices/workspaces', format('{0}-prj', variables('ResourcePrefix'))), resourceId('Microsoft.Authorization/roleDefinitions', '64702f94-c441-49e6-a78b-ef80e0188fee'))]", + "properties": { + "roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', '64702f94-c441-49e6-a78b-ef80e0188fee')]", + "principalId": "[reference(resourceId('Microsoft.App/containerApps', toLower(format('{0}Bck-ca', variables('ResourcePrefix')))), '2023-05-01', 'full').identity.principalId]" + }, + "dependsOn": [ + "[resourceId('Microsoft.App/containerApps', toLower(format('{0}Bck-ca', variables('ResourcePrefix'))))]" + ] + }, { "type": "Microsoft.Resources/deployments", "apiVersion": "2022-09-01", @@ -374,7 +387,7 @@ "_generator": { "name": "bicep", "version": "0.34.44.8038", - "templateHash": "17863870312619064541" + "templateHash": "107965290127824528" } }, "parameters": { @@ -472,9 +485,7 @@ "_generator": { "name": "bicep", "version": "0.34.44.8038", - "templateHash": "10664495342911727649" - "version": "0.34.44.8038", - "templateHash": "10664495342911727649" + "templateHash": "7479964703030361933" } }, "parameters": { @@ -610,7 +621,7 @@ "_generator": { "name": "bicep", "version": "0.34.44.8038", - "templateHash": "8087543237770345715" + "templateHash": "13939205582736222851" } }, "parameters": { From 70a1ec24c685aa3c6a2fd565959d289689ed879b Mon Sep 17 00:00:00 2001 From: "Kanchan Nagshetti (Persistent Systems Inc)" Date: Wed, 23 Apr 2025 11:21:37 +0530 Subject: [PATCH 31/47] added AzureAIAgent path --- src/backend/sql_agents/helpers/agents_manager.py | 2 +- src/backend/sql_agents/process_batch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/sql_agents/helpers/agents_manager.py b/src/backend/sql_agents/helpers/agents_manager.py index 8767b796..e335a302 100644 --- a/src/backend/sql_agents/helpers/agents_manager.py +++ b/src/backend/sql_agents/helpers/agents_manager.py @@ -2,7 +2,7 @@ import logging -from semantic_kernel.agents import AzureAIAgent # pylint: disable=E0611 +from semantic_kernel.agents.azure_ai import AzureAIAgent # pylint: disable=E0611 from sql_agents.agents.agent_config import AgentBaseConfig from sql_agents.agents.fixer.setup import setup_fixer_agent diff --git a/src/backend/sql_agents/process_batch.py b/src/backend/sql_agents/process_batch.py index 132c574f..177b08f5 100644 --- a/src/backend/sql_agents/process_batch.py +++ b/src/backend/sql_agents/process_batch.py @@ -23,7 +23,7 @@ from fastapi import HTTPException -from semantic_kernel.agents import AzureAIAgent # pylint: disable=E0611 +from semantic_kernel.agents.azure_ai import AzureAIAgent # pylint: disable=E0611 from semantic_kernel.contents import AuthorRole from semantic_kernel.exceptions.service_exceptions import ServiceResponseException From 9ed85849c8776fdfef161ab083e479d726d36ea1 Mon Sep 17 00:00:00 2001 From: "Kanchan Nagshetti (Persistent Systems Inc)" Date: Wed, 23 Apr 2025 11:32:37 +0530 Subject: [PATCH 32/47] edit --- src/backend/sql_agents/helpers/agents_manager.py | 1 + src/backend/sql_agents/process_batch.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/sql_agents/helpers/agents_manager.py b/src/backend/sql_agents/helpers/agents_manager.py index e335a302..e7a8f871 100644 --- a/src/backend/sql_agents/helpers/agents_manager.py +++ b/src/backend/sql_agents/helpers/agents_manager.py @@ -4,6 +4,7 @@ from semantic_kernel.agents.azure_ai import AzureAIAgent # pylint: disable=E0611 + from sql_agents.agents.agent_config import AgentBaseConfig from sql_agents.agents.fixer.setup import setup_fixer_agent from sql_agents.agents.migrator.setup import setup_migrator_agent diff --git a/src/backend/sql_agents/process_batch.py b/src/backend/sql_agents/process_batch.py index 177b08f5..ccd84d84 100644 --- a/src/backend/sql_agents/process_batch.py +++ b/src/backend/sql_agents/process_batch.py @@ -27,7 +27,6 @@ from semantic_kernel.contents import AuthorRole from semantic_kernel.exceptions.service_exceptions import ServiceResponseException - from sql_agents.agents.agent_config import AgentBaseConfig from sql_agents.convert_script import convert_script from sql_agents.helpers.agents_manager import SqlAgents From 61517c021e543fa5c7f4f439757c2d69164d07ce Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Wed, 23 Apr 2025 12:48:56 +0530 Subject: [PATCH 33/47] added agent_config file --- .../sql_agents/agents/agent_config_test.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 src/tests/backend/sql_agents/agents/agent_config_test.py diff --git a/src/tests/backend/sql_agents/agents/agent_config_test.py b/src/tests/backend/sql_agents/agents/agent_config_test.py new file mode 100644 index 00000000..8250a235 --- /dev/null +++ b/src/tests/backend/sql_agents/agents/agent_config_test.py @@ -0,0 +1,42 @@ +import importlib +from unittest.mock import AsyncMock, patch + +import pytest + + +@pytest.fixture +def mock_project_client(): + return AsyncMock() + + +@patch.dict("os.environ", { + "MIGRATOR_AGENT_MODEL_DEPLOY": "migrator-model", + "PICKER_AGENT_MODEL_DEPLOY": "picker-model", + "FIXER_AGENT_MODEL_DEPLOY": "fixer-model", + "SEMANTIC_VERIFIER_AGENT_MODEL_DEPLOY": "semantic-verifier-model", + "SYNTAX_CHECKER_AGENT_MODEL_DEPLOY": "syntax-checker-model", + "SELECTION_MODEL_DEPLOY": "selection-model", + "TERMINATION_MODEL_DEPLOY": "termination-model", +}) +def test_agent_model_type_mapping_and_instance(mock_project_client): + # Re-import to re-evaluate class variable with patched env + from sql_agents.agents import agent_config + importlib.reload(agent_config) + + AgentType = agent_config.AgentType + AgentBaseConfig = agent_config.AgentBaseConfig + + # Test model_type mapping + assert AgentBaseConfig.model_type[AgentType.MIGRATOR] == "migrator-model" + assert AgentBaseConfig.model_type[AgentType.PICKER] == "picker-model" + assert AgentBaseConfig.model_type[AgentType.FIXER] == "fixer-model" + assert AgentBaseConfig.model_type[AgentType.SEMANTIC_VERIFIER] == "semantic-verifier-model" + assert AgentBaseConfig.model_type[AgentType.SYNTAX_CHECKER] == "syntax-checker-model" + assert AgentBaseConfig.model_type[AgentType.SELECTION] == "selection-model" + assert AgentBaseConfig.model_type[AgentType.TERMINATION] == "termination-model" + + # Test __init__ stores params correctly + config = AgentBaseConfig(mock_project_client, sql_from="sql1", sql_to="sql2") + assert config.ai_project_client == mock_project_client + assert config.sql_from == "sql1" + assert config.sql_to == "sql2" From 9dd852630c239a3992ab2f40570f2ba44333cf69 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Wed, 23 Apr 2025 12:56:45 +0530 Subject: [PATCH 34/47] updated import path --- src/backend/sql_agents/helpers/agents_manager.py | 3 +-- src/backend/sql_agents/process_batch.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend/sql_agents/helpers/agents_manager.py b/src/backend/sql_agents/helpers/agents_manager.py index e7a8f871..af5d6365 100644 --- a/src/backend/sql_agents/helpers/agents_manager.py +++ b/src/backend/sql_agents/helpers/agents_manager.py @@ -2,8 +2,7 @@ import logging -from semantic_kernel.agents.azure_ai import AzureAIAgent # pylint: disable=E0611 - +from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent # pylint: disable=E0611 from sql_agents.agents.agent_config import AgentBaseConfig from sql_agents.agents.fixer.setup import setup_fixer_agent diff --git a/src/backend/sql_agents/process_batch.py b/src/backend/sql_agents/process_batch.py index ccd84d84..b93ef3c1 100644 --- a/src/backend/sql_agents/process_batch.py +++ b/src/backend/sql_agents/process_batch.py @@ -23,7 +23,7 @@ from fastapi import HTTPException -from semantic_kernel.agents.azure_ai import AzureAIAgent # pylint: disable=E0611 +from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent # pylint: disable=E0611 from semantic_kernel.contents import AuthorRole from semantic_kernel.exceptions.service_exceptions import ServiceResponseException From 85cfeeae6615b3b9840261a93969bf041145bd4f Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Wed, 23 Apr 2025 13:07:46 +0530 Subject: [PATCH 35/47] edit2 --- .github/workflows/test.yml | 2 +- src/backend/sql_agents/helpers/agents_manager.py | 2 +- src/backend/sql_agents/process_batch.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 66a60b3e..9829b7b6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -96,7 +96,7 @@ jobs: - name: Run Backend Tests with Coverage if: env.skip_backend_tests == 'false' run: | - cd src/tests/backend + cd src/backend pytest --cov=. --cov-report=term-missing --cov-report=xml diff --git a/src/backend/sql_agents/helpers/agents_manager.py b/src/backend/sql_agents/helpers/agents_manager.py index af5d6365..e335a302 100644 --- a/src/backend/sql_agents/helpers/agents_manager.py +++ b/src/backend/sql_agents/helpers/agents_manager.py @@ -2,7 +2,7 @@ import logging -from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent # pylint: disable=E0611 +from semantic_kernel.agents.azure_ai import AzureAIAgent # pylint: disable=E0611 from sql_agents.agents.agent_config import AgentBaseConfig from sql_agents.agents.fixer.setup import setup_fixer_agent diff --git a/src/backend/sql_agents/process_batch.py b/src/backend/sql_agents/process_batch.py index b93ef3c1..b51e88d9 100644 --- a/src/backend/sql_agents/process_batch.py +++ b/src/backend/sql_agents/process_batch.py @@ -23,7 +23,7 @@ from fastapi import HTTPException -from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent # pylint: disable=E0611 +from semantic_kernel.agents.azure_ai import AzureAIAgent # pylint: disable=E0611 from semantic_kernel.contents import AuthorRole from semantic_kernel.exceptions.service_exceptions import ServiceResponseException From ba5a529f48b182828105fb199256e7148642509e Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Wed, 23 Apr 2025 13:10:06 +0530 Subject: [PATCH 36/47] edit workflow --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9829b7b6..ba2388ff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -96,7 +96,7 @@ jobs: - name: Run Backend Tests with Coverage if: env.skip_backend_tests == 'false' run: | - cd src/backend + cd src pytest --cov=. --cov-report=term-missing --cov-report=xml From 10d636cc42ac8eb100cb2ff4599479de29c8381d Mon Sep 17 00:00:00 2001 From: Shreyas-Microsoft Date: Wed, 23 Apr 2025 20:24:43 +0530 Subject: [PATCH 37/47] bring consistency for cancel behaviour --- src/frontend/src/components/uploadButton.tsx | 2 +- src/frontend/src/pages/batchView.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/frontend/src/components/uploadButton.tsx b/src/frontend/src/components/uploadButton.tsx index e294a0b4..47d5569e 100644 --- a/src/frontend/src/components/uploadButton.tsx +++ b/src/frontend/src/components/uploadButton.tsx @@ -366,7 +366,7 @@ const FileUploadZone: React.FC = ({ onConfirm={cancelAllUploads} onCancel={() => setShowLogoCancelDialog(false)} confirmText="Leave and lose progress" - cancelText="Stay here" + cancelText="Continue" /> { onConfirm={handleLeave} onCancel={() => setShowLeaveDialog(false)} confirmText="Return to home and lose progress" - cancelText="Stay here" + cancelText="Continue" />
); From 7085a79dc0b3e6d42d0a29707444e7cc281dccc0 Mon Sep 17 00:00:00 2001 From: Harmanpreet Kaur Date: Thu, 24 Apr 2025 09:56:49 +0530 Subject: [PATCH 38/47] edited file path --- src/backend/sql_agents/helpers/agents_manager.py | 2 +- src/backend/sql_agents/process_batch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/sql_agents/helpers/agents_manager.py b/src/backend/sql_agents/helpers/agents_manager.py index e335a302..af5d6365 100644 --- a/src/backend/sql_agents/helpers/agents_manager.py +++ b/src/backend/sql_agents/helpers/agents_manager.py @@ -2,7 +2,7 @@ import logging -from semantic_kernel.agents.azure_ai import AzureAIAgent # pylint: disable=E0611 +from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent # pylint: disable=E0611 from sql_agents.agents.agent_config import AgentBaseConfig from sql_agents.agents.fixer.setup import setup_fixer_agent diff --git a/src/backend/sql_agents/process_batch.py b/src/backend/sql_agents/process_batch.py index b51e88d9..b93ef3c1 100644 --- a/src/backend/sql_agents/process_batch.py +++ b/src/backend/sql_agents/process_batch.py @@ -23,7 +23,7 @@ from fastapi import HTTPException -from semantic_kernel.agents.azure_ai import AzureAIAgent # pylint: disable=E0611 +from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent # pylint: disable=E0611 from semantic_kernel.contents import AuthorRole from semantic_kernel.exceptions.service_exceptions import ServiceResponseException From fa9dfe89adaf6e0fdd8bea0790444a191cad33a6 Mon Sep 17 00:00:00 2001 From: "Kanchan Nagshetti (Persistent Systems Inc)" Date: Thu, 24 Apr 2025 11:43:40 +0530 Subject: [PATCH 39/47] app_logger --- .../backend/common/logger/app_logger_test.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 src/tests/backend/common/logger/app_logger_test.py diff --git a/src/tests/backend/common/logger/app_logger_test.py b/src/tests/backend/common/logger/app_logger_test.py new file mode 100644 index 00000000..730685ab --- /dev/null +++ b/src/tests/backend/common/logger/app_logger_test.py @@ -0,0 +1,116 @@ +# test_app_logger.py + +import logging +import json +import pytest +from unittest.mock import patch, MagicMock + +from src.backend.common.logger.app_logger import AppLogger, LogLevel # replace 'your_module_name' with the correct one + + +@pytest.fixture +def logger_name(): + return "test_logger" + + +@pytest.fixture +def app_logger(logger_name): + return AppLogger(logger_name) + + +def test_log_level_constants(): + assert LogLevel.NONE == logging.NOTSET + assert LogLevel.DEBUG == logging.DEBUG + assert LogLevel.INFO == logging.INFO + assert LogLevel.WARNING == logging.WARNING + assert LogLevel.ERROR == logging.ERROR + assert LogLevel.CRITICAL == logging.CRITICAL + + +@patch("src.backend.common.logger.app_logger.logging.getLogger") +def test_app_logger_init(mock_get_logger, logger_name): + mock_logger = MagicMock() + mock_get_logger.return_value = mock_logger + + logger = AppLogger(logger_name) + + assert logger.logger == mock_logger + mock_logger.setLevel.assert_called_once_with(logging.DEBUG) + + # New Correct Check: Check that addHandler was called with StreamHandler + assert mock_logger.addHandler.called + handler_arg = mock_logger.addHandler.call_args[0][0] + assert isinstance(handler_arg, logging.StreamHandler) + + + +def test_format_message_without_kwargs(app_logger): + message = "test message" + formatted = app_logger._format_message(message) + expected = json.dumps({"message": message}) + assert formatted == expected + + +def test_format_message_with_kwargs(app_logger): + message = "test message" + context = {"user": "john", "action": "login"} + formatted = app_logger._format_message(message, **context) + expected = json.dumps({"message": message, "context": context}) + assert formatted == expected + + +@patch.object(logging.Logger, "debug") +def test_debug(mock_debug, app_logger): + app_logger.debug("Debug Message", user="test") + assert mock_debug.called + args, kwargs = mock_debug.call_args + log_entry = json.loads(args[0]) + assert log_entry["message"] == "Debug Message" + assert "context" in log_entry + assert log_entry["context"]["user"] == "test" + + +@patch.object(logging.Logger, "info") +def test_info(mock_info, app_logger): + app_logger.info("Info Message", user="test") + assert mock_info.called + args, kwargs = mock_info.call_args + log_entry = json.loads(args[0]) + assert log_entry["message"] == "Info Message" + + +@patch.object(logging.Logger, "warning") +def test_warning(mock_warning, app_logger): + app_logger.warning("Warning Message", user="test") + assert mock_warning.called + args, kwargs = mock_warning.call_args + log_entry = json.loads(args[0]) + assert log_entry["message"] == "Warning Message" + + +@patch.object(logging.Logger, "error") +def test_error(mock_error, app_logger): + app_logger.error("Error Message", user="test") + assert mock_error.called + args, kwargs = mock_error.call_args + log_entry = json.loads(args[0]) + assert log_entry["message"] == "Error Message" + + +@patch.object(logging.Logger, "critical") +def test_critical(mock_critical, app_logger): + app_logger.critical("Critical Message", user="test") + assert mock_critical.called + args, kwargs = mock_critical.call_args + log_entry = json.loads(args[0]) + assert log_entry["message"] == "Critical Message" + + +@patch("src.backend.common.logger.app_logger.logging.getLogger") +def test_set_min_log_level(mock_get_logger): + mock_logger = MagicMock() + mock_get_logger.return_value = mock_logger + + AppLogger.set_min_log_level(LogLevel.ERROR) + + mock_logger.setLevel.assert_called_with(LogLevel.ERROR) From d94f66e98d844050baa9173f49305394d9f268bc Mon Sep 17 00:00:00 2001 From: "Kanchan Nagshetti (Persistent Systems Inc)" Date: Thu, 24 Apr 2025 12:10:06 +0530 Subject: [PATCH 40/47] edit --- src/tests/backend/common/logger/app_logger_test.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/tests/backend/common/logger/app_logger_test.py b/src/tests/backend/common/logger/app_logger_test.py index 730685ab..d584ed0d 100644 --- a/src/tests/backend/common/logger/app_logger_test.py +++ b/src/tests/backend/common/logger/app_logger_test.py @@ -1,11 +1,9 @@ -# test_app_logger.py - -import logging import json +import logging import pytest from unittest.mock import patch, MagicMock -from src.backend.common.logger.app_logger import AppLogger, LogLevel # replace 'your_module_name' with the correct one +from src.backend.common.logger.app_logger import AppLogger, LogLevel @pytest.fixture @@ -113,4 +111,4 @@ def test_set_min_log_level(mock_get_logger): AppLogger.set_min_log_level(LogLevel.ERROR) - mock_logger.setLevel.assert_called_with(LogLevel.ERROR) + mock_logger.setLevel.assert_called_with(LogLevel.ERROR) \ No newline at end of file From be464967c8dfb005caacdf402dd601f671b43a48 Mon Sep 17 00:00:00 2001 From: "Vishal Shinde (Persistent Systems Inc)" Date: Thu, 24 Apr 2025 12:24:30 +0530 Subject: [PATCH 41/47] Pylint issue fix --- src/backend/common/database/database_base.py | 38 ++++---- .../backend/common/config/config_test.py | 23 ++--- .../common/database/database_base_test.py | 95 +++++++++++-------- .../common/database/database_factory_test.py | 14 +-- .../backend/common/storage/blob_azure_test.py | 14 +-- 5 files changed, 97 insertions(+), 87 deletions(-) diff --git a/src/backend/common/database/database_base.py b/src/backend/common/database/database_base.py index 318ffecc..66d36f42 100644 --- a/src/backend/common/database/database_base.py +++ b/src/backend/common/database/database_base.py @@ -17,54 +17,54 @@ class DatabaseBase(ABC): @abstractmethod async def initialize_cosmos(self) -> None: """Initialize the cosmosdb client and create container if needed""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def create_batch(self, user_id: str, batch_id: uuid.UUID) -> BatchRecord: """Create a new conversion batch""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def get_file_logs(self, file_id: str) -> Dict: """Retrieve all logs for a file""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def get_batch_from_id(self, batch_id: str) -> Dict: """Retrieve all logs for a file""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def get_batch_files(self, batch_id: str) -> List[Dict]: """Retrieve all files for a batch""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def delete_file_logs(self, file_id: str) -> None: """Delete all logs for a file""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def get_user_batches(self, user_id: str) -> Dict: """Retrieve all batches for a user""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def add_file( self, batch_id: uuid.UUID, file_id: uuid.UUID, file_name: str, storage_path: str ) -> FileRecord: """Add a file entry to the database""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def get_batch(self, user_id: str, batch_id: str) -> Optional[Dict]: """Retrieve a batch and its associated files""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def get_file(self, file_id: str) -> Optional[Dict]: """Retrieve a file entry along with its logs""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def add_file_log( @@ -77,39 +77,39 @@ async def add_file_log( author_role: AuthorRole, ) -> None: """Log a file status update""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def update_file(self, file_record: FileRecord) -> None: - """update file record""" - pass # pragma: no cover + """Update file record""" + pass # pragma: no cover @abstractmethod async def update_batch(self, batch_record: BatchRecord) -> BatchRecord: """Update a batch record""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def delete_all(self, user_id: str) -> None: """Delete all batches, files, and logs for a user""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def delete_batch(self, user_id: str, batch_id: str) -> None: """Delete a batch along with its files and logs""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def delete_file(self, user_id: str, batch_id: str, file_id: str) -> None: """Delete a file and its logs, and update batch file count""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def get_batch_history(self, user_id: str, batch_id: str) -> List[Dict]: """Retrieve all logs for a batch""" - pass # pragma: no cover + pass # pragma: no cover @abstractmethod async def close(self) -> None: """Close database connection""" - pass # pragma: no cover + pass # pragma: no cover diff --git a/src/tests/backend/common/config/config_test.py b/src/tests/backend/common/config/config_test.py index dc4306d8..6984ae8f 100644 --- a/src/tests/backend/common/config/config_test.py +++ b/src/tests/backend/common/config/config_test.py @@ -1,9 +1,6 @@ -import os -import sys import pytest - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..', 'backend'))) - + + @pytest.fixture(autouse=True) def clear_env(monkeypatch): # Clear environment variables that might affect tests. @@ -21,7 +18,8 @@ def clear_env(monkeypatch): ] for key in keys: monkeypatch.delenv(key, raising=False) - + + def test_config_initialization(monkeypatch): # Set the full configuration environment variables. monkeypatch.setenv("AZURE_TENANT_ID", "test-tenant-id") @@ -34,11 +32,11 @@ def test_config_initialization(monkeypatch): monkeypatch.setenv("COSMOSDB_LOG_CONTAINER", "test-log-container") monkeypatch.setenv("AZURE_BLOB_CONTAINER_NAME", "test-blob-container-name") monkeypatch.setenv("AZURE_BLOB_ACCOUNT_NAME", "test-blob-account-name") - + # Local import to avoid triggering circular imports during module collection. from common.config.config import Config config = Config() - + assert config.azure_tenant_id == "test-tenant-id" assert config.azure_client_id == "test-client-id" assert config.azure_client_secret == "test-client-secret" @@ -49,7 +47,8 @@ def test_config_initialization(monkeypatch): assert config.cosmosdb_log_container == "test-log-container" assert config.azure_blob_container_name == "test-blob-container-name" assert config.azure_blob_account_name == "test-blob-account-name" - + + def test_cosmosdb_config_initialization(monkeypatch): # Set only cosmosdb-related environment variables. monkeypatch.setenv("COSMOSDB_ENDPOINT", "test-cosmosdb-endpoint") @@ -57,14 +56,12 @@ def test_cosmosdb_config_initialization(monkeypatch): monkeypatch.setenv("COSMOSDB_BATCH_CONTAINER", "test-batch-container") monkeypatch.setenv("COSMOSDB_FILE_CONTAINER", "test-file-container") monkeypatch.setenv("COSMOSDB_LOG_CONTAINER", "test-log-container") - + from common.config.config import Config config = Config() - + assert config.cosmosdb_endpoint == "test-cosmosdb-endpoint" assert config.cosmosdb_database == "test-database" assert config.cosmosdb_batch_container == "test-batch-container" assert config.cosmosdb_file_container == "test-file-container" assert config.cosmosdb_log_container == "test-log-container" - - \ No newline at end of file diff --git a/src/tests/backend/common/database/database_base_test.py b/src/tests/backend/common/database/database_base_test.py index 8dd57d81..325cf7e9 100644 --- a/src/tests/backend/common/database/database_base_test.py +++ b/src/tests/backend/common/database/database_base_test.py @@ -1,67 +1,66 @@ -import asyncio import uuid -import pytest -import os -import sys -from datetime import datetime from enum import Enum - -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../backend'))) - + + from common.database.database_base import DatabaseBase -from common.models.api import BatchRecord, FileRecord, ProcessStatus - +from common.models.api import ProcessStatus + +import pytest + + # Allow instantiation of the abstract base class by clearing its abstract methods. DatabaseBase.__abstractmethods__ = set() - + + @pytest.fixture def db_instance(): # Create a concrete implementation of DatabaseBase using async methods. class ConcreteDatabase(DatabaseBase): async def create_batch(self, user_id, batch_id): pass - + async def get_file_logs(self, file_id): pass - + async def get_batch_files(self, user_id, batch_id): pass - + async def delete_file_logs(self, file_id): pass - + async def get_user_batches(self, user_id): pass - + async def add_file(self, batch_id, file_id, file_name, file_path): pass - + async def get_batch(self, user_id, batch_id): pass - + async def get_file(self, file_id): pass - + async def log_file_status(self, file_id, status, description, log_type): pass - + async def log_batch_status(self, batch_id, status, file_count): pass - + async def delete_all(self, user_id): pass - + async def delete_batch(self, user_id, batch_id): pass - + async def delete_file(self, user_id, batch_id, file_id): pass - + async def close(self): pass - + return ConcreteDatabase() - + + def get_dummy_status(): """ Try to use a specific ProcessStatus value (e.g. PROCESSING). @@ -76,76 +75,90 @@ def get_dummy_status(): # If the enum is empty, create a dummy one. DummyStatus = Enum("DummyStatus", {"DUMMY": "dummy"}) return DummyStatus.DUMMY - + + @pytest.mark.asyncio async def test_create_batch(db_instance): result = await db_instance.create_batch("user1", uuid.uuid4()) # Since the method is implemented as pass, result is None. assert result is None - + + @pytest.mark.asyncio async def test_get_file_logs(db_instance): result = await db_instance.get_file_logs("file1") assert result is None - + + @pytest.mark.asyncio async def test_get_batch_files(db_instance): result = await db_instance.get_batch_files("user1", "batch1") assert result is None - + + @pytest.mark.asyncio async def test_delete_file_logs(db_instance): result = await db_instance.delete_file_logs("file1") assert result is None - + + @pytest.mark.asyncio async def test_get_user_batches(db_instance): result = await db_instance.get_user_batches("user1") assert result is None - + + @pytest.mark.asyncio async def test_add_file(db_instance): result = await db_instance.add_file(uuid.uuid4(), uuid.uuid4(), "test.txt", "/dummy/path") assert result is None - + + @pytest.mark.asyncio async def test_get_batch(db_instance): result = await db_instance.get_batch("user1", "batch1") assert result is None - + + @pytest.mark.asyncio async def test_get_file(db_instance): result = await db_instance.get_file("file1") assert result is None - + + @pytest.mark.asyncio async def test_log_file_status(db_instance): # Using ProcessStatus.COMPLETED as an example. result = await db_instance.log_file_status("file1", ProcessStatus.COMPLETED, "desc", "log_type") assert result is None - + + @pytest.mark.asyncio async def test_log_batch_status(db_instance): dummy_status = get_dummy_status() result = await db_instance.log_batch_status("batch1", dummy_status, 5) assert result is None - + + @pytest.mark.asyncio async def test_delete_all(db_instance): result = await db_instance.delete_all("user1") assert result is None - + + @pytest.mark.asyncio async def test_delete_batch(db_instance): result = await db_instance.delete_batch("user1", "batch1") assert result is None - + + @pytest.mark.asyncio async def test_delete_file(db_instance): result = await db_instance.delete_file("user1", "batch1", "file1") assert result is None - + + @pytest.mark.asyncio async def test_close(db_instance): result = await db_instance.close() - assert result is None \ No newline at end of file + assert result is None diff --git a/src/tests/backend/common/database/database_factory_test.py b/src/tests/backend/common/database/database_factory_test.py index 67ad35ab..43d98fff 100644 --- a/src/tests/backend/common/database/database_factory_test.py +++ b/src/tests/backend/common/database/database_factory_test.py @@ -1,10 +1,8 @@ -import os -import sys -import pytest -import asyncio from unittest.mock import AsyncMock, patch -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..', 'backend'))) + +import pytest + @pytest.fixture(autouse=True) def patch_config(monkeypatch): @@ -22,10 +20,10 @@ def dummy_init(self): monkeypatch.setattr(Config, "__init__", dummy_init) # Replace the init method + @pytest.fixture(autouse=True) def patch_cosmosdb_client(monkeypatch): """Patch CosmosDBClient to use a dummy implementation.""" - from common.database.database_factory import CosmosDBClient class DummyCosmosDBClient: def __init__(self, endpoint, credential, database_name, batch_container, file_container, log_container): @@ -53,6 +51,7 @@ async def close(self): monkeypatch.setattr("common.database.database_factory.CosmosDBClient", DummyCosmosDBClient) + @pytest.mark.asyncio async def test_get_database(): """Test database retrieval using the factory.""" @@ -67,11 +66,12 @@ async def test_get_database(): assert db_instance.file_container == "dummy_file" assert db_instance.log_container == "dummy_log" + @pytest.mark.asyncio async def test_main_function(): """Test the main function in database factory.""" with patch("common.database.database_factory.DatabaseFactory.get_database", new_callable=AsyncMock, return_value=AsyncMock()) as mock_get_database, patch("builtins.print") as mock_print: - + from common.database.database_factory import main await main() diff --git a/src/tests/backend/common/storage/blob_azure_test.py b/src/tests/backend/common/storage/blob_azure_test.py index 573cd085..68e5ad0d 100644 --- a/src/tests/backend/common/storage/blob_azure_test.py +++ b/src/tests/backend/common/storage/blob_azure_test.py @@ -1,16 +1,14 @@ import json -import os -import sys -import pytest -from unittest.mock import AsyncMock, patch, MagicMock from io import BytesIO +from unittest.mock import MagicMock, patch -# Add backend directory to sys.path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../..', 'backend'))) from common.storage.blob_azure import AzureBlobStorage +import pytest + + @pytest.fixture def mock_blob_service(): """Fixture to mock Azure Blob Storage service client""" @@ -118,6 +116,7 @@ async def test_list_files(blob_storage, mock_blob_service): class AsyncIterator: """Helper class to create an async iterator""" + def __init__(self, items): self._items = items @@ -186,6 +185,7 @@ async def test_close(blob_storage, mock_blob_service): service_client.close.assert_called_once() + @pytest.mark.asyncio async def test_blob_storage_init_exception(): """Test that an exception during initialization logs the error message""" @@ -222,4 +222,4 @@ async def test_blob_storage_init_exception(): mock_logger_instance.error.assert_called_once_with(expected_error_log) # Assert that debug log is written for container existence - mock_logger_instance.debug.assert_called_once_with(expected_debug_log) \ No newline at end of file + mock_logger_instance.debug.assert_called_once_with(expected_debug_log) From 89fed7a4aac75e930471c60934391ee9469acd70 Mon Sep 17 00:00:00 2001 From: "Vishal Shinde (Persistent Systems Inc)" Date: Thu, 24 Apr 2025 12:38:11 +0530 Subject: [PATCH 42/47] fixed app_logger.py test cases and pylint issue. --- .../common/database/database_factory_test.py | 2 +- .../backend/common/logger/app_logger_test.py | 134 ++++++++---------- 2 files changed, 58 insertions(+), 78 deletions(-) diff --git a/src/tests/backend/common/database/database_factory_test.py b/src/tests/backend/common/database/database_factory_test.py index 43d98fff..27d98105 100644 --- a/src/tests/backend/common/database/database_factory_test.py +++ b/src/tests/backend/common/database/database_factory_test.py @@ -71,7 +71,7 @@ async def test_get_database(): async def test_main_function(): """Test the main function in database factory.""" with patch("common.database.database_factory.DatabaseFactory.get_database", new_callable=AsyncMock, return_value=AsyncMock()) as mock_get_database, patch("builtins.print") as mock_print: - + from common.database.database_factory import main await main() diff --git a/src/tests/backend/common/logger/app_logger_test.py b/src/tests/backend/common/logger/app_logger_test.py index d584ed0d..9301eb30 100644 --- a/src/tests/backend/common/logger/app_logger_test.py +++ b/src/tests/backend/common/logger/app_logger_test.py @@ -1,9 +1,10 @@ import json import logging -import pytest -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch + +from common.logger.app_logger import AppLogger, LogLevel # Adjust the import based on your actual path -from src.backend.common.logger.app_logger import AppLogger, LogLevel +import pytest @pytest.fixture @@ -12,11 +13,16 @@ def logger_name(): @pytest.fixture -def app_logger(logger_name): - return AppLogger(logger_name) +def logger_instance(logger_name): + """Fixture to return AppLogger with mocked handler""" + with patch("common.logger.app_logger.logging.getLogger") as mock_get_logger: + mock_logger = MagicMock() + mock_get_logger.return_value = mock_logger + yield AppLogger(logger_name) -def test_log_level_constants(): +def test_log_levels(): + """Ensure log levels are set correctly""" assert LogLevel.NONE == logging.NOTSET assert LogLevel.DEBUG == logging.DEBUG assert LogLevel.INFO == logging.INFO @@ -25,90 +31,64 @@ def test_log_level_constants(): assert LogLevel.CRITICAL == logging.CRITICAL -@patch("src.backend.common.logger.app_logger.logging.getLogger") -def test_app_logger_init(mock_get_logger, logger_name): - mock_logger = MagicMock() - mock_get_logger.return_value = mock_logger - - logger = AppLogger(logger_name) - - assert logger.logger == mock_logger - mock_logger.setLevel.assert_called_once_with(logging.DEBUG) - - # New Correct Check: Check that addHandler was called with StreamHandler - assert mock_logger.addHandler.called - handler_arg = mock_logger.addHandler.call_args[0][0] - assert isinstance(handler_arg, logging.StreamHandler) - - - -def test_format_message_without_kwargs(app_logger): - message = "test message" - formatted = app_logger._format_message(message) - expected = json.dumps({"message": message}) - assert formatted == expected +def test_format_message_basic(logger_instance): + result = logger_instance._format_message("Test message") + parsed = json.loads(result) + assert parsed["message"] == "Test message" + assert "context" not in parsed -def test_format_message_with_kwargs(app_logger): - message = "test message" - context = {"user": "john", "action": "login"} - formatted = app_logger._format_message(message, **context) - expected = json.dumps({"message": message, "context": context}) - assert formatted == expected +def test_format_message_with_context(logger_instance): + result = logger_instance._format_message("Contextual message", key1="value1", key2="value2") + parsed = json.loads(result) + assert parsed["message"] == "Contextual message" + assert parsed["context"] == {"key1": "value1", "key2": "value2"} -@patch.object(logging.Logger, "debug") -def test_debug(mock_debug, app_logger): - app_logger.debug("Debug Message", user="test") - assert mock_debug.called - args, kwargs = mock_debug.call_args - log_entry = json.loads(args[0]) - assert log_entry["message"] == "Debug Message" - assert "context" in log_entry - assert log_entry["context"]["user"] == "test" +def test_debug_log(logger_instance): + with patch.object(logger_instance.logger, "debug") as mock_debug: + logger_instance.debug("Debug log", user="tester") + mock_debug.assert_called_once() + log_json = json.loads(mock_debug.call_args[0][0]) + assert log_json["message"] == "Debug log" + assert log_json["context"]["user"] == "tester" -@patch.object(logging.Logger, "info") -def test_info(mock_info, app_logger): - app_logger.info("Info Message", user="test") - assert mock_info.called - args, kwargs = mock_info.call_args - log_entry = json.loads(args[0]) - assert log_entry["message"] == "Info Message" +def test_info_log(logger_instance): + with patch.object(logger_instance.logger, "info") as mock_info: + logger_instance.info("Info log", module="log_module") + mock_info.assert_called_once() + log_json = json.loads(mock_info.call_args[0][0]) + assert log_json["message"] == "Info log" + assert log_json["context"]["module"] == "log_module" -@patch.object(logging.Logger, "warning") -def test_warning(mock_warning, app_logger): - app_logger.warning("Warning Message", user="test") - assert mock_warning.called - args, kwargs = mock_warning.call_args - log_entry = json.loads(args[0]) - assert log_entry["message"] == "Warning Message" +def test_warning_log(logger_instance): + with patch.object(logger_instance.logger, "warning") as mock_warning: + logger_instance.warning("Warning log") + mock_warning.assert_called_once() -@patch.object(logging.Logger, "error") -def test_error(mock_error, app_logger): - app_logger.error("Error Message", user="test") - assert mock_error.called - args, kwargs = mock_error.call_args - log_entry = json.loads(args[0]) - assert log_entry["message"] == "Error Message" +def test_error_log(logger_instance): + with patch.object(logger_instance.logger, "error") as mock_error: + logger_instance.error("Error log", error_code=500) + mock_error.assert_called_once() + log_json = json.loads(mock_error.call_args[0][0]) + assert log_json["message"] == "Error log" + assert log_json["context"]["error_code"] == 500 -@patch.object(logging.Logger, "critical") -def test_critical(mock_critical, app_logger): - app_logger.critical("Critical Message", user="test") - assert mock_critical.called - args, kwargs = mock_critical.call_args - log_entry = json.loads(args[0]) - assert log_entry["message"] == "Critical Message" +def test_critical_log(logger_instance): + with patch.object(logger_instance.logger, "critical") as mock_critical: + logger_instance.critical("Critical log") + mock_critical.assert_called_once() -@patch("src.backend.common.logger.app_logger.logging.getLogger") -def test_set_min_log_level(mock_get_logger): - mock_logger = MagicMock() - mock_get_logger.return_value = mock_logger +def test_set_min_log_level(): + with patch("common.logger.app_logger.logging.getLogger") as mock_get_logger: + mock_logger = MagicMock() + mock_get_logger.return_value = mock_logger - AppLogger.set_min_log_level(LogLevel.ERROR) + AppLogger.set_min_log_level(LogLevel.ERROR) - mock_logger.setLevel.assert_called_with(LogLevel.ERROR) \ No newline at end of file + mock_logger.setLevel.assert_called_once_with(LogLevel.ERROR) From 06eca57af3597ef70cb8bd1b7c899aaee75fe121 Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Thu, 24 Apr 2025 13:43:13 +0530 Subject: [PATCH 43/47] feat: updated unit test cases for cosmosdb_test file --- .../backend/common/database/cosmosdb_test.py | 435 +++++++++++++++++- 1 file changed, 414 insertions(+), 21 deletions(-) diff --git a/src/tests/backend/common/database/cosmosdb_test.py b/src/tests/backend/common/database/cosmosdb_test.py index a6e7f1ed..d4970111 100644 --- a/src/tests/backend/common/database/cosmosdb_test.py +++ b/src/tests/backend/common/database/cosmosdb_test.py @@ -1,34 +1,31 @@ -import pytest -import asyncio import os import sys -from unittest import mock - -from unittest.mock import AsyncMock, patch -from uuid import uuid4 -from datetime import datetime, timezone -from azure.cosmos.exceptions import CosmosResourceExistsError - # Add backend directory to sys.path sys.path.insert( 0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../..", "backend")), ) +from datetime import datetime, timezone # noqa: E402 +from unittest import mock # noqa: E402 +from unittest.mock import AsyncMock # noqa: E402 +from uuid import uuid4 # noqa: E402 -from common.models.api import ( +from azure.cosmos.aio import CosmosClient # noqa: E402 +from azure.cosmos.exceptions import CosmosResourceExistsError # noqa: E402 + +from common.database.cosmosdb import ( # noqa: E402 + CosmosDBClient, +) +from common.models.api import ( # noqa: E402 AgentType, + AuthorRole, BatchRecord, - FileLog, + FileRecord, LogType, ProcessStatus, - FileRecord, - AuthorRole, -) -from common.logger.app_logger import AppLogger -from common.database.cosmosdb import ( - CosmosDBClient, -) -from azure.cosmos.aio import CosmosClient +) # noqa: E402 + +import pytest # noqa: E402 # Mocked data for the test endpoint = "https://fake.cosmosdb.azure.com" @@ -51,8 +48,6 @@ def cosmos_db_client(): ) - - @pytest.mark.asyncio async def test_initialize_cosmos(cosmos_db_client, mocker): # Mocking CosmosClient and its methods @@ -160,6 +155,7 @@ async def test_create_batch_new(cosmos_db_client, mocker): mock_batch_container.create_item.assert_called_once_with(body=batch.dict()) + @pytest.mark.asyncio async def test_create_batch_exists(cosmos_db_client, mocker): user_id = "user_1" @@ -192,6 +188,32 @@ async def test_create_batch_exists(cosmos_db_client, mocker): mock_get_batch.assert_called_once_with(user_id, str(batch_id)) +@pytest.mark.asyncio +async def test_create_batch_exception(cosmos_db_client, mocker): + user_id = "user_1" + batch_id = uuid4() + + # Mock the batch_container and make create_item raise a general Exception + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mock_batch_container.create_item = AsyncMock(side_effect=Exception("Unexpected Error")) + + # Mock the logger to verify logging + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Call the method and assert it raises the exception + with pytest.raises(Exception, match="Unexpected Error"): + await cosmos_db_client.create_batch(user_id, batch_id) + + # Ensure logger.error was called with expected message and error + mock_logger.error.assert_called_once() + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to create batch" + assert "error" in called_kwargs + assert "Unexpected Error" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_add_file(cosmos_db_client, mocker): batch_id = uuid4() @@ -219,6 +241,33 @@ async def test_add_file(cosmos_db_client, mocker): mock_file_container.create_item.assert_called_once_with(body=file_record.dict()) +@pytest.mark.asyncio +async def test_add_file_exception(cosmos_db_client, mocker): + batch_id = uuid4() + file_id = uuid4() + file_name = "document.pdf" + storage_path = "/files/document.pdf" + + # Mock file_container.create_item to raise a general exception + mock_file_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + mock_file_container.create_item = AsyncMock(side_effect=Exception("Insert failed")) + + # Mock logger to capture error logs + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Expect an exception when calling add_file + with pytest.raises(Exception, match="Insert failed"): + await cosmos_db_client.add_file(batch_id, file_id, file_name, storage_path) + + # Check that logger.error was called properly + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to add file" + assert "error" in called_kwargs + assert "Insert failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_update_file(cosmos_db_client, mocker): file_id = uuid4() @@ -249,6 +298,41 @@ async def test_update_file(cosmos_db_client, mocker): mock_file_container.replace_item.assert_called_once_with(item=str(file_id), body=file_record.dict()) +@pytest.mark.asyncio +async def test_update_file_exception(cosmos_db_client, mocker): + # Create a sample FileRecord + file_record = FileRecord( + file_id=uuid4(), + batch_id=uuid4(), + original_name="file.txt", + blob_path="/storage/file.txt", + translated_path="", + status=ProcessStatus.READY_TO_PROCESS, + error_count=0, + syntax_count=0, + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + ) + + # Mock file_container.replace_item to raise an exception + mock_file_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + mock_file_container.replace_item = AsyncMock(side_effect=Exception("Update failed")) + + # Mock logger + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Expect an exception when update_file is called + with pytest.raises(Exception, match="Update failed"): + await cosmos_db_client.update_file(file_record) + + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to update file" + assert "error" in called_kwargs + assert "Update failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_update_batch(cosmos_db_client, mocker): batch_record = BatchRecord( @@ -274,6 +358,37 @@ async def test_update_batch(cosmos_db_client, mocker): mock_batch_container.replace_item.assert_called_once_with(item=str(batch_record.batch_id), body=batch_record.dict()) +@pytest.mark.asyncio +async def test_update_batch_exception(cosmos_db_client, mocker): + # Create a sample BatchRecord + batch_record = BatchRecord( + batch_id=uuid4(), + user_id="user_1", + file_count=3, + created_at=datetime.now(timezone.utc), + updated_at=datetime.now(timezone.utc), + status=ProcessStatus.READY_TO_PROCESS, + ) + + # Mock batch_container.replace_item to raise an exception + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mock_batch_container.replace_item = AsyncMock(side_effect=Exception("Update batch failed")) + + # Mock logger to verify logging + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Expect an exception when update_batch is called + with pytest.raises(Exception, match="Update batch failed"): + await cosmos_db_client.update_batch(batch_record) + + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to update batch" + assert "error" in called_kwargs + assert "Update batch failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_get_batch(cosmos_db_client, mocker): user_id = "user_1" @@ -313,6 +428,33 @@ async def mock_query_items(query, parameters): ) +@pytest.mark.asyncio +async def test_get_batch_exception(cosmos_db_client, mocker): + user_id = "user_1" + batch_id = str(uuid4()) + + # Mock batch_container.query_items to raise an exception + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mock_batch_container.query_items = mock.MagicMock( + side_effect=Exception("Get batch failed") + ) + + # Patch logger + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Call get_batch and expect it to raise an exception + with pytest.raises(Exception, match="Get batch failed"): + await cosmos_db_client.get_batch(user_id, batch_id) + + # Ensure logger.error was called with the expected error message + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to get batch" + assert "error" in called_kwargs + assert "Get batch failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_get_file(cosmos_db_client, mocker): file_id = str(uuid4()) @@ -346,6 +488,31 @@ async def mock_query_items(query, parameters): mock_file_container.query_items.assert_called_once() +@pytest.mark.asyncio +async def test_get_file_exception(cosmos_db_client, mocker): + file_id = str(uuid4()) + + # Mock file_container.query_items to raise an exception + mock_file_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + mock_file_container.query_items = mock.MagicMock( + side_effect=Exception("Get file failed") + ) + + # Mock logger to verify logging + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Call get_file and expect an exception + with pytest.raises(Exception, match="Get file failed"): + await cosmos_db_client.get_file(file_id) + + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to get file" + assert "error" in called_kwargs + assert "Get file failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_get_batch_files(cosmos_db_client, mocker): batch_id = str(uuid4()) @@ -389,6 +556,32 @@ async def mock_query_items(query, parameters): mock_file_container.query_items.assert_called_once() +@pytest.mark.asyncio +async def test_get_batch_files_exception(cosmos_db_client, mocker): + batch_id = str(uuid4()) + + # Mock file_container.query_items to raise an exception + mock_file_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + mock_file_container.query_items = mock.MagicMock( + side_effect=Exception("Get batch file failed") + ) + + # Mock logger to verify logging + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Expect the exception to be raised + with pytest.raises(Exception, match="Get batch file failed"): + await cosmos_db_client.get_batch_files(batch_id) + + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to get files" + assert "error" in called_kwargs + assert "Get batch file failed" in called_kwargs["error"] + + + @pytest.mark.asyncio async def test_get_batch_from_id(cosmos_db_client, mocker): batch_id = str(uuid4()) @@ -421,6 +614,31 @@ async def mock_query_items(query, parameters): mock_batch_container.query_items.assert_called_once() +@pytest.mark.asyncio +async def test_get_batch_from_id_exception(cosmos_db_client, mocker): + batch_id = str(uuid4()) + + # Mock batch_container.query_items to raise an exception + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mock_batch_container.query_items = mock.MagicMock( + side_effect=Exception("Get batch from id failed") + ) + + # Mock logger to verify logging + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Call the method and expect it to raise an exception + with pytest.raises(Exception, match="Get batch from id failed"): + await cosmos_db_client.get_batch_from_id(batch_id) + + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to get batch from ID" + assert "error" in called_kwargs + assert "Get batch from id failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_get_user_batches(cosmos_db_client, mocker): user_id = "user_123" @@ -454,6 +672,32 @@ async def mock_query_items(query, parameters): mock_batch_container.query_items.assert_called_once() +@pytest.mark.asyncio +async def test_get_user_batches_exception(cosmos_db_client, mocker): + user_id = "user_" + str(uuid4()) + + # Mock batch_container.query_items to raise an exception + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mock_batch_container.query_items = mock.MagicMock( + side_effect=Exception("Get user batch failed") + ) + + # Mock logger to capture the error + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Call the method and expect it to raise the exception + with pytest.raises(Exception, match="Get user batch failed"): + await cosmos_db_client.get_user_batches(user_id) + + # Ensure logger.error was called with the expected message and error + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to get user batches" + assert "error" in called_kwargs + assert "Get user batch failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_get_file_logs(cosmos_db_client, mocker): file_id = str(uuid4()) @@ -509,6 +753,32 @@ async def mock_query_items(query, parameters): mock_log_container.query_items.assert_called_once() +@pytest.mark.asyncio +async def test_get_file_logs_exception(cosmos_db_client, mocker): + file_id = str(uuid4()) + + # Mock log_container.query_items to raise an exception + mock_log_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'log_container', mock_log_container) + mock_log_container.query_items = mock.MagicMock( + side_effect=Exception("Get file log failed") + ) + + # Mock logger to verify error logging + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Call the method and expect it to raise the exception + with pytest.raises(Exception, match="Get file log failed"): + await cosmos_db_client.get_file_logs(file_id) + + # Assert logger.error was called with correct arguments + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to get file logs" + assert "error" in called_kwargs + assert "Get file log failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_delete_all(cosmos_db_client, mocker): user_id = str(uuid4()) @@ -536,6 +806,38 @@ async def test_delete_all(cosmos_db_client, mocker): mock_log_container.delete_item.assert_called_once() +@pytest.mark.asyncio +async def test_delete_all_exception(cosmos_db_client, mocker): + user_id = f"user_{uuid4()}" + + # Mock batch_container to raise an exception on delete + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mock_batch_container.delete_item = mock.AsyncMock( + side_effect=Exception("Delete failed") + ) + + # Also mock file_container and log_container to avoid accidental execution + mock_file_container = mock.MagicMock() + mock_log_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + mocker.patch.object(cosmos_db_client, 'log_container', mock_log_container) + + # Mock logger to verify error handling + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Call the method and expect it to raise the exception + with pytest.raises(Exception, match="Delete failed"): + await cosmos_db_client.delete_all(user_id) + + # Check that logger.error was called with expected error message + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to delete all user data" + assert "error" in called_kwargs + assert "Delete failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_delete_logs(cosmos_db_client, mocker): file_id = str(uuid4()) @@ -568,6 +870,32 @@ async def mock_query_items(query, parameters): mock_log_container.query_items.assert_called_once() +@pytest.mark.asyncio +async def test_delete_logs_exception(cosmos_db_client, mocker): + file_id = str(uuid4()) + + # Mock log_container.query_items to raise an exception + mock_log_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'log_container', mock_log_container) + mock_log_container.query_items = mock.MagicMock( + side_effect=Exception("Query failed") + ) + + # Mock logger to verify error handling + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Call the method and expect it to raise the exception + with pytest.raises(Exception, match="Query failed"): + await cosmos_db_client.delete_logs(file_id) + + # Check that logger.error was called with expected error message + called_args, called_kwargs = mock_logger.error.call_args + assert called_args[0] == "Failed to delete all user data" + assert "error" in called_kwargs + assert "Query failed" in called_kwargs["error"] + + @pytest.mark.asyncio async def test_delete_batch(cosmos_db_client, mocker): user_id = str(uuid4()) @@ -583,6 +911,42 @@ async def test_delete_batch(cosmos_db_client, mocker): mock_batch_container.delete_item.assert_called_once() +@pytest.mark.asyncio +async def test_delete_batch_exception(cosmos_db_client, mocker): + user_id = f"user_{uuid4()}" + batch_id = str(uuid4()) + + # Mock batch_container.delete_item to raise an exception + mock_batch_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'batch_container', mock_batch_container) + mock_batch_container.delete_item = mock.AsyncMock( + side_effect=Exception("Delete failed") + ) + + # Mock logger to verify error logging + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Expect the exception to be raised from the inner try block + with pytest.raises(Exception, match="Delete failed"): + await cosmos_db_client.delete_batch(user_id, batch_id) + + # Check that both error logs were triggered + assert mock_logger.error.call_count == 2 + + # First log: failed to delete the specific batch + first_call_args, first_call_kwargs = mock_logger.error.call_args_list[0] + assert f"Failed to delete batch with ID: {batch_id}" in first_call_args[0] + assert "error" in first_call_kwargs + assert "Delete failed" in first_call_kwargs["error"] + + # Second log: higher-level operation failed + second_call_args, second_call_kwargs = mock_logger.error.call_args_list[1] + assert second_call_args[0] == "Failed to perform delete batch operation" + assert "error" in second_call_kwargs + assert "Delete failed" in second_call_kwargs["error"] + + @pytest.mark.asyncio async def test_delete_file(cosmos_db_client, mocker): user_id = str(uuid4()) @@ -607,6 +971,35 @@ async def test_delete_file(cosmos_db_client, mocker): mock_file_container.delete_item.assert_called_once_with(file_id, partition_key=file_id) +@pytest.mark.asyncio +async def test_delete_file_exception(cosmos_db_client, mocker): + user_id = f"user_{uuid4()}" + file_id = str(uuid4()) + + # Mock delete_logs to raise an exception + mocker.patch.object( + cosmos_db_client, + 'delete_logs', + mock.AsyncMock(side_effect=Exception("Delete file failed")) + ) + + # Mock file_container to ensure delete_item is not accidentally called + mock_file_container = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'file_container', mock_file_container) + + # Mock logger to verify error logging + mock_logger = mock.MagicMock() + mocker.patch.object(cosmos_db_client, 'logger', mock_logger) + + # Expect an exception to be raised from delete_logs + with pytest.raises(Exception, match="Delete file failed"): + await cosmos_db_client.delete_file(user_id, file_id) + + mock_logger.error.assert_called_once() + called_args, _ = mock_logger.error.call_args + assert f"Failed to delete file and logs for file_id {file_id}" in called_args[0] + + @pytest.mark.asyncio async def test_add_file_log(cosmos_db_client, mocker): file_id = uuid4() From 204deae75067264e95f26f47aba7b753631cef52 Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Thu, 24 Apr 2025 13:46:16 +0530 Subject: [PATCH 44/47] fix: pylint issues of cosmosdb_test file --- .../backend/common/database/cosmosdb_test.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/tests/backend/common/database/cosmosdb_test.py b/src/tests/backend/common/database/cosmosdb_test.py index d4970111..df53fde1 100644 --- a/src/tests/backend/common/database/cosmosdb_test.py +++ b/src/tests/backend/common/database/cosmosdb_test.py @@ -212,8 +212,8 @@ async def test_create_batch_exception(cosmos_db_client, mocker): assert called_args[0] == "Failed to create batch" assert "error" in called_kwargs assert "Unexpected Error" in called_kwargs["error"] - - + + @pytest.mark.asyncio async def test_add_file(cosmos_db_client, mocker): batch_id = uuid4() @@ -266,7 +266,7 @@ async def test_add_file_exception(cosmos_db_client, mocker): assert called_args[0] == "Failed to add file" assert "error" in called_kwargs assert "Insert failed" in called_kwargs["error"] - + @pytest.mark.asyncio async def test_update_file(cosmos_db_client, mocker): @@ -387,7 +387,7 @@ async def test_update_batch_exception(cosmos_db_client, mocker): assert called_args[0] == "Failed to update batch" assert "error" in called_kwargs assert "Update batch failed" in called_kwargs["error"] - + @pytest.mark.asyncio async def test_get_batch(cosmos_db_client, mocker): @@ -511,7 +511,7 @@ async def test_get_file_exception(cosmos_db_client, mocker): assert called_args[0] == "Failed to get file" assert "error" in called_kwargs assert "Get file failed" in called_kwargs["error"] - + @pytest.mark.asyncio async def test_get_batch_files(cosmos_db_client, mocker): @@ -581,7 +581,6 @@ async def test_get_batch_files_exception(cosmos_db_client, mocker): assert "Get batch file failed" in called_kwargs["error"] - @pytest.mark.asyncio async def test_get_batch_from_id(cosmos_db_client, mocker): batch_id = str(uuid4()) @@ -637,7 +636,7 @@ async def test_get_batch_from_id_exception(cosmos_db_client, mocker): assert called_args[0] == "Failed to get batch from ID" assert "error" in called_kwargs assert "Get batch from id failed" in called_kwargs["error"] - + @pytest.mark.asyncio async def test_get_user_batches(cosmos_db_client, mocker): @@ -836,7 +835,7 @@ async def test_delete_all_exception(cosmos_db_client, mocker): assert called_args[0] == "Failed to delete all user data" assert "error" in called_kwargs assert "Delete failed" in called_kwargs["error"] - + @pytest.mark.asyncio async def test_delete_logs(cosmos_db_client, mocker): @@ -894,7 +893,7 @@ async def test_delete_logs_exception(cosmos_db_client, mocker): assert called_args[0] == "Failed to delete all user data" assert "error" in called_kwargs assert "Query failed" in called_kwargs["error"] - + @pytest.mark.asyncio async def test_delete_batch(cosmos_db_client, mocker): From cd098d4038b4e31725c8202b441803b16494d558 Mon Sep 17 00:00:00 2001 From: Prashant-Microsoft Date: Thu, 24 Apr 2025 14:29:35 +0530 Subject: [PATCH 45/47] feat: added pytest-mock package --- src/backend/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/backend/requirements.txt b/src/backend/requirements.txt index 9b9a37c0..c5d6b636 100644 --- a/src/backend/requirements.txt +++ b/src/backend/requirements.txt @@ -20,6 +20,7 @@ azure-functions # Development tools pytest +pytest-mock black pylint flake8 From 10036102bbde7b9f1154b1260668d753c27ebeb8 Mon Sep 17 00:00:00 2001 From: "Vishal Shinde (Persistent Systems Inc)" Date: Thu, 24 Apr 2025 15:40:56 +0530 Subject: [PATCH 46/47] fixed blob_base.py and blob_factory.py file test cases --- src/backend/common/storage/blob_factory.py | 18 +- src/backend/sql_agents/process_batch.py | 2 +- .../backend/common/storage/blob_base_test.py | 140 +++----- .../common/storage/blob_factory_test.py | 302 +++--------------- 4 files changed, 107 insertions(+), 355 deletions(-) diff --git a/src/backend/common/storage/blob_factory.py b/src/backend/common/storage/blob_factory.py index fc855635..d20c2de8 100644 --- a/src/backend/common/storage/blob_factory.py +++ b/src/backend/common/storage/blob_factory.py @@ -1,3 +1,4 @@ +import asyncio from typing import Optional from common.config.config import Config # Load config @@ -31,15 +32,14 @@ async def close_storage() -> None: # Local testing of config and code -if __name__ == "__main__": - # Example usage - import asyncio +async def main(): + storage = await BlobStorageFactory.get_storage() + + # Use the storage instance + blob = await storage.get_file("q1_informix.sql") + print("Blob content:", blob) - async def main(): - storage = await BlobStorageFactory.get_storage() - # Use the storage instance... - blob = await storage.get_file("q1_informix.sql") - print(blob) - await BlobStorageFactory.close_storage() + await BlobStorageFactory.close_storage() +if __name__ == "__main__": asyncio.run(main()) diff --git a/src/backend/sql_agents/process_batch.py b/src/backend/sql_agents/process_batch.py index b93ef3c1..1434fba5 100644 --- a/src/backend/sql_agents/process_batch.py +++ b/src/backend/sql_agents/process_batch.py @@ -23,7 +23,7 @@ from fastapi import HTTPException -from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent # pylint: disable=E0611 +from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent # pylint: disable=E0611 from semantic_kernel.contents import AuthorRole from semantic_kernel.exceptions.service_exceptions import ServiceResponseException diff --git a/src/tests/backend/common/storage/blob_base_test.py b/src/tests/backend/common/storage/blob_base_test.py index 561007ed..d7e2383d 100644 --- a/src/tests/backend/common/storage/blob_base_test.py +++ b/src/tests/backend/common/storage/blob_base_test.py @@ -1,128 +1,86 @@ -from datetime import datetime -from typing import Any, BinaryIO, Dict +from io import BytesIO +from typing import Any, BinaryIO, Dict, Optional + + +from common.storage.blob_base import BlobStorageBase # Adjust import path as needed -# Import the abstract base class from the production code. -from common.storage.blob_base import BlobStorageBase import pytest -# Create a dummy concrete subclass of BlobStorageBase that calls the parent's abstract methods. -class DummyBlobStorage(BlobStorageBase): - async def initialize(self) -> None: - # Call the parent (which is just a pass) - await super().initialize() - # Return a dummy value so we can verify our override is called. - return "initialized" +class MockBlobStorage(BlobStorageBase): + """Mock implementation of BlobStorageBase for testing""" async def upload_file( self, file_content: BinaryIO, blob_path: str, - content_type: str = None, - metadata: Dict[str, str] = None, + content_type: Optional[str] = None, + metadata: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: - await super().upload_file(file_content, blob_path, content_type, metadata) - # Return a dummy dictionary that simulates upload details. return { - "url": "https://dummy.blob.core.windows.net/dummy_container/" + blob_path, - "size": len(file_content), - "etag": "dummy_etag", + "path": blob_path, + "size": len(file_content.read()), + "content_type": content_type or "application/octet-stream", + "metadata": metadata or {}, + "url": f"https://mockstorage.com/{blob_path}", } async def get_file(self, blob_path: str) -> BinaryIO: - await super().get_file(blob_path) - # Return dummy binary content. - return b"dummy content" + return BytesIO(b"mock data") async def delete_file(self, blob_path: str) -> bool: - await super().delete_file(blob_path) - # Simulate a successful deletion. return True - async def list_files(self, prefix: str = None) -> list[Dict[str, Any]]: - await super().list_files(prefix) + async def list_files(self, prefix: Optional[str] = None) -> list[Dict[str, Any]]: return [ - { - "name": "dummy.txt", - "size": 123, - "created_at": datetime.now(), - "content_type": "text/plain", - "metadata": {"dummy": "value"}, - } + {"name": "file1.txt", "size": 100, "content_type": "text/plain"}, + {"name": "file2.jpg", "size": 200, "content_type": "image/jpeg"}, ] -# tests cases with each method. +@pytest.fixture +def mock_blob_storage(): + """Fixture to provide a MockBlobStorage instance""" + return MockBlobStorage() @pytest.mark.asyncio -async def test_initialize(): - storage = DummyBlobStorage() - result = await storage.initialize() - # Since the dummy override returns "initialized" after calling super(), - # we assert that the result equals that string. - assert result == "initialized" +async def test_upload_file(mock_blob_storage): + """Test upload_file method""" + file_content = BytesIO(b"dummy data") + result = await mock_blob_storage.upload_file(file_content, "test_blob.txt", "text/plain") - -@pytest.mark.asyncio -async def test_upload_file(): - storage = DummyBlobStorage() - content = b"hello world" - blob_path = "folder/hello.txt" - content_type = "text/plain" - metadata = {"key": "value"} - result = await storage.upload_file(content, blob_path, content_type, metadata) - # Verify that our dummy return value is as expected. - assert ( - result["url"] - == "https://dummy.blob.core.windows.net/dummy_container/" + blob_path - ) - assert result["size"] == len(content) - assert result["etag"] == "dummy_etag" + assert result["path"] == "test_blob.txt" + assert result["size"] == len(b"dummy data") + assert result["content_type"] == "text/plain" + assert "url" in result @pytest.mark.asyncio -async def test_get_file(): - storage = DummyBlobStorage() - result = await storage.get_file("folder/hello.txt") - # Verify that we get the dummy binary content. - assert result == b"dummy content" +async def test_get_file(mock_blob_storage): + """Test get_file method""" + result = await mock_blob_storage.get_file("test_blob.txt") - -@pytest.mark.asyncio -async def test_delete_file(): - storage = DummyBlobStorage() - result = await storage.delete_file("folder/hello.txt") - # Verify that deletion returns True. - assert result is True + assert isinstance(result, BytesIO) + assert result.read() == b"mock data" @pytest.mark.asyncio -async def test_list_files(): - storage = DummyBlobStorage() - result = await storage.list_files("dummy") - # Verify that we receive a list with one item having a 'name' key. - assert isinstance(result, list) - assert len(result) == 1 - assert "dummy.txt" in result[0]["name"] - assert result[0]["size"] == 123 - assert result[0]["content_type"] == "text/plain" - assert result[0]["metadata"] == {"dummy": "value"} +async def test_delete_file(mock_blob_storage): + """Test delete_file method""" + result = await mock_blob_storage.delete_file("test_blob.txt") + + assert result is True @pytest.mark.asyncio -async def test_smoke_all_methods(): - storage = DummyBlobStorage() - init_val = await storage.initialize() - assert init_val == "initialized" - upload_val = await storage.upload_file( - b"data", "file.txt", "text/plain", {"a": "b"} - ) - assert upload_val["size"] == 4 - file_val = await storage.get_file("file.txt") - assert file_val == b"dummy content" - delete_val = await storage.delete_file("file.txt") - assert delete_val is True - list_val = await storage.list_files("file") - assert isinstance(list_val, list) +async def test_list_files(mock_blob_storage): + """Test list_files method""" + result = await mock_blob_storage.list_files() + + assert len(result) == 2 + assert result[0]["name"] == "file1.txt" + assert result[1]["name"] == "file2.jpg" + assert result[0]["size"] == 100 + assert result[1]["size"] == 200 diff --git a/src/tests/backend/common/storage/blob_factory_test.py b/src/tests/backend/common/storage/blob_factory_test.py index 47e344ff..70ed7ecf 100644 --- a/src/tests/backend/common/storage/blob_factory_test.py +++ b/src/tests/backend/common/storage/blob_factory_test.py @@ -1,284 +1,78 @@ -import asyncio -import os -import sys -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock, patch -# Adjust sys.path so that the project root is found. -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../"))) -# Set required environment variables (dummy values) -os.environ["COSMOSDB_ENDPOINT"] = "https://dummy-endpoint" -os.environ["COSMOSDB_KEY"] = "dummy-key" -os.environ["COSMOSDB_DATABASE"] = "dummy-database" -os.environ["COSMOSDB_CONTAINER"] = "dummy-container" -os.environ["AZURE_OPENAI_DEPLOYMENT_NAME"] = "dummy-deployment" -os.environ["AZURE_OPENAI_API_VERSION"] = "2023-01-01" -os.environ["AZURE_OPENAI_ENDPOINT"] = "https://dummy-openai-endpoint" +from common.storage.blob_factory import BlobStorageFactory -# Patch missing azure module so that event_utils imports without error. -sys.modules["azure.monitor.events.extension"] = MagicMock() -# --- Import the module under test --- -from common.storage.blob_base import BlobStorageBase # noqa: E402 -from common.storage.blob_factory import BlobStorageFactory # noqa: E402 - -import pytest # noqa: E402 - -# --- Dummy configuration for testing --- - - -class DummyConfig: - azure_blob_connection_string = "dummy_connection_string" - azure_blob_container_name = "dummy_container" - -# --- Fixture to patch Config in our tests --- - - -@pytest.fixture(autouse=True) -def patch_config(monkeypatch): - # Import the real Config from your project. - from common.config.config import Config - - def dummy_init(self): - self.azure_blob_connection_string = DummyConfig.azure_blob_connection_string - self.azure_blob_container_name = DummyConfig.azure_blob_container_name - monkeypatch.setattr(Config, "__init__", dummy_init) - # Reset the BlobStorageFactory singleton before each test. - BlobStorageFactory._instance = None - - -class DummyAzureBlobStorage(BlobStorageBase): - def __init__(self, connection_string: str, container_name: str): - self.connection_string = connection_string - self.container_name = container_name - self.initialized = False - self.files = {} # maps blob_path to tuple(file_content, content_type, metadata) - - async def initialize(self): - self.initialized = True - - async def upload_file(self, file_content: bytes, blob_path: str, content_type: str, metadata: dict): - self.files[blob_path] = (file_content, content_type, metadata) - return { - "url": f"https://dummy.blob.core.windows.net/{self.container_name}/{blob_path}", - "size": len(file_content), - "etag": "dummy_etag" - } - - async def get_file(self, blob_path: str): - if blob_path in self.files: - return self.files[blob_path][0] - else: - raise FileNotFoundError(f"File {blob_path} not found") - - async def delete_file(self, blob_path: str): - if blob_path in self.files: - del self.files[blob_path] - # No error if file does not exist. - - async def list_files(self, prefix: str = ""): - return [path for path in self.files if path.startswith(prefix)] - - async def close(self): - self.initialized = False - -# --- Fixture to patch AzureBlobStorage --- - - -@pytest.fixture(autouse=True) -def patch_azure_blob_storage(monkeypatch): - monkeypatch.setattr("common.storage.blob_factory.AzureBlobStorage", DummyAzureBlobStorage) - BlobStorageFactory._instance = None - -# -------------------- Tests for BlobStorageFactory -------------------- +import pytest @pytest.mark.asyncio -async def test_get_storage_success(): - """Test that get_storage returns an initialized DummyAzureBlobStorage instance and is a singleton.""" - storage = await BlobStorageFactory.get_storage() - assert isinstance(storage, DummyAzureBlobStorage) - assert storage.initialized is True - - # Call get_storage again; it should return the same instance. - storage2 = await BlobStorageFactory.get_storage() - assert storage is storage2 +async def test_get_storage_logs_on_init(): + """Test that logger logs on initialization""" + # Force reset the singleton before test + BlobStorageFactory._instance = None + mock_storage_instance = MagicMock() -@pytest.mark.asyncio -async def test_get_storage_missing_config(monkeypatch): - """ - Test that get_storage raises a ValueError when configuration is missing. + with patch("common.storage.blob_factory.AzureBlobStorage", return_value=mock_storage_instance), \ + patch("common.storage.blob_factory.Config") as mock_config, \ + patch.object(BlobStorageFactory, "_logger") as mock_logger: - We simulate missing connection string and container name. - """ - from common.config.config import Config + mock_config_instance = MagicMock() + mock_config_instance.azure_blob_account_name = "account" + mock_config_instance.azure_blob_container_name = "container" + mock_config.return_value = mock_config_instance - def dummy_init_missing(self): - self.azure_blob_connection_string = "" - self.azure_blob_container_name = "" - monkeypatch.setattr(Config, "__init__", dummy_init_missing) - with pytest.raises(ValueError, match="Azure Blob Storage configuration is missing"): await BlobStorageFactory.get_storage() - -@pytest.mark.asyncio -async def test_close_storage_success(): - """Test that close_storage calls close() on the storage instance and resets the singleton.""" - storage = await BlobStorageFactory.get_storage() - # Patch close() method with an async mock. - storage.close = AsyncMock() - await BlobStorageFactory.close_storage() - storage.close.assert_called_once() - assert BlobStorageFactory._instance is None - -# -------------------- File Upload Tests -------------------- - - -@pytest.mark.asyncio -async def test_upload_file_success(): - """Test that upload_file successfully uploads a file and returns metadata.""" - storage = DummyAzureBlobStorage("dummy", "container") - await storage.initialize() - file_content = b"Hello, Blob!" - blob_path = "folder/blob.txt" - content_type = "text/plain" - metadata = {"meta": "data"} - result = await storage.upload_file(file_content, blob_path, content_type, metadata) - assert "url" in result - assert result["size"] == len(file_content) - assert blob_path in storage.files - - -@pytest.mark.asyncio -async def test_upload_file_error(monkeypatch): - """Test that an exception during file upload is propagated.""" - storage = DummyAzureBlobStorage("dummy", "container") - await storage.initialize() - monkeypatch.setattr(storage, "upload_file", AsyncMock(side_effect=Exception("Upload failed"))) - with pytest.raises(Exception, match="Upload failed"): - await storage.upload_file(b"data", "file.txt", "text/plain", {}) - -# -------------------- File Retrieval Tests -------------------- + mock_logger.info.assert_called_once_with("Initialized Azure Blob Storage: container") @pytest.mark.asyncio -async def test_get_file_success(): - """Test that get_file retrieves the correct file content.""" - storage = DummyAzureBlobStorage("dummy", "container") - await storage.initialize() - blob_path = "folder/data.bin" - file_content = b"BinaryData" - storage.files[blob_path] = (file_content, "application/octet-stream", {}) - result = await storage.get_file(blob_path) - assert result == file_content - +async def test_close_storage_resets_instance(): + """Test that close_storage resets the singleton instance""" + # Setup instance first + mock_storage_instance = MagicMock() -@pytest.mark.asyncio -async def test_get_file_not_found(): - """Test that get_file raises FileNotFoundError when file does not exist.""" - storage = DummyAzureBlobStorage("dummy", "container") - await storage.initialize() - with pytest.raises(FileNotFoundError): - await storage.get_file("nonexistent.file") + with patch("common.storage.blob_factory.AzureBlobStorage", return_value=mock_storage_instance), \ + patch("common.storage.blob_factory.Config") as mock_config: -# -------------------- File Deletion Tests -------------------- + mock_config_instance = MagicMock() + mock_config_instance.azure_blob_account_name = "account" + mock_config_instance.azure_blob_container_name = "container" + mock_config.return_value = mock_config_instance + instance = await BlobStorageFactory.get_storage() + assert instance is not None -@pytest.mark.asyncio -async def test_delete_file_success(): - """Test that delete_file removes an existing file.""" - storage = DummyAzureBlobStorage("dummy", "container") - await storage.initialize() - blob_path = "folder/remove.txt" - storage.files[blob_path] = (b"To remove", "text/plain", {}) - await storage.delete_file(blob_path) - assert blob_path not in storage.files - - -@pytest.mark.asyncio -async def test_delete_file_nonexistent(): - """Test that deleting a non-existent file does not raise an error.""" - storage = DummyAzureBlobStorage("dummy", "container") - await storage.initialize() - # Should not raise any exception. - await storage.delete_file("nonexistent.file") - assert True + await BlobStorageFactory.close_storage() -# -------------------- File Listing Tests -------------------- + assert BlobStorageFactory._instance is None @pytest.mark.asyncio -async def test_list_files_with_prefix(): - """Test that list_files returns files that match the given prefix.""" - storage = DummyAzureBlobStorage("dummy", "container") - await storage.initialize() - storage.files = { - "folder/a.txt": (b"A", "text/plain", {}), - "folder/b.txt": (b"B", "text/plain", {}), - "other/c.txt": (b"C", "text/plain", {}), - } - result = await storage.list_files("folder/") - assert set(result) == {"folder/a.txt", "folder/b.txt"} - - -@pytest.mark.asyncio -async def test_list_files_no_files(): - """Test that list_files returns an empty list when no files match the prefix.""" - storage = DummyAzureBlobStorage("dummy", "container") - await storage.initialize() - storage.files = {} - result = await storage.list_files("prefix/") - assert result == [] - -# -------------------- Additional Basic Tests -------------------- - - -@pytest.mark.asyncio -async def test_dummy_azure_blob_storage_initialize(): - """Test that initializing DummyAzureBlobStorage sets the initialized flag.""" - storage = DummyAzureBlobStorage("dummy_conn", "dummy_container") - assert storage.initialized is False - await storage.initialize() - assert storage.initialized is True - - -@pytest.mark.asyncio -async def test_dummy_azure_blob_storage_upload_and_retrieve(): - """Test that a file uploaded to DummyAzureBlobStorage can be retrieved.""" - storage = DummyAzureBlobStorage("dummy_conn", "dummy_container") - await storage.initialize() - content = b"Sample file content" - blob_path = "folder/sample.txt" - metadata = {"author": "tester"} - result = await storage.upload_file(content, blob_path, "text/plain", metadata) - assert "url" in result - assert result["size"] == len(content) - retrieved = await storage.get_file(blob_path) - assert retrieved == content - +async def test_get_storage_after_close_reinitializes(): + """Test that get_storage reinitializes after close_storage is called""" + # Force reset before test + BlobStorageFactory._instance = None -@pytest.mark.asyncio -async def test_dummy_azure_blob_storage_close(): - """Test that close() sets initialized to False.""" - storage = DummyAzureBlobStorage("dummy_conn", "dummy_container") - await storage.initialize() - await storage.close() - assert storage.initialized is False + with patch("common.storage.blob_factory.AzureBlobStorage") as mock_storage, \ + patch("common.storage.blob_factory.Config") as mock_config: -# -------------------- Test for BlobStorageFactory Singleton Usage -------------------- + mock_storage.side_effect = [MagicMock(name="instance1"), MagicMock(name="instance2")] + mock_config_instance = MagicMock() + mock_config_instance.azure_blob_account_name = "account" + mock_config_instance.azure_blob_container_name = "container" + mock_config.return_value = mock_config_instance -def test_common_usage_of_blob_factory(): - """Test that manually setting the singleton in BlobStorageFactory works as expected.""" - # Create a dummy storage instance. - dummy_storage = DummyAzureBlobStorage("dummy", "container") - dummy_storage.initialized = True - BlobStorageFactory._instance = dummy_storage - storage = asyncio.run(BlobStorageFactory.get_storage()) - assert storage is dummy_storage + # First init + instance1 = await BlobStorageFactory.get_storage() + await BlobStorageFactory.close_storage() + # Re-init + instance2 = await BlobStorageFactory.get_storage() -if __name__ == "__main__": - # Run tests when this file is executed directly. - asyncio.run(pytest.main()) + assert instance1 is not instance2 + assert mock_storage.call_count == 2 From 4cedd0fa85ba556741f89a22b75bebea2e32bd31 Mon Sep 17 00:00:00 2001 From: "Vishal Shinde (Persistent Systems Inc)" Date: Thu, 24 Apr 2025 18:07:31 +0530 Subject: [PATCH 47/47] removed psl-backend-unit-test line --- .github/workflows/test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ba2388ff..34a2f24d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,6 @@ on: - main - dev - demo - - psl-backend-unit-test pull_request: types: - opened