diff --git a/pylint/testutils/functional/test_file.py b/pylint/testutils/functional/test_file.py index 342ced44bf..fedc155d72 100644 --- a/pylint/testutils/functional/test_file.py +++ b/pylint/testutils/functional/test_file.py @@ -101,6 +101,10 @@ def module(self) -> str: package = basename(self._directory) return ".".join([package, self.base]) + @property + def expected_output_base(self) -> str: + return join(self._directory, self.base + ".txt") + @property def expected_output(self) -> str: files = [ diff --git a/pylint/testutils/lint_module_test.py b/pylint/testutils/lint_module_test.py index 3444ea7adf..82b8cd0d65 100644 --- a/pylint/testutils/lint_module_test.py +++ b/pylint/testutils/lint_module_test.py @@ -250,13 +250,7 @@ def check_messages(self) -> list[OutputLine]: __tracebackhide__ = True # pylint: disable=unused-variable modules_to_check = [self._test_file.source] self._linter.check(modules_to_check) - expected_messages, _ = self._get_expected() - actual_messages, actual_output = self._get_actual() - assert ( - expected_messages == actual_messages - ), self.error_msg_for_unequal_messages( - actual_messages, expected_messages, actual_output - ) + _, actual_output = self._get_actual() return actual_output def _runTest(self) -> None: diff --git a/requirements_test_min.txt b/requirements_test_min.txt index 989cfc90ac..f746610b94 100644 --- a/requirements_test_min.txt +++ b/requirements_test_min.txt @@ -5,6 +5,7 @@ typing-extensions~=4.15 py~=1.11.0 pytest>=8.4,<10.0 pytest-benchmark~=5.1 +pytest-remaster==0.0.5 pytest-timeout~=2.4 towncrier>=24.8,<26.0 requests diff --git a/tests/test_functional.py b/tests/test_functional.py index 918bb1c47d..95f38f0a34 100644 --- a/tests/test_functional.py +++ b/tests/test_functional.py @@ -6,6 +6,7 @@ from __future__ import annotations +import sys from collections.abc import Iterator from pathlib import Path from typing import TYPE_CHECKING @@ -67,5 +68,10 @@ def test_functional( actual_output = lint_test.check_messages() golden_master.check( - lint_test.serialize_output(actual_output), test_file.expected_output + lint_test.serialize_output(actual_output), + test_file.expected_output_base, + dimensions={ + "version": f"{sys.version_info[0]}{sys.version_info[1]}", + "platform": sys.platform, + }, ) diff --git a/tests/testutils/test_lint_module_output_update.py b/tests/testutils/test_lint_module_output_update.py deleted file mode 100644 index 60819ac74f..0000000000 --- a/tests/testutils/test_lint_module_output_update.py +++ /dev/null @@ -1,141 +0,0 @@ -# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html -# For details: https://github.com/pylint-dev/pylint/blob/main/LICENSE -# Copyright (c) https://github.com/pylint-dev/pylint/blob/main/CONTRIBUTORS.txt - -# pylint: disable=redefined-outer-name - -from __future__ import annotations - -import shutil -from collections.abc import Callable -from pathlib import Path - -import pytest - -from pylint.testutils import FunctionalTestFile, LintModuleTest -from pylint.testutils.functional import LintModuleOutputUpdate - -FIXTURE_DIRECTORY = Path(__file__).parent / "data/functional" -DIRECTORIES = list(FIXTURE_DIRECTORY.iterdir()) - - -@pytest.fixture() -def lint_module_fixture( - tmp_path: Path, -) -> Callable[[str], tuple[Path, Path, LintModuleOutputUpdate]]: - def inner(base: str) -> tuple[Path, Path, LintModuleOutputUpdate]: - filename = tmp_path / f"{base}.py" - expected_output_file = tmp_path / f"{base}.txt" - lmou = LintModuleOutputUpdate( - test_file=FunctionalTestFile(str(tmp_path), str(filename)) - ) - return filename, expected_output_file, lmou - - return inner - - -def test_lint_module_output_update_fail_before( - lint_module_fixture: Callable[[str], tuple[Path, Path, LintModuleOutputUpdate]], -) -> None: - """There is a fail before the output need to be updated.""" - filename, expected_output_file, lmou = lint_module_fixture("foo") - filename.write_text("", encoding="utf8") - assert not expected_output_file.exists() - with pytest.raises(AssertionError, match="1: disallowed-name"): - lmou.runTest() - assert not expected_output_file.exists() - - -def test_lint_module_output_update_effective( - lint_module_fixture: Callable[[str], tuple[Path, Path, LintModuleOutputUpdate]], -) -> None: - """The file is updated following a successful tests with wrong output.""" - filename, expected_output_file, lmou = lint_module_fixture("foo") - filename.write_text("# [disallowed-name]\n", encoding="utf8") - lmou.runTest() - assert (expected_output_file).exists() - assert ( - expected_output_file.read_text(encoding="utf8") - == 'disallowed-name:1:0:None:None::"Disallowed name ""foo""":HIGH\n' - ) - - -def test_lint_module_output_update_remove_useless_txt( - lint_module_fixture: Callable[[str], tuple[Path, Path, LintModuleOutputUpdate]], -) -> None: - """The file is updated following a successful tests with wrong output.""" - filename, expected_output_file, lmou = lint_module_fixture("fine_name") - expected_output_file.write_text("", encoding="utf8") - filename.write_text("", encoding="utf8") - lmou.runTest() - assert not expected_output_file.exists() - - -@pytest.mark.parametrize( - "directory_path", DIRECTORIES, ids=[str(p) for p in DIRECTORIES] -) -def test_update_of_functional_output(directory_path: Path, tmp_path: Path) -> None: - """Functional test for the functional tests' helper.""" - - def _check_expected_output(_ftf: FunctionalTestFile) -> None: - new_output_path = _ftf.expected_output - assert Path( - new_output_path - ).exists(), "The expected output file does not exists" - with open(new_output_path, encoding="utf8") as f: - new_output = f.read() - assert ( - new_output == "exec-used:7:0:7:14::Use of exec:UNDEFINED\n" - ), f"The content was wrongly updated in {new_output_path}" - - def _assert_behavior_is_correct( - _ftf: FunctionalTestFile, - _lint_module: LintModuleTest, - _lint_module_output_update: LintModuleOutputUpdate, - _new_path: Path, - ) -> None: - new_path_str = str(_new_path) - if "wrong_test" in new_path_str: - expected = r'Wrong message\(s\) raised for "exec_used.py"' - with pytest.raises(AssertionError, match=expected): - _lint_module.runTest() - # When the tests are wrong we do not update the output at all - # and the test should fail - with pytest.raises(AssertionError, match=expected): - _lint_module_output_update.runTest() - elif "ok_test" in new_path_str: - if any(f"{x}_output" in new_path_str for x in ("wrong", "no", "broken")): - with pytest.raises( - AssertionError, match=r'Wrong output for "exec_used\.txt"' - ): - _lint_module.runTest() - elif "ok_output" in new_path_str: - _lint_module.runTest() - _check_expected_output(_ftf) - else: - raise AssertionError(f"Unhandled test case: {new_path_str}") - - # When the tests are ok we update the output whatever it's state - # was originally - _lint_module_output_update.runTest() - _check_expected_output(_ftf) - else: - raise AssertionError( - f"Do not pollute '{FIXTURE_DIRECTORY}' with unrelated " - f"or badly named test files." - ) - - new_path = tmp_path / directory_path.name - shutil.copytree(directory_path, new_path) - for filename in new_path.iterdir(): - if filename.suffix != ".py": - continue - ftf = FunctionalTestFile(directory=str(new_path), filename=filename.name) - # Standard functional test helper - lint_module = LintModuleTest(ftf) - # Functional test helper that automatically update the output - lint_module_output_update = LintModuleOutputUpdate(ftf) - - _assert_behavior_is_correct( - ftf, lint_module, lint_module_output_update, new_path - )