Fix for unittest with testscenarios (#23230)

Fixes #17641

---------

Co-authored-by: eleanorjboyd <eleanorboyd@microsoft.com>

Author: karthiknadig

build/test-requirements.txt

@@ -12,6 +12,8 @@ flask
 fastapi
 uvicorn
 django
+testresources
+testscenarios
 
 # Integrated TensorBoard tests
 tensorboard

python_files/tests/unittestadapter/.data/test_scenarios/tests/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import os
+
+import testresources
+from testscenarios import generate_scenarios
+
+def load_tests(loader, tests, pattern):
+    this_dir = os.path.dirname(__file__)
+    mytests = loader.discover(start_dir=this_dir, pattern=pattern)
+    result = testresources.OptimisingTestSuite()
+    result.addTests(generate_scenarios(mytests))
+    result.addTests(generate_scenarios(tests))
+    return result

python_files/tests/unittestadapter/.data/test_scenarios/tests/test_scene.py

@@ -0,0 +1,25 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+from testscenarios import TestWithScenarios
+
+class TestMathOperations(TestWithScenarios):
+    scenarios = [
+        ('add', {'test_id': 'test_add', 'a': 5, 'b': 3, 'expected': 8}),
+        ('subtract', {'test_id': 'test_subtract', 'a': 5, 'b': 3, 'expected': 2}),
+        ('multiply', {'test_id': 'test_multiply', 'a': 5, 'b': 3, 'expected': 15})
+    ]
+    a: int = 0
+    b: int = 0
+    expected: int = 0
+    test_id: str = ""
+
+    def test_operations(self):
+        result = None
+        if self.test_id == 'test_add':
+            result = self.a + self.b
+        elif self.test_id == 'test_subtract':
+            result = self.a - self.b
+        elif self.test_id == 'test_multiply':
+            result = self.a * self.b
+        self.assertEqual(result, self.expected)

python_files/tests/unittestadapter/test_execution.py

@@ -141,7 +141,7 @@ def test_subtest_run(mock_send_run_data) -> None:
                 "pattern_a_test.DiscoveryA.test_one_a",
                 "pattern_a_test.DiscoveryA.test_two_a",
             ],
-            "*test",
+            "*test.py",
             os.fspath(TEST_DATA_PATH / "two_patterns"),
             "success",
         ),
@@ -174,6 +174,16 @@ def test_subtest_run(mock_send_run_data) -> None:
             os.fspath(TEST_DATA_PATH),
             "success",
         ),
+        (
+            [
+                "test_scene.TestMathOperations.test_operations(add)",
+                "test_scene.TestMathOperations.test_operations(subtract)",
+                "test_scene.TestMathOperations.test_operations(multiply)",
+            ],
+            "*",
+            os.fspath(TEST_DATA_PATH / "test_scenarios" / "tests"),
+            "success",
+        ),
     ],
 )
 def test_multiple_ids_run(mock_send_run_data, test_ids, pattern, cwd, expected_outcome) -> None:

python_files/unittestadapter/execution.py

@@ -143,6 +143,35 @@ def formatResult(
         send_run_data(result, test_run_pipe)
 
 
+def filter_tests(suite: unittest.TestSuite, test_ids: List[str]) -> unittest.TestSuite:
+    """Filter the tests in the suite to only run the ones with the given ids."""
+    filtered_suite = unittest.TestSuite()
+    for test in suite:
+        if isinstance(test, unittest.TestCase):
+            if test.id() in test_ids:
+                filtered_suite.addTest(test)
+        else:
+            filtered_suite.addTest(filter_tests(test, test_ids))
+    return filtered_suite
+
+
+def get_all_test_ids(suite: unittest.TestSuite) -> List[str]:
+    """Return a list of all test ids in the suite."""
+    test_ids = []
+    for test in suite:
+        if isinstance(test, unittest.TestCase):
+            test_ids.append(test.id())
+        else:
+            test_ids.extend(get_all_test_ids(test))
+    return test_ids
+
+
+def find_missing_tests(test_ids: List[str], suite: unittest.TestSuite) -> List[str]:
+    """Return a list of test ids that are not in the suite."""
+    all_test_ids = get_all_test_ids(suite)
+    return [test_id for test_id in test_ids if test_id not in all_test_ids]
+
+
 # Args: start_path path to a directory or a file, list of ids that may be empty.
 # Edge cases:
 # - if tests got deleted since the VS Code side last ran discovery and the current test run,
@@ -169,16 +198,6 @@ def run_tests(
             start_dir = os.path.dirname(cwd)
             pattern = os.path.basename(cwd)
 
-        # Discover tests at path with the file name as a pattern (if any).
-        loader = unittest.TestLoader()
-
-        args = {  # noqa: F841
-            "start_dir": start_dir,
-            "pattern": pattern,
-            "top_level_dir": top_level_dir,
-        }
-        suite = loader.discover(start_dir, pattern, top_level_dir)  # noqa: F841
-
         if failfast is None:
             failfast = False
         if locals is None:
@@ -191,9 +210,20 @@ def run_tests(
             failfast=failfast,
             verbosity=verbosity,
         )
-        # lets try to tailer our own suite so we can figure out running only the ones we want
+
+        # Discover tests at path with the file name as a pattern (if any).
         loader = unittest.TestLoader()
-        tailor: unittest.TestSuite = loader.loadTestsFromNames(test_ids)
+        suite = loader.discover(start_dir, pattern, top_level_dir)
+
+        # lets try to tailer our own suite so we can figure out running only the ones we want
+        tailor: unittest.TestSuite = filter_tests(suite, test_ids)
+
+        # If any tests are missing, add them to the payload.
+        not_found = find_missing_tests(test_ids, tailor)
+        if not_found:
+            missing_suite = loader.loadTestsFromNames(not_found)
+            tailor.addTests(missing_suite)
+
         result: UnittestTestResult = runner.run(tailor)  # type: ignore
 
         payload["result"] = result.formatted