Handle error tests as a different test icon (#21630)

fixes #21625

adds a few things:
- returns error on any caught errors in pytest that aren't assertion
errors
- add error node type handling to result resolver
- update tests for both files

Author: eleanorjboyd

pythonFiles/tests/pytestadapter/expected_execution_test_output.py

@@ -157,7 +157,7 @@
 error_raised_exception_execution_expected_output = {
     "error_raise_exception.py::TestSomething::test_a": {
         "test": "error_raise_exception.py::TestSomething::test_a",
-        "outcome": "failure",
+        "outcome": "error",
         "message": "ERROR MESSAGE",
         "traceback": "TRACEBACK",
         "subtest": None,

pythonFiles/tests/pytestadapter/test_execution.py

@@ -174,7 +174,10 @@ def test_pytest_execution(test_ids, expected_const):
         assert a["cwd"] == os.fspath(TEST_DATA_PATH)
         actual_result_dict.update(a["result"])
     for key in actual_result_dict:
-        if actual_result_dict[key]["outcome"] == "failure":
+        if (
+            actual_result_dict[key]["outcome"] == "failure"
+            or actual_result_dict[key]["outcome"] == "error"
+        ):
             actual_result_dict[key]["message"] = "ERROR MESSAGE"
         if actual_result_dict[key]["traceback"] != None:
             actual_result_dict[key]["traceback"] = "TRACEBACK"

pythonFiles/vscode_pytest/__init__.py

@@ -82,7 +82,9 @@ def pytest_exception_interact(node, call, report):
             )
     else:
         # if execution, send this data that the given node failed.
-        report_value = "failure"
+        report_value = "error"
+        if call.excinfo.typename == "AssertionError":
+            report_value = "failure"
         node_id = str(node.nodeid)
         if node_id not in collected_tests_so_far:
             collected_tests_so_far.append(node_id)
@@ -119,7 +121,7 @@ class TestOutcome(Dict):
     """
 
     test: str
-    outcome: Literal["success", "failure", "skipped"]
+    outcome: Literal["success", "failure", "skipped", "error"]
     message: Union[str, None]
     traceback: Union[str, None]
     subtest: Optional[str]

src/client/testing/testController/common/resultResolver.ts

@@ -102,7 +102,29 @@ export class PythonResultResolver implements ITestResultResolver {
                     testCases.push(...tempArr);
                 });
 
-                if (
+                if (rawTestExecData.result[keyTemp].outcome === 'error') {
+                    const rawTraceback = rawTestExecData.result[keyTemp].traceback ?? '';
+                    const traceback = splitLines(rawTraceback, {
+                        trim: false,
+                        removeEmptyEntries: true,
+                    }).join('\r\n');
+                    const text = `${rawTestExecData.result[keyTemp].test} failed with error: ${
+                        rawTestExecData.result[keyTemp].message ?? rawTestExecData.result[keyTemp].outcome
+                    }\r\n${traceback}\r\n`;
+                    const message = new TestMessage(text);
+
+                    const grabVSid = this.runIdToVSid.get(keyTemp);
+                    // search through freshly built array of testItem to find the failed test and update UI.
+                    testCases.forEach((indiItem) => {
+                        if (indiItem.id === grabVSid) {
+                            if (indiItem.uri && indiItem.range) {
+                                message.location = new Location(indiItem.uri, indiItem.range);
+                                runInstance.errored(indiItem, message);
+                                runInstance.appendOutput(fixLogLines(text));
+                            }
+                        }
+                    });
+                } else if (
                     rawTestExecData.result[keyTemp].outcome === 'failure' ||
                     rawTestExecData.result[keyTemp].outcome === 'passed-unexpected'
                 ) {

src/test/testing/testController/resultResolver.unit.test.ts

@@ -390,6 +390,45 @@ suite('Result Resolver tests', () => {
             // verify that the passed function was called for the single test item
             runInstance.verify((r) => r.skipped(typemoq.It.isAny()), typemoq.Times.once());
         });
+        test('resolveExecution handles error correctly as test outcome', async () => {
+            // test specific constants used expected values
+            testProvider = 'pytest';
+            workspaceUri = Uri.file('/foo/bar');
+            resultResolver = new ResultResolver.PythonResultResolver(
+                testControllerMock.object,
+                testProvider,
+                workspaceUri,
+            );
+            // add a mock test item to the map of known VSCode ids to run ids
+            resultResolver.runIdToVSid.set('mockTestItem1', 'mockTestItem1');
+            resultResolver.runIdToVSid.set('mockTestItem2', 'mockTestItem2');
+
+            // add this mock test to the map of known test items
+            resultResolver.runIdToTestItem.set('mockTestItem1', mockTestItem1);
+            resultResolver.runIdToTestItem.set('mockTestItem2', mockTestItem2);
+
+            // create a successful payload with a single test called mockTestItem1
+            const successPayload: ExecutionTestPayload = {
+                cwd: workspaceUri.fsPath,
+                status: 'success',
+                result: {
+                    mockTestItem1: {
+                        test: 'test',
+                        outcome: 'error', // failure, passed-unexpected, skipped, success, expected-failure, subtest-failure, subtest-succcess
+                        message: 'message',
+                        traceback: 'traceback',
+                        subtest: 'subtest',
+                    },
+                },
+                error: '',
+            };
+
+            // call resolveExecution
+            resultResolver.resolveExecution(successPayload, runInstance.object);
+
+            // verify that the passed function was called for the single test item
+            runInstance.verify((r) => r.errored(typemoq.It.isAny(), typemoq.It.isAny()), typemoq.Times.once());
+        });
         test('resolveExecution handles success correctly', async () => {
             // test specific constants used expected values
             testProvider = 'pytest';