Fixed two bugs in the conformance test: (#1763)

1. The "conformance_automated" was not written to the output in some cases.
2. The output file was not written if it didn't previously exist and the tests passed.
This doesn't change the results of any of the tests.

Author: erictraut

conformance/results/mypy/constructors_consistency.toml

@@ -2,3 +2,8 @@ conformant = "Pass"
 notes = """
 Does not report inconsistency between __new__ and __init__ (optional).
 """
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+"""

conformance/results/mypy/exceptions_context_managers.toml

@@ -3,3 +3,4 @@ errors_diff = """
 """
 output = """
 """
+conformance_automated = "Pass"

conformance/results/pyre/constructors_call_type.toml

@@ -11,3 +11,4 @@ constructors_call_type.py:72:4 Missing argument [20]: Call `Meta1.__call__` expe
 constructors_call_type.py:81:4 Missing argument [20]: Call `Class2.__new__` expects argument `y`.
 constructors_call_type.py:82:11 Incompatible parameter type [6]: In call `Class2.__new__`, for 2nd positional argument, expected `str` but got `int`.
 """
+conformance_automated = "Pass"

conformance/results/pyre/constructors_consistency.toml

@@ -2,3 +2,8 @@ conformant = "Pass"
 notes = """
 Does not report inconsistency between __new__ and __init__ (optional).
 """
+conformance_automated = "Pass"
+errors_diff = """
+"""
+output = """
+"""

conformance/results/pyright/constructors_call_new.toml

@@ -10,3 +10,4 @@ constructors_call_new.py:145:1 - error:
     Type "type[Class11[str]]" is incompatible with type "type[Class11[int]]"
       Type parameter "T@Class11" is invariant, but "str" is not the same as "int" (reportGeneralTypeIssues)
 """
+conformance_automated = "Pass"

conformance/results/pyright/constructors_consistency.toml

@@ -6,3 +6,4 @@ constructors_consistency.py:25:9 - error: Mismatch between signature of __new__
   Signature of __init__ is "(x: str)"
   Signature of __new__ is "()" (reportInconsistentConstructor)
 """
+conformance_automated = "Pass"

conformance/results/pyright/exceptions_context_managers.toml

@@ -3,3 +3,4 @@ errors_diff = """
 """
 output = """
 """
+conformance_automated = "Pass"

conformance/results/pytype/constructors_consistency.toml

@@ -7,3 +7,4 @@ errors_diff = """
 output = """
 
 """
+conformance_automated = "Pass"

conformance/src/main.py

@@ -141,6 +141,7 @@ def update_output_for_test(
 
     results_file = results_dir / f"{test_name}.toml"
     results_file.parent.mkdir(parents=True, exist_ok=True)
+    should_write = False
 
     # Read the existing results file if present.
     try:
@@ -153,7 +154,6 @@ def update_output_for_test(
         print(f"Error decoding {results_file}")
         existing_results = {}
 
-    should_write = False
     ignored_errors = existing_results.get("ignore_errors", [])
     errors_diff = "\n" + diff_expected_errors(type_checker, test_case, output, ignored_errors)
     old_errors_diff = "\n" + existing_results.get("errors_diff", "")
@@ -165,8 +165,10 @@ def update_output_for_test(
         print(f"New output: {errors_diff}")
         print("")
 
-        existing_results["conformance_automated"] = "Fail" if errors_diff.strip() else "Pass"
-
+    conformance_automated = "Fail" if errors_diff.strip() else "Pass"
+    if existing_results.get("conformance_automated") != conformance_automated:
+        should_write = True
+        existing_results["conformance_automated"] = conformance_automated
 
     old_output = existing_results.get("output", "")
     old_output = f"\n{old_output}"