Update Label_Stats to avoid creating unnecessary diffs in CSV output

Author: frreiss

scripts/Label_Stats.ipynb

@@ -51,8 +51,8 @@
     "_CONLL_4_NOT_IN_GOLD_FILE       = os.path.join(data_dir, \"CoNLL_4_not_in_gold.csv\")\n",
     "_CONLL_4_TRAIN_NOT_IN_GOLD_FILE = os.path.join(data_dir, \"CoNLL_4_train_not_in_gold.csv\")\n",
     "\n",
-    "# Optionally output figures generated\n",
-    "save_figures = True\n",
+    "# Change this constant to True to regenerate PDF/EPS/PNG files with figures.\n",
+    "save_figures = False\n",
     "figure_dir = os.path.join(data_dir, \"label_stats_images\")\n",
     "if save_figures and not os.path.exists(figure_dir):\n",
     "    os.mkdir(figure_dir)"
@@ -8655,10 +8655,22 @@
     "write_file.loc[:, \"hand_labelled\"] = write_file[\"agreeing_models\"].isna()\n",
     "write_file = write_file[write_file.error_type != \"None\"]\n",
     "\n",
-    "write_file.rename(columns={\"conll_2\": \"Original entrants ensemble\", \"conll_3\": \"custom models ensemble\", \"conll_4\": \"cross validation ensemble\"}, inplace=True)\n",
+    "write_file = write_file.rename(columns={\"conll_2\": \"Original entrants ensemble\", \n",
+    "                                        \"conll_3\": \"Custom models ensemble\", \n",
+    "                                        \"conll_4\": \"Cross validation ensemble\"})\n",
     "\n",
-    "write_file.to_csv(ALL_LABELS_OUTPUT_FILE_NAME)\n",
-    "print(\"Done\")"
+    "# Drop \"agreeing_models\" and boolean columns to make diffs smaller.\n",
+    "write_file = write_file.drop(columns=[\"agreeing_models\",\n",
+    "                                      \"hand_labelled\",\n",
+    "                                      \"Original entrants ensemble\",\n",
+    "                                      \"Custom models ensemble\",\n",
+    "                                      \"Cross validation ensemble\"])\n",
+    "\n",
+    "# TODO: Sort so that order is consistent across Python versions.\n",
+    "#write_file = write_file.sort_values([\"fold\", \"doc_offset\", \"corpus_span\", \"corpus_ent_type\", \"error_type\"])\n",
+    "\n",
+    "write_file.to_csv(ALL_LABELS_OUTPUT_FILE_NAME, index=False)\n",
+    "print(f\"Wrote merged labels to {ALL_LABELS_OUTPUT_FILE_NAME}\")"
    ]
   },
   {
@@ -10189,7 +10201,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.6"
+   "version": "3.8.5"
   }
  },
  "nbformat": 4,