vera license is weird, don't collect by default. Also clean up some types

Signed-off-by: John Seekins <john@robot-house.us>

Author: johnseekins

ice_scrapers/field_offices.py

@@ -41,7 +41,7 @@ def scrape_field_offices() -> dict:
     return office_data
 
 
-def _scrape_page(page_url: str) -> list:
+def _scrape_page(page_url: str) -> list[dict]:
     """Scrape a single page of facilities using BeautifulSoup"""
     logger.debug("  Fetching: %s", page_url)
     try:

ice_scrapers/spreadsheet_load.py

@@ -17,7 +17,6 @@
     facility_schema,
     field_office_schema,
 )
-from typing import Tuple
 from utils import (
     logger,
     session,
@@ -59,7 +58,7 @@
 ]
 
 
-def _download_sheet(keep_sheet: bool = True, force_download: bool = True) -> Tuple[polars.DataFrame, str]:
+def _download_sheet(keep_sheet: bool = True, force_download: bool = True) -> tuple[polars.DataFrame, str]:
     """Download the detention stats sheet from ice.gov"""
     resp = session.get(base_xlsx_url, timeout=120)
     resp.raise_for_status()

ice_scrapers/utils.py

@@ -1,6 +1,5 @@
 from bs4 import BeautifulSoup
 import re
-from typing import Tuple
 from utils import (
     logger,
     session,
@@ -35,7 +34,7 @@ def special_facilities(facility: dict) -> dict:
     return facility
 
 
-def repair_name(name: str, locality: str) -> Tuple[str, bool]:
+def repair_name(name: str, locality: str) -> tuple[str, bool]:
     """Even facility names are occasionally bad"""
     matches = [
         {"match": "ALEXANDRIA STAGING FACILI", "replace": "Alexandria Staging Facility", "locality": "ALEXANDRIA"},
@@ -96,7 +95,7 @@ def repair_name(name: str, locality: str) -> Tuple[str, bool]:
     return name, cleaned
 
 
-def repair_street(street: str, locality: str = "") -> Tuple[str, bool]:
+def repair_street(street: str, locality: str = "") -> tuple[str, bool]:
     """Generally, we'll let the spreadsheet win arguments just to be consistent"""
     street_filters = [
         # address mismatch between site and spreadsheet
@@ -217,7 +216,7 @@ def repair_street(street: str, locality: str = "") -> Tuple[str, bool]:
     return street, cleaned
 
 
-def repair_zip(zip_code: int, locality: str) -> Tuple[str, bool]:
+def repair_zip(zip_code: int, locality: str) -> tuple[str, bool]:
     """
     Excel does a cool thing where it strips leading 0s
     Also, many zip codes are mysteriously discordant
@@ -248,7 +247,7 @@ def repair_zip(zip_code: int, locality: str) -> Tuple[str, bool]:
     return zcode, cleaned
 
 
-def repair_locality(locality: str, administrative_area: str) -> Tuple[str, bool]:
+def repair_locality(locality: str, administrative_area: str) -> tuple[str, bool]:
     """
     There is no consistency with any address.
     How the post office ever successfully delivered a letter is beyond me

ice_scrapers/vera_data.py

@@ -3,7 +3,6 @@
 import os
 import polars
 from schemas import facility_schema
-from typing import Tuple
 from utils import (
     logger,
     session,
@@ -17,7 +16,7 @@
 filename = f"{SCRIPT_DIR}{os.sep}vera_facilities.csv"
 
 
-def _vera_name_fixes(name: str, city: str) -> Tuple[str, bool]:
+def _vera_name_fixes(name: str, city: str) -> tuple[str, bool]:
     """Match Vera names with ice.gov names"""
     matches = [
         {"match": "Adams County", "replace": "Adams County Courthouse", "city": "Ritzville"},
@@ -199,7 +198,7 @@ def _vera_name_fixes(name: str, city: str) -> Tuple[str, bool]:
     return name, fixed
 
 
-def _vera_city_fixes(city: str, state: str) -> Tuple[str, bool]:
+def _vera_city_fixes(city: str, state: str) -> tuple[str, bool]:
     """There are a few cases where getting a state match requires some munging"""
     matches = [
         {"match": "Saipan", "replace": "Susupe, Saipan", "city": "MP"},

main.py

@@ -35,76 +35,79 @@ def main() -> None:
         description="ICE Detention Facilities Data Scraper and Enricher",
         formatter_class=ArgumentDefaultsHelpFormatter,
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--scrape",
         action="store_true",
         default=False,
         help="Scrape initial facility data from ICE.gov",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--enrich",
         action="store_true",
         default=False,
         help="enrich collected data",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--load-existing",
         action="store_true",
         default=False,
         help="load data from local files",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--file-type",
         choices=supported_output_types,
+        type=str,
         help="type of file to export",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--output-file-name",
         "-o",
         default="ice_detention_facilities",
+        type=str,
         help="The file we'll write data out to (excluding the suffix)",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--debug",
         action="store_true",
         help="Full debug information and logging",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--enrich-workers",
         type=int,
         default=3,
         help="Number of concurrent processes to allow while enriching data",
     )
     # todo these need more attention, but should now be accepted as command line options now.
-    parser.add_argument(
+    _ = parser.add_argument(
         "--debug-wikipedia",
         action="store_true",
         help="Add another column on export for Wikipedia debugging details and redirects",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--debug-wikidata",
         action="store_true",
         help="Add another column on export for Wikidata debugging details and redirects",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--debug-osm",
         action="store_true",
         help="Add another column on export for OpenStreetMap debugging details and redirects",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--skip-downloads",
         action="store_true",
         help="Skip downloading sheet data",
     )
-    parser.add_argument(
+    _ = parser.add_argument(
         "--delete-sheets",
         action="store_true",
         help="Remove any sheets we downloaded",
     )
-    parser.add_argument(
-        "--skip-vera",
+    _ = parser.add_argument(
+        "--use-vera",
         action="store_true",
-        help="Don't collect vera.org data",
+        default=False,
+        help="Collect vera.org data",
     )
 
     args = parser.parse_args()
@@ -131,7 +134,7 @@ def main() -> None:
         facilities_data = facilities_scrape_wrapper(
             keep_sheet=not args.delete_sheets,
             force_download=not args.skip_downloads,
-            skip_vera=args.skip_vera,
+            skip_vera=not args.use_vera,
         )
     elif args.load_existing:
         facilities_data = copy.deepcopy(default_data.facilities_data)

schemas.py

@@ -124,4 +124,4 @@
     "wikidata_found": 0,
 }
 
-supported_output_types: list = ["csv", "json", "parquet", "xlsx"]
+supported_output_types: list[str] = ["csv", "json", "parquet", "xlsx"]