fix conflicts and matching for JTF again

Signed-off-by: John Seekins <john@robot-house.us>

Author: johnseekins

ice_scrapers/__init__.py

@@ -137,6 +137,7 @@
     repair_name,  # noqa: F401
     repair_street,  # noqa: F401
     repair_zip,  # noqa: F401
+    special_facilities,  # noqa: F401
     update_facility,  # noqa: F401
 )
 from .facilities_scraper import scrape_facilities  # noqa: F401,E402

ice_scrapers/facilities_scraper.py

@@ -7,6 +7,7 @@
     repair_locality,
     repair_street,
     repair_zip,
+    special_facilities,
     update_facility,
 )
 from schemas import facility_schema
@@ -27,7 +28,7 @@ def scrape_facilities(facilities_data: dict) -> dict:
     logger.info("Starting to scrape ICE.gov detention facilities...")
     facilities_data["scraped_date"] = datetime.datetime.now(datetime.UTC)
     urls = get_ice_scrape_pages(base_scrape_url)
-
+    scraped_count = 0
     for page_num, url in enumerate(urls):
         logger.info("Scraping page %s/%s...", page_num + 1, len(urls))
         try:
@@ -36,7 +37,9 @@ def scrape_facilities(facilities_data: dict) -> dict:
             logger.error("Error scraping page %s: %s", page_num + 1, e)
         logger.debug("Found %s facilities on page %s", len(facilities), page_num + 1)
         time.sleep(1)  # Be respectful to the server
+        scraped_count += len(facilities)
         for facility in facilities:
+            facility = special_facilities(facility)
             addr = facility["address"]
             street, cleaned = repair_street(addr["street"], addr["locality"])
             if cleaned:
@@ -75,7 +78,7 @@ def scrape_facilities(facilities_data: dict) -> dict:
                 facilities_data["facilities"][facility["name"]] = facility  # type: ignore [index]
 
     facilities_data["scrape_runtime"] = time.time() - start_time
-    logger.info("Total facilities scraped: %s", len(list(facilities_data["facilities"].keys())))  # type: ignore [attr-defined]
+    logger.info("Total facilities scraped: %s", scraped_count)
     logger.info(" Completed in %s seconds", facilities_data["scrape_runtime"])
     return facilities_data
 

ice_scrapers/spreadsheet_load.py

@@ -8,6 +8,7 @@
     repair_name,
     repair_street,
     repair_zip,
+    special_facilities,
 )
 import os
 import polars
@@ -131,12 +132,20 @@ def load_sheet(keep_sheet: bool = True, force_download: bool = True) -> dict:
         name, cleaned = repair_name(row["Name"], row["City"])
         if cleaned:
             details["_repaired_record"] = True
-        full_address = ",".join([street, locality, row["State"], zcode]).upper()
         details["address"]["administrative_area"] = row["State"]
         details["address"]["locality"] = locality
         details["address"]["postal_code"] = zcode
         details["address"]["street"] = street
         details["name"] = name
+        details = special_facilities(details)
+        full_address = ",".join(
+            [
+                details["address"]["street"],
+                details["address"]["locality"],
+                details["address"]["administrative_area"],
+                details["address"]["postal_code"],
+            ]
+        ).upper()
 
         """
         population statistics

ice_scrapers/utils.py

@@ -7,6 +7,34 @@
 )
 
 
+def special_facilities(facility: dict) -> dict:
+    """
+    Some very specific facilities have unique fixes
+    that are hard to fit into our normal repair_* pattern.
+
+    Please don't expand this function unless it's necessary
+    """
+    match facility["name"]:
+        case "Naval Station Guantanamo Bay (JTF Camp Six and Migrant Ops Center Main A)":
+            """
+            First special case? JTF Camp Six is purely a mess.
+            While we work on getting a consistent address for this facility,
+            we'll need to make the two records converge.
+            """
+            facility["address"]["country"] = "Cuba"
+            facility["address"]["administrative_area"] = "FPO"
+            facility["address"]["locality"] = "FPO"
+            facility["address"]["postal_code"] = "34009"
+            facility["address"]["street"] = "AVENUE C PSC 1005 BOX 55"
+        case "JTF CAMP SIX":
+            facility["address"]["country"] = "Cuba"
+            facility["address"]["administrative_area"] = "FPO"
+            facility["name"] = "Naval Station Guantanamo Bay (JTF Camp Six and Migrant Ops Center Main A)"
+        case _:
+            pass
+    return facility
+
+
 def repair_street(street: str, locality: str = "") -> Tuple[str, bool]:
     """Generally, we'll let the spreadsheet win arguments just to be consistent"""
     street_filters = [

ice_scrapers/vera_data.py

@@ -240,15 +240,21 @@ def collect_vera_facility_data(facilities_data: dict, keep_sheet: bool = True, f
             continue
         found = False
         facility_name, fixed_name = _vera_name_fixes(row["detention_facility_name"], row["city"])
+        row["name"] = facility_name
         city, fixed_city = _vera_city_fixes(row["city"], row["state"])
+        row["city"] = city
         if fixed_name or fixed_city:
             fixed += 1
-        addr_str = f"{facility_name},{city},{row['state']},United States"
+        if row["name"] == "JTF Camp Six":
+            row["state"] = "FPO"
+            row["city"] = "FPO"
+            row["name"] = "Naval Station Guantanamo Bay (JTF Camp Six and Migrant Ops Center Main A)"
+        addr_str = f"{row['name']},{row['city']},{row['state']}"
         for k, v in facilities_data["facilities"].items():
             if (
-                v["name"].upper() == facility_name.upper()
+                v["name"].upper() == row["name"].upper()
                 and v["address"]["administrative_area"].upper() == row["state"].upper()
-                and v["address"]["locality"].upper() == city.upper()
+                and v["address"]["locality"].upper() == row["city"].upper()
             ):
                 logger.debug("  Found matching facility %s...", v["name"])
                 facilities_data["facilities"][k]["osm"]["latitude"] = row["latitude"]
@@ -261,10 +267,9 @@ def collect_vera_facility_data(facilities_data: dict, keep_sheet: bool = True, f
         if not found:
             facilities_data["facilities"][addr_str] = copy.deepcopy(facility_schema)
             facilities_data["facilities"][addr_str]["source_urls"].append(base_url)
-            facilities_data["facilities"][addr_str]["name"] = facility_name
+            facilities_data["facilities"][addr_str]["name"] = row["name"]
             facilities_data["facilities"][addr_str]["address"]["administrative_area"] = row["state"]
-            facilities_data["facilities"][addr_str]["address"]["locality"] = city
-            facilities_data["facilities"][addr_str]["address"]["country"] = "United States"
+            facilities_data["facilities"][addr_str]["address"]["locality"] = row["city"]
             facilities_data["facilities"][addr_str]["address_str"] = addr_str
             facilities_data["facilities"][addr_str]["osm"]["latitude"] = row["latitude"]
             facilities_data["facilities"][addr_str]["osm"]["longitude"] = row["longitude"]