rough fuzzy matching

Signed-off-by: John Seekins <john@robot-house.us>

Author: johnseekins

ice_scrapers/general.py

@@ -1,4 +1,5 @@
 import copy
+from thefuzz import fuzz  # type: ignore [import-untyped]
 from schemas import facilities_schema
 from .agencies import scrape_agencies
 from .custom_facilities import insert_additional_facilities
@@ -10,16 +11,40 @@
 from .inspections import find_inspections
 from .spreadsheet_load import load_sheet
 from .vera_data import collect_vera_facility_data
+from utils import logger
 
 
 def facilities_scrape_wrapper(
-    keep_sheet: bool = True, force_download: bool = True, skip_vera: bool = False
+    keep_sheet: bool = True,
+    force_download: bool = True,
+    skip_vera: bool = False,
+    inspection_text: bool = False,
 ) -> tuple[dict, dict]:
-    _ = find_inspections()
     agencies = scrape_agencies(keep_sheet, force_download)
     facilities_data = copy.deepcopy(facilities_schema)
     facilities = load_sheet(keep_sheet, force_download)
     facilities_data["facilities"] = copy.deepcopy(facilities)
+    facility_name_map = {v["name"].lower(): k for k, v in facilities_data["facilities"].items()}
+    inspections = find_inspections(keep_text=inspection_text)
+
+    # actually attach inspections to facilities
+    for facility, inspect in inspections.items():
+        logger.debug("  Matching %s for inspection details...", facility)
+        # exact match (extremely unlikely)
+        if facility.lower() in facility_name_map:
+            facilities_data["facilities"][facility_name_map[facility.lower()]]["inspection"]["details"] = copy.deepcopy(
+                inspect
+            )
+            break
+        logger.debug("    Checking fuzzy matches:")
+        for k, v in facility_name_map.items():
+            r = fuzz.partial_ratio(facility, k)
+            logger.debug("    %s === %s, ratio: %s", facility, k, r)
+            if r > 80:
+                logger.info("  Probably the right facility %s => %s, (ratio %s)", k, facility, r)
+                facilities_data["facilities"][facility_name_map[k]]["inspection"]["details"] = copy.deepcopy(inspect)
+                break
+
     facilities_data = scrape_facilities(facilities_data)
     if not skip_vera:
         facilities_data = collect_vera_facility_data(facilities_data, keep_sheet, force_download)

ice_scrapers/inspections.py

@@ -34,7 +34,7 @@ def _extract_txt(url: str) -> str:
     return full_text
 
 
-def find_inspections() -> dict:
+def find_inspections(keep_text: bool = True) -> dict:
     os.makedirs(storage_dir, exist_ok=True)
     inspections: dict = {}
     logger.info("Collecting inspection reports from %s", root_url)
@@ -55,15 +55,16 @@ def find_inspections() -> dict:
         # fifth capture group should be the inspection date
         date: str = matches.group(5)  # type: ignore [union-attr]
         obj["date"] = date
-        text = zstd.compress(_extract_txt(str(url)).encode("utf-8"))
-        logger.debug(
-            "    Facility: %s, date: %s, url: %s, report length (compressed): %s",
-            location,
-            date,
-            url,
-            sys.getsizeof(text),
-        )
-        obj["text"] = text
+        if keep_text:
+            text = zstd.compress(_extract_txt(str(url)).encode("utf-8"))
+            logger.debug(
+                "    Facility: %s, date: %s, url: %s, report length (compressed): %s",
+                location,
+                date,
+                url,
+                sys.getsizeof(text),
+            )
+            obj["text"] = text
         if location in inspections:
             inspections[location].append(obj)
         else:

pyproject.toml

@@ -12,6 +12,7 @@ dependencies = [
     "polars>=1.33.0",
     "pyarrow>=21.0.0",
     "requests>=2.32.5",
+    "thefuzz>=0.22.1",
     "xlsxwriter>=3.2.5",
 ]