reduce logging of inspections

Signed-off-by: John Seekins <john@robot-house.us>

Author: johnseekins

ice_scrapers/agencies.py

@@ -12,7 +12,7 @@
 from utils import (
     logger,
     output_folder,
-    session,
+    req_get,
 )
 from .utils import download_file
 
@@ -22,8 +22,7 @@
 def scrape_agencies(keep_sheet: bool = True, force_download: bool = True) -> dict:
     """Collect data on participating agencies"""
     start_time = time.time()
-    resp = session.get(base_xlsx_url, timeout=120)
-    resp.raise_for_status()
+    resp = req_get(base_xlsx_url, timeout=120)
     soup = BeautifulSoup(resp.content, "html.parser")
     links = [link["href"] for link in soup.findAll("a", href=re.compile("^https://www.ice.gov/doclib.*xlsx"))]
     if not links:

ice_scrapers/facilities_scraper.py

@@ -95,7 +95,7 @@ def _scrape_updated(url: str) -> datetime.datetime:
         return datetime.datetime.strptime(default_timestamp, timestamp_format)
     logger.debug("  Fetching: %s", url)
     try:
-        response = req_get(url, timeout=30)
+        response = req_get(url, timeout=30, wait_time=0.1)
     except Exception as e:
         logger.error("  Error parsing %s: %s", url, e)
         return datetime.datetime.strptime(default_timestamp, timestamp_format)
@@ -117,7 +117,7 @@ def _scrape_page(page_url: str) -> list:
     """Scrape a single page of facilities using BeautifulSoup"""
     logger.debug("  Fetching: %s", page_url)
     try:
-        response = req_get(page_url, timeout=30)
+        response = req_get(page_url, timeout=30, wait_time=0.1)
     except Exception as e:
         logger.error("  Error parsing %s: %s", page_url, e)
         return []

ice_scrapers/field_offices.py

@@ -14,7 +14,7 @@
 import time
 from utils import (
     logger,
-    session,
+    req_get,
 )
 from .utils import get_ice_scrape_pages
 
@@ -45,8 +45,7 @@ def _scrape_page(page_url: str) -> list[dict]:
     """Scrape a single page of facilities using BeautifulSoup"""
     logger.debug("  Fetching: %s", page_url)
     try:
-        response = session.get(page_url, timeout=30)
-        response.raise_for_status()
+        response = req_get(page_url, timeout=30)
     except Exception as e:
         logger.error("  Error parsing %s: %s", page_url, e)
         return []

ice_scrapers/general.py

@@ -26,6 +26,7 @@ def facilities_scrape_wrapper(
     facilities_data["facilities"] = copy.deepcopy(facilities)
     facility_name_map = {v["name"].lower(): k for k, v in facilities_data["facilities"].items()}
     inspections = find_inspections(keep_text=inspection_text)
+    facilities_data = scrape_facilities(facilities_data)
 
     # actually attach inspections to facilities
     for facility, inspect in inspections.items():
@@ -36,16 +37,15 @@ def facilities_scrape_wrapper(
                 inspect
             )
             break
-        logger.debug("    Checking fuzzy matches:")
+        # logger.debug("    Checking fuzzy matches:")
         for k, v in facility_name_map.items():
             r = fuzz.partial_ratio(facility, k)
-            logger.debug("    %s === %s, ratio: %s", facility, k, r)
+            # logger.debug("    %s === %s, ratio: %s", facility, k, r)
             if r > 80:
-                logger.info("  Probably the right facility %s => %s, (ratio %s)", k, facility, r)
+                logger.debug("  Probably the right facility %s => %s, (ratio %s)", k, facility, r)
                 facilities_data["facilities"][facility_name_map[k]]["inspection"]["details"] = copy.deepcopy(inspect)
                 break
 
-    facilities_data = scrape_facilities(facilities_data)
     if not skip_vera:
         facilities_data = collect_vera_facility_data(facilities_data, keep_sheet, force_download)
     field_offices = scrape_field_offices()

ice_scrapers/inspections.py

@@ -8,7 +8,7 @@
 from utils import (
     logger,
     output_folder,
-    session,
+    req_get,
 )
 from .utils import download_file
 
@@ -38,7 +38,7 @@ def find_inspections(keep_text: bool = True) -> dict:
     os.makedirs(storage_dir, exist_ok=True)
     inspections: dict = {}
     logger.info("Collecting inspection reports from %s", root_url)
-    resp = session.get(root_url, timeout=120)
+    resp = req_get(root_url, timeout=120)
     resp.raise_for_status()
     soup = BeautifulSoup(resp.content, "html.parser")
     content = soup.select_one("div.facility-inspections")

ice_scrapers/spreadsheet_load.py

@@ -15,7 +15,7 @@
 from utils import (
     logger,
     output_folder,
-    session,
+    req_get,
 )
 from .utils import (
     download_file,
@@ -64,8 +64,7 @@
 
 def _download_sheet(keep_sheet: bool = True, force_download: bool = True) -> tuple[polars.DataFrame, str]:
     """Download the detention stats sheet from ice.gov"""
-    resp = session.get(base_xlsx_url, timeout=120)
-    resp.raise_for_status()
+    resp = req_get(base_xlsx_url, timeout=120)
     soup = BeautifulSoup(resp.content, "html.parser")
     links = soup.findAll("a", href=re.compile("^https://www.ice.gov/doclib.*xlsx"))
     if not links:

ice_scrapers/utils.py

@@ -304,7 +304,7 @@ def get_ice_scrape_pages(url: str) -> list[str]:
     This _may_ be generic to Drupal's pagination code...
     """
     try:
-        resp = req_get(url, timeout=30)
+        resp = req_get(url, timeout=30, wait_time=0.1)
     except Exception:
         return []
     soup = BeautifulSoup(resp.content, "html.parser")

ice_scrapers/vera_data.py

@@ -6,7 +6,7 @@
 from utils import (
     logger,
     output_folder,
-    session,
+    req_get,
 )
 
 # Github can aggressively rate-limit requests, so this may fail in surprising ways!
@@ -216,8 +216,7 @@ def _vera_city_fixes(city: str, state: str) -> tuple[str, bool]:
 def collect_vera_facility_data(facilities_data: dict, keep_sheet: bool = True, force_download: bool = True) -> dict:
     logger.info("Collecting and extracting data from vera.org facility data...")
     if force_download or not os.path.exists(filename):
-        res = session.get(base_url, timeout=120, stream=True)
-        res.raise_for_status()
+        res = req_get(base_url, timeout=120, stream=True)
         size = len(res.content)
         with open(filename, "wb") as f:
             for chunk in res.iter_content(chunk_size=1024):