From a78e8b4329fb96e4d960ba756997ab6e2ca39999 Mon Sep 17 00:00:00 2001 From: khushthecoder Date: Wed, 15 Apr 2026 23:44:53 +0530 Subject: [PATCH 1/2] fix: add retry, timeout, and backoff defaults for GCS, S3, and HTTP filesystem access Configure sensible retry and timeout defaults in _init_filesystem() so that transient network errors (503, 429, DNS failures, timeouts) are automatically retried instead of immediately crashing with a cryptic OSError. Changes: - GCS (gcsfs): retry=3, timeout=60s - S3 (s3fs): retries=3, connect_timeout=60s, read_timeout=60s - HTTP/HTTPS (fsspec): timeout=60s - Local filesystems are unaffected All defaults use setdefault() so users can still override via storage_options if needed. Closes #1303 --- malariagen_data/util.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/malariagen_data/util.py b/malariagen_data/util.py index aa8b2d6f0..958e96aeb 100644 --- a/malariagen_data/util.py +++ b/malariagen_data/util.py @@ -479,6 +479,12 @@ def _init_filesystem(url, **kwargs): kwargs.setdefault("token", credentials) + # Set retry and timeout defaults for GCS to handle transient errors + # on unreliable networks (e.g., field stations in endemic regions). + # gcsfs supports retry (number of retries) and timeout (seconds). + kwargs.setdefault("retry", 3) + kwargs.setdefault("timeout", 60) + # Ensure options are passed through to gcsfs, even if URL is chained. if url.startswith("gs://") or url.startswith("gcs://"): storage_options = kwargs @@ -501,6 +507,13 @@ def _init_filesystem(url, **kwargs): kwargs.setdefault("endpoint_url", "https://cog.sanger.ac.uk") kwargs.setdefault("config_kwargs", config) + # Set retry and timeout defaults for S3 to handle transient errors. + # s3fs supports retries (max retry attempts) and + # config_kwargs for connect/read timeouts. + kwargs.setdefault("retries", 3) + config.setdefault("connect_timeout", 60) + config.setdefault("read_timeout", 60) + if url.startswith("s3://"): storage_options = kwargs else: @@ -509,6 +522,9 @@ def _init_filesystem(url, **kwargs): else: # Some other kind of URL, pass through kwargs as-is. + # Set a default timeout for remote HTTP/HTTPS filesystems. + if url.startswith("http://") or url.startswith("https://"): + kwargs.setdefault("timeout", 60) storage_options = kwargs if simplecache_options is not None: From 8b9b8d958980f79eff93ad1e8cc27abcbb5bb5b6 Mon Sep 17 00:00:00 2001 From: khushthecoder Date: Thu, 16 Apr 2026 00:13:18 +0530 Subject: [PATCH 2/2] fix: guard heatmap non-unique index test against single-row DataFrames When the frequency DataFrame has only one row (e.g., due to sample query filtering), the "contig" column is unique, so passing index="contig" does not raise ValueError. Only assert the error when the contig values are actually non-unique. This fixes a flaky test failure in test_gene_cnv_frequencies_with_str_cohorts_and_sample_query. --- tests/anoph/test_frq.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/anoph/test_frq.py b/tests/anoph/test_frq.py index dfebb9bf6..05c29060c 100644 --- a/tests/anoph/test_frq.py +++ b/tests/anoph/test_frq.py @@ -16,7 +16,7 @@ def check_plot_frequencies_heatmap(api, frq_df): # Test index parameter - if None, should use dataframe index. fig = api.plot_frequencies_heatmap(frq_df, show=False, index=None, max_len=None) - if "contig" in list(frq_df.columns): + if "contig" in list(frq_df.columns) and not frq_df["contig"].is_unique: # Not unique. with pytest.raises(ValueError): api.plot_frequencies_heatmap(