Merge branch 'master' into GH995-fix-sample-query-options-forwarding-in-pairwise-average-fst

Author: jonbrenas

.codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+    patch:
+      default:
+        target: 80%
+        threshold: 0%

.github/workflows/tests.yml

@@ -12,6 +12,10 @@ jobs:
       fail-fast: true
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
+        numpy-spec:
+          # Keep this aligned with pyproject.toml: numpy = ">=2.0.2,<2.1"
+          - "==2.0.2"       # locked baseline
+          - ">=2.0.2,<2.1"  # latest allowed in declared range
     runs-on: ubuntu-latest
 
     steps:
@@ -23,8 +27,26 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Verify NumPy version
-        run: poetry run python -c "import numpy; print('NumPy version:', numpy.__version__)"
+      - name: Install matrix NumPy version
+        run: poetry run pip install --upgrade --no-deps "numpy${{ matrix.numpy-spec }}"
+
+      - name: Verify NumPy version and spec
+        env:
+          NUMPY_SPEC: ${{ matrix.numpy-spec }}
+        run: |
+          poetry run python - <<'PY'
+          import os
+          import numpy
+          from packaging.specifiers import SpecifierSet
+
+          spec = SpecifierSet(os.environ["NUMPY_SPEC"])
+          version = numpy.__version__
+          if version not in spec:
+              raise RuntimeError(
+                  f"NumPy version {version} does not satisfy matrix spec {spec}"
+              )
+          print("NumPy version:", version, "| spec:", spec)
+          PY
 
       - name: Run unit tests
         run: poetry run pytest -v tests --ignore tests/integration --typeguard-packages=malariagen_data,malariagen_data.anoph

malariagen_data/adir1.py

@@ -19,6 +19,10 @@
     "dirus": TAXON_PALETTE[0],
 }
 
+XPEHH_GWSS_CACHE_NAME = "adir1_xpehh_gwss_v1"
+IHS_GWSS_CACHE_NAME = "adir1_ihs_gwss_v1"
+ROH_HMM_CACHE_NAME = "adir1_roh_hmm_v1"
+
 
 class Adir1(AnophelesDataResource):
     """Provides access to data from Adir1.0 releases.
@@ -71,6 +75,10 @@ class Adir1(AnophelesDataResource):
 
     """
 
+    _xpehh_gwss_cache_name = XPEHH_GWSS_CACHE_NAME
+    _ihs_gwss_cache_name = IHS_GWSS_CACHE_NAME
+    _roh_hmm_cache_name = ROH_HMM_CACHE_NAME
+
     def __init__(
         self,
         url=None,

malariagen_data/af1.py

@@ -21,6 +21,10 @@
     "funestus": TAXON_PALETTE[0],
 }
 
+XPEHH_GWSS_CACHE_NAME = "af1_xpehh_gwss_v1"
+IHS_GWSS_CACHE_NAME = "af1_ihs_gwss_v1"
+ROH_HMM_CACHE_NAME = "af1_roh_hmm_v1"
+
 
 class Af1(AnophelesDataResource):
     """Provides access to data from Af1.x releases.
@@ -75,6 +79,7 @@ class Af1(AnophelesDataResource):
 
     _xpehh_gwss_cache_name = XPEHH_GWSS_CACHE_NAME
     _ihs_gwss_cache_name = IHS_GWSS_CACHE_NAME
+    _roh_hmm_cache_name = ROH_HMM_CACHE_NAME
 
     def __init__(
         self,

malariagen_data/ag3.py

@@ -95,6 +95,10 @@ def _setup_aim_palettes():
     "aim_species": "object",
 }
 
+XPEHH_GWSS_CACHE_NAME = "ag3_xpehh_gwss_v1"
+IHS_GWSS_CACHE_NAME = "ag3_ihs_gwss_v1"
+ROH_HMM_CACHE_NAME = "ag3_roh_hmm_v1"
+
 
 class Ag3(AnophelesDataResource):
     """Provides access to data from Ag3.x releases.
@@ -153,6 +157,7 @@ class Ag3(AnophelesDataResource):
 
     _xpehh_gwss_cache_name = XPEHH_GWSS_CACHE_NAME
     _ihs_gwss_cache_name = IHS_GWSS_CACHE_NAME
+    _roh_hmm_cache_name = ROH_HMM_CACHE_NAME
 
     def __init__(
         self,

malariagen_data/amin1.py

@@ -19,6 +19,10 @@
     "dirus": TAXON_PALETTE[0],
 }
 
+XPEHH_GWSS_CACHE_NAME = "amin1_xpehh_gwss_v1"
+IHS_GWSS_CACHE_NAME = "amin1_ihs_gwss_v1"
+ROH_HMM_CACHE_NAME = "amin1_roh_hmm_v1"
+
 
 class Amin1(AnophelesDataResource):
     """Provides access to data from Amin1.0 releases.
@@ -71,8 +75,9 @@ class Amin1(AnophelesDataResource):
 
     """
 
-    #    _xpehh_gwss_cache_name = XPEHH_GWSS_CACHE_NAME
-    #    _ihs_gwss_cache_name = IHS_GWSS_CACHE_NAME
+    _xpehh_gwss_cache_name = XPEHH_GWSS_CACHE_NAME
+    _ihs_gwss_cache_name = IHS_GWSS_CACHE_NAME
+    _roh_hmm_cache_name = ROH_HMM_CACHE_NAME
 
     def __init__(
         self,

malariagen_data/anoph/base.py

@@ -562,6 +562,13 @@ def _sample_set_has_unrestricted_use(self, *, sample_set: str):
         release_manifest_df = self._read_sample_sets_manifest(
             single_release=sample_set_release
         )
+
+        if "unrestricted_use" not in release_manifest_df.columns:
+            raise ValueError(
+                f"Column 'unrestricted_use' missing from manifest for sample set '{sample_set}'. "
+                "This indicates a data integrity issue in the release manifest."
+            )
+
         sample_set_records_srs = release_manifest_df.loc[
             release_manifest_df["sample_set"] == sample_set, "unrestricted_use"
         ]
@@ -824,12 +831,19 @@ def lookup_study_info(self, sample_set: base_params.sample_set) -> dict:
     def lookup_terms_of_use_info(self, sample_set: base_params.sample_set) -> dict:
         if self._cache_sample_set_to_terms_of_use_info is None:
             df_sample_sets = self._available_sample_sets().set_index("sample_set")
+            expected_cols = [
+                "terms_of_use_expiry_date",
+                "terms_of_use_url",
+                "unrestricted_use",
+            ]
+            missing_cols = [c for c in expected_cols if c not in df_sample_sets.columns]
+            if missing_cols:
+                raise ValueError(
+                    f"Terms-of-use columns missing from manifest: {missing_cols}. "
+                    "This indicates a data integrity issue in the release manifest."
+                )
             self._cache_sample_set_to_terms_of_use_info = df_sample_sets[
-                [
-                    "terms_of_use_expiry_date",
-                    "terms_of_use_url",
-                    "unrestricted_use",
-                ]
+                expected_cols
             ].to_dict(orient="index")
         try:
             return self._cache_sample_set_to_terms_of_use_info[sample_set]

malariagen_data/anoph/cnv_data.py

@@ -296,7 +296,7 @@ def open_cnv_coverage_calls(
             marker = path + "/.zmetadata"
             if not self._fs.exists(marker):
                 raise ValueError(
-                    f"CNV coverage calls analysis f{analysis!r} not implemented for sample set {sample_set!r}"
+                    f"CNV coverage calls analysis {analysis!r} not implemented for sample set {sample_set!r}"
                 )
             store = _init_zarr_store(fs=self._fs, path=path)
             root = zarr.open_consolidated(store=store)

malariagen_data/anoph/cnv_frq.py

@@ -446,6 +446,7 @@ def gene_cnv_frequencies_advanced(
         chunks: base_params.chunks = base_params.native_chunks,
         inline_array: base_params.inline_array = base_params.inline_array_default,
         taxon_by: frq_params.taxon_by = frq_params.taxon_by_default,
+        filter_unassigned: Optional[frq_params.filter_unassigned] = None,
     ) -> xr.Dataset:
         regions: List[Region] = _parse_multi_region(self, region)
         del region
@@ -468,6 +469,7 @@ def gene_cnv_frequencies_advanced(
                     chunks=chunks,
                     inline_array=inline_array,
                     taxon_by=taxon_by,
+                    filter_unassigned=filter_unassigned,
                 )
                 for r in regions
             ],
@@ -497,6 +499,7 @@ def _gene_cnv_frequencies_advanced(
         chunks,
         inline_array,
         taxon_by,
+        filter_unassigned,
     ):
         debug = self._log.debug
 
@@ -527,6 +530,7 @@ def _gene_cnv_frequencies_advanced(
             area_by=area_by,
             period_by=period_by,
             taxon_by=taxon_by,
+            filter_unassigned=filter_unassigned,
         )
 
         debug("group samples to make cohorts")

malariagen_data/anoph/distance.py

@@ -217,6 +217,9 @@ def _biallelic_diplotype_pairwise_distances(
         n_snps = gn.shape[0]
 
         # Prepare data for pairwise distance calculation.
+        # Mask missing calls (-127) before computing distances.
+        gn = gn.astype(float)
+        gn[gn == -127] = np.nan
         X = np.ascontiguousarray(gn.T)
 
         # Look up distance function.