Merge pull request #855 from adilraza99/fix-numpy-slicing-compat

Stabilize CI matrix and pin NumPy baseline to 1.26.4

Author: jonbrenas

.github/workflows/coverage.yml

@@ -1,34 +1,34 @@
 name: coverage
 on:
-    push:
-        branches:
-            - master
-    pull_request:
-        branches:
-            - master
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
 jobs:
-    coverage:
-        strategy:
-            fail-fast: true
-        runs-on: ubuntu-latest
-        steps:
-            - name: Checkout source
-              uses: actions/checkout@v4
+  coverage:
+    strategy:
+      fail-fast: true
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout source
+        uses: actions/checkout@v4
 
-            - name: Setup python
-              uses: actions/setup-python@v5
-              with:
-                  python-version: '3.12'
-                  cache: 'pip'
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+          cache: "pip"
 
-            - name: Install package
-              run: pip install .[dev]
+      - name: Install package
+        run: pip install .[dev]
 
-            - name: Run unit tests with coverage
-              run: pytest -v tests --ignore tests/integration --cov malariagen_data/anoph --cov-report=xml
+      - name: Run unit tests with coverage
+        run: pytest -v tests --ignore tests/integration --cov malariagen_data/anoph --cov-report=xml
 
-            - name: Upload coverage report
-              uses: codecov/codecov-action@v3
-              with:
-                  files: ./coverage.xml
-                  verbose: true
+      - name: Upload coverage report
+        uses: codecov/codecov-action@v3
+        with:
+          files: ./coverage.xml
+          verbose: true

.github/workflows/tests.yml

@@ -11,12 +11,8 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.10", "3.11", "3.12"]
-        numpy-version:
-          - "numpy==1.26.4"  # current colab version
-          - "numpy~=2.0"  # 2.0 series
-          # numba (0.60.0) does not yet support numpy 2.1, disable this for now
-          # - "numpy~=2.1"  # 2.1 series
+        python-version: ["3.10"]
+        numpy-version: ["numpy==1.26.4"]
     runs-on: ubuntu-latest
 
     steps:
@@ -32,5 +28,8 @@ jobs:
       - name: Install package
         run: pip install "${{ matrix.numpy-version }}" .[dev]
 
+      - name: Verify NumPy version
+        run: python -c "import numpy; print('NumPy version:', numpy.__version__)"
+
       - name: Run unit tests
         run: pytest -v tests --ignore tests/integration --typeguard-packages=malariagen_data,malariagen_data.anoph

malariagen_data/anoph/cnv_data.py

@@ -240,7 +240,9 @@ def cnv_hmm(
                     # noinspection PyArgumentList
                     other = pd.Interval(r.start, r.end, closed="both")
                     loc_region = index.overlaps(other)  # type: ignore
-                    x = x.isel(variants=loc_region)
+                    # Convert boolean mask to integer indices for NumPy 2.x compatibility
+                    variant_indices = np.where(loc_region)[0]
+                    x = x.isel(variants=variant_indices)
 
                 lx.append(x)
 
@@ -267,7 +269,9 @@ def cnv_hmm(
             if max_coverage_variance is not None:
                 cov_var = ds["sample_coverage_variance"].values
                 loc_pass_samples = cov_var <= max_coverage_variance
-                ds = ds.isel(samples=loc_pass_samples)
+                # Convert boolean mask to integer indices for NumPy 2.x compatibility
+                sample_indices = np.where(loc_pass_samples)[0]
+                ds = ds.isel(samples=sample_indices)
 
         return ds
 
@@ -445,7 +449,9 @@ def cnv_coverage_calls(
                 # noinspection PyArgumentList
                 other = pd.Interval(r.start, r.end, closed="both")
                 loc_region = index.overlaps(other)  # type: ignore
-                x = x.isel(variants=loc_region)
+                # Convert boolean mask to integer indices for NumPy 2.x compatibility
+                variant_indices = np.where(loc_region)[0]
+                x = x.isel(variants=variant_indices)
 
             lx.append(x)
         ds = _simple_xarray_concat(lx, dim=DIM_VARIANT)

malariagen_data/anoph/cnv_frq.py

@@ -573,8 +573,8 @@ def _gene_cnv_frequencies_advanced(
             cohort_is_called = np.take(is_called, sample_indices, axis=1)
 
             # compute cohort allele counts
-            np.sum(cohort_is_amp, axis=1, out=count[::2, cohort_index])
-            np.sum(cohort_is_del, axis=1, out=count[1::2, cohort_index])
+            count[::2, cohort_index] = np.sum(cohort_is_amp, axis=1)
+            count[1::2, cohort_index] = np.sum(cohort_is_del, axis=1)
 
             # compute cohort allele numbers
             cohort_n_called = np.sum(cohort_is_called, axis=1)
@@ -635,13 +635,17 @@ def _gene_cnv_frequencies_advanced(
         debug("deal with invariants")
         if drop_invariant:
             loc_variant = df_variants["max_af"].values > 0
-            ds_out = ds_out.isel(variants=loc_variant)
+            # Convert boolean mask to integer indices for NumPy 2.x compatibility
+            variant_indices = np.where(loc_variant)[0]
+            ds_out = ds_out.isel(variants=variant_indices)
             df_variants = df_variants.loc[loc_variant].reset_index(drop=True)
 
         debug("apply variant query")
         if variant_query is not None:
             loc_variants = df_variants.eval(variant_query).values
-            ds_out = ds_out.isel(variants=loc_variants)
+            # Convert boolean mask to integer indices for NumPy 2.x compatibility
+            variant_indices = np.where(loc_variants)[0]
+            ds_out = ds_out.isel(variants=variant_indices)
 
         debug("add confidence intervals")
         _add_frequency_ci(ds=ds_out, ci_method=ci_method)

malariagen_data/anoph/frq_base.py

@@ -53,10 +53,10 @@ def _prep_samples_for_cohort_grouping(*, df_samples, area_by, period_by, taxon_b
             )
 
         # Raise a ValueError if the specified period_by column does not contain instances pd.Period.
-        if not all(
-            df_samples[period_by].apply(
-                lambda value: pd.isnull(value) or isinstance(value, pd.Period)
-            )
+        if (
+            not df_samples[period_by]
+            .apply(lambda value: pd.isnull(value) or isinstance(value, pd.Period))
+            .all()
         ):
             raise TypeError(
                 f"Invalid values in {period_by!r} column. Must be either pandas.Period or null."

malariagen_data/anoph/hap_data.py

@@ -416,7 +416,9 @@ def haplotypes(
                 raise ValueError(
                     f"No samples found for phasing analysis {analysis!r} and query {sample_query_prepped!r}"
                 )
-            ds = ds.isel(samples=loc_samples)
+            # Convert boolean mask to integer indices for NumPy 2.x compatibility
+            sample_indices = np.where(loc_samples)[0]
+            ds = ds.isel(samples=sample_indices)
 
         if cohort_size is not None:
             # Handle cohort size - overrides min and max.

malariagen_data/anoph/snp_frq.py

@@ -630,7 +630,9 @@ def snp_allele_frequencies_advanced(
                     f"No SNPs remaining after applying variant query {variant_query!r}."
                 )
 
-            ds_out = ds_out.isel(variants=loc_variants)
+            # Convert boolean mask to integer indices for NumPy 2.x compatibility
+            variant_indices = np.where(loc_variants)[0]
+            ds_out = ds_out.isel(variants=variant_indices)
 
         # Add confidence intervals.
         _add_frequency_ci(ds=ds_out, ci_method=ci_method)
@@ -768,7 +770,9 @@ def aa_allele_frequencies_advanced(
                     f"No SNPs remaining after applying variant query {variant_query!r}."
                 )
 
-            ds_aa_frq = ds_aa_frq.isel(variants=loc_variants)
+            # Convert boolean mask to integer indices for NumPy 2.x compatibility
+            variant_indices = np.where(loc_variants)[0]
+            ds_aa_frq = ds_aa_frq.isel(variants=variant_indices)
 
         # Compute new confidence intervals.
         _add_frequency_ci(ds=ds_aa_frq, ci_method=ci_method)

poetry.lock

@@ -21,15 +21,13 @@ license = "MIT"
 
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"
-numpy = "<2.2"
+numpy = "==1.26.4"
 numba = ">=0.60.0"
 llvmlite = "*"
 scipy = "*"
 pandas = "*"
-zarr = ">=2.11, <3.0.0"
-# zarr<3.0.0 is not compatible with numcodecs 0.16.0
-# https://github.com/zarr-developers/zarr-python/issues/2963
-numcodecs = "<0.16"
+zarr = ">=2.18.3, <3.0.0"
+numcodecs = ">=0.13, <0.16"
 dask = {version="*", extras=["array"]}
 distributed = "*"
 fsspec = "*"
@@ -44,7 +42,7 @@ plotly = "*"
 # https://github.com/bokeh/bokeh/issues/14412
 # https://github.com/malariagen/malariagen-data-python/issues/734
 bokeh = "<3.7.0"
-statsmodels = "*"
+statsmodels = ">=0.14.2"
 ipyleaflet = ">=0.17.0"
 ipywidgets = "*"
 # https://stackoverflow.com/questions/73929564/entrypoints-object-has-no-attribute-get-digital-ocean