Merge pull request #786 from malariagen/fix-windows-path-mainrepo

Fix Windows path compatibility in cloud storage URLs

Author: jonbrenas

malariagen_data/anoph/frq_base.py

@@ -210,7 +210,7 @@ def plot_frequencies_heatmap(
 
         # Indexing.
         if index is None:
-            index = list(df.index.names)
+            index = [str(name) for name in df.index.names]
         df = df.reset_index().copy()
         if isinstance(index, list):
             index_col = (

malariagen_data/anoph/sample_metadata.py

@@ -1,6 +1,18 @@
 import io
 from itertools import cycle
-from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    Hashable,
+    cast,
+)
 
 import ipyleaflet  # type: ignore
 import numpy as np
@@ -39,7 +51,7 @@ def __init__(
         # data resources, and so column names and dtype need to be
         # passed in as parameters.
         self._aim_metadata_columns: Optional[List[str]] = None
-        self._aim_metadata_dtype: Dict[str, Any] = dict()
+        self._aim_metadata_dtype: Dict[str, Union[str, type, np.dtype]] = dict()
         if isinstance(aim_metadata_dtype, Mapping):
             self._aim_metadata_columns = list(aim_metadata_dtype.keys())
             self._aim_metadata_dtype.update(aim_metadata_dtype)
@@ -140,7 +152,19 @@ def _parse_general_metadata(
                 "longitude": "float64",
                 "sex_call": "object",
             }
-            df = pd.read_csv(io.BytesIO(data), dtype=dtype, na_values="")
+            # Mapping of string dtypes to actual dtypes
+            dtype_map = {
+                "object": str,
+                "int64": np.int64,
+                "float64": np.float64,
+            }
+
+            # Convert string dtypes to actual dtypes
+            dtype_fixed: Mapping[Hashable, Union[str, np.dtype, type]] = {
+                col: dtype_map.get(dtype[col], str) for col in dtype
+            }
+
+            df = pd.read_csv(io.BytesIO(data), dtype=dtype_fixed, na_values="")
 
             # Ensure all column names are lower case.
             df.columns = [c.lower() for c in df.columns]  # type: ignore
@@ -460,7 +484,12 @@ def _parse_aim_metadata(
         if isinstance(data, bytes):
             # Parse CSV data.
             df = pd.read_csv(
-                io.BytesIO(data), dtype=self._aim_metadata_dtype, na_values=""
+                io.BytesIO(data),
+                dtype=cast(
+                    Mapping[Hashable, Union[str, type, np.dtype]],
+                    self._aim_metadata_dtype,
+                ),
+                na_values="",
             )
 
             # Ensure all column names are lower case.

malariagen_data/anoph/snp_frq.py

@@ -3,6 +3,7 @@
 
 import allel  # type: ignore
 import numpy as np
+import numpy.typing as npt
 import pandas as pd
 from numpydoc_decorator import doc  # type: ignore
 import xarray as xr
@@ -518,8 +519,8 @@ def snp_allele_frequencies_advanced(
 
         # Set up main event variables.
         n_variants, n_cohorts = len(variant_position), len(df_cohorts)
-        count = np.zeros((n_variants, n_cohorts), dtype=int)
-        nobs = np.zeros((n_variants, n_cohorts), dtype=int)
+        count: npt.NDArray[np.float64] = np.zeros((n_variants, n_cohorts), dtype=int)
+        nobs: npt.NDArray[np.float64] = np.zeros((n_variants, n_cohorts), dtype=int)
 
         # Build event count and nobs for each cohort.
         cohorts_iterator = self._progress(

malariagen_data/plasmodium.py

@@ -1,5 +1,4 @@
 import json
-import os
 
 import dask.array as da
 import pandas as pd
@@ -60,7 +59,7 @@ def sample_metadata(self):
             One row per sample.
         """
         if self._cache_sample_metadata is None:
-            path = os.path.join(self._path, self.CONF["metadata_path"])
+            path = f"{self._path}/{self.CONF['metadata_path']}"
             with self._fs.open(path) as f:
                 self._cache_sample_metadata = pd.read_csv(f, sep="\t", na_values="")
         return self._cache_sample_metadata
@@ -75,7 +74,7 @@ def _open_variant_calls_zarr(self):
 
         """
         if self._cache_variant_calls_zarr is None:
-            path = os.path.join(self._path, self.CONF["variant_calls_zarr_path"])
+            path = f"{self._path}/{self.CONF['variant_calls_zarr_path']}"
             store = init_zarr_store(fs=self._fs, path=path)
             self._cache_variant_calls_zarr = zarr.open_consolidated(store=store)
         return self._cache_variant_calls_zarr
@@ -205,7 +204,7 @@ def open_genome(self):
 
         """
         if self._cache_genome is None:
-            path = os.path.join(self._path, self.CONF["reference_path"])
+            path = f"{self._path}/{self.CONF['reference_path']}"
             store = init_zarr_store(fs=self._fs, path=path)
             self._cache_genome = zarr.open_consolidated(store=store)
         return self._cache_genome
@@ -317,7 +316,7 @@ def genome_features(self, attributes=("ID", "Parent", "Name")):
         try:
             df = self._cache_genome_features[attributes]
         except KeyError:
-            path = os.path.join(self._path, self.CONF["annotations_path"])
+            path = f"{self._path}/{self.CONF['annotations_path']}"
             with self._fs.open(path, mode="rb") as f:
                 df = read_gff3(f, compression="gzip")
             if attributes is not None:

tests/anoph/test_hap_data.py

@@ -605,7 +605,7 @@ def test_haplotypes_virtual_contigs(
 
         # Test with region.
         seq = api.genome_sequence(region=chrom)
-        start, stop = sorted(np.random.randint(low=1, high=len(seq), size=2))
+        start, stop = sorted(map(int, np.random.randint(low=1, high=len(seq), size=2)))
         region = f"{chrom}:{start:,}-{stop:,}"
 
         # Standard checks.

tests/anoph/test_snp_frq.py

@@ -153,7 +153,7 @@ def test_snp_effects(fixture, api: AnophelesSnpFrequencyAnalysis):
 
     # Check some values.
     assert np.all(df["contig"] == transcript["contig"])
-    position = df["position"].values
+    position = df["position"].to_numpy()
     assert np.all(position >= transcript["start"])
     assert np.all(position <= transcript["end"])
     assert np.all(position[1:] >= position[:-1])