Merge branch 'master' into GH391_add_params_to_allele_frequencies_advanced

Author: leehart

README.md

@@ -35,8 +35,7 @@ for release notes.
 
 ## Developer setup
 
-To get setup for development, see [this
-video](https://youtu.be/QniQi-Hoo9A) and the instructions below.
+To get setup for development, see [this video if you prefer VS Code](https://youtu.be/zddl3n1DCFM), or [this older video if you prefer PyCharm](https://youtu.be/QniQi-Hoo9A), and the instructions below.
 
 Fork and clone this repo:
 
@@ -48,27 +47,27 @@ Install Python, e.g.:
 
 ```bash
 sudo add-apt-repository ppa:deadsnakes/ppa
-sudo apt install python3.9 python3.9-venv
+sudo apt install python3.10 python3.10-venv
 ```
 
 Install pipx, e.g.:
 
 ```bash
-python3.9 -m pip install --user pipx
-python3.9 -m pipx ensurepath
+python3.10 -m pip install --user pipx
+python3.10 -m pipx ensurepath
 ```
 
 Install [poetry](https://python-poetry.org/docs/#installation), e.g.:
 
 ```bash
-pipx install poetry==1.8.2 --python=/usr/bin/python3.9
+pipx install poetry
 ```
 
 Create development environment:
 
 ```bash
 cd malariagen-data-python
-poetry use 3.9
+poetry use 3.10
 poetry install
 ```
 
@@ -81,7 +80,7 @@ poetry shell
 Install pre-commit and pre-commit hooks:
 
 ```bash
-pipx install pre-commit --python=/usr/bin/python3.9
+pipx install pre-commit
 pre-commit install
 ```
 
@@ -97,7 +96,9 @@ Run fast unit tests using simulated data:
 poetry run pytest -v tests/anoph
 ```
 
-To run legacy tests which read data from GCS, you'll need to [install the Google Cloud CLI](https://cloud.google.com/sdk/docs/install). E.g., if on Linux:
+To run legacy tests which read data from GCS, you'll need to [request access to MalariaGEN data on GCS](https://malariagen.github.io/vector-data/vobs/vobs-data-access.html).
+
+Once access has been granted, [install the Google Cloud CLI](https://cloud.google.com/sdk/docs/install). E.g., if on Linux:
 
 ```bash
 ./install_gcloud.sh
@@ -128,6 +129,17 @@ the documentation via GitHub Actions.
 The version switcher for the documentation can then be updated by
 modifying the `docs/source/_static/switcher.json` file accordingly.
 
+## Citation
+
+If you use the `malariagen_data` package in a publication
+or include any of its functions or code in other materials (_e.g._ training resources),
+please cite: [doi.org/10.5281/zenodo.11173411](doi.org/10.5281/zenodo.11173411)
+
+Some functions may require additional citations to acknowledge specific contributions. These are indicated in the description for each relevant function.
+
+For any questions, please feel free to contact us at: [support@malariagen.net](mailto:support@malariagen.net)
+
+
 ## Sponsorship
 
 This project is currently supported by the following grants:

docs/source/_static/switcher.json

@@ -1,47 +1,82 @@
 [
     {
-        "name": "13.0.0",
-        "version": "v13.0.0",
-        "url": "https:///malariagen.github.io/malariagen-data-python/v13.0.0/",
+        "name": "15.0.1",
+        "version": "v15.0.1",
+        "url": "https://malariagen.github.io/malariagen-data-python/v15.0.1/",
         "preferred": true
     },
+    {
+        "name": "14.0.0",
+        "version": "v14.0.0",
+        "url": "https://malariagen.github.io/malariagen-data-python/v14.0.0/"
+    },
+    {
+        "name": "13.5.0",
+        "version": "v13.5.0",
+        "url": "https://malariagen.github.io/malariagen-data-python/v13.5.0/"
+    },
+    {
+        "name": "13.4.0",
+        "version": "v13.4.0",
+        "url": "https://malariagen.github.io/malariagen-data-python/v13.4.0/"
+    },
+    {
+        "name": "13.3.0",
+        "version": "v13.3.0",
+        "url": "https://malariagen.github.io/malariagen-data-python/v13.3.0/"
+    },
+    {
+        "name": "13.2.1",
+        "version": "v13.2.1",
+        "url": "https://malariagen.github.io/malariagen-data-python/v13.2.0/"
+    },
+    {
+        "name": "13.1.0",
+        "version": "v13.1.0",
+        "url": "https://malariagen.github.io/malariagen-data-python/v13.1.0/"
+    },
+    {
+        "name": "13.0.4",
+        "version": "v13.0.4",
+        "url": "https://malariagen.github.io/malariagen-data-python/v13.0.4/"
+    },
     {
         "name": "12.0.0",
         "version": "v12.0.0",
-        "url": "https:///malariagen.github.io/malariagen-data-python/v12.0.0/"
+        "url": "https://malariagen.github.io/malariagen-data-python/v12.0.0/"
     },
     {
         "name": "11.0.0",
         "version": "v11.0.0",
-        "url": "https:///malariagen.github.io/malariagen-data-python/v11.0.0/"
+        "url": "https://malariagen.github.io/malariagen-data-python/v11.0.0/"
     },
     {
         "name": "10.0.0",
         "version": "v10.0.0",
-        "url": "https:///malariagen.github.io/malariagen-data-python/v10.0.0/"
+        "url": "https://malariagen.github.io/malariagen-data-python/v10.0.0/"
     },
     {
         "name": "9.0.0",
         "version": "v9.0.0",
-        "url": "https:///malariagen.github.io/malariagen-data-python/v9.0.0/"
+        "url": "https://malariagen.github.io/malariagen-data-python/v9.0.0/"
     },
     {
         "name": "8.0.0",
         "version": "v8.0.0",
-        "url": "https:///malariagen.github.io/malariagen-data-python/v8.0.0/"
+        "url": "https://malariagen.github.io/malariagen-data-python/v8.0.0/"
     },
     {
         "name": "7.15.0",
         "version": "v7.15.0",
-        "url": "https:///malariagen.github.io/malariagen-data-python/v7.15.0/"
+        "url": "https://malariagen.github.io/malariagen-data-python/v7.15.0/"
     },
     {
         "name": "7.14.0",
         "version": "v7.14.0",
-        "url": "https:///malariagen.github.io/malariagen-data-python/v7.14.0/"
+        "url": "https://malariagen.github.io/malariagen-data-python/v7.14.0/"
     },
     {
         "version": "dev",
-        "url": "https:///malariagen.github.io/malariagen-data-python/latest/"
+        "url": "https://malariagen.github.io/malariagen-data-python/latest/"
     }
 ]

malariagen_data/af1.py

@@ -9,6 +9,7 @@
 MAJOR_VERSION_PATH = "v1.0"
 CONFIG_PATH = "v1.0-config.json"
 GCS_DEFAULT_URL = "gs://vo_afun_release_master_us_central1/"
+GCS_DEFAULT_PUBLIC_URL = "gs://vo_anoph_temp_us_central1/vo_afun_release/"
 GCS_REGION_URLS = {
     "us-central1": "gs://vo_afun_release_master_us_central1",
 }
@@ -78,6 +79,7 @@ class Af1(AnophelesDataResource):
     def __init__(
         self,
         url=None,
+        public_url=GCS_DEFAULT_PUBLIC_URL,
         bokeh_output_notebook=True,
         results_cache=None,
         log=sys.stdout,
@@ -93,6 +95,7 @@ def __init__(
     ):
         super().__init__(
             url=url,
+            public_url=public_url,
             config_path=CONFIG_PATH,
             cohorts_analysis=cohorts_analysis,
             aim_analysis=None,

malariagen_data/ag3.py

@@ -15,6 +15,7 @@
 MAJOR_VERSION_PATH = "v3"
 CONFIG_PATH = "v3-config.json"
 GCS_DEFAULT_URL = "gs://vo_agam_release_master_us_central1/"
+GCS_DEFAULT_PUBLIC_URL = "gs://vo_anoph_temp_us_central1/vo_agam_release/"
 GCS_REGION_URLS = {
     "us-central1": "gs://vo_agam_release_master_us_central1",
 }
@@ -138,6 +139,7 @@ class Ag3(AnophelesDataResource):
     def __init__(
         self,
         url=None,
+        public_url=GCS_DEFAULT_PUBLIC_URL,
         bokeh_output_notebook=True,
         results_cache=None,
         log=sys.stdout,
@@ -154,6 +156,7 @@ def __init__(
     ):
         super().__init__(
             url=url,
+            public_url=public_url,
             config_path=CONFIG_PATH,
             cohorts_analysis=cohorts_analysis,
             aim_analysis=aim_analysis,

malariagen_data/anoph/base.py

@@ -47,6 +47,7 @@ def __init__(
         self,
         *,
         url: str,
+        public_url: str,
         config_path: str,
         pre: bool,
         major_version_number: int,
@@ -117,6 +118,8 @@ def __init__(
             raise ValueError("A value for the `url` parameter must be provided.")
         del url
 
+        self._public_url = public_url
+
         # Set up fsspec filesystem. N.B., we use fsspec here to allow for
         # accessing different types of storage - fsspec will automatically
         # detect which type of storage to use based on the URL provided.

malariagen_data/anoph/dipclust.py

@@ -44,7 +44,11 @@ def __init__(
 
     @check_types
     @doc(
-        summary="Hierarchically cluster diplotypes in region and produce an interactive plot.",
+        summary=""""
+        Hierarchically cluster diplotypes in region and produce an interactive plot.
+
+        If you use this function in a publication, please cite both this package and the original manuscript: doi.org/10.1093/molbev/msae140
+        """,
     )
     def plot_diplotype_clustering(
         self,
@@ -591,7 +595,11 @@ def _insert_dipclust_snp_trace(
         return figures, subplot_heights
 
     @doc(
-        summary="Perform diplotype clustering, annotated with heterozygosity, gene copy number and amino acid variants.",
+        summary=""""
+        Perform diplotype clustering, annotated with heterozygosity, gene copy number and amino acid variants.
+
+        If you use this function in a publication, please cite both this package and the original manuscript: doi.org/10.1093/molbev/msae140
+        """,
         parameters=dict(
             heterozygosity="Plot heterozygosity track.",
             snp_transcript="Plot amino acid variants for these transcripts.",

malariagen_data/anoph/frq_base.py

@@ -235,7 +235,7 @@ def plot_frequencies_heatmap(
 
         # Indexing.
         if index is None:
-            index = list(df.index.names)
+            index = [str(name) for name in df.index.names]
         df = df.reset_index().copy()
         if isinstance(index, list):
             index_col = (

malariagen_data/anoph/fst.py

@@ -539,8 +539,11 @@ def plot_pairwise_average_fst(
             if annotation == "standard error":
                 fig_df.loc[cohort1, cohort2] = se
             elif annotation == "Z score":
-                zs = fst / se
-                fig_df.loc[cohort1, cohort2] = zs
+                try:
+                    zs = fst / se
+                    fig_df.loc[cohort1, cohort2] = zs
+                except ZeroDivisionError:
+                    fig_df.loc[cohort1, cohort2] = np.nan
             else:
                 fig_df.loc[cohort1, cohort2] = fst
 

malariagen_data/anoph/igv.py

@@ -31,20 +31,21 @@ def _igv_config(
             "reference": {
                 "id": self._genome_ref_id,
                 "name": self._genome_ref_name,
-                "fastaURL": f"{self._url}{self._genome_fasta_path}",
-                "indexURL": f"{self._url}{self._genome_fai_path}",
+                "fastaURL": f"{self._public_url}{self._genome_fasta_path}",
+                "indexURL": f"{self._public_url}{self._genome_fai_path}",
                 "tracks": [
                     {
                         "name": "Genes",
                         "type": "annotation",
                         "format": "gff3",
-                        "url": f"{self._url}{self._geneset_gff3_path}",
+                        "url": f"{self._public_url}{self._geneset_gff3_path}",
                         "indexed": False,
                     }
                 ],
             },
             "locus": str(region),
         }
+
         if tracks:
             config["tracks"] = tracks
 
@@ -58,7 +59,7 @@ def _igv_site_filters_tracks(
     ):
         tracks = []
         for site_mask in self.site_mask_ids:
-            site_filters_vcf_url = f"{self._url}{self._major_version_path}/site_filters/{self._site_filters_analysis}/vcf/{site_mask}/{contig}_sitefilters.vcf.gz"  # noqa
+            site_filters_vcf_url = f"{self._public_url}{self._major_version_path}/site_filters/{self._site_filters_analysis}/vcf/{site_mask}/{contig}_sitefilters.vcf.gz"  # f"{self._url}{self._major_version_path}/site_filters/{self._site_filters_analysis}/vcf/{site_mask}/{contig}_sitefilters.vcf.gz"  # noqa
             track_config = {
                 "name": f"Filters - {site_mask}",
                 "url": site_filters_vcf_url,

malariagen_data/anoph/sample_metadata.py

@@ -1,6 +1,18 @@
 import io
 from itertools import cycle
-from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple, Union
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    Hashable,
+    cast,
+)
 
 import ipyleaflet  # type: ignore
 import numpy as np
@@ -39,7 +51,7 @@ def __init__(
         # data resources, and so column names and dtype need to be
         # passed in as parameters.
         self._aim_metadata_columns: Optional[List[str]] = None
-        self._aim_metadata_dtype: Dict[str, Any] = dict()
+        self._aim_metadata_dtype: Dict[str, Union[str, type, np.dtype]] = dict()
         if isinstance(aim_metadata_dtype, Mapping):
             self._aim_metadata_columns = list(aim_metadata_dtype.keys())
             self._aim_metadata_dtype.update(aim_metadata_dtype)
@@ -140,7 +152,19 @@ def _parse_general_metadata(
                 "longitude": "float64",
                 "sex_call": "object",
             }
-            df = pd.read_csv(io.BytesIO(data), dtype=dtype, na_values="")
+            # Mapping of string dtypes to actual dtypes
+            dtype_map = {
+                "object": str,
+                "int64": np.int64,
+                "float64": np.float64,
+            }
+
+            # Convert string dtypes to actual dtypes
+            dtype_fixed: Mapping[Hashable, Union[str, np.dtype, type]] = {
+                col: dtype_map.get(dtype[col], str) for col in dtype
+            }
+
+            df = pd.read_csv(io.BytesIO(data), dtype=dtype_fixed, na_values="")
 
             # Ensure all column names are lower case.
             df.columns = [c.lower() for c in df.columns]  # type: ignore
@@ -460,7 +484,12 @@ def _parse_aim_metadata(
         if isinstance(data, bytes):
             # Parse CSV data.
             df = pd.read_csv(
-                io.BytesIO(data), dtype=self._aim_metadata_dtype, na_values=""
+                io.BytesIO(data),
+                dtype=cast(
+                    Mapping[Hashable, Union[str, type, np.dtype]],
+                    self._aim_metadata_dtype,
+                ),
+                na_values="",
             )
 
             # Ensure all column names are lower case.