Merge branch 'master' into GH1005-fix-cnv-fixed-denominator

Author: adilraza99

.codecov.yml

@@ -0,0 +1,9 @@
+coverage:
+  status:
+    project:
+      default:
+        target: auto
+    patch:
+      default:
+        target: 80%
+        threshold: 0%

malariagen_data/anoph/base.py

@@ -562,6 +562,13 @@ def _sample_set_has_unrestricted_use(self, *, sample_set: str):
         release_manifest_df = self._read_sample_sets_manifest(
             single_release=sample_set_release
         )
+
+        if "unrestricted_use" not in release_manifest_df.columns:
+            raise ValueError(
+                f"Column 'unrestricted_use' missing from manifest for sample set '{sample_set}'. "
+                "This indicates a data integrity issue in the release manifest."
+            )
+
         sample_set_records_srs = release_manifest_df.loc[
             release_manifest_df["sample_set"] == sample_set, "unrestricted_use"
         ]
@@ -824,12 +831,19 @@ def lookup_study_info(self, sample_set: base_params.sample_set) -> dict:
     def lookup_terms_of_use_info(self, sample_set: base_params.sample_set) -> dict:
         if self._cache_sample_set_to_terms_of_use_info is None:
             df_sample_sets = self._available_sample_sets().set_index("sample_set")
+            expected_cols = [
+                "terms_of_use_expiry_date",
+                "terms_of_use_url",
+                "unrestricted_use",
+            ]
+            missing_cols = [c for c in expected_cols if c not in df_sample_sets.columns]
+            if missing_cols:
+                raise ValueError(
+                    f"Terms-of-use columns missing from manifest: {missing_cols}. "
+                    "This indicates a data integrity issue in the release manifest."
+                )
             self._cache_sample_set_to_terms_of_use_info = df_sample_sets[
-                [
-                    "terms_of_use_expiry_date",
-                    "terms_of_use_url",
-                    "unrestricted_use",
-                ]
+                expected_cols
             ].to_dict(orient="index")
         try:
             return self._cache_sample_set_to_terms_of_use_info[sample_set]

tests/anoph/test_base.py

@@ -256,3 +256,113 @@ def test_lookup_study(fixture, api):
 
     with pytest.raises(ValueError):
         api.lookup_study("foobar")
+
+
+def _strip_terms_of_use_from_manifest(manifest_path):
+    """Rewrite a manifest TSV file without terms-of-use columns."""
+    df = pd.read_csv(manifest_path, sep="\t")
+    cols_to_drop = [c for c in df.columns if c.startswith("terms_of_use")]
+    df = df.drop(columns=cols_to_drop)
+    df.to_csv(manifest_path, index=False, sep="\t")
+
+
+def test_lookup_terms_of_use_info_missing_columns(ag3_sim_fixture):
+    import shutil
+
+    manifest_paths = [
+        ag3_sim_fixture.bucket_path / "v3" / "manifest.tsv",
+        ag3_sim_fixture.bucket_path / "v3.1" / "manifest.tsv",
+    ]
+    backups = []
+    for mp in manifest_paths:
+        bp = mp.parent / "manifest.tsv.bak"
+        shutil.copy2(mp, bp)
+        backups.append(bp)
+
+    try:
+        for mp in manifest_paths:
+            _strip_terms_of_use_from_manifest(mp)
+
+        api = AnophelesBase(
+            url=ag3_sim_fixture.url,
+            public_url=ag3_sim_fixture.url,
+            config_path=_ag3.CONFIG_PATH,
+            major_version_number=_ag3.MAJOR_VERSION_NUMBER,
+            major_version_path=_ag3.MAJOR_VERSION_PATH,
+            pre=True,
+        )
+
+        sample_set = "1177-VO-ML-LEHMANN-VMF00004"
+        with pytest.raises(ValueError, match="Terms-of-use columns missing"):
+            api.lookup_terms_of_use_info(sample_set)
+    finally:
+        for mp, bp in zip(manifest_paths, backups):
+            shutil.move(bp, mp)
+
+
+def test_sample_set_has_unrestricted_use_missing_column(ag3_sim_fixture):
+    import shutil
+
+    manifest_paths = [
+        ag3_sim_fixture.bucket_path / "v3" / "manifest.tsv",
+        ag3_sim_fixture.bucket_path / "v3.1" / "manifest.tsv",
+    ]
+    backups = []
+    for mp in manifest_paths:
+        bp = mp.parent / "manifest.tsv.bak"
+        shutil.copy2(mp, bp)
+        backups.append(bp)
+
+    try:
+        for mp in manifest_paths:
+            _strip_terms_of_use_from_manifest(mp)
+
+        api = AnophelesBase(
+            url=ag3_sim_fixture.url,
+            public_url=ag3_sim_fixture.url,
+            config_path=_ag3.CONFIG_PATH,
+            major_version_number=_ag3.MAJOR_VERSION_NUMBER,
+            major_version_path=_ag3.MAJOR_VERSION_PATH,
+            pre=True,
+        )
+
+        sample_set = "1177-VO-ML-LEHMANN-VMF00004"
+        with pytest.raises(ValueError, match="unrestricted_use.*missing"):
+            api._sample_set_has_unrestricted_use(sample_set=sample_set)
+    finally:
+        for mp, bp in zip(manifest_paths, backups):
+            shutil.move(bp, mp)
+
+
+def test_sample_sets_no_terms_of_use(ag3_sim_fixture):
+    import shutil
+
+    manifest_paths = [
+        ag3_sim_fixture.bucket_path / "v3" / "manifest.tsv",
+        ag3_sim_fixture.bucket_path / "v3.1" / "manifest.tsv",
+    ]
+    backups = []
+    for mp in manifest_paths:
+        bp = mp.parent / "manifest.tsv.bak"
+        shutil.copy2(mp, bp)
+        backups.append(bp)
+
+    try:
+        for mp in manifest_paths:
+            _strip_terms_of_use_from_manifest(mp)
+
+        api = AnophelesBase(
+            url=ag3_sim_fixture.url,
+            public_url=ag3_sim_fixture.url,
+            config_path=_ag3.CONFIG_PATH,
+            major_version_number=_ag3.MAJOR_VERSION_NUMBER,
+            major_version_path=_ag3.MAJOR_VERSION_PATH,
+            pre=True,
+        )
+
+        df = api.sample_sets(release="3.1")
+        assert isinstance(df, pd.DataFrame)
+        assert len(df) > 0
+    finally:
+        for mp, bp in zip(manifest_paths, backups):
+            shutil.move(bp, mp)