chore: use scverse-misc for data downloading

benchmarks/benchmarks/_utils.py

@@ -71,6 +71,7 @@ def _bmmc(n_obs: int = 4000) -> AnnData:
     registry = pooch.create(
         path=pooch.os_cache("pooch"),
         base_url="doi:10.6084/m9.figshare.22716739.v1/",
+        retry_if_failed=3,
     )
     registry.load_registry_from_doi()
     samples = {smp: f"{smp}_filtered_feature_bc_matrix.h5" for smp in ("s1d1", "s1d3")}
@@ -106,6 +107,7 @@ def _lung93k() -> AnnData:
     registry = pooch.create(
         path=pooch.os_cache("pooch"),
         base_url="doi:10.6084/m9.figshare.25664775.v1/",
+        retry_if_failed=3,
     )
     registry.load_registry_from_doi()
     path = registry.fetch("adata.raw_compressed.h5ad")

docs/conf.py

@@ -169,6 +169,7 @@
     python=("https://docs.python.org/3", None),
     rapids_singlecell=("https://rapids-singlecell.readthedocs.io/en/latest/", None),
     scipy=("https://docs.scipy.org/doc/scipy/", None),
+    scverse_misc=("https://scverse-misc.readthedocs.io/stable/", None),
     seaborn=("https://seaborn.pydata.org/", None),
     session_info2=("https://session-info2.readthedocs.io/en/stable/", None),
     squidpy=("https://squidpy.readthedocs.io/en/stable/", None),

docs/release-notes/1.13.0a1.md

@@ -18,7 +18,7 @@
 - Add {func}`scanpy.pp.harmony_integrate` with Harmony1 and Harmony2 support for batch correction {smaller}`S Dicks, P Angerer` ({pr}`3953`)
 - Add support for {class}`numpy.random.Generator` to all functions previously accepting a `random_state` parameter {smaller}`P Angerer` ({pr}`3983`)
 - Add `layer` parameter to {func}`~scanpy.tl.filter_rank_genes_groups` {smaller}`P Angerer` ({pr}`3999`)
-- add `sparse_format` parameter to `~scanpy.read_10x_mtx` defaulting to CSR {smaller}`E. Provo` ({pr}`4017`)
+- add `sparse_format` parameter to {func}`~scanpy.io.read_10x_mtx` defaulting to CSR {smaller}`E. Provo` ({pr}`4017`)
 - Add `mean_in_log_space` argument to {func}`scanpy.tl.rank_genes_groups` for customizing how log-fold-change is calculated {user}`ilan-gold` ({pr}`4037`)
 - Introduce blazing-fast [`illico`][] as a new `wilcoxon` `method` in {func}`~scanpy.tl.rank_genes_groups`. Use it via either `method="wilcoxon_illico"` or via {attr}`~scanpy.Preset.ScanpyV2Preview` (preferred) {smaller}`I Gold`
 

docs/release-notes/4310.chore.md

@@ -0,0 +1,2 @@
+Fetch the bundled datasets through {func}`scverse_misc.datasets.fetch`, adding hash verification, retries, and fall back to their pre-S3 upstream URLs.
+{func}`~scanpy.io.read_10x_mtx` and friends now use `pooch` for downloading, also gaining retries {smaller}`P Angerer`

pyproject.toml

@@ -52,7 +52,6 @@ classifiers = [
 dynamic = [ "version" ]
 dependencies = [
     "anndata>=0.12.14",
-    "certifi",
     "fast-array-utils[accel,sparse]>=1.4",
     "h5py>=3.11",
     "joblib",
@@ -67,7 +66,7 @@ dependencies = [
     "pynndescent>=0.5.13",
     "scikit-learn>=1.6",
     "scipy>=1.15",
-    "scverse-misc[settings]>=0.1.1",
+    "scverse-misc[datasets,settings]>=0.1.4",
     "seaborn>=0.13.2",
     "session-info2",
     "statsmodels>=0.14.5",

src/scanpy/_utils/__init__.py

@@ -37,7 +37,6 @@
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable, KeysView, Mapping
-    from pathlib import Path
     from typing import Any
 
     from anndata import AnnData
@@ -67,7 +66,6 @@
     "axis_nnz",
     "check_array_function_arguments",
     "check_nonnegative_integers",
-    "check_presence_download",
     "check_use_raw",
     "compute_association_matrix_of_groups",
     "descend_classes_and_funcs",
@@ -842,14 +840,6 @@ def select_groups(
     return groups_order_subset, groups_masks_obs
 
 
-def check_presence_download(filename: Path, backup_url: str):
-    """Check if file is present otherwise download."""
-    if not filename.is_file():
-        from ..io._download import download
-
-        download(backup_url, filename)
-
-
 # --------------------------------------------------------------------------------
 # Neighbors
 # --------------------------------------------------------------------------------

src/scanpy/datasets/_datasets.py

@@ -18,8 +18,9 @@
 from .._utils import _doc_params
 from .._utils._doctests import doctest_internet, doctest_needs, doctest_skipif
 from .._utils.random import _accepts_legacy_random_state, _legacy_random_state
+from ..io._download import download
 from ..io._read import read, read_zarr
-from ._utils import check_datasetdir_exists
+from ._utils import check_datasetdir_exists, fetch_dataset
 
 if TYPE_CHECKING:
     from typing import Literal
@@ -120,7 +121,6 @@ def blobs(
 
 @_doctest_skipif_old_anndata
 @doctest_internet
-@check_datasetdir_exists
 def burczynski06() -> AnnData:
     """Bulk data with conditions ulcerative colitis (UC) and Crohn’s disease (CD) :cite:p:`Burczynski2006`.
 
@@ -143,9 +143,7 @@ def burczynski06() -> AnnData:
         layers: None (.X)
 
     """
-    filename = settings.datasetdir / "burczynski06/GDS1615_full.soft.gz"
-    url = "https://exampledata.scverse.org/scanpy/GDS1615_full.soft.gz"
-    return read(filename, backup_url=url)
+    return read(fetch_dataset("burczynski06"))
 
 
 @_doctest_skipif_old_anndata
@@ -195,7 +193,6 @@ def krumsiek11() -> AnnData:
 @_doctest_skipif_old_anndata
 @doctest_internet
 @doctest_needs("openpyxl")
-@check_datasetdir_exists
 def moignard15() -> AnnData:
     r"""Hematopoiesis in early mouse embryos :cite:p:`Moignard2015`.
 
@@ -223,11 +220,7 @@ def moignard15() -> AnnData:
         layers: None (.X)
 
     """
-    filename = settings.datasetdir / "moignard15/nbt.3154-S3.xlsx"
-    backup_url = (
-        "https://exampledata.scverse.org/scanpy/41587_2015_BFnbt3154_MOESM4_ESM.xlsx"
-    )
-    adata = read(filename, sheet="dCt_values.txt", backup_url=backup_url)
+    adata = read(fetch_dataset("moignard15"), sheet="dCt_values.txt")
     # filter out 4 genes as in Haghverdi et al. (2016)
     gene_subset = ~np.isin(adata.var_names, ["Eif2b1", "Mrpl19", "Polr2a", "Ubc"])
     adata = adata[:, gene_subset].copy()  # retain non-removed genes
@@ -256,7 +249,6 @@ def moignard15() -> AnnData:
 
 @_doctest_skipif_old_anndata
 @doctest_internet
-@check_datasetdir_exists
 def paul15() -> AnnData:
     """Development of Myeloid Progenitors :cite:p:`Paul2015`.
 
@@ -281,11 +273,7 @@ def paul15() -> AnnData:
     """
     import h5py
 
-    filename = settings.datasetdir / "paul15/paul15.h5"
-    filename.parent.mkdir(exist_ok=True)
-    backup_url = "https://exampledata.scverse.org/scanpy/paul15.h5"
-    _utils.check_presence_download(filename, backup_url)
-    with h5py.File(filename, "r") as f:
+    with h5py.File(fetch_dataset("paul15"), "r") as f:
         # Coercing to float32 for backwards compatibility
         x = f["data.debatched"][()].astype(np.float32)
         gene_names = f["data.debatched_rownames"][()].astype(str)
@@ -429,7 +417,6 @@ def pbmc68k_reduced() -> AnnData:
 
 @_doctest_skipif_old_anndata
 @doctest_internet
-@check_datasetdir_exists
 def pbmc3k() -> AnnData:
     r"""3k PBMCs from 10x Genomics.
 
@@ -475,16 +462,13 @@ def pbmc3k() -> AnnData:
         layers: None (.X)
 
     """
-    url = "https://exampledata.scverse.org/scanpy/pbmc3k_raw.h5ad"
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=OldFormatWarning)
-        adata = read(settings.datasetdir / "pbmc3k_raw.h5ad", backup_url=url)
-    return adata
+        return read(fetch_dataset("pbmc3k"))
 
 
 @_doctest_skipif_old_anndata
 @doctest_internet
-@check_datasetdir_exists
 def pbmc3k_processed() -> AnnData:
     """Processed 3k PBMCs from 10x Genomics.
 
@@ -517,12 +501,10 @@ def pbmc3k_processed() -> AnnData:
         layers: None (.X)
 
     """  # noqa: D401
-    url = "https://exampledata.scverse.org/scanpy/pbmc3k.h5ad"
-
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=OldFormatWarning)
         warnings.filterwarnings("ignore", r"Moving.*from.*uns.*to.*obsp", FutureWarning)
-        return read(settings.datasetdir / "pbmc3k_processed.h5ad", backup_url=url)
+        return read(fetch_dataset("pbmc3k_processed"))
 
 
 def _download_visium_dataset(
@@ -556,27 +538,22 @@ def _download_visium_dataset(
     # Download spatial data
     tar_filename = f"{sample_id}_spatial.tar.gz"
     tar_pth = sample_dir / tar_filename
-    _utils.check_presence_download(
-        filename=tar_pth, backup_url=f"{url_prefix}/{tar_filename}"
-    )
+    download(f"{url_prefix}/{tar_filename}", tar_pth)
     with tarfile.open(tar_pth) as f:
         f.extraction_filter = tarfile.data_filter
         for el in f:
             if not (sample_dir / el.name).exists():
                 f.extract(el, sample_dir)
 
     # Download counts
-    _utils.check_presence_download(
-        filename=sample_dir / "filtered_feature_bc_matrix.h5",
-        backup_url=f"{url_prefix}/{sample_id}_filtered_feature_bc_matrix.h5",
+    download(
+        f"{url_prefix}/{sample_id}_filtered_feature_bc_matrix.h5",
+        sample_dir / "filtered_feature_bc_matrix.h5",
     )
 
     # Download image
     if download_image:
-        _utils.check_presence_download(
-            filename=sample_dir / "image.tif",
-            backup_url=f"{url_prefix}/{sample_id}_image.tif",
-        )
+        download(f"{url_prefix}/{sample_id}_image.tif", sample_dir / "image.tif")
 
     return sample_dir
 

src/scanpy/datasets/_utils.py

@@ -1,13 +1,19 @@
 from __future__ import annotations
 
-from functools import wraps
+from functools import cache, wraps
+from importlib.resources import as_file, files
+from pathlib import Path
 from typing import TYPE_CHECKING
 
+from scverse_misc.datasets import fetch, parse_registry, register_loader
+
 from .._settings import settings
 
 if TYPE_CHECKING:
     from collections.abc import Callable
 
+    from scverse_misc.datasets import DatasetEntry, DownloadCB
+
 
 def check_datasetdir_exists[**P, R](f: Callable[P, R]) -> Callable[P, R]:
     @wraps(f)
@@ -16,3 +22,23 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
         return f(*args, **kwargs)
 
     return wrapper
+
+
+@register_loader("scanpy")
+def _load_file(entry: DatasetEntry, target: Path, download: DownloadCB, /) -> Path:
+    """Download the single file making up `entry` and return its path."""
+    (file,) = entry.files
+    # `target` is `datasetdir/scanpy`; its parent keeps scanpy’s flat layout.
+    return Path(download(file, dest=target.parent))
+
+
+@cache
+def _registry() -> tuple[str | None, dict[str, DatasetEntry]]:
+    with as_file(files(__package__) / "registry.yaml") as path:
+        return parse_registry(path)
+
+
+def fetch_dataset(name: str) -> Path:
+    """Download `name` into :attr:`~scanpy.settings.datasetdir` if needed, return its path."""
+    base_url, datasets = _registry()
+    return fetch(datasets[name], settings.datasetdir, base_url=base_url)

src/scanpy/datasets/registry.yaml

@@ -0,0 +1,55 @@
+# Datasets scanpy hosts itself, fetched via `scverse_misc.datasets.fetch`.
+#
+# Add a `sha256` for every file: it is what lets a truncated or corrupted
+# download be detected (and retried) instead of surfacing as a parse error.
+#
+# `fallback_urls` are the upstream locations we used before our own S3 bucket
+# (see #4011); they are used if the bucket is down or serves garbage.
+base_url: https://exampledata.scverse.org/scanpy
+
+datasets:
+
+  burczynski06:
+    type: scanpy
+    files:
+      - name: burczynski06.soft.gz
+        s3_key: GDS1615_full.soft.gz
+        sha256: 86126c1a3c163ea20abb14c1a9711aaff34e6c492ef6dd86298bbaf18cc3f5f3
+        fallback_urls:
+          - ftp://ftp.ncbi.nlm.nih.gov/geo/datasets/GDS1nnn/GDS1615/soft/GDS1615_full.soft.gz
+
+  moignard15:
+    type: scanpy
+    files:
+      - name: moignard15.xlsx
+        s3_key: 41587_2015_BFnbt3154_MOESM4_ESM.xlsx
+        sha256: e7b358fef2b6da115b7ee3ab4c7fc55fd80e210bba427902e4afc9118992747c
+        fallback_urls:
+          - https://static-content.springer.com/esm/art%3A10.1038%2Fnbt.3154/MediaObjects/41587_2015_BFnbt3154_MOESM4_ESM.xlsx
+
+  paul15:
+    type: scanpy
+    files:
+      - name: paul15.h5
+        s3_key: paul15.h5
+        sha256: 6161984f758dd464992edc23f1a8ab89b2081600130c6aa93a1ab12b3ada5bfb
+        fallback_urls:
+          - https://falexwolf.de/data/paul15.h5
+
+  pbmc3k:
+    type: scanpy
+    files:
+      - name: pbmc3k_raw.h5ad
+        s3_key: pbmc3k_raw.h5ad
+        sha256: 89a96f1beaa2dd83a687666d3f19a4513ac27a2a2d12581fcd77afed7ea653a1
+        fallback_urls:
+          - https://falexwolf.de/data/pbmc3k_raw.h5ad
+
+  pbmc3k_processed:
+    type: scanpy
+    files:
+      - name: pbmc3k_processed.h5ad
+        s3_key: pbmc3k.h5ad
+        sha256: 0db367b991dd95809732b218539ede489bea99113807f62ebd7ccc970025fe38
+        fallback_urls:
+          - https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad