diff --git a/.codacy/codacy.yaml b/.codacy/codacy.yaml new file mode 100644 index 0000000..15365c7 --- /dev/null +++ b/.codacy/codacy.yaml @@ -0,0 +1,15 @@ +runtimes: + - dart@3.7.2 + - go@1.22.3 + - java@17.0.10 + - node@22.2.0 + - python@3.11.11 +tools: + - dartanalyzer@3.7.2 + - eslint@8.57.0 + - lizard@1.17.31 + - pmd@7.11.0 + - pylint@3.3.6 + - revive@1.7.0 + - semgrep@1.78.0 + - trivy@0.66.0 diff --git a/README.md b/README.md index 0715bc6..423db45 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ You can: - download public and private PRIDE files -- download public MassIVE datasets directly from `MSV...` accessions +- download public MassIVE (`MSV...`), JPOST (`JPST...`), and iProX (`IPX...`) datasets directly. MassIVE goes through FTPS at `massive-ftp.ucsd.edu`, with an automatic HTTPS fallback (via the GNPS2 file index and the `massive.ucsd.edu` ProteoSAFe endpoint) for networks that block FTP/FTPS; JPOST uses the JSON PROXI endpoint at `repository.jpostdb.org` for listings and `ftp.jpostdb.org` for transfers; iProX fetches the dataset's ProteomeXchange XML from `download.iprox.org` and downloads files over anonymous HTTP - download by category (`RAW`, `SEARCH`, `RESULT`, etc.) - stream project and file metadata - search projects by keyword and filters @@ -45,231 +45,82 @@ pip install --upgrade pridepy pridepy --help ``` -### Option 3: Install from source (development) +### Option 3: Install the latest code directly from GitHub -```bash -git clone https://github.com/PRIDE-Archive/pridepy -cd pridepy -uv sync --extra dev -uv run pridepy --help -``` - -## Quick Start (New Users) +To get features that have not been released to PyPI yet, install straight from a +branch. `master` holds the latest stable code; `dev` holds the newest (and +potentially unstable) development work. -### 1) Download all raw files for a project (robust mode) +With `uv`: ```bash -pridepy download-all-public-raw-files \ - -a PXD008644 \ - -o ./downloads/PXD008644 \ - --checksum-check -``` - -What this does: -- default `ftp` starts with FTP and falls back (`ftp -> aspera -> s3 -> globus`) -- `--checksum-check` downloads project checksums and validates files -- empty/corrupt files are retried automatically +# Latest stable (master) +uv tool install "git+https://github.com/PRIDE-Archive/pridepy@master" -### 2) Continue interrupted downloads safely - -```bash -pridepy download-all-public-raw-files \ - -a PXD008644 \ - -o ./downloads/PXD008644 \ - --skip-if-downloaded-already \ - --checksum-check +# Bleeding edge (dev) +uv tool install "git+https://github.com/PRIDE-Archive/pridepy@dev" ``` -### 3) Download a public MassIVE dataset directly +Or with `pip`: ```bash -pridepy download-all-public-raw-files \ - -a MSV000082297 \ - -o ./downloads/MSV000082297 -``` - -For direct `MSV...` downloads, `pridepy` enumerates the dataset from MassIVE's public FTP tree. Raw downloads follow MassIVE's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. +# Latest stable (master) +pip install --upgrade "git+https://github.com/PRIDE-Archive/pridepy@master" -### 4) Download only selected categories - -```bash -pridepy download-all-public-category-files \ - -a PXD022105 \ - -o ./downloads/PXD022105 \ - -c RAW,SEARCH +# Bleeding edge (dev) +pip install --upgrade "git+https://github.com/PRIDE-Archive/pridepy@dev" ``` -You can also request a specific MassIVE collection through the same category interface: - -```bash -pridepy download-all-public-category-files \ - -a MSV000082297 \ - -o ./downloads/MSV000082297-results \ - -c RESULT -``` - -### 5) Download one file by name - -```bash -pridepy download-file-by-name \ - -a PXD022105 \ - -f checksum.txt \ - -o ./downloads/PXD022105 \ - --checksum-check -``` - -### 6) Download raw files from ProteomeXchange - -```bash -pridepy download-px-raw-files \ - -a PXD039236 \ - -o ./downloads/PXD039236 -``` - -### 6) Download a named subset of files (manifest) - -```bash -pridepy download-files-by-list \ - -a PXD001819 \ - -F files.txt \ - -o ./downloads/PXD001819 \ - --checksum-check -``` - -`files.txt` is one filename per line (blank lines and `#` comments are -ignored). Internally each filename is resolved against the project metadata -API and downloaded via the same batch + protocol-fallback engine as -`download-all-public-raw-files`. Use `-f a.raw,b.raw,c.raw` instead of -`-F` for a small inline list. - -Useful options: - -- `-p globus` — use the globus download strategy (HTTP Range + resume) -- `-w 3` — download up to 3 files in parallel (globus only, max 3) -- `--checksum-check` — validate files against PRIDE checksums after download +You can pin to any branch, tag, or commit by changing the part after `@` (e.g. +`@v0.0.16` or `@`). -### 7) Download files from raw URLs +### Option 4: Install from source (development) ```bash -pridepy download-files-by-url \ - -F urls.txt \ - -o ./downloads/urls +git clone https://github.com/PRIDE-Archive/pridepy +cd pridepy +uv sync --extra dev +uv run pridepy --help ``` -`urls.txt` is one fully-qualified URL per line. Schemes `http`, `https`, and -`ftp` are dispatched to the matching downloader. Use `-u/--urls` for one or -more comma-separated URLs, e.g. `--urls https://a.com/x.raw,ftp://b.com/y.raw`. -Note: URLs containing literal commas are not supported with `--urls`; use a -manifest file (`-F`) instead. +## Usage -Useful options: - -- `-p globus` — use globus download strategy for http/https URLs (resume-capable) -- `-w 3` — download up to 3 files in parallel (globus only, max 3) -- `--checksum-check` — validate against PRIDE checksums (accession inferred - from PRIDE URL paths; only PRIDE archive URLs are supported) - -## CLI Command Overview +See the **[usage guide](docs/usage.md)** for detailed instructions and examples: +downloading data (PRIDE, MassIVE, JPOST, iProX, ProteomeXchange), category and +manifest downloads, private files, streaming metadata, searching projects, and +the Python API. ```bash pridepy --help ``` -Main commands: -- `download-all-public-raw-files` -- `download-all-public-category-files` -- `download-file-by-name` -- `download-files-by-list` -- `download-files-by-url` -- `download-px-raw-files` -- `list-private-files` -- `stream-files-metadata` -- `stream-projects-metadata` -- `search-projects-by-keywords-and-filters` - -## More CLI Examples - -### Search projects - -```bash -pridepy search-projects-by-keywords-and-filters \ - -k human \ - -f projectTags==ProteomeTools,organismsPart==Pancreas \ - -sd DESC \ - -sf accession \ - -sf submissionDate -``` - -### Stream all project metadata to JSON - -```bash -pridepy stream-projects-metadata -o all_pride_projects.json -``` - -### Stream all file metadata for one accession +| Command | Purpose | +| --- | --- | +| `download-all-public-raw-files` | Download every public RAW file of a dataset | +| `download-all-public-category-files` | Download files of one or more categories (RAW, SEARCH, …) | +| `download-file-by-name` | Download a single file (public or private) | +| `download-files-by-list` | Download a named subset of files from a manifest/CSV | +| `download-files-by-url` | Download files from raw `http`/`https`/`ftp` URLs | +| `download-px-raw-files` | Download RAW files resolved from a ProteomeXchange accession | +| `list-private-files` | List files of a private project (needs credentials) | +| `stream-files-metadata` | Stream file metadata (one project or all) to JSON | +| `stream-projects-metadata` | Stream all project metadata to JSON | +| `search-projects-by-keywords-and-filters` | Search projects by keyword and filters | -```bash -pridepy stream-files-metadata -a PXD005011 -o PXD005011_files.json -``` - -### Download private files - -List files: - -```bash -pridepy list-private-files -a PXD022105 -u YOUR_USER -p YOUR_PASSWORD -``` - -Download a private file: +Quick examples: ```bash -pridepy download-file-by-name \ - -a PXD022105 \ - -f checksum.txt \ - -o ./downloads/private \ - --username YOUR_USER \ - --password YOUR_PASSWORD -``` - -## Python API Examples - -### Example: get raw files for a project - -```python -from pridepy.files.files import Files +# Download all public RAW files of a dataset (any repository) +pridepy download-all-public-raw-files -a PXD008644 -o ./downloads/PXD008644 --checksum-check -files = Files() -raw_files = files.get_all_raw_file_list("PXD008644") -print(f"RAW files: {len(raw_files)}") -print(raw_files[0]["fileName"]) -``` - -For MassIVE accessions, the same method returns the files found under the dataset's `raw/` collection: +# Download a ProteomeXchange dataset by its PXD accession +pridepy download-px-raw-files -a PXD039236 -o ./downloads/PXD039236 -```python -from pridepy.files.files import Files - -files = Files() -raw_files = files.get_all_raw_file_list("MSV000082297") -print(f"MassIVE raw files: {len(raw_files)}") +# Download a native MassIVE / JPOST / iProX dataset +pridepy download-all-public-raw-files -a MSV000082297 -o ./downloads/MSV000082297 ``` -### Example: search projects - -```python -from pridepy.project.project import Project - -project = Project() -results = project.search_by_keywords_and_filters( - keyword="PXD009476", - query_filter="", - page_size=25, - page=0, - sort_direction="DESC", - sort_fields="accession", -) -print(f"Hits: {len(results)}") -``` +Full option tables and more examples are in [docs/usage.md](docs/usage.md). ## Development and Release (uv) diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..005d5e0 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,350 @@ +# pridepy usage guide + +This guide covers how to download data and query metadata with `pridepy`. For +installation, see the [README](../README.md#installation). + +`pridepy` works with PRIDE accessions and, transparently, with native MassIVE +(`MSV…`), JPOST (`JPST…`), and iProX (`IPX…`) accessions. The downloader +supports `ftp`, `aspera`, `s3`, and `globus`: by default it starts with FTP, +falls back across the remaining protocols when a transfer fails, and validates +downloaded files (non-empty, and checksum validation when enabled). + +## Contents + +- [Command overview](#command-overview) +- [PRIDE file downloads](#pride-file-downloads) +- [Metadata and search](#metadata-and-search) +- [Download from ProteomeXchange and other repositories](#download-from-proteomexchange-and-other-repositories) +- [Python API examples](#python-api-examples) + +## Command overview + +```bash +pridepy --help +``` + +| Command | Purpose | +| --- | --- | +| `download-all-public-raw-files` | Download every public RAW file of a dataset | +| `download-all-public-category-files` | Download files of one or more categories (RAW, SEARCH, …) | +| `download-file-by-name` | Download a single file (public or private) | +| `download-files-by-list` | Download a named subset of files from a manifest/CSV | +| `download-files-by-url` | Download files from raw `http`/`https`/`ftp` URLs | +| `download-px-raw-files` | Download RAW files resolved from a ProteomeXchange accession | +| `list-private-files` | List files of a private project (needs credentials) | +| `stream-files-metadata` | Stream file metadata (one project or all) to JSON | +| `stream-projects-metadata` | Stream all project metadata to JSON | +| `search-projects-by-keywords-and-filters` | Search projects by keyword and filters | + +The download commands work for PRIDE accessions and, transparently, for native +MassIVE (`MSV…`), JPOST (`JPST…`), and iProX (`IPX…`) accessions — see +[Download from ProteomeXchange and other repositories](#download-from-proteomexchange-and-other-repositories). + +## PRIDE file downloads + +PRIDE downloads start with FTP and fall back across the remaining protocols +(`ftp -> aspera -> s3 -> globus`) when a transfer fails. They support resume, +per-file retries, parallel workers, and optional checksum validation. Empty or +corrupt files are retried automatically. + +### Common download options + +These options are shared by `download-all-public-raw-files`, +`download-all-public-category-files`, `download-file-by-name`, and +`download-files-by-list`: + +| Option | Description | Default | +| --- | --- | --- | +| `-a, --accession` | Dataset accession (e.g. `PXD008644`) | required | +| `-o, --output-folder` | Destination directory | required | +| `-p, --protocol` | Transfer protocol: `ftp`, `aspera`, `globus`, `s3` (FTP-first with fallback) | `ftp` | +| `-w, --parallel-files` | Download 1–3 files concurrently — primarily for `globus`; not available on `download-file-by-name` | `1` | +| `--skip-if-downloaded-already` | Resume: skip files already present locally | off | +| `--checksum-check` | Download PRIDE checksums and validate each file | off | +| `--aspera-maximum-bandwidth` | Aspera cap, e.g. `50M`, `100M`, `200M` (Aspera only) | `100M` | +| `--preserve-structure` | Recreate the dataset's subdirectory layout (e.g. `raw/…/`) under the output folder instead of downloading flat | off | + +By default, files are downloaded **flat** into the output folder (no +`raw/…/` subdirectories). When two files would collapse to the same name, +later ones get a numeric suffix (`run.raw`, `run_1.raw`). Pass +`--preserve-structure` to keep the dataset's original directory layout. + +### Download all raw files (robust mode) + +```bash +pridepy download-all-public-raw-files \ + -a PXD008644 \ + -o ./downloads/PXD008644 \ + --checksum-check +``` + +Continue an interrupted download safely by adding `--skip-if-downloaded-already`: + +```bash +pridepy download-all-public-raw-files \ + -a PXD008644 \ + -o ./downloads/PXD008644 \ + --skip-if-downloaded-already \ + --checksum-check +``` + +### Download only selected categories + +```bash +pridepy download-all-public-category-files \ + -a PXD022105 \ + -o ./downloads/PXD022105 \ + -c RAW,SEARCH +``` + +`-c, --category` takes one or more comma-separated categories. Valid values: +`RAW`, `PEAK`, `SEARCH`, `RESULT`, `SPECTRUM_LIBRARY`, `OTHER`, `FASTA`. + +### Download one file by name + +```bash +pridepy download-file-by-name \ + -a PXD022105 \ + -f checksum.txt \ + -o ./downloads/PXD022105 \ + --checksum-check +``` + +`-f, --file-name` is the file to download. + +### Download a named subset of files (manifest) + +```bash +pridepy download-files-by-list \ + -a PXD001819 \ + -F files.txt \ + -o ./downloads/PXD001819 \ + --checksum-check +``` + +`files.txt` is one filename per line (blank lines and `#` comments are +ignored). Each filename is resolved against the project metadata and downloaded +via the same batch + protocol-fallback engine as `download-all-public-raw-files`. +Use `-f a.raw,b.raw,c.raw` instead of `-F` for a small inline list (you can +combine both). + +### Download files from raw URLs + +```bash +pridepy download-files-by-url \ + -F urls.txt \ + -o ./downloads/urls +``` + +`urls.txt` is one fully-qualified URL per line. Schemes `http`, `https`, and +`ftp` are dispatched to the matching downloader. Use `-u, --urls` for one or +more comma-separated URLs, e.g. `--urls https://a.com/x.raw,ftp://b.com/y.raw` +(URLs containing literal commas must use a manifest file instead). + +Command-specific options: + +| Option | Description | Default | +| --- | --- | --- | +| `-F, --url-list` | Manifest file, one URL per line | — | +| `-u, --urls` | Comma-separated URL(s) | — | +| `-p, --protocol` | `ftp` (per-scheme) or `globus` (resume-capable http/https) | `ftp` | +| `-w, --parallel-files` | Download 1–3 files concurrently (any scheme) | `1` | +| `--checksum-check` | Validate against PRIDE checksums (accession inferred from PRIDE URL paths; only PRIDE archive URLs supported) | off | + +### Private (restricted) files + +List the files of a private project with your PRIDE credentials: + +```bash +pridepy list-private-files -a PXD022105 -u YOUR_USER -p YOUR_PASSWORD +``` + +Download a private file by passing `--username`/`--password` to +`download-file-by-name`: + +```bash +pridepy download-file-by-name \ + -a PXD022105 \ + -f checksum.txt \ + -o ./downloads/private \ + --username YOUR_USER \ + --password YOUR_PASSWORD +``` + +## Metadata and search + +### Stream all project metadata to JSON + +```bash +pridepy stream-projects-metadata -o all_pride_projects.json +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-o, --output-file` | JSON file to write all project metadata to | required | + +### Stream file metadata + +```bash +# All file metadata for one accession +pridepy stream-files-metadata -a PXD005011 -o PXD005011_files.json +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-o, --output-file` | JSON file to write file metadata to | required | +| `-a, --accession` | Limit to one project (omit to stream all files) | optional | + +### Search projects by keywords and filters + +```bash +pridepy search-projects-by-keywords-and-filters \ + -k human \ + -f projectTags==ProteomeTools,organismsPart==Pancreas \ + -sd DESC \ + -sf accession \ + -sf submissionDate +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-k, --keyword` | Keyword searched across project fields | required | +| `-f, --filters` | `field==value` filters, comma-separated (e.g. `accession==PRD000001`) | — | +| `-ps, --page-size` | Results per page (1–1000) | `100` | +| `-p, --page` | Page number (0-based) | `0` | +| `-sd, --sort-direction` | `ASC` or `DESC` | `DESC` | +| `-sf, --sort-fields` | Sort field(s), repeatable. One of: `accession`, `submissionDate`, `diseases`, `organismsPart`, `organisms`, `instruments`, `softwares`, `avgDownloadsPerFile`, `downloadCount`, `publicationDate` | `submissionDate` | + +## Download from ProteomeXchange and other repositories + +A ProteomeXchange (`PXD…` / `PRD…`) accession is a cross-repository identifier: +the dataset may be hosted at PRIDE, MassIVE, JPOST, iProX, or elsewhere. +`pridepy` lets you start from the ProteomeXchange accession, or go straight to +the hosting repository using its **native** accession. + +### Start from a ProteomeXchange accession + +`download-px-raw-files` resolves the dataset's ProteomeXchange XML and downloads +the RAW files it references, regardless of which repository hosts them: + +```bash +pridepy download-px-raw-files \ + -a PXD039236 \ + -o ./downloads/PXD039236 +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-a, --accession` | ProteomeXchange accession (e.g. `PXD039236`). `--px` is a deprecated alias | required | +| `-o, --output-folder` | Destination directory | required | +| `--skip-if-downloaded-already` | Skip files already present locally | off | + +### Go directly to the hosting repository (native MassIVE / JPOST / iProX accessions) + +Datasets that do not have a ProteomeXchange accession — or where you already +know the native accession — can be downloaded directly. The standard download +commands accept MassIVE, JPOST, and iProX accessions transparently: + +```bash +# MassIVE (FTPS at massive-ftp.ucsd.edu) +pridepy download-all-public-raw-files \ + -a MSV000082297 \ + -o ./downloads/MSV000082297 + +# JPOST (PROXI listing + ftp.jpostdb.org) +pridepy download-all-public-raw-files \ + -a JPST002311 \ + -o ./downloads/JPST002311 + +# iProX (ProteomeXchange XML + anonymous HTTP at download.iprox.org) +pridepy download-all-public-raw-files \ + -a IPX0017413000 \ + -o ./downloads/IPX0017413000 +``` + +How each repository is enumerated: + +- **MassIVE** walks the FTPS tree at `massive-ftp.ucsd.edu` (the server requires TLS). MassIVE distributes datasets across versioned root directories (`/v01`–`/vNN`); `pridepy` discovers the correct root automatically. If FTP/FTPS is blocked by the network, `pridepy` falls back to HTTPS: it lists the dataset from the GNPS2 file index (`datasetcache.gnps2.org`) and downloads each file from the ProteoSAFe endpoint at `massive.ucsd.edu` (byte-identical to the FTPS copy). +- **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. +- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTP (with `Range` support for resume). iProX also exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTP endpoint so no iProX credentials are required. + +`download-all-public-raw-files` retrieves the files stored under the dataset's +`raw/` collection. These direct downloads support resume (REST for FTP, +byte-Range for HTTP), per-file retries, parallel workers (`-w` up to 3), and +post-transfer size verification against the server-reported size. By default +files are written flat into the output folder; pass `--preserve-structure` to +keep the dataset's sub-directory layout. + +You can also request a specific collection from these repositories through the +same category interface: + +```bash +pridepy download-all-public-category-files \ + -a MSV000082297 \ + -o ./downloads/MSV000082297-results \ + -c RESULT +``` + +## Python API examples + +> **Breaking change (0.0.16):** the legacy `pridepy.files.files.Files` class has been +> removed. Replace `from pridepy.files.files import Files` with +> `from pridepy.download.client import Client`; `Client` exposes the same public +> methods (`get_all_raw_file_list`, `download_all_raw_files`, +> `get_submitted_file_path_prefix`, `download_file_by_name`, +> `download_all_category_files`, `download_px_raw_files`, …). + +### Get raw files for a project + +```python +from pridepy.download.client import Client + +client = Client() +raw_files = client.get_all_raw_file_list("PXD008644") +print(f"RAW files: {len(raw_files)}") +print(raw_files[0]["fileName"]) +``` + +For MassIVE / JPOST / iProX accessions, the same method returns the files found under the dataset's `raw/` collection: + +```python +from pridepy.download.client import Client + +client = Client() +for accession in ("MSV000082297", "JPST002311", "IPX0017413000"): + raw_files = client.get_all_raw_file_list(accession) + print(f"{accession} raw files: {len(raw_files)}") +``` + +### Download all raw files for a project + +```python +from pridepy.download.client import Client + +client = Client() +client.download_all_raw_files( + accession="PXD008644", + output_folder="./downloads/PXD008644", + skip_if_downloaded_already=True, + protocol="ftp", + aspera_maximum_bandwidth="100M", + checksum_check=True, +) +``` + +### Search projects + +```python +from pridepy.project.project import Project + +project = Project() +results = project.search_by_keywords_and_filters( + keyword="PXD009476", + query_filter="", + page_size=25, + page=0, + sort_direction="DESC", + sort_fields="accession", +) +print(f"Hits: {len(results)}") +``` diff --git a/pridepy/__init__.py b/pridepy/__init__.py index ca4e345..9fee806 100644 --- a/pridepy/__init__.py +++ b/pridepy/__init__.py @@ -1 +1 @@ -from .pridepy import main +from .pridepy import main # noqa: F401 (re-exported for the `pridepy` console script) diff --git a/pridepy/download/__init__.py b/pridepy/download/__init__.py new file mode 100644 index 0000000..854ca75 --- /dev/null +++ b/pridepy/download/__init__.py @@ -0,0 +1,20 @@ +"""The pridepy download subsystem. + +This package holds everything involved in turning an accession (or URL) into +downloaded files: + +- Repository adapters — one module per repository (``pride``, ``massive``, + ``jpost``, ``iprox``, ``proteomexchange``). Each subclasses + :class:`pridepy.download.base.Provider` and implements at least ``matches`` + + ``list_files``. Direct-download adapters (MassIVE / JPOST / iProX) inherit + the whole download workflow from the base; ``PrideProvider`` additionally + overrides ``get_download_url``, ``download_files``, and ``download_by_name`` + for its multi-protocol fallback and public/private split. +- :mod:`registry` — maps an accession to the right adapter. +- :mod:`transport` — shared FTP/FTPS/HTTPS plumbing (resume, retry, parallel). +- :mod:`util` — checksum and record helpers. +- :mod:`by_url` — cross-cutting download command that takes raw URLs rather + than an accession. +- :mod:`client` — the :class:`~pridepy.download.client.Client` facade the CLI + drives; dispatches to adapters via the registry. +""" diff --git a/pridepy/download/base.py b/pridepy/download/base.py new file mode 100644 index 0000000..3382f08 --- /dev/null +++ b/pridepy/download/base.py @@ -0,0 +1,325 @@ +"""Abstract base class for pridepy providers. + +The :class:`Provider` base implements the download *workflow* via the +Template Method pattern: most adapters only fill in the holes +(:meth:`matches`, :meth:`list_files`) while the shared listing-filter and +download-orchestration methods live here. Adapters that need different +behaviour override the relevant hooks — e.g. PRIDE overrides +:meth:`get_download_url`, :meth:`download_files` (multi-protocol fallback), +and :meth:`download_by_name` (public/private split). Everything not +overridden routes through the inherited default that partitions record URLs +by scheme. +""" +import logging +from abc import ABC, abstractmethod +from typing import ClassVar, Dict, List, Optional +from urllib.parse import urlparse + +from pridepy.download import transport +from pridepy.download.util import flatten_relative_paths + + +class Provider(ABC): + """Abstract base for every repository pridepy can list and download from.""" + + name: ClassVar[str] # "pride", "massive", "jpost", "iprox" + use_tls: ClassVar[bool] = False + + # ------------------------------------------------------------------ + # Abstract holes — adapters must implement these. + # ------------------------------------------------------------------ + + @staticmethod + @abstractmethod + def matches(accession: str) -> bool: + """Return True if this provider should handle ``accession``.""" + + @abstractmethod + def list_files(self, accession: str) -> List[Dict]: + """Return pridepy file records for the dataset. + + Each record is a dict shaped like the PRIDE V3 API file response, + with at minimum: ``accession``, ``fileName``, ``fileCategory`` + (with nested ``value``), ``publicFileLocations`` (list of + ``{"name": ..., "value": }``). + """ + + # ------------------------------------------------------------------ + # Hook with default — adapters may override. + # ------------------------------------------------------------------ + + def get_download_url(self, record: Dict, protocol: str = "ftp") -> str: + """Resolve the download URL for ``record``. + + Default: return the ``"FTP Protocol"`` public-file-location value + (most direct-download adapters store their public URL there — ftp:// + for MassIVE/JPOST, http:// for iProX). Records that use a different + location name (e.g. MassIVE's HTTPS fallback uses ``"HTTPS"``) fall + through to the first location. Adapters with richer, protocol-aware + resolution (PRIDE: aspera/globus/s3) override this. + """ + locations = record.get("publicFileLocations", []) + if not locations: + raise ValueError("No public file locations present") + for location in locations: + if location.get("name") == "FTP Protocol": + return location.get("value") + return locations[0].get("value") + + # ------------------------------------------------------------------ + # Shared listing filters. + # ------------------------------------------------------------------ + + def _list_files_checked(self, accession: str) -> List[Dict]: + """Call :meth:`list_files` and fail clearly if it yields no listing. + + The PRIDE API helper returns ``None`` on a network error (e.g. a read + timeout), so guard here to raise an actionable error rather than a + cryptic ``TypeError: 'NoneType' object is not iterable`` downstream. + """ + records = self.list_files(accession) + if records is None: + raise RuntimeError( + f"Could not list files for {accession}: the repository API " + f"returned no data (it may be unreachable, or the accession " + f"may be invalid)." + ) + return records + + @staticmethod + def _category_value(record: Dict) -> Optional[str]: + """Safely read ``fileCategory.value`` from a (possibly partial) record.""" + return (record.get("fileCategory") or {}).get("value") + + def get_raw_files(self, accession: str) -> List[Dict]: + """Return records whose ``fileCategory.value`` is ``"RAW"``.""" + records = self._list_files_checked(accession) + return [r for r in records if self._category_value(r) == "RAW"] + + def get_category_files( + self, accession: str, categories: "str | List[str]" + ) -> List[Dict]: + """Return records belonging to the given category (or categories).""" + if isinstance(categories, str): + categories = [categories] + category_set = {c.upper() for c in categories} + records = self._list_files_checked(accession) + return [r for r in records if self._category_value(r) in category_set] + + def find_file(self, accession: str, file_name: str) -> List[Dict]: + """Return records whose ``fileName`` equals ``file_name``.""" + records = self._list_files_checked(accession) + return [r for r in records if r.get("fileName") == file_name] + + # ------------------------------------------------------------------ + # Shared download workflow (Template Method). + # ------------------------------------------------------------------ + + def download_all_raw( + self, + accession: str, + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str, + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + parallel_files: int = 1, + flatten: bool = True, + ) -> None: + """Download all RAW files for the dataset.""" + self.download_files( + accession=accession, + records=self.get_raw_files(accession), + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + flatten=flatten, + ) + + def download_category( + self, + accession: str, + output_folder: str, + categories: "str | List[str]", + skip_if_downloaded_already: bool, + protocol: str, + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + parallel_files: int = 1, + flatten: bool = True, + ) -> None: + """Download all files of the given categories for the dataset.""" + self.download_files( + accession=accession, + records=self.get_category_files(accession, categories), + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + flatten=flatten, + ) + + def download_by_name( + self, + accession: str, + file_name: str, + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str, + username: Optional[str] = None, + password: Optional[str] = None, + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + ) -> None: + """Download a single file by name from the dataset.""" + records = self.find_file(accession, file_name) + if not records: + raise Exception( + f"File name {file_name} not found in dataset {accession}" + ) + self.download_files( + accession=accession, + records=records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + ) + + def download_by_filenames( + self, + accession: str, + file_names: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str = "ftp", + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + parallel_files: int = 1, + flatten: bool = True, + ) -> None: + """Download a subset of project files identified by a filename list. + + :raises ValueError: if ``file_names`` is empty or none match. + """ + if not file_names: + raise ValueError("file_names must contain at least one filename") + + all_files = self._list_files_checked(accession) + requested = set(file_names) + matched = [f for f in all_files if f.get("fileName") in requested] + missing = sorted(requested - {f.get("fileName") for f in matched}) + if missing: + logging.warning("Files not found in project %s: %s", accession, missing) + if not matched: + raise ValueError( + f"No matching files in project {accession} for: {sorted(requested)}" + ) + + self.download_files( + accession=accession, + records=matched, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + flatten=flatten, + ) + + # ------------------------------------------------------------------ + # Default transport — adapters may override (e.g. PrideProvider). + # ------------------------------------------------------------------ + + def download_files( + self, + accession: str, + records: List[Dict], + output_folder: str, + skip_if_downloaded_already: bool = False, + protocol: str = "ftp", + parallel_files: int = 1, + checksum_check: bool = False, + aspera_maximum_bandwidth: str = "100M", + username: Optional[str] = None, + password: Optional[str] = None, + flatten: bool = True, + ) -> None: + """Partition record URLs by scheme and route to the matching transport. + + ``ftp://`` URLs are handed to :func:`transport.download_ftp_urls` + (with this provider's :attr:`use_tls`); ``http(s)://`` URLs go to + :func:`transport.download_http_urls`. + + When ``flatten`` is True (the default) every file is written directly + into ``output_folder`` by its basename, de-duplicating colliding + basenames across the whole set (they share one folder). When False the + dataset's subdirectory layout is preserved via each record's + ``relativePath``. + """ + if protocol not in ("ftp", "https", "http"): + logging.warning( + "Direct downloads currently use ftp / http(s) only. " + f"Ignoring requested protocol '{protocol}' for {accession}." + ) + + # Collect transfer entries in one pass, keeping order stable. + entries = [] # list of (scheme, url, relpath) + for record in records: + url = self.get_download_url(record, protocol) + relpath = record.get("relativePath") + lowered = url.lower() + if lowered.startswith("ftp://"): + entries.append(("ftp", url, relpath)) + elif lowered.startswith(("http://", "https://")): + entries.append(("http", url, relpath)) + if not entries: + logging.info( + f"No files matched for direct-download dataset {accession}" + ) + return + + if flatten: + # All files share one output folder, so dedup basenames globally; + # fall back to the URL path when a record carries no relativePath. + sources = [rel if rel else urlparse(url).path for _, url, rel in entries] + dest_paths: List[Optional[str]] = flatten_relative_paths(sources) + else: + dest_paths = [rel for _, _, rel in entries] + + ftp_urls: List[str] = [] + ftp_relpaths: List[Optional[str]] = [] + http_urls: List[str] = [] + http_relpaths: List[Optional[str]] = [] + for (scheme, url, _), dest in zip(entries, dest_paths): + if scheme == "ftp": + ftp_urls.append(url) + ftp_relpaths.append(dest) + else: + http_urls.append(url) + http_relpaths.append(dest) + + if ftp_urls: + transport.download_ftp_urls( + ftp_urls=ftp_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=self.use_tls, + parallel_files=parallel_files, + relative_paths=ftp_relpaths, + ) + if http_urls: + transport.download_http_urls( + http_urls=http_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + parallel_files=parallel_files, + relative_paths=http_relpaths, + ) diff --git a/pridepy/download/by_url.py b/pridepy/download/by_url.py new file mode 100644 index 0000000..0b4d03a --- /dev/null +++ b/pridepy/download/by_url.py @@ -0,0 +1,217 @@ +"""Download a list of explicit URLs (ftp/http/https). + +Each URL is dispatched to the matching transport based on its scheme. +PRIDE checksum validation is supported when the accession can be +inferred from the URL path. +""" +import ftplib +import logging +import os +from concurrent.futures import ThreadPoolExecutor, as_completed +from ftplib import FTP +from typing import List, Tuple +from urllib.parse import urlparse + +from tqdm import tqdm + +from pridepy.download import transport +from pridepy.download import util as _provider_util +from pridepy.download.pride import PrideProvider +from pridepy.util.api_handling import Util + + +def _http_download_url(url: str, target: str) -> None: + """Stream an http/https URL into ``target`` with a progress bar.""" + session = Util.create_session_with_retries() + with session.get(url, stream=True, timeout=60) as response: + response.raise_for_status() + total = int(response.headers.get("Content-Length", 0)) + # When the server applied Content-Encoding (gzip/deflate), requests + # decompresses transparently, so the on-disk size is the decompressed + # size while Content-Length is the compressed size — skip the size + # check to avoid a false "incomplete" on an intact file. + content_encoding = response.headers.get("Content-Encoding") + with open(target, "wb") as out, tqdm( + total=total, + unit="B", + unit_scale=True, + desc=os.path.basename(target), + ) as pbar: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + out.write(chunk) + pbar.update(len(chunk)) + if total and not content_encoding: + actual = os.path.getsize(target) + if actual != total: + raise RuntimeError( + f"Incomplete download for {target}: got {actual} bytes, " + f"expected {total}" + ) + + +def _ftp_download_url(parsed, target: str) -> None: + """Download a single file from an ftp:// URL with a progress bar.""" + host = parsed.hostname + if not host: + raise ValueError(f"FTP URL missing host: {parsed.geturl()}") + port = parsed.port or 21 + user = parsed.username or "anonymous" + pwd = parsed.password or "anonymous@" + remote_path = parsed.path + with FTP() as ftp: + ftp.connect(host, port, timeout=60) + ftp.login(user, pwd) + try: + total = ftp.size(remote_path) or 0 + except ftplib.error_perm: + total = 0 + with open(target, "wb") as out, tqdm( + total=total, + unit="B", + unit_scale=True, + desc=os.path.basename(target), + ) as pbar: + + def _callback(data: bytes) -> None: + out.write(data) + pbar.update(len(data)) + + ftp.retrbinary(f"RETR {remote_path}", _callback) + if total: + actual = os.path.getsize(target) + if actual != total: + raise RuntimeError( + f"Incomplete download for {target}: got {actual} bytes, " + f"expected {total}" + ) + + +def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: int = 0) -> None: + """Route a parsed URL to its protocol-specific downloader. + + ``protocol='globus'`` swaps the http/https single-connection streamer + for :func:`pridepy.download.transport._parallel_download` (single-connection + with progress bar). ftp:// URLs are unaffected. + """ + scheme = (parsed.scheme or "").lower() + if scheme in ("http", "https"): + if protocol == "globus": + transport._parallel_download(parsed.geturl(), target, position=position) + else: + _http_download_url(parsed.geturl(), target) + elif scheme == "ftp": + _ftp_download_url(parsed, target) + else: + raise ValueError(f"Unsupported URL scheme: {scheme}") + + +def _download_single_url( + url: str, + output_folder: str, + skip_if_exists: bool = False, + protocol: str = "ftp", + position: int = 0, +) -> str: + """Download one URL, dispatched by scheme; return the local file path.""" + parsed = urlparse(url) + if not (parsed.scheme or "").lower(): + raise ValueError(f"URL missing scheme: {url}") + + file_name = os.path.basename(parsed.path) + if not file_name: + raise ValueError(f"Cannot derive filename from URL: {url}") + + target = os.path.join(output_folder, file_name) + if skip_if_exists and os.path.isfile(target) and os.path.getsize(target) > 0: + logging.info("Skipping %s: already downloaded", file_name) + return target + + try: + _dispatch_url_scheme(parsed, target, protocol, position=position) + except Exception: + # Don't leave a truncated/partial file behind — a non-empty partial + # would otherwise be wrongly skipped on the next run. + _provider_util._remove_if_exists(target) + raise + + ok, reason = _provider_util.validate_download(target) + if not ok: + _provider_util._remove_if_exists(target) + raise RuntimeError(f"Download invalid: {reason} ({target})") + return target + + +def download_files_by_url( + urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool = False, + protocol: str = "ftp", + parallel_files: int = 1, + checksum_check: bool = False, +) -> None: + """Download files from a list of raw URLs, dispatched by URL scheme. + + Supported schemes: ``http``, ``https``, ``ftp``. Each URL is downloaded + independently; per-URL errors are logged, then aggregated and re-raised + as a single :class:`RuntimeError` so callers see a complete failure + summary. + + :param urls: fully-qualified URLs (each contains its scheme) + :param output_folder: directory to write downloaded files into + :param skip_if_downloaded_already: skip URLs whose target file exists + :param protocol: ``ftp`` (default) for single-connection per URL scheme; + ``globus`` for resume-capable http/https downloads (single-connection stream) + (no effect on ftp:// URLs which always use single-connection FTP) + :param checksum_check: validate downloads against PRIDE checksum API; + accessions are inferred from URL paths (only PRIDE URLs supported) + :raises ValueError: if ``urls`` is empty + :raises RuntimeError: if one or more URLs failed + """ + if not urls: + raise ValueError("urls must contain at least one URL") + + os.makedirs(output_folder, exist_ok=True) + + parallel_files = min(parallel_files, 3, len(urls)) + failures: List[Tuple[str, str]] = [] + + if parallel_files < 2: + for url in urls: + try: + _download_single_url( + url, output_folder, skip_if_downloaded_already, protocol, + ) + except Exception as exc: # pylint: disable=broad-except + logging.error("Failed to download %s: %s", url, exc) + failures.append((url, str(exc))) + else: + logging.info( + "Downloading %d URL(s) with %d parallel workers", + len(urls), parallel_files, + ) + with ThreadPoolExecutor(max_workers=parallel_files) as executor: + futures = { + executor.submit( + _download_single_url, + url, output_folder, skip_if_downloaded_already, protocol, + position=idx, + ): url + for idx, url in enumerate(urls) + } + for future in as_completed(futures): + url = futures[future] + try: + future.result() + except Exception as exc: # pylint: disable=broad-except + logging.error("Failed to download %s: %s", url, exc) + failures.append((url, str(exc))) + + if failures: + summary = ", ".join(f"{u} ({e})" for u, e in failures) + raise RuntimeError( + f"Failed to download {len(failures)} URL(s): {summary}" + ) + + if checksum_check: + PrideProvider.validate_urls_checksums(urls, output_folder) diff --git a/pridepy/download/client.py b/pridepy/download/client.py new file mode 100644 index 0000000..979c650 --- /dev/null +++ b/pridepy/download/client.py @@ -0,0 +1,315 @@ +#!/usr/bin/env python +"""Public ``Client`` facade — thin surface over the modular adapter +architecture in :mod:`pridepy.download`. + +The adapter classes own all transport/listing logic; this module exposes +a small set of high-level operations (CLI entry points + a handful of +one-line shims for downstream Python users). +""" +import logging +from typing import Dict, List, Optional, Tuple + +import requests # noqa: F401 — kept as a patch target for tests + +from pridepy.download import registry, transport +from pridepy.download import util as _provider_util +from pridepy.download.iprox import IproxProvider +from pridepy.download.jpost import JpostProvider +from pridepy.download.massive import MASSIVE_CATEGORY_MAP, MassiveProvider +from pridepy.download.pride import PrideProvider +from pridepy.download.proteomexchange import ProteomeXchangeProvider +from pridepy.download import by_url + +# Re-export Progress so `from pridepy.download.client import Progress` works. +from pridepy.download.util import Progress # noqa: F401 + + +class Client: + """High-level facade over the per-repository adapters.""" + + # PRIDE class-attribute re-exports (kept here for back-compat). + V3_API_BASE_URL = PrideProvider.V3_API_BASE_URL + API_BASE_URL = PrideProvider.API_BASE_URL + API_PRIVATE_URL = PrideProvider.API_PRIVATE_URL + PRIDE_ARCHIVE_FTP = PrideProvider.ARCHIVE_FTP + PRIDE_ARCHIVE_FTP_URL_PREFIX = PrideProvider.ARCHIVE_FTP_URL_PREFIX + PRIDE_ARCHIVE_HTTPS_URL_PREFIX = PrideProvider.ARCHIVE_HTTPS_URL_PREFIX + S3_URL = PrideProvider.S3_URL + S3_BUCKET = PrideProvider.S3_BUCKET + PROTOCOL_ORDER = PrideProvider.PROTOCOL_ORDER + + # MassIVE class-attribute re-exports. + MASSIVE_ARCHIVE_FTP = MassiveProvider.ARCHIVE_FTP + MASSIVE_ARCHIVE_FTP_URL_PREFIX = MassiveProvider.ARCHIVE_FTP_URL_PREFIX + + # JPOST class-attribute re-exports. + JPOST_ARCHIVE_FTP = JpostProvider.ARCHIVE_FTP + JPOST_ARCHIVE_FTP_URL_PREFIX = JpostProvider.ARCHIVE_FTP_URL_PREFIX + JPOST_PROXI_BASE_URL = JpostProvider.PROXI_BASE_URL + JPOST_PROXI_CATEGORY_MAP = JpostProvider.PROXI_CATEGORY_MAP + + # iProX class-attribute re-exports. + IPROX_DOWNLOAD_BASE_URL = IproxProvider.DOWNLOAD_BASE_URL + IPROX_PX_XML_URL_TEMPLATE = IproxProvider.PX_XML_URL_TEMPLATE + IPROX_PX_CATEGORY_MAP = IproxProvider.PX_CATEGORY_MAP + + # MassIVE category map re-exported. + MASSIVE_CATEGORY_MAP = MASSIVE_CATEGORY_MAP + + logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + + def __init__(self): + pass + + # Pure delegating shims kept for backward compatibility. + + @staticmethod + def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: + """Shim — see :func:`pridepy.download.util.compute_md5`.""" + return _provider_util.compute_md5(file_path, chunk_size) + + @staticmethod + def validate_download(file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, str]: + """Shim — see :func:`pridepy.download.util.validate_download`.""" + return _provider_util.validate_download(file_path, expected_checksum) + + @staticmethod + def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: + """Shim — see :func:`pridepy.download.util.read_checksum_file`.""" + return _provider_util.read_checksum_file(checksum_file_path) + + @staticmethod + def download_ftp_urls( + ftp_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + max_connection_retries: int = 3, + max_download_retries: int = 3, + use_tls: bool = False, + parallel_files: int = 1, + ) -> None: + """Shim — see :func:`pridepy.download.transport.download_ftp_urls`.""" + return transport.download_ftp_urls( + ftp_urls=ftp_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + use_tls=use_tls, + parallel_files=parallel_files, + ) + + @staticmethod + def download_http_urls( + http_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + parallel_files: int = 1, + max_retries: int = 3, + ) -> None: + """Shim — see :func:`pridepy.download.transport.download_http_urls`.""" + return transport.download_http_urls( + http_urls=http_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + parallel_files=parallel_files, + max_retries=max_retries, + ) + + # Accession-matcher convenience helpers (useful public API). + + @staticmethod + def is_massive_accession(accession: str) -> bool: + return MassiveProvider.matches(accession) + + @staticmethod + def is_jpost_accession(accession: str) -> bool: + return JpostProvider.matches(accession) + + @staticmethod + def is_iprox_accession(accession: str) -> bool: + return IproxProvider.matches(accession) + + @staticmethod + def is_direct_download_accession(accession: str) -> bool: + """True for MassIVE / JPOST / iProX (explicitly excludes PRIDE).""" + try: + provider = registry.resolve(accession) + except ValueError: + return False + return provider.name != "pride" + + @staticmethod + def _repo_uses_tls(accession: str) -> bool: + """Return the resolved provider's ``use_tls`` flag (False if unknown).""" + try: + provider = registry.resolve(accession) + except ValueError: + return False + return getattr(provider, "use_tls", False) + + # Listing / metadata. + + async def stream_all_files_metadata(self, output_file, accession=None): + """Shim — see :meth:`PrideProvider.stream_all_files_metadata`.""" + return await PrideProvider().stream_all_files_metadata(output_file, accession) + + def get_all_raw_file_list(self, accession): + """Get raw file list for any registered provider (records with fileCategory == "RAW").""" + return registry.resolve(accession).get_raw_files(accession) + + def get_all_category_file_list( + self, accession: str, categories: "str | List[str]" + ) -> List[Dict]: + """Retrieve project files belonging to the given categories.""" + return registry.resolve(accession).get_category_files(accession, categories) + + def get_submitted_file_path_prefix(self, accession): + """Shim — see :meth:`PrideProvider.get_submitted_file_path_prefix`.""" + return PrideProvider().get_submitted_file_path_prefix(accession) + + def get_file_from_api(self, accession, file_name) -> List[Dict]: + """Return records matching ``file_name`` from the provider's listing.""" + try: + return registry.resolve(accession).find_file(accession, file_name) + except Exception as e: + raise Exception("File not found " + str(e)) from e + + # Download entry points. + + def download_all_raw_files( + self, + accession, + output_folder, + skip_if_downloaded_already, + protocol, + aspera_maximum_bandwidth: str, + checksum_check: bool = False, + parallel_files: int = 1, + flatten: bool = True, + ): + """Download all RAW files for any registered provider.""" + return registry.resolve(accession).download_all_raw( + accession, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, + flatten=flatten, + ) + + def download_all_category_files( + self, + accession: str, + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str, + aspera_maximum_bandwidth: str, + checksum_check: bool, + categories: List[str] = None, + category: str = None, + parallel_files: int = 1, + flatten: bool = True, + ): + """Download all files of the given categories from a project.""" + if categories is None: + categories = [category] if category else ["RAW"] + return registry.resolve(accession).download_category( + accession, + output_folder, + categories, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, + flatten=flatten, + ) + + def download_file_by_name( + self, + accession, + file_name, + output_folder, + skip_if_downloaded_already, + protocol, + username, + password, + aspera_maximum_bandwidth, + checksum_check, + ): + """Download a single file by name. + + Dispatches to the resolved provider. PRIDE overrides this to handle + its public / private split via the V2 private API; the direct-download + providers (MassIVE / JPOST / iProX) use the inherited public path. + """ + return registry.resolve(accession).download_by_name( + accession, + file_name, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + username=username, + password=password, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + ) + + def download_files_by_list( + self, + accession: str, + file_names: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str = "ftp", + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + parallel_files: int = 1, + flatten: bool = True, + ) -> None: + """Download a subset of project files identified by a filename list.""" + return registry.resolve(accession).download_by_filenames( + accession, + file_names, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, + flatten=flatten, + ) + + @staticmethod + def download_files_by_url( + urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool = False, + protocol: str = "ftp", + parallel_files: int = 1, + checksum_check: bool = False, + ) -> None: + """Delegate to :func:`pridepy.download.by_url.download_files_by_url`.""" + return by_url.download_files_by_url( + urls=urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + ) + + def download_px_raw_files( + self, + px_id_or_url: str, + output_folder: str, + skip_if_downloaded_already: bool = True, + flatten: bool = True, + ) -> None: + """Delegate to :meth:`ProteomeXchangeProvider.download_from_accession_or_url`.""" + return ProteomeXchangeProvider().download_from_accession_or_url( + px_id_or_url, output_folder, skip_if_downloaded_already, flatten=flatten + ) diff --git a/pridepy/download/iprox.py b/pridepy/download/iprox.py new file mode 100644 index 0000000..db5e7fc --- /dev/null +++ b/pridepy/download/iprox.py @@ -0,0 +1,132 @@ +"""iProX direct-download provider. + +iProX publishes the ProteomeXchange XML for each dataset at a +deterministic path on its anonymous HTTP download server:: + + http://download.iprox.org//PX_.xml + +We fetch the XML, walk every ````'s ``cvParam`` entries, and +turn each ``Associated raw file URI`` (and sibling URIs for search-engine +output, result files, etc.) into a pridepy file record. File downloads +themselves go through plain HTTP on the same host, which supports +``Range`` requests for resume. +""" +import logging +import os +import re +import defusedxml.ElementTree as ET +from typing import ClassVar, Dict, List, Optional +from urllib.parse import urlparse + +import requests + +from pridepy.download import registry +from pridepy.download.base import Provider +from pridepy.download.jpost import JpostProvider + + +@registry.register +class IproxProvider(Provider): + name: ClassVar[str] = "iprox" + use_tls: ClassVar[bool] = False # download.iprox.org serves over plain HTTP + + DOWNLOAD_BASE_URL: ClassVar[str] = "http://download.iprox.org/" + PX_XML_URL_TEMPLATE: ClassVar[str] = ( + "http://download.iprox.org/{accession}/PX_{accession}.xml" + ) + # iProX PX XML uses the same PSI-MS cvParam "name" values as JPOST PROXI, + # so we reuse JpostProvider's category map. + PX_CATEGORY_MAP: ClassVar[Dict[str, str]] = JpostProvider.PROXI_CATEGORY_MAP + + @staticmethod + def matches(accession: str) -> bool: + """Return True when ``accession`` looks like an iProX dataset accession.""" + if not accession: + return False + return bool(re.fullmatch(r"IPX\d{7,10}", accession.upper())) + + @staticmethod + def _get_public_root(accession: str) -> str: + return f"/{accession.upper()}" + + @classmethod + def _get_public_ftp_url(cls, accession: str, remote_path: str) -> str: + # NOTE: name kept as `_get_public_ftp_url` for parity with other providers, + # but iProX URLs are http(s) not ftp. The dispatcher routes by scheme. + root_path = cls._get_public_root(accession).rstrip("/") + relative_path = remote_path + if remote_path.startswith(root_path): + relative_path = remote_path[len(root_path):].lstrip("/") + return f"{cls.DOWNLOAD_BASE_URL}{accession.upper()}/{relative_path}" + + @classmethod + def _build_file_record( + cls, accession: str, file_url: str, category_from_px: Optional[str] = None + ) -> Dict: + """Build a pridepy file record for an iProX file. + + ``file_url`` is the file URI from the PX XML (``http://`` on + download.iprox.org; ``https://`` is also accepted if present). + ``category_from_px`` is the ``cvParam`` ``name`` from the dataset's + ProteomeXchange XML (e.g. ``"Associated raw file URI"``). + """ + from pridepy.download.massive import MassiveProvider + parsed = urlparse(file_url) + root_prefix = f"/{accession.upper()}/" + relative_path = parsed.path + if relative_path.startswith(root_prefix): + relative_path = relative_path[len(root_prefix):] + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + if category_from_px and category_from_px in cls.PX_CATEGORY_MAP: + category = cls.PX_CATEGORY_MAP[category_from_px] + else: + category = MassiveProvider._map_collection_to_category(collection) + return { + "accession": accession.upper(), + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": category}, + # "FTP Protocol" is the existing label the download dispatcher uses + # to locate a file URL; here it actually points at HTTP + # (download.iprox.org). Provider.download_files routes by URL scheme. + "publicFileLocations": [{"name": "FTP Protocol", "value": file_url}], + "relativePath": relative_path, + "collection": collection, + "source": "iProX", + } + + def list_files(self, accession: str) -> List[Dict]: + normalized = accession.upper() + xml_url = self.PX_XML_URL_TEMPLATE.format(accession=normalized) + logging.info(f"Fetching iProX PX XML: {xml_url}") + response = requests.get(xml_url, timeout=30) + response.raise_for_status() + try: + root = ET.fromstring(response.content) + except ET.ParseError as parse_error: + raise RuntimeError( + f"Unable to parse iProX PX XML for {normalized}: {parse_error}" + ) from parse_error + + records: List[Dict] = [] + for dataset_file in root.iter("DatasetFile"): + for cv in dataset_file.findall("cvParam"): + name = cv.attrib.get("name") + value = cv.attrib.get("value") + if not value or not name or not name.endswith("URI"): + continue + if not value.lower().startswith(("http://", "https://")): + continue + records.append( + self._build_file_record( + normalized, + value, + category_from_px=name, + ) + ) + if not records: + raise RuntimeError( + f"iProX PX XML for {normalized} contained no downloadable " + f"HTTP/HTTPS URIs" + ) + return records diff --git a/pridepy/download/jpost.py b/pridepy/download/jpost.py new file mode 100644 index 0000000..a2abef1 --- /dev/null +++ b/pridepy/download/jpost.py @@ -0,0 +1,150 @@ +"""JPOST direct-download provider. + +PRIMARY listing: PROXI JSON at repository.jpostdb.org. The PROXI endpoint +returns ``datasetFiles[*].value`` as ``ftp://`` URLs alongside CV labels +(Associated raw file URI, Search engine output file URI, etc.) which map +cleanly to PRIDE file categories. + +FALLBACK listing: when PROXI fails, walk the FTP tree at ftp.jpostdb.org. +This is needed because JPOST's FTP server rate-limits aggressively per +source IP (sticky 421-too-many-connections); the PROXI path lets us avoid +walking the FTP tree just for a listing. +""" +import logging +import os +import re +from typing import ClassVar, Dict, List, Optional +from urllib.parse import urlparse + +import requests + +from pridepy.download import registry +from pridepy.download.base import Provider + + +@registry.register +class JpostProvider(Provider): + name: ClassVar[str] = "jpost" + use_tls: ClassVar[bool] = False + + ARCHIVE_FTP: ClassVar[str] = "ftp.jpostdb.org" + ARCHIVE_FTP_URL_PREFIX: ClassVar[str] = "ftp://ftp.jpostdb.org/" + PROXI_BASE_URL: ClassVar[str] = "https://repository.jpostdb.org/proxi/datasets/" + + PROXI_CATEGORY_MAP: ClassVar[Dict[str, str]] = { + "Associated raw file URI": "RAW", + "Result file URI": "RESULT", + "Search engine output file URI": "SEARCH", + "Peak list file URI": "PEAK", + "Spectrum library file URI": "SPECTRUM_LIBRARY", + "Sequence database URI": "FASTA", + "Quantification file URI": "RESULT", + } + + @staticmethod + def matches(accession: str) -> bool: + if not accession: + return False + return bool(re.fullmatch(r"JPST\d{6}", accession.upper())) + + @staticmethod + def _get_public_root(accession: str) -> str: + return f"/{accession.upper()}" + + @classmethod + def _get_public_ftp_url(cls, accession: str, remote_path: str) -> str: + root_path = cls._get_public_root(accession).rstrip("/") + relative_path = remote_path + if remote_path.startswith(root_path): + relative_path = remote_path[len(root_path):].lstrip("/") + return f"{cls.ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" + + @classmethod + def _build_file_record( + cls, accession: str, ftp_url: str, category_from_proxi: Optional[str] = None + ) -> Dict: + """Build a pridepy file record from an FTP URL. + + When ``category_from_proxi`` is provided (e.g. ``"Associated raw file URI"``), + the PROXI CV name takes precedence over the heuristic collection-from-path + mapping. Falls back to the same path-segment heuristic used for MassIVE + when the category isn't known. + """ + # Import the MassIVE collection->category map for the fallback heuristic. + from pridepy.download.massive import MassiveProvider + parsed = urlparse(ftp_url) + root_prefix = f"/{accession.upper()}/" + relative_path = parsed.path + if relative_path.startswith(root_prefix): + relative_path = relative_path[len(root_prefix):] + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + if category_from_proxi and category_from_proxi in cls.PROXI_CATEGORY_MAP: + category = cls.PROXI_CATEGORY_MAP[category_from_proxi] + else: + category = MassiveProvider._map_collection_to_category(collection) + return { + "accession": accession.upper(), + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": category}, + "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], + "relativePath": relative_path, + "collection": collection, + "source": "JPOST", + } + + def list_files(self, accession: str) -> List[Dict]: + """PRIMARY: PROXI JSON. FALLBACK: FTP tree walk.""" + normalized = accession.upper() + try: + return self._list_via_proxi(normalized) + except Exception as proxi_error: + logging.warning( + f"JPOST PROXI listing failed for {normalized} " + f"({proxi_error}); falling back to FTP tree walk." + ) + from pridepy.download import transport + remote_root = self._get_public_root(normalized) + remote_files = transport._list_ftp_repo_files( + host=self.ARCHIVE_FTP, + remote_root=remote_root, + error_label=f"JPOST dataset {normalized}", + ) + return [ + self._build_file_record( + normalized, + self._get_public_ftp_url(normalized, remote_file), + ) + for remote_file in remote_files + ] + + def _list_via_proxi(self, accession: str) -> List[Dict]: + """Fetch JPOST PROXI dataset metadata and turn each datasetFiles entry into a file record.""" + import json as _json + proxi_url = f"{self.PROXI_BASE_URL}{accession}" + logging.info(f"Fetching JPOST PROXI metadata: {proxi_url}") + response = requests.get( + proxi_url, + headers={"Accept": "application/json"}, + timeout=30, + ) + response.raise_for_status() + data = _json.loads(response.content) + dataset_files = data.get("datasetFiles") or [] + records: List[Dict] = [] + for entry in dataset_files: + value = (entry or {}).get("value") + if not value or not value.startswith("ftp://"): + continue + records.append( + self._build_file_record( + accession, + value, + category_from_proxi=(entry or {}).get("name"), + ) + ) + if not records: + raise RuntimeError( + f"JPOST PROXI returned no FTP file URIs for {accession}" + ) + return records diff --git a/pridepy/download/massive.py b/pridepy/download/massive.py new file mode 100644 index 0000000..56d36de --- /dev/null +++ b/pridepy/download/massive.py @@ -0,0 +1,198 @@ +"""MassIVE direct-download provider. + +Primary path: list files by walking the FTPS tree at massive-ftp.ucsd.edu +(TLS is required by the server) and download them over FTPS. MassIVE spreads +datasets across versioned root directories (``/v01`` … ``/vNN``) plus auxiliary +roots (``x01`` / ``z01``) that may hold only a partial, peak-only copy; the +version is not derivable from the accession, so the correct root is discovered +at listing time and the versioned roots are preferred over the auxiliary ones. + +HTTPS fallback: some networks block FTP/FTPS entirely. When the FTPS +listing fails, fall back to the HTTPS file index at datasetcache.gnps2.org +and download each file from the ProteoSAFe HTTPS endpoint at +massive.ucsd.edu (same bytes as FTPS, verified by checksum). The fallback +keeps everything over HTTPS so it works on FTPS-blocked networks. +""" +import logging +import os +import re +from typing import ClassVar, Dict, List +from urllib.parse import quote, urlparse + +import requests + +from pridepy.download import registry +from pridepy.download.base import Provider + + +MASSIVE_CATEGORY_MAP = { + "raw": "RAW", + "peak": "PEAK", + "ccms_peak": "PEAK", + "search": "SEARCH", + "result": "RESULT", + "ccms_result": "RESULT", + "quant": "RESULT", + "fasta": "FASTA", + "spectrum_library": "SPECTRUM_LIBRARY", + "library": "SPECTRUM_LIBRARY", +} + + +@registry.register +class MassiveProvider(Provider): + name: ClassVar[str] = "massive" + use_tls: ClassVar[bool] = True + + ARCHIVE_FTP: ClassVar[str] = "massive-ftp.ucsd.edu" + ARCHIVE_FTP_URL_PREFIX: ClassVar[str] = "ftp://massive-ftp.ucsd.edu/v01/" + + # HTTPS fallback for FTPS-blocked networks. + HTTPS_DOWNLOAD_URL: ClassVar[str] = ( + "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile" + ) + # GNPS2 dataset cache: HTTPS file index (datasette CSV stream). + HTTPS_FILE_INDEX_URL: ClassVar[str] = ( + "https://datasetcache.gnps2.org/datasette/database/filename.csv" + ) + + @staticmethod + def matches(accession: str) -> bool: + """Return True when ``accession`` is a MassIVE dataset accession.""" + if not accession: + return False + return bool(re.fullmatch(r"R?MSV\d{9}", accession.upper())) + + @classmethod + def _get_public_ftp_url(cls, accession: str, remote_path: str) -> str: + """Build the FTPS URL for an absolute server path inside the dataset. + + ``remote_path`` is the absolute path returned by the tree walk + (e.g. ``/v04/MSV000088302/ccms_peak/run.mzML``). MassIVE distributes + datasets across versioned roots, so the version is preserved as-is + rather than assumed to be ``v01``. + """ + if not remote_path.startswith("/"): + remote_path = "/" + remote_path + return f"ftp://{cls.ARCHIVE_FTP}{remote_path}" + + @staticmethod + def _map_collection_to_category(collection: str) -> str: + return MASSIVE_CATEGORY_MAP.get(collection.lower(), "OTHER") + + @classmethod + def _build_file_record(cls, accession: str, ftp_url: str) -> Dict: + """Build a pridepy file record from an FTP URL inside the dataset.""" + parsed = urlparse(ftp_url) + # The version root differs per dataset (/v01../vNN), so derive the + # dataset-relative path from the accession marker rather than a fixed + # ``/v01//`` prefix. + marker = f"/{accession.upper()}/" + relative_path = parsed.path + marker_index = relative_path.find(marker) + if marker_index != -1: + relative_path = relative_path[marker_index + len(marker):] + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + return { + "accession": accession.upper(), + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": cls._map_collection_to_category(collection)}, + "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], + "relativePath": relative_path, + "collection": collection, + "source": "MassIVE", + } + + @classmethod + def _get_https_url(cls, accession: str, relative_path: str) -> str: + """ProteoSAFe HTTPS download URL for a dataset-relative file path. + + Mirrors the FTPS file: ``f./`` in the + ProteoSAFe ftp file-space. Verified to return byte-identical content + to the FTPS copy. + """ + file_param = f"f.{accession.upper()}/{relative_path.lstrip('/')}" + return ( + f"{cls.HTTPS_DOWNLOAD_URL}?forceDownload=true" + f"&file={quote(file_param, safe='/.')}" + ) + + @classmethod + def _build_https_file_record(cls, accession: str, relative_path: str) -> Dict: + """Build a file record whose download location is the HTTPS endpoint.""" + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + return { + "accession": accession.upper(), + "fileName": os.path.basename(relative_path), + "fileCategory": {"value": cls._map_collection_to_category(collection)}, + # base.Provider.download_files routes by URL scheme; an https:// + # value here sends the file through the HTTPS transport. + "publicFileLocations": [ + {"name": "HTTPS", "value": cls._get_https_url(accession, relative_path)} + ], + "relativePath": relative_path, + "collection": collection, + "source": "MassIVE", + } + + def _list_via_https(self, accession: str) -> List[Dict]: + """List dataset files over HTTPS via the GNPS2 dataset cache. + + Used when FTPS is unavailable (blocked network). Streams the file + index as CSV and builds HTTPS-download records. + """ + import csv + normalized = accession.upper() + logging.info(f"Listing MassIVE dataset {normalized} via HTTPS file index") + response = requests.get( + self.HTTPS_FILE_INDEX_URL, + params={"dataset__exact": normalized, "_stream": "on", "_col": "filepath"}, + timeout=60, + stream=True, + ) + response.raise_for_status() + lines = (line.decode("utf-8") for line in response.iter_lines() if line) + records: List[Dict] = [] + for row in csv.DictReader(lines): + file_path = (row.get("filepath") or "").strip() + if file_path: + records.append(self._build_https_file_record(normalized, file_path)) + if not records: + raise RuntimeError( + f"No files found via HTTPS file index for MassIVE dataset {normalized}" + ) + return records + + def list_files(self, accession: str) -> List[Dict]: + from pridepy.download import transport + normalized = accession.upper() + try: + # The version root (/v01../vNN) is not derivable from the + # accession, so discover which root holds the dataset instead of + # assuming /v01. + remote_files = transport._resolve_and_walk_ftp_dataset( + host=self.ARCHIVE_FTP, + accession=normalized, + error_label=f"MassIVE dataset {normalized}", + use_tls=True, + # /vNN roots hold the complete dataset; x01/z01 hold only + # partial (peak-only) copies, so prefer the versioned roots. + prefer_prefix="v", + ) + except Exception as ftps_error: + logging.warning( + "MassIVE FTPS listing failed for %s (%s); " + "falling back to the HTTPS file index.", + normalized, + ftps_error, + ) + return self._list_via_https(normalized) + return [ + self._build_file_record( + normalized, + self._get_public_ftp_url(normalized, remote_file), + ) + for remote_file in remote_files + ] diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py new file mode 100644 index 0000000..8ab017f --- /dev/null +++ b/pridepy/download/pride.py @@ -0,0 +1,978 @@ +"""PRIDE Archive provider. + +PRIDE has the richest behaviour of all providers: multi-protocol batch +download with aspera/s3/ftp/globus fallback, private-dataset path with +username/password auth, checksum TSV validation, and submitter-path +helpers. This module owns all of that logic; the +:class:`~pridepy.download.client.Client` facade exposes a thin public +surface for downstream callers. + +Implementation note: PRIDE provider methods route through other +PrideProvider methods (``PrideProvider.X(...)``) or directly through the +shared ``transport`` / ``util`` helpers — they do NOT call back into the +``Client`` facade. Tests patch the canonical locations +(``PrideProvider.X``, ``transport.X``, ``util.X``) directly. +""" +import importlib.resources +import logging +import os +import platform +import re +import subprocess +import time +import urllib +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import ClassVar, Dict, List, Optional +from urllib.parse import urlparse + +import boto3 +import botocore +import requests +from botocore.config import Config +from tqdm import tqdm + +from pridepy.authentication.authentication import Authentication +from pridepy.download import registry, transport +from pridepy.download import util as _provider_util +from pridepy.download.base import Provider +from pridepy.download.util import Progress +from pridepy.util.api_handling import Util + + +@registry.register +class PrideProvider(Provider): + """PRIDE Archive provider with multi-protocol fallback orchestration.""" + + name: ClassVar[str] = "pride" + + V3_API_BASE_URL: ClassVar[str] = "https://www.ebi.ac.uk/pride/ws/archive/v3" + API_BASE_URL: ClassVar[str] = "https://www.ebi.ac.uk/pride/ws/archive/v3" + API_PRIVATE_URL: ClassVar[str] = "https://www.ebi.ac.uk/pride/private/ws/archive/v2" + ARCHIVE_FTP: ClassVar[str] = "ftp.pride.ebi.ac.uk" + ARCHIVE_FTP_URL_PREFIX: ClassVar[str] = "ftp://ftp.pride.ebi.ac.uk/" + ARCHIVE_HTTPS_URL_PREFIX: ClassVar[str] = "https://ftp.pride.ebi.ac.uk/" + S3_URL: ClassVar[str] = "https://hh.fire.sdo.ebi.ac.uk" + S3_BUCKET: ClassVar[str] = "pride-public" + PROTOCOL_ORDER: ClassVar[List[str]] = ["aspera", "s3", "ftp", "globus"] + + @staticmethod + def matches(accession: str) -> bool: + """Return True when ``accession`` is a PRIDE dataset accession.""" + if not accession: + return False + return bool(re.fullmatch(r"(?:PXD|PRD)\d+", accession.upper())) + + # ------------------------------------------------------------------ + # Listing + # ------------------------------------------------------------------ + + async def stream_all_files_metadata(self, output_file, accession=None): + """ + get stream all project files from PRIDE API in JSON format + """ + if accession is None: + request_url = f"{self.V3_API_BASE_URL}/files/all" + count_request_url = f"{self.V3_API_BASE_URL}/files/count" + else: + request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/all" + count_request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/count" + headers = {"Accept": "application/JSON"} + response = Util.get_api_call(count_request_url, headers) + total_records = response.json() + + regex_search_pattern = '"fileName"' + await Util.stream_response_to_file( + output_file, total_records, regex_search_pattern, request_url, headers + ) + + def stream_all_files_by_project(self, accession) -> List[Dict]: + """ + get stream all project files from PRIDE API in JSON format + """ + request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/all" + headers = {"Accept": "application/JSON"} + record_files = Util.read_json_stream(api_url=request_url, headers=headers) + return record_files + + def list_files(self, accession: str) -> List[Dict]: + """Return PRIDE file records for the dataset.""" + return self.stream_all_files_by_project(accession) + + def get_submitted_file_path_prefix(self, accession): + """ + At pride repository, public data is disseminated according to a proper structure. + I.e. base/path/ + yyyy/mm/accession/ + submitted/ + This extracts the yyyy/mm/accession path fragment from the API by examine the file path + of a public file. + I.e. ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2018/10/PXD008644/7550GI_Y.raw + :param accession: PRIDE accession + :return: path fragment (eg: 2018/10/PXD008644) + """ + records = self._list_files_checked(accession) + raw_files = [ + r + for r in records + if (r.get("fileCategory") or {}).get("value") == "RAW" + and r.get("publicFileLocations") + ] + if not raw_files: + raise ValueError( + f"Cannot determine submitted path prefix for {accession}: " + f"no RAW file with a public location was found." + ) + first_file = raw_files[0]["publicFileLocations"][0]["value"] + match = re.search(r"\d{4}/\d{2}/(?:PXD|PRD)\d*", first_file) + if match is None: + raise ValueError( + f"Cannot determine submitted path prefix for {accession}: " + f"unexpected file path layout ({first_file!r})." + ) + return match.group() + + # ------------------------------------------------------------------ + # Static utilities + # ------------------------------------------------------------------ + + @staticmethod + def _protocol_sequence(protocol: str) -> List[str]: + """ + Build the ordered list of protocols to try for a requested download mode. + """ + if protocol not in PrideProvider.PROTOCOL_ORDER: + return [] + return [protocol] + [p for p in PrideProvider.PROTOCOL_ORDER if p != protocol] + + @staticmethod + def get_ascp_binary(): + """ + Detect the OS and architecture, and return the appropriate ascp binary path. + + Returns: + str: Path to the correct ascp binary. + """ + os_type = platform.system().lower() + arch, _ = platform.architecture() + aspera_dir = importlib.resources.files("pridepy").joinpath("aspera/") + + if os_type == "linux": + if arch == "32bit": + return os.path.join(aspera_dir, "linux-32", "ascp") + elif arch == "64bit": + return os.path.join(aspera_dir, "linux-64", "ascp") + elif os_type == "darwin": # macOS (intel-based) + return os.path.join(aspera_dir, "mac-intel", "ascp") + elif os_type == "windows": + if arch == "32bit": + return os.path.join(aspera_dir, "windows-32", "ascp.exe") + elif arch == "64bit": + return os.path.join(aspera_dir, "windows-64", "ascp.exe") + else: + raise OSError(f"Unsupported OS or architecture: {os_type}, {arch}") + + @staticmethod + def get_output_file_name(download_url, file, output_folder): + """Build the local output path for ``download_url`` inside ``output_folder``.""" + public_filepath_part = download_url.rsplit("/", 1) + accession = file.get("accession", "unknown-accession") + logging.debug(accession + " -> " + public_filepath_part[1]) + new_file_path = os.path.join(output_folder, f"{public_filepath_part[1]}") + return new_file_path + + @staticmethod + def _get_download_url(file_record: Dict, protocol: str) -> str: + """Resolve the PRIDE public download URL for a file and protocol. + + Raises ValueError when the requested protocol has no suitable location. + Aspera requires a dedicated "Aspera Protocol" entry; ftp/s3/globus + derive their URL from the "FTP Protocol" entry (falling back to an + arbitrary non-Aspera location would produce a URL the caller cannot + actually transfer with). The globus URL is the FTP path rewritten to + the PRIDE archive HTTPS prefix. + """ + locations = file_record.get("publicFileLocations", []) + if not locations: + raise ValueError("No public file locations present") + + aspera_url = None + ftp_url = None + for location in locations: + name = location.get("name") + if name == "Aspera Protocol": + aspera_url = location.get("value") + elif name == "FTP Protocol": + ftp_url = location.get("value") + + if protocol == "aspera": + if not aspera_url: + raise ValueError("Aspera URL not available") + return aspera_url + + if not ftp_url: + raise ValueError("FTP URL not available") + if protocol == "ftp": + return ftp_url + if protocol == "globus": + return ftp_url.replace( + PrideProvider.ARCHIVE_FTP_URL_PREFIX, + PrideProvider.ARCHIVE_HTTPS_URL_PREFIX, + 1, + ) + if protocol == "s3": + return ftp_url + raise ValueError(f"Unsupported protocol: {protocol}") + + def get_download_url(self, record: Dict, protocol: str = "ftp") -> str: + """Override the base hook with PRIDE's multi-protocol resolution.""" + return PrideProvider._get_download_url(record, protocol) + + @staticmethod + def _resolve_local_path(file_record: Dict, output_folder: str) -> str: + """Compute the canonical local path for a file regardless of protocol.""" + try: + canonical_url = PrideProvider._get_download_url(file_record, "ftp") + except ValueError: + canonical_url = "" + if canonical_url: + return PrideProvider.get_output_file_name(canonical_url, file_record, output_folder) + return os.path.join(output_folder, file_record["fileName"]) + + @staticmethod + def extract_accession_from_url(url: str) -> Optional[str]: + """Extract a PRIDE accession (PXD/PRD followed by digits) from a URL. + + PRIDE archive URLs follow the pattern + ``…/pride/data/archive/YYYY/MM//filename``. + Returns ``None`` when no accession can be identified. + """ + match = re.search(r"((?:PXD|PRD)\d{4,})", url) + return match.group(1) if match else None + + @staticmethod + def validate_urls_checksums(urls: List[str], output_folder: str) -> None: + """Validate downloaded files against the PRIDE checksum API. + + Accessions are inferred from URL paths via + :meth:`extract_accession_from_url`. URLs that do not contain a + recognisable PRIDE accession are skipped with a warning. + + :raises RuntimeError: if one or more files fail validation + """ + accession_urls: Dict[str, List[str]] = {} + for url in urls: + acc = PrideProvider.extract_accession_from_url(url) + if acc: + accession_urls.setdefault(acc, []).append(url) + else: + logging.warning( + "Cannot infer PRIDE accession from URL, skipping checksum: %s", url + ) + + validation_failures: List[str] = [] + for acc, acc_urls in accession_urls.items(): + checksum_file_path = PrideProvider.save_checksum_file(acc, output_folder) + checksum_map = _provider_util.read_checksum_file(checksum_file_path) + logging.info( + "Loaded checksums for %d files (project %s)", + len(checksum_map), acc, + ) + for url in acc_urls: + file_name = os.path.basename(urlparse(url).path) + target = os.path.join(output_folder, file_name) + expected = checksum_map.get(file_name) + logging.info("Validating %s", file_name) + valid, reason = _provider_util.validate_download(target, expected) + if not valid: + logging.error("Validation failed for %s: %s", file_name, reason) + validation_failures.append(f"{file_name} ({reason})") + else: + logging.info("Checksum OK: %s", file_name) + + if validation_failures: + raise RuntimeError( + f"Checksum validation failed for {len(validation_failures)} file(s): " + + ", ".join(validation_failures) + ) + + @staticmethod + def save_checksum_file(accession, output_folder): + """ + Download and persist the checksum manifest for a PRIDE accession. + """ + os.makedirs(output_folder, exist_ok=True) + url = f"{PrideProvider.V3_API_BASE_URL}/files/checksum/{accession}" + headers = {"accept": "text/plain"} + request = urllib.request.Request(url, headers=headers, method="GET") + logging.info(f"Fetching checksum file from {url}") + with urllib.request.urlopen(request, timeout=60) as response: + data = response.read().decode("utf-8") + # Save the data to a .tsv file + output_path = os.path.join(output_folder, f"{accession}-checksum.tsv") + with open(output_path, "w", encoding="utf-8") as file: + file.write(data) + return output_path + + # ------------------------------------------------------------------ + # Per-protocol single-file workers + # ------------------------------------------------------------------ + + @staticmethod + def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_retries=6, position=0): + """Download a single file via globus; used as a worker target.""" + download_url = PrideProvider._get_download_url(file, "globus") + new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) + + if skip_if_downloaded_already and os.path.exists(new_file_path): + logging.info(f"Skipping download as file already exists: {new_file_path}") + return + + for attempt in range(1, max_retries + 1): + try: + transport._parallel_download(download_url, new_file_path, position=position) + return + except Exception as e: + logging.warning(f"Attempt {attempt}/{max_retries} failed for {file.get('fileName', '?')}: {e}") + if attempt == max_retries: + raise + + # ------------------------------------------------------------------ + # Per-protocol batch helpers + # ------------------------------------------------------------------ + + @staticmethod + def download_files_from_aspera( + file_list_json: List[Dict], + output_folder: str, + skip_if_downloaded_already, + maximum_bandwidth: str = "100M", + ): + """ + Download files using aspera transfer url + :param file_list_json: file list in json format + :param output_folder: folder to download the files + :param maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. + :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + """ + ascp_path = PrideProvider.get_ascp_binary() + key_full_path = importlib.resources.files("pridepy").joinpath( + "aspera/key/asperaweb_id_dsa.openssh" + ) + key_path = os.path.abspath(key_full_path) + failed: List[str] = [] + for file in file_list_json: + if file["publicFileLocations"][0]["name"] == "Aspera Protocol": + download_url = file["publicFileLocations"][0]["value"] + else: + download_url = file["publicFileLocations"][1]["value"] + + # Create a clean filename to save the downloaded file + logging.debug(f"Downloading via Aspera: {download_url}") + new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) + + if skip_if_downloaded_already and os.path.exists(new_file_path): + logging.info("Skipping download as file already exists") + continue + + try: + # Execute the ascp command using subprocess + subprocess.run( + [ + ascp_path, + "-QT", + "-P", + "33001", + "-l", + maximum_bandwidth, # Options for Aspera: adjust as necessary + "-i", + key_path, + download_url, + new_file_path, # Source and destination + ], + check=True, + ) + logging.info(f"Successfully downloaded {new_file_path} via Aspera") + except subprocess.CalledProcessError as e: + logging.error(f"Aspera download failed for {new_file_path}: {str(e)}") + failed.append(file.get("fileName", new_file_path)) + if failed: + raise RuntimeError( + f"Aspera download failed for {len(failed)} file(s): {failed}" + ) + + @staticmethod + def download_files_from_globus( + file_list_json: List[Dict], output_folder, skip_if_downloaded_already, + parallel_files: int = 1, + checksum_map: Optional[Dict[str, str]] = None, + ): + """ + Download files using globus transfer url with progress bar for each file. + When skip_if_downloaded_already is True, files are pre-filtered so that + only missing or incomplete files are submitted to the worker pool, + ensuring the -w parallel_files parameter is fully utilised. + When checksum_map is provided, existing files are validated against + their expected checksum; corrupted files are re-downloaded. + :param file_list_json: file list in json format + :param output_folder: folder to download the files + :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + :param parallel_files: number of files to download simultaneously + :param checksum_map: mapping of file name to expected MD5 checksum + """ + if checksum_map is None: + checksum_map = {} + + if not (os.path.isdir(output_folder)): + os.makedirs(output_folder, exist_ok=True) + + # --- Phase 0: pre-filter files that need downloading ----------------- + files_to_download: List[Dict] = [] + for file in file_list_json: + download_url = PrideProvider._get_download_url(file, "globus") + new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) + if skip_if_downloaded_already and os.path.exists(new_file_path): + expected_cs = checksum_map.get(file.get("fileName", "")) + if expected_cs: + valid, reason = _provider_util.validate_download(new_file_path, expected_cs) + if not valid: + logging.warning(f"Corrupted file detected ({reason}), will re-download: {new_file_path}") + files_to_download.append(file) + continue + logging.info(f"Skipping download as file already exists: {new_file_path}") + continue + files_to_download.append(file) + + if not files_to_download: + logging.info("All files already downloaded, nothing to do.") + return + + logging.info( + f"{len(file_list_json) - len(files_to_download)} file(s) skipped, " + f"{len(files_to_download)} file(s) to download" + ) + + # --- Phase 1: download (skip check already done, pass False) --------- + parallel_files = min(parallel_files, 3, len(files_to_download)) + failed: List[str] = [] + if parallel_files < 2: + for file in files_to_download: + try: + PrideProvider._globus_download_one( + file, output_folder, False + ) + new_file_path = PrideProvider.get_output_file_name( + PrideProvider._get_download_url(file, "globus"), file, output_folder + ) + logging.info(f"Successfully downloaded {new_file_path}") + except Exception as e: + logging.error(f"Download from Globus failed: {str(e)}") + failed.append(file.get("fileName", "")) + else: + logging.info(f"Downloading {len(files_to_download)} file(s) with {parallel_files} parallel workers") + with ThreadPoolExecutor(max_workers=parallel_files) as executor: + futures = { + executor.submit( + PrideProvider._globus_download_one, + file, output_folder, False, + position=idx, + ): file + for idx, file in enumerate(files_to_download) + } + for future in as_completed(futures): + try: + future.result() + except Exception as e: + logging.error(f"Download from Globus failed: {str(e)}") + failed.append(futures[future].get("fileName", "")) + if failed: + raise RuntimeError( + f"Globus download failed for {len(failed)} file(s): {failed}" + ) + + @staticmethod + def download_files_from_s3( + file_list_json: List[Dict], output_folder: str, skip_if_downloaded_already + ): + """ + Download files using S3 transfer URL with a progress bar and retry logic. + :param file_list_json: file list in JSON format + :param output_folder: folder to download the files + :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + """ + if not os.path.isdir(output_folder): + os.makedirs(output_folder, exist_ok=True) + + # Retry and timeout config + retry_config = Config( + retries={"max_attempts": 5, "mode": "standard"}, + connect_timeout=120, # Increase timeout to 120 seconds + read_timeout=120, # Timeout for reading data + signature_version=botocore.UNSIGNED, # Unsigned requests for public data + ) + + s3_resource = boto3.resource( + "s3", + config=retry_config, + endpoint_url=PrideProvider.S3_URL, + ) + bucket = s3_resource.Bucket(PrideProvider.S3_BUCKET) + + failed: List[str] = [] + for file in file_list_json: + try: + # Determine S3 or FTP path + download_url = ( + file["publicFileLocations"][0]["value"] + if file["publicFileLocations"][0]["name"] == "FTP Protocol" + else file["publicFileLocations"][1]["value"] + ) + + ftp_base_url = "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/" + s3_path = download_url.replace(ftp_base_url, "") + new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) + + if skip_if_downloaded_already and os.path.exists(new_file_path): + logging.info("Skipping download as file already exists") + continue + + logging.debug(f"Downloading From S3: {s3_path}") + + # Get file size for progress tracking + obj = bucket.Object(s3_path) + total_size = obj.content_length + + # Initialize progress bar + progress = Progress(total_size, new_file_path) + + # Download with progress bar and retry handling + for attempt in range(5): + try: + bucket.download_file(s3_path, new_file_path, Callback=progress) + progress.close() + logging.info(f"Successfully downloaded {new_file_path}") + break + except botocore.exceptions.ClientError as e: + if e.response["Error"]["Code"] == "404": + logging.error("The object does not exist.") + break + else: + logging.error(f"Download failed: {e}") + if attempt < 4: + time.sleep(2**attempt) # Exponential backoff + logging.info(f"Retrying... ({attempt + 1}/5)") + else: + raise + except Exception as e: + logging.error(f"Failed to download {file.get('fileName')}: {e}") + failed.append(file.get("fileName", "")) + if failed: + raise RuntimeError( + f"S3 download failed for {len(failed)} file(s): {failed}" + ) + + # ------------------------------------------------------------------ + # Private dataset download + # ------------------------------------------------------------------ + + def download_private_file_name(self, accession, file_name, output_folder, username, password): + """ + Get the information for a given private file to be downloaded from the api. + :param accession: Project accession + :param file_name: The file name to be downloaded + :param username: Username with access to the dataset + :param password: Password for user with access to the dataset + """ + + auth = Authentication() + auth_token = auth.get_token(username, password) + validate_token = auth.validate_token(auth_token) + logging.info("Valid token after login: {}".format(validate_token)) + + url = self.API_PRIVATE_URL + "/projects/{}/files?search={}".format(accession, file_name) + content = requests.get( + url, + headers={"Authorization": "Bearer {}".format(auth_token)}, + timeout=(10, 60), + ) + if content.ok and content.status_code == 200: + json_file = content.json() + if ( + "_embedded" in json_file + and "files" in json_file["_embedded"] + and len(json_file["_embedded"]["files"]) == 1 + ): + download_url = json_file["_embedded"]["files"][0]["_links"]["download"]["href"] + logging.info(download_url) + + # Create a clean filename to save the downloaded file + new_file_path = os.path.join(output_folder, f"{file_name}") + + session = Util.create_session_with_retries() # Create session with retries + # Check if the file already exists + if os.path.exists(new_file_path): + resume_header = {"Range": f"bytes={os.path.getsize(new_file_path)}-"} + mode = "ab" # Append to file + resume_size = os.path.getsize(new_file_path) + else: + resume_header = {} + mode = "wb" # Write new file + resume_size = 0 + + with session.get( + download_url, stream=True, headers=resume_header, timeout=(10, 60) + ) as r: + r.raise_for_status() + total_size = int(r.headers.get("content-length", 0)) + resume_size + block_size = 1024 * 1024 # 1 MB chunks + + with tqdm( + total=total_size, + unit="B", + unit_scale=True, + desc=new_file_path, + initial=resume_size, + ) as pbar: + with open(new_file_path, mode) as f: + for chunk in r.iter_content(chunk_size=block_size): + if chunk: + f.write(chunk) + pbar.update(len(chunk)) + + logging.info(f"Successfully downloaded {new_file_path}") + + else: + logging.info( + "File name {} found more than once for the given project {}".format( + file_name, accession + ) + ) + else: + logging.info( + f"File name {file_name} now found in the project {accession}, or user don't have access" + ) + raise Exception( + f"File name {file_name} now found in the project {accession}, or user don't have access" + ) + + # ------------------------------------------------------------------ + # Multi-protocol orchestrator + # ------------------------------------------------------------------ + + @staticmethod + def _batch_download_by_protocol( + file_list: List[Dict], + output_folder: str, + protocol: str, + skip_if_downloaded_already: bool, + aspera_maximum_bandwidth: str, + parallel_files: int = 1, + checksum_map: Optional[Dict[str, str]] = None, + ) -> None: + """ + Transfer a batch of files with one protocol, reusing a single + connection where the underlying helper supports it (FTP, S3). + """ + if not file_list: + return + if protocol == "ftp": + # Route through the shared transport, which downloads each file on + # its own connection with REST-based resume, post-transfer size + # checks, and per-file reconnect — so one slow/timed-out large file + # no longer poisons the connection and cascade-fails the rest of the + # batch (issue #107). Files land flat by basename (PRIDE archive is + # flat within a dataset), matching ``_resolve_local_path``. + ftp_urls = [ + PrideProvider._get_download_url(record, "ftp") for record in file_list + ] + transport.download_ftp_urls( + ftp_urls=ftp_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=False, + parallel_files=parallel_files, + ) + return + if protocol == "aspera": + PrideProvider.download_files_from_aspera( + file_list, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + maximum_bandwidth=aspera_maximum_bandwidth, + ) + return + if protocol == "globus": + PrideProvider.download_files_from_globus( + file_list, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + parallel_files=parallel_files, + checksum_map=checksum_map or {}, + ) + return + if protocol == "s3": + PrideProvider.download_files_from_s3( + file_list, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + ) + return + raise ValueError(f"Unsupported protocol: {protocol}") + + @staticmethod + def _download_with_fallback( + file_record: Dict, + output_folder: str, + protocol_sequence: List[str], + expected_checksum: Optional[str], + aspera_maximum_bandwidth: str, + max_protocol_retries: int = 2, + parallel_files: int = 1, + ) -> bool: + """ + Download one file by trying each protocol in sequence, validating + after every attempt. Intended as the per-file fallback path; batch + download of the primary protocol is handled separately. + """ + local_path = PrideProvider._resolve_local_path(file_record, output_folder) + + for protocol in protocol_sequence: + for attempt in range(1, max_protocol_retries + 1): + logging.info( + f"Downloading {file_record['fileName']} via {protocol} " + f"(attempt {attempt}/{max_protocol_retries})" + ) + try: + _provider_util._remove_if_exists(local_path) + PrideProvider._batch_download_by_protocol( + [file_record], + output_folder, + protocol, + skip_if_downloaded_already=False, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + parallel_files=parallel_files, + ) + except Exception as error: + logging.error( + f"Protocol {protocol} failed for {file_record['fileName']}: {error}" + ) + + valid, reason = _provider_util.validate_download(local_path, expected_checksum) + if valid: + logging.info( + f"File {file_record['fileName']} downloaded successfully via {protocol}" + ) + return True + + logging.warning( + f"Validation failed for {file_record['fileName']} via {protocol}: {reason}" + ) + _provider_util._remove_if_exists(local_path) + + logging.warning( + f"Protocol {protocol} exhausted for {file_record['fileName']}, switching protocol." + ) + + logging.error(f"All protocol attempts failed for {file_record['fileName']}") + return False + + def download_files( + self, + accession, + records: List[Dict], + output_folder: str, + skip_if_downloaded_already: bool = False, + protocol: str = "ftp", + parallel_files: int = 1, + checksum_check: bool = False, + aspera_maximum_bandwidth: str = "100M", + username: Optional[str] = None, + password: Optional[str] = None, + flatten: bool = True, + ): + """Override Provider.download_files with the multi-protocol orchestrator. + + Phase 1 batches the requested protocol (FTP routes through the shared + transport with per-file reconnect + REST resume); Phase 2 validates + every file and falls back per-file across the remaining protocols. + + ``flatten`` is accepted for interface parity but is a no-op for PRIDE: + a dataset's files live flat in its archive directory (no sub-tree to + preserve), so they always land directly in ``output_folder`` by + basename, which is what ``_resolve_local_path`` expects for Phase 2. + """ + PrideProvider._download_files_batch( + file_list_json=records, + accession=accession, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, + ) + + def download_by_name( + self, + accession, + file_name, + output_folder, + skip_if_downloaded_already, + protocol, + username=None, + password=None, + aspera_maximum_bandwidth="100M", + checksum_check=False, + ): + """Download a single file by name, honouring PRIDE's public/private split. + + PRIDE exposes private datasets via the V2 private API (username + + password); public datasets route through the standard listing + + multi-protocol download path inherited from :class:`Provider`. + """ + public_project = False + project_status = Util.get_api_call( + self.API_BASE_URL + "/status/{}".format(accession) + ) + + if project_status.status_code == 200: + if project_status.text == "PRIVATE": + public_project = False + elif project_status.text == "PUBLIC": + public_project = True + else: + raise Exception( + "Dataset {} is not present in PRIDE Archive".format(accession) + ) + + if public_project: + logging.info("Downloading file from public dataset {}".format(accession)) + super().download_by_name( + accession=accession, + file_name=file_name, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + username=username, + password=password, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + ) + elif not public_project and (username is not None and password is not None): + logging.info("Downloading file from private dataset {}".format(accession)) + self.download_private_file_name( + accession=accession, + file_name=file_name, + output_folder=output_folder, + username=username, + password=password, + ) + else: + logging.error( + "For a private dataset {} you must provide a username and password".format( + accession + ) + ) + raise Exception( + "For a private dataset {} you must provide a username and password".format( + accession + ) + ) + + @staticmethod + def _download_files_batch( + file_list_json: List[Dict], + accession, + output_folder: str, + skip_if_downloaded_already, + protocol: str = "ftp", + aspera_maximum_bandwidth: str = "100M", # Aspera maximum bandwidth + checksum_check=False, + parallel_files: int = 1, + ): + """ + Download files using the ftp, aspera, globus, or s3 transfer protocol. + :param file_list_json: File list in JSON format + :param accession: Project accession + :param output_folder: Folder to download the files + :param protocol: ftp, aspera, globus, s3 + :param aspera_maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. + :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + """ + protocols_supported = ["ftp", "aspera", "globus", "s3"] + if protocol not in protocols_supported: + logging.error("Protocol should be one of ftp, aspera, globus, s3") + return + + os.makedirs(output_folder, exist_ok=True) + + checksum_map: Dict[str, str] = {} + if checksum_check: + checksum_file_path = PrideProvider.save_checksum_file(accession, output_folder) + checksum_map = _provider_util.read_checksum_file(checksum_file_path) + logging.info(f"Loaded checksums for {len(checksum_map)} files") + + if not file_list_json: + return + + protocol_sequence = PrideProvider._protocol_sequence(protocol) + primary_protocol = protocol_sequence[0] + # Retry with the primary protocol first, then fall back to others + fallback_sequence = protocol_sequence + + # Phase 1: batch download with the requested protocol. Reuses a single + # FTP/S3 connection for all files (the previous behaviour) instead of + # paying the per-file reconnect cost in the common happy path. + logging.info( + f"Downloading {len(file_list_json)} file(s) via {primary_protocol} (batch)" + ) + try: + PrideProvider._batch_download_by_protocol( + file_list_json, + output_folder, + primary_protocol, + skip_if_downloaded_already=skip_if_downloaded_already, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + parallel_files=parallel_files, + checksum_map=checksum_map, + ) + except Exception as exc: + logging.warning( + f"Batch {primary_protocol} run hit an error; will retry individual failures: {exc}" + ) + + # Phase 2: validate every file and fall back per-file for the ones + # that are missing or invalid. + logging.info("Phase 2: validating %d downloaded file(s)", len(file_list_json)) + failed_files: List[str] = [] + for i, file_record in enumerate(file_list_json, 1): + expected_checksum = checksum_map.get(file_record["fileName"]) + local_path = PrideProvider._resolve_local_path(file_record, output_folder) + logging.info("Validating [%d/%d] %s", i, len(file_list_json), file_record["fileName"]) + valid, reason = _provider_util.validate_download(local_path, expected_checksum) + if valid: + continue + + logging.warning( + f"{file_record['fileName']} invalid after {primary_protocol} ({reason})" + ) + if "checksum mismatch" in reason: + _provider_util._remove_if_exists(local_path) + + if not fallback_sequence: + failed_files.append(file_record.get("fileName", "")) + continue + + success = PrideProvider._download_with_fallback( + file_record=file_record, + output_folder=output_folder, + protocol_sequence=fallback_sequence, + expected_checksum=expected_checksum, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + parallel_files=parallel_files, + ) + if not success: + failed_files.append(file_record.get("fileName", "")) + + if failed_files: + failed_summary = ", ".join(failed_files) + logging.error(f"Failed to download {len(failed_files)} file(s): {failed_summary}") + raise RuntimeError(f"Failed to download {len(failed_files)} file(s): {failed_summary}") diff --git a/pridepy/download/proteomexchange.py b/pridepy/download/proteomexchange.py new file mode 100644 index 0000000..32ea72e --- /dev/null +++ b/pridepy/download/proteomexchange.py @@ -0,0 +1,192 @@ +"""ProteomeXchange provider. + +ProteomeXchange is a meta-repository: a PXD/PRD accession routes through +the cross-repository XML at ``proteomecentral.proteomexchange.org``, and +the XML's ``Associated raw file URI`` cvParams point at the actual hosting +repository (PRIDE / MassIVE / JPOST / iProX / etc.). + +Unlike the other providers in this package, ``ProteomeXchangeProvider`` is +NOT auto-registered with :mod:`pridepy.download.registry`. PXD/PRD +accessions would otherwise be ambiguous between PRIDE's V3 API listing and +ProteomeXchange's XML listing; the registry continues to route PXD/PRD via +:class:`pridepy.download.pride.PrideProvider`. ``ProteomeXchangeProvider`` +is the explicit gateway invoked by the ``download-px-raw-files`` CLI +command and by ``Client.download_px_raw_files`` — callers who specifically +want the cross-repository XML view. + +The class accepts either: + +- a plain accession (``PXD039236``) +- a ProteomeCentral dataset URL (``https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=...``) + +…and resolves it to the XML endpoint via :meth:`_normalize_px_xml_url`. +""" +import logging +import os +import posixpath +import re +import defusedxml.ElementTree as ET +from typing import ClassVar, Dict, List +from urllib.parse import urlparse + +from pridepy.download.base import Provider +from pridepy.util.api_handling import Util + + +class ProteomeXchangeProvider(Provider): + name: ClassVar[str] = "proteomexchange" + + @staticmethod + def matches(accession: str) -> bool: + """Return True for PXD/PRD accessions or ProteomeCentral URLs. + + Not used by :mod:`pridepy.download.registry` (this provider is + deliberately not auto-registered). Provided for parity with the + ``Provider`` interface and so direct callers can introspect whether + a given input looks like something ProteomeXchange knows how to + handle. + """ + if not accession: + return False + if accession.lower().startswith(("http://", "https://")): + return "proteomexchange" in accession.lower() or "cgi/GetDataset" in accession + return bool(re.fullmatch(r"(?:PXD|PRD)\d+", accession.upper())) + + @staticmethod + def _normalize_px_xml_url(px_id_or_url: str) -> str: + """Build the ProteomeXchange XML endpoint URL from an accession or URL. + + Examples accepted: + - ``PXD039236`` + - ``https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236`` + - ``https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236&anything`` + """ + if px_id_or_url.startswith("http://") or px_id_or_url.startswith("https://"): + parsed = urlparse(px_id_or_url) + query = parsed.query or "" + if "ID=" in query: + id_value = [ + q.split("=", 1)[1] for q in query.split("&") if q.startswith("ID=") + ] + if id_value: + return ( + "https://proteomecentral.proteomexchange.org/cgi/GetDataset" + f"?ID={id_value[0]}&outputMode=XML&test=no" + ) + if parsed.path.endswith("/cgi/GetDataset"): + return ( + "https://proteomecentral.proteomexchange.org/cgi/GetDataset" + f"?{query}&outputMode=XML&test=no" + ) + return ( + "https://proteomecentral.proteomexchange.org/cgi/GetDataset" + f"?ID={px_id_or_url}&outputMode=XML&test=no" + ) + + @staticmethod + def _parse_px_xml_for_raw_file_urls(px_xml_url: str) -> List[str]: + """Fetch the PX XML and return every ``Associated raw file URI`` value.""" + headers = {"Accept": "application/xml"} + response = Util.get_api_call(px_xml_url, headers) + response.raise_for_status() + root = ET.fromstring(response.content) + + urls: List[str] = [] + for dataset_file in root.iter("DatasetFile"): + for cv in dataset_file.findall("cvParam"): + name = cv.attrib.get("name") + value = cv.attrib.get("value") + if name == "Associated raw file URI" and value: + urls.append(value) + return urls + + @staticmethod + def _relative_paths_for_urls(urls: List[str]) -> List[str]: + """Compute a dataset-relative destination path for each URL. + + The PX XML's raw-file URIs point at arbitrary directories on the + hosting repository, so flattening to the URL basename would let + same-named files in different directories overwrite each other. + Strip the common parent directory shared by all URIs and keep the + remainder, so e.g. ``.../run1/x.raw`` and ``.../run2/x.raw`` become + ``run1/x.raw`` and ``run2/x.raw``. A single file (or one with no + shared prefix) falls back to its basename. + """ + paths = [urlparse(url).path for url in urls] + if not paths: + return [] + dirs = [posixpath.dirname(p) for p in paths] + try: + common = dirs[0] if len(paths) == 1 else posixpath.commonpath(dirs) + except ValueError: + common = "" + rels: List[str] = [] + for path in paths: + # ``common`` is the shared ancestor of every path's directory, so + # each path starts with it; strip it to keep the disambiguating + # remainder. ``common`` can legitimately be ``"/"`` (files in + # different top-level dirs) — handle that by stripping it too, + # rather than collapsing to the (colliding) basename. + if common: + rel = path[len(common):].lstrip("/") + else: + rel = posixpath.basename(path) + rels.append(rel or posixpath.basename(path)) + return rels + + def list_files(self, accession: str) -> List[Dict]: + """Return the dataset's raw-file URIs as minimal file records. + + The PX XML doesn't expose checksums or rich category labels, so + each record carries just enough to drive the downloader. A + ``relativePath`` is derived per file (see + :meth:`_relative_paths_for_urls`) so the transport layer preserves + directory structure instead of colliding on duplicate basenames. + """ + px_xml_url = self._normalize_px_xml_url(accession) + logging.info(f"Fetching PX XML: {px_xml_url}") + urls = self._parse_px_xml_for_raw_file_urls(px_xml_url) + relative_paths = self._relative_paths_for_urls(urls) + records: List[Dict] = [] + for url, relative_path in zip(urls, relative_paths): + parsed = urlparse(url) + records.append( + { + "accession": accession, + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": url} + ], + "relativePath": relative_path, + "source": "ProteomeXchange", + } + ) + return records + + def download_from_accession_or_url( + self, + px_id_or_url: str, + output_folder: str, + skip_if_downloaded_already: bool = True, + flatten: bool = True, + ) -> None: + """End-to-end: resolve XML, list files, partition by scheme, download. + + Convenience for the ``download-px-raw-files`` CLI command — combines + :meth:`list_files` and :meth:`download_files` with the original + ``download_px_raw_files`` defaults (skip-if-downloaded-already + defaults to ``True``, no parallel workers). + """ + records = self.list_files(px_id_or_url) + if not records: + logging.info("No Associated raw file URIs found in PX XML") + return + self.download_files( + accession=px_id_or_url, + records=records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol="ftp", + flatten=flatten, + ) diff --git a/pridepy/download/registry.py b/pridepy/download/registry.py new file mode 100644 index 0000000..47d2786 --- /dev/null +++ b/pridepy/download/registry.py @@ -0,0 +1,38 @@ +"""Accession-to-provider resolution. + +Providers are tried in priority order; direct-download repositories +(MassIVE / JPOST / iProX) are tried first because their accession patterns +are unambiguous. PRIDE is tried last and acts as the catch-all for +``PXD\\d+`` / ``PRD\\d+`` accessions. +""" +from typing import List, Type + +from pridepy.download.base import Provider + +_PROVIDERS: List[Type[Provider]] = [] # populated by individual provider modules + + +def register(provider_cls: Type[Provider]) -> Type[Provider]: + """Register a provider class. Usable as a decorator.""" + if provider_cls not in _PROVIDERS: + _PROVIDERS.append(provider_cls) + return provider_cls + + +def resolve(accession: str) -> Provider: + """Return a provider instance that matches ``accession``. + + :raises ValueError: when no registered provider matches. + """ + for cls in _PROVIDERS: + if cls.matches(accession): + return cls() + raise ValueError(f"No provider registered for accession {accession!r}") + + +def is_known(accession: str) -> bool: + """Return True if any registered provider matches ``accession``.""" + for cls in _PROVIDERS: + if cls.matches(accession): + return True + return False diff --git a/pridepy/download/transport.py b/pridepy/download/transport.py new file mode 100644 index 0000000..122e7de --- /dev/null +++ b/pridepy/download/transport.py @@ -0,0 +1,696 @@ +"""Shared FTP / FTPS / HTTPS download transport. + +Stateless helpers used by the per-repository adapters (and re-exported on +:class:`pridepy.download.client.Client` for downstream callers that use +``Client.download_ftp_urls`` etc.). +""" +import ftplib +import logging +import os +import socket +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from ftplib import FTP +from typing import Dict, List, Optional, Tuple +from urllib.parse import urlparse + +import requests +from tqdm import tqdm + +from pridepy.util.api_handling import Util + + +def _safe_join(output_folder: str, relative_path: str) -> str: + """Join ``output_folder`` with a dataset-relative path. + + Preserves sub-directory structure (so identically-named files in + different collections don't collide). Guards against absolute paths or + ``..`` traversal that would escape ``output_folder`` by falling back to + the basename — provider relative paths are already dataset-relative, so + this is purely defensive. + """ + relative_path = (relative_path or "").lstrip("/") + if not relative_path: + return output_folder + local_path = os.path.normpath(os.path.join(output_folder, relative_path)) + out_abs = os.path.abspath(output_folder) + local_abs = os.path.abspath(local_path) + if local_abs != out_abs and not local_abs.startswith(out_abs + os.sep): + return os.path.join(output_folder, os.path.basename(relative_path)) + return local_path + + +def _dest_path( + output_folder: str, url_path: str, relative_path: Optional[str] +) -> str: + """Resolve the local destination for a download. + + Uses the dataset-relative path when available (preserving layout), + otherwise falls back to the URL basename. + """ + if relative_path: + return _safe_join(output_folder, relative_path) + return os.path.join(output_folder, os.path.basename(url_path)) + + +def _open_ftp_connection(host: str, use_tls: bool, timeout: int = 30) -> FTP: + """ + Open an anonymous FTP connection, transparently using FTPS when the + server requires TLS (e.g., MassIVE). When ``use_tls`` is False but the + server replies ``421 TLS is required`` to ``login``, transparently + retry with FTPS so callers don't need to know the policy in advance. + """ + if use_tls: + ftp: FTP = ftplib.FTP_TLS(host, timeout=timeout) + ftp.login() + ftp.prot_p() + else: + ftp = FTP(host, timeout=timeout) + try: + ftp.login() + except ftplib.error_temp as e: + if "TLS" in str(e).upper(): + try: + ftp.close() + except Exception: + pass + ftp = ftplib.FTP_TLS(host, timeout=timeout) + ftp.login() + ftp.prot_p() + else: + raise + ftp.set_pasv(True) + return ftp + + +def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: + """ + Recursively list files under a remote FTP directory. + """ + import posixpath + file_paths: List[str] = [] + try: + entries = list(ftp.mlsd(remote_dir)) + for name, facts in entries: + if name in {".", ".."}: + continue + child_path = posixpath.join(remote_dir.rstrip("/"), name) + if facts.get("type") == "dir": + file_paths.extend(_walk_ftp_tree(ftp, child_path)) + elif facts.get("type") == "file": + file_paths.append(child_path) + return file_paths + except (AttributeError, ftplib.error_perm): + pass + + current_dir = ftp.pwd() + listing: List[str] = [] + try: + ftp.cwd(remote_dir) + ftp.retrlines("LIST", listing.append) + for entry in listing: + parts = entry.split(maxsplit=8) + if len(parts) < 9: + continue + name = parts[8] + if name in {".", ".."}: + continue + child_path = posixpath.join(remote_dir.rstrip("/"), name) + if entry.startswith("d"): + file_paths.extend(_walk_ftp_tree(ftp, child_path)) + else: + file_paths.append(child_path) + finally: + ftp.cwd(current_dir) + return file_paths + + +def _list_ftp_repo_files( + host: str, + remote_root: str, + error_label: str, + use_tls: bool = False, +) -> List[str]: + """ + Connect to an anonymous FTP host (FTP or FTPS), walk a directory tree, + and return file paths. + + ``use_tls`` should be True for servers that reject plain FTP (e.g. + MassIVE). Centralizes connection lifecycle so a constructor failure + doesn't mask the underlying error in ``finally`` (PR #98 review). + """ + ftp: Optional[FTP] = None + try: + ftp = _open_ftp_connection(host, use_tls=use_tls) + logging.info(f"Connected to FTP host: {host} (tls={use_tls})") + return _walk_ftp_tree(ftp, remote_root) + except Exception as error: + raise RuntimeError( + f"Unable to list public files for {error_label}: {error}" + ) from error + finally: + if ftp is not None: + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + + +def _resolve_and_walk_ftp_dataset( + host: str, + accession: str, + error_label: str, + use_tls: bool = False, + prefer_prefix: str = "", +) -> List[str]: + """ + Find which top-level directory on ``host`` holds ``accession`` and walk it. + + Some repositories (e.g. MassIVE) distribute datasets across several + versioned root directories (``/v01`` … ``/vNN``, plus auxiliary roots like + ``x01`` / ``z01`` that may hold only a partial, derived copy) and the + version is not derivable from the accession. Probe each top-level directory + for ``/`` and walk the first match, reusing a single + connection for both discovery and listing. + + ``prefer_prefix`` lets the caller try the canonical roots first: roots + whose name starts with the prefix (e.g. ``"v"`` for MassIVE versioned + storage) are probed before any others, so a complete copy is chosen over + an auxiliary partial one when a dataset exists under both. + + :raises RuntimeError: on connection failure or when the accession is not + found under any top-level directory. + """ + ftp: Optional[FTP] = None + try: + ftp = _open_ftp_connection(host, use_tls=use_tls) + logging.info(f"Connected to FTP host: {host} (tls={use_tls})") + roots: List[str] = [] + ftp.retrlines("NLST /", roots.append) + # Servers may return bare names or absolute paths; keep the leaf name. + candidates = [] + for entry in roots: + name = entry.strip().strip("/").split("/")[-1] + if name and name not in {".", ".."}: + candidates.append(name) + if prefer_prefix: + prefix = prefer_prefix.lower() + candidates.sort( + key=lambda n: (not n.lower().startswith(prefix), n) + ) + for name in candidates: + dataset_root = f"/{name}/{accession}" + try: + ftp.cwd(dataset_root) + except ftplib.error_perm: + continue + logging.info(f"Found {accession} under {dataset_root} on {host}") + return _walk_ftp_tree(ftp, dataset_root) + raise RuntimeError( + f"{accession} not found under any top-level directory on {host}" + ) + except Exception as error: + raise RuntimeError( + f"Unable to list public files for {error_label}: {error}" + ) from error + finally: + if ftp is not None: + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + + +def _download_one_ftp_path( + ftp: FTP, + ftp_path: str, + local_path: str, + skip_if_downloaded_already: bool, + max_download_retries: int, + position: int = 0, +) -> None: + """ + Download a single FTP path over an existing connection, with REST resume + and per-file retry. Raises on giving up so the caller can decide what to do. + """ + if skip_if_downloaded_already and os.path.exists(local_path): + logging.info(f"Skipping download as file already exists: {local_path}") + return + + attempt = 0 + last_error: Optional[Exception] = None + while attempt < max_download_retries: + try: + total_size = ftp.size(ftp_path) + if os.path.exists(local_path): + current_size = os.path.getsize(local_path) + mode = "ab" + else: + current_size = 0 + mode = "wb" + + with open(local_path, mode) as f, tqdm( + total=total_size, + unit="B", + unit_scale=True, + desc=local_path, + initial=current_size, + position=position, + leave=True, + ) as pbar: + def callback(data): + f.write(data) + pbar.update(len(data)) + + if current_size: + try: + ftp.sendcmd(f"REST {current_size}") + except Exception: + current_size = 0 + f.seek(0) + f.truncate() + ftp.retrbinary(f"RETR {ftp_path}", callback) + + # Post-transfer integrity check: server-reported size must match + # the local size. Catches half-finished transfers that retrbinary + # didn't raise on (e.g. server closed the data channel early). + # The next iteration will REST-resume from where we left off. + if total_size: + final_size = os.path.getsize(local_path) + if final_size != total_size: + attempt += 1 + logging.error( + f"Size mismatch for {local_path}: " + f"got {final_size} bytes, expected {total_size} " + f"(attempt {attempt})" + ) + continue + logging.info(f"Successfully downloaded {local_path}") + return + except (socket.timeout, ftplib.error_temp, ftplib.error_perm) as e: + attempt += 1 + last_error = e + logging.error( + f"Download failed for {local_path} (attempt {attempt}): {e}" + ) + raise RuntimeError( + f"Giving up on {local_path} after {max_download_retries} attempts" + ) from last_error + + +def _download_ftp_paths_serial( + host: str, + items: List[Tuple[str, str]], + skip_if_downloaded_already: bool, + use_tls: bool, + max_connection_retries: int, + max_download_retries: int, +) -> List[str]: + """Download all paths from one host over a single (reused) connection. + + ``items`` is a list of ``(ftp_path, local_path)`` pairs; ``local_path`` + is the precomputed destination (already including any sub-directories). + + Returns the list of ``ftp_path`` values that could not be downloaded + (connection never established, or per-file giving up) so the caller can + surface a failure instead of reporting false success. + """ + connection_attempt = 0 + while connection_attempt < max_connection_retries: + failed: List[str] = [] + try: + ftp = _open_ftp_connection(host, use_tls=use_tls) + logging.info(f"Connected to FTP host: {host} (tls={use_tls})") + for ftp_path, local_path in items: + parent = os.path.dirname(local_path) + if parent: + os.makedirs(parent, exist_ok=True) + try: + _download_one_ftp_path( + ftp=ftp, + ftp_path=ftp_path, + local_path=local_path, + skip_if_downloaded_already=skip_if_downloaded_already, + max_download_retries=max_download_retries, + ) + except Exception as e: + logging.error( + f"Failed to download {ftp_path} from {host}: {e}" + ) + failed.append(ftp_path) + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + logging.info(f"Disconnected from FTP host: {host}") + return failed + except (socket.timeout, ftplib.error_temp, ftplib.error_perm, OSError) as e: + connection_attempt += 1 + logging.error( + f"FTP connection failed (attempt {connection_attempt}): {e}" + ) + if connection_attempt < max_connection_retries: + logging.info("Retrying connection...") + time.sleep(5) + else: + logging.error( + f"Giving up after {max_connection_retries} failed connection attempts to {host}." + ) + return [ftp_path for ftp_path, _ in items] + return [ftp_path for ftp_path, _ in items] + + +def _download_ftp_paths_parallel( + host: str, + items: List[Tuple[str, str]], + skip_if_downloaded_already: bool, + use_tls: bool, + max_connection_retries: int, + max_download_retries: int, + parallel_files: int, +) -> List[str]: + """ + Download paths concurrently using ``parallel_files`` workers; each + worker opens its own FTP connection so transfers don't serialize. + + ``items`` is a list of ``(ftp_path, local_path)`` pairs. Returns the list + of ``ftp_path`` values that failed so the caller can surface a failure. + """ + def worker(item: Tuple[str, str], position: int) -> Optional[str]: + ftp_path, local_path = item + if skip_if_downloaded_already and os.path.exists(local_path): + logging.info(f"Skipping download as file already exists: {local_path}") + return None + parent = os.path.dirname(local_path) + if parent: + os.makedirs(parent, exist_ok=True) + connection_attempt = 0 + while connection_attempt < max_connection_retries: + try: + ftp = _open_ftp_connection(host, use_tls=use_tls) + try: + _download_one_ftp_path( + ftp=ftp, + ftp_path=ftp_path, + local_path=local_path, + skip_if_downloaded_already=False, + max_download_retries=max_download_retries, + position=position, + ) + return None + finally: + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + except (socket.timeout, ftplib.error_temp, ftplib.error_perm, OSError) as e: + connection_attempt += 1 + logging.error( + f"FTP connection failed for {ftp_path} (attempt {connection_attempt}): {e}" + ) + if connection_attempt < max_connection_retries: + time.sleep(5) + except Exception as e: + logging.error(f"Failed to download {ftp_path} from {host}: {e}") + return ftp_path + logging.error(f"Giving up on {ftp_path} from {host}") + return ftp_path + + failed: List[str] = [] + with ThreadPoolExecutor(max_workers=parallel_files) as executor: + future_to_path = { + executor.submit(worker, item, idx): item[0] + for idx, item in enumerate(items) + } + for future in as_completed(future_to_path): + try: + result = future.result() + if result is not None: + failed.append(result) + except Exception as e: + logging.error(f"Parallel FTP download error: {e}") + failed.append(future_to_path[future]) + return failed + + +def download_ftp_urls( + ftp_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + max_connection_retries: int = 3, + max_download_retries: int = 3, + use_tls: bool = False, + parallel_files: int = 1, + relative_paths: Optional[List[str]] = None, +) -> None: + """ + Download a list of FTP URLs with retries, REST-based resume, and + optional parallel workers. + + :param use_tls: Open the FTP connection with TLS (FTP_TLS / PROT P). + Required for hosts that reject plain anonymous FTP (e.g. MassIVE). + When False but the server replies ``421 TLS is required``, the + connection is transparently retried over TLS. + :param parallel_files: When >1, downloads run concurrently with that + many worker connections per host (capped at the number of files). + :param relative_paths: Optional per-URL dataset-relative destination + paths (parallel to ``ftp_urls``). When given, files are written to + ``output_folder/`` so identically-named files in + different collections don't collide. When omitted, the URL basename + is used (legacy flat layout). + :raises RuntimeError: after attempting every file, if one or more failed. + """ + if not os.path.isdir(output_folder): + os.makedirs(output_folder, exist_ok=True) + + host_to_items: Dict[str, List[Tuple[str, str]]] = {} + for idx, url in enumerate(ftp_urls): + parsed = urlparse(url) + remote_path = parsed.path.lstrip("/") + relpath = ( + relative_paths[idx] + if relative_paths and idx < len(relative_paths) + else None + ) + if not parsed.hostname: + raise ValueError( + f"Cannot download FTP URL with no host: {url!r}" + ) + local_path = _dest_path(output_folder, remote_path, relpath) + host_to_items.setdefault(parsed.hostname, []).append((remote_path, local_path)) + + failed: List[str] = [] + for host, items in host_to_items.items(): + workers = max(1, min(parallel_files, len(items))) + if workers > 1: + failed.extend(_download_ftp_paths_parallel( + host=host, + items=items, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=use_tls, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + parallel_files=workers, + )) + else: + failed.extend(_download_ftp_paths_serial( + host=host, + items=items, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=use_tls, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + )) + + if failed: + raise RuntimeError( + f"Failed to download {len(failed)} FTP file(s): {failed}" + ) + + +def _parallel_download(url, file_path, position=0): + """Download a file via a single-connection HTTP stream with optional resume. + If a partial file exists and the server supports Range requests, resumes + from where it left off; otherwise restarts from scratch.""" + parent = os.path.dirname(file_path) + if parent: + os.makedirs(parent, exist_ok=True) + + session = Util.create_session_with_retries() + try: + head = session.head(url, timeout=(30, 30)) + head.raise_for_status() + total_size = int(head.headers.get("content-length", 0)) + accept_ranges = head.headers.get("accept-ranges", "none").strip().lower() + except (requests.RequestException, ValueError) as exc: + logging.info(f"HEAD request failed, falling back to single connection: {exc}") + total_size = 0 + accept_ranges = "none" + + resume_size = 0 + if os.path.exists(file_path) and accept_ranges == "bytes" and total_size > 0: + resume_size = os.path.getsize(file_path) + if resume_size >= total_size: + logging.info(f"File already complete: {file_path}") + return + if resume_size > 0: + logging.info(f"Resuming download from {resume_size} bytes: {file_path}") + + headers = {"Range": f"bytes={resume_size}-"} if resume_size > 0 else {} + content_encoding = None + with session.get(url, headers=headers, stream=True, timeout=(30, 60)) as r: + r.raise_for_status() + content_encoding = r.headers.get("Content-Encoding") + if resume_size > 0 and r.status_code != 206: + logging.warning("Server did not honor Range request (status %s), restarting download", r.status_code) + resume_size = 0 + with tqdm(total=total_size, unit="B", unit_scale=True, desc=file_path, + initial=resume_size, position=position, leave=True) as pbar: + mode = "ab" if resume_size > 0 else "wb" + with open(file_path, mode, buffering=8 * 1024 * 1024) as f: + for chunk in r.iter_content(chunk_size=8 * 1024 * 1024): + if chunk: + f.write(chunk) + pbar.update(len(chunk)) + + # Post-transfer integrity check mirroring the FTP path: the written size + # must match the server-reported Content-Length. A server that closes the + # data channel mid-stream without raising leaves a truncated file; raising + # here lets the caller's retry loop re-download (Range-resuming when able). + # Skipped when the server applied Content-Encoding (gzip/deflate): requests + # decompresses transparently, so on-disk size won't match Content-Length. + if total_size and not content_encoding: + actual_size = os.path.getsize(file_path) + if actual_size != total_size: + raise RuntimeError( + f"Incomplete download for {file_path}: got {actual_size} bytes, " + f"expected {total_size}" + ) + + +def _http_download_one( + url: str, + output_folder: str, + skip_if_downloaded_already: bool, + max_retries: int = 3, + position: int = 0, + relative_path: Optional[str] = None, +) -> None: + """ + Download a single HTTP(S) URL with HEAD-then-Range resume and retry. + Used as the worker target for both the serial loop and the parallel + ThreadPoolExecutor path. Reuses :meth:`_parallel_download` so the same + resume / restart-on-non-206 behaviour is shared with globus downloads. + + ``relative_path`` (when given) is the dataset-relative destination, so + files keep their collection layout instead of being flattened to the + URL basename. + """ + local_path = _dest_path(output_folder, urlparse(url).path, relative_path) + if skip_if_downloaded_already and os.path.exists(local_path): + logging.info(f"Skipping download as file already exists: {local_path}") + return + last_error: Optional[Exception] = None + for attempt in range(1, max_retries + 1): + try: + _parallel_download(url, local_path, position=position) + logging.info(f"Successfully downloaded {local_path}") + return + except Exception as e: + last_error = e + logging.warning( + f"HTTP download attempt {attempt}/{max_retries} failed for {url}: {e}" + ) + raise RuntimeError( + f"Giving up on {local_path} after {max_retries} HTTP attempts" + ) from last_error + + +def download_http_urls( + http_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + parallel_files: int = 1, + max_retries: int = 3, + relative_paths: Optional[List[str]] = None, +) -> None: + """ + Download a list of HTTP(S) URLs with HEAD-then-Range resume, per-file + retries, and an optional ``parallel_files`` worker pool. + + When ``parallel_files`` > 1, downloads run concurrently using a + :class:`ThreadPoolExecutor`. Each worker manages its own file (a new + ``requests`` session is opened inside ``_parallel_download``) so the + only shared resource is the output directory. + + :param relative_paths: Optional per-URL dataset-relative destination + paths (parallel to ``http_urls``); see :func:`download_ftp_urls`. + :raises RuntimeError: after attempting every URL, if one or more failed. + """ + if not os.path.isdir(output_folder): + os.makedirs(output_folder, exist_ok=True) + + if not http_urls: + return + + def _rel(idx: int) -> Optional[str]: + if relative_paths and idx < len(relative_paths): + return relative_paths[idx] + return None + + failed: List[str] = [] + workers = max(1, min(parallel_files, len(http_urls))) + if workers > 1: + logging.info( + f"Downloading {len(http_urls)} HTTP(S) file(s) with {workers} parallel workers" + ) + with ThreadPoolExecutor(max_workers=workers) as executor: + future_to_url = { + executor.submit( + _http_download_one, + url, + output_folder, + skip_if_downloaded_already, + max_retries, + idx, + _rel(idx), + ): url + for idx, url in enumerate(http_urls) + } + for future in as_completed(future_to_url): + url = future_to_url[future] + try: + future.result() + except Exception as e: + logging.error(f"HTTP download failed for {url}: {e}") + failed.append(url) + else: + for idx, url in enumerate(http_urls): + try: + _http_download_one( + url, + output_folder, + skip_if_downloaded_already, + max_retries, + relative_path=_rel(idx), + ) + except Exception as e: + logging.error(f"HTTP download failed for {url}: {e}") + failed.append(url) + + if failed: + raise RuntimeError( + f"Failed to download {len(failed)} HTTP(S) file(s): {failed}" + ) diff --git a/pridepy/download/util.py b/pridepy/download/util.py new file mode 100644 index 0000000..11a201d --- /dev/null +++ b/pridepy/download/util.py @@ -0,0 +1,153 @@ +"""Cross-cutting utilities used by providers and the Client facade. + +Pure functions (and one tiny Progress class) for checksums, record-shape +helpers, and download progress. Originally on the facade as @staticmethods; +moved here so providers can use them without depending on the facade at +import time, while :class:`~pridepy.download.client.Client` keeps shim +re-exports for backward compatibility with existing test patches. +""" +import hashlib +import logging +import os +from typing import Dict, List, Optional, Tuple + +from tqdm import tqdm + + +def flatten_relative_paths(relative_paths: List[str]) -> List[str]: + """Map dataset-relative paths to flat, de-duplicated basenames. + + Used when downloading into a single output folder without recreating the + dataset's subdirectory tree. Files keep their basename; when two or more + source paths collapse to the same basename, the first one (by sorted source + path) keeps the bare name and later ones get a numeric suffix inserted + before the final extension (``run.raw`` -> ``run_1.raw``). + + Suffixing is decided by the *sorted* source paths so the mapping is + deterministic across runs (independent of disk state or input order), + keeping skip-if-downloaded and resume stable. The returned list is + positionally aligned with ``relative_paths`` so callers can zip it back to + their records. + """ + assigned: Dict[str, str] = {} + seen_basenames: Dict[str, int] = {} + # Decide names in sorted-source order so the result is deterministic. + for source in sorted(relative_paths): + basename = os.path.basename((source or "").lstrip("/")) + count = seen_basenames.get(basename, 0) + if count == 0: + assigned[source] = basename + else: + stem, ext = os.path.splitext(basename) + assigned[source] = f"{stem}_{count}{ext}" + seen_basenames[basename] = count + 1 + return [assigned[source] for source in relative_paths] + + +class Progress: + def __init__(self, total_size, file_name): + self.pbar = tqdm( + total=total_size, + unit="B", + unit_scale=True, + desc="Downloading {}".format(file_name), + ) + + def __call__(self, bytes_amount): + self.pbar.update(bytes_amount) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.pbar.close() + + def close(self): + self.pbar.close() + + +def _find_tsv_columns(header: str) -> Optional[Tuple[int, int]]: + """Return (name_idx, checksum_idx) from a TSV header, or None.""" + cols = [col.strip().lower() for col in header.split("\t")] + required_cols = {"file-name", "file-md5checksum", "file-size"} + if not required_cols.issubset(set(cols)): + return None + return cols.index("file-name"), cols.index("file-md5checksum") + + +def _is_md5_checksum(value: str) -> bool: + return len(value) == 32 and all(char in "0123456789abcdef" for char in value) + + +def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: + """ + Read PRIDE API checksum TSV and build {file_name: md5} map. + Expected format: File-Name\tFile-MD5Checksum\tFile-Size + """ + checksums: Dict[str, str] = {} + if not checksum_file_path or not os.path.exists(checksum_file_path): + return checksums + + with open(checksum_file_path, "r", encoding="utf-8") as f: + header = f.readline().strip() + if not header: + return checksums + + col_indices = _find_tsv_columns(header) + if col_indices is None: + logging.warning(f"Unrecognized checksum file format: {header}") + return checksums + + name_idx, checksum_idx = col_indices + min_cols = max(name_idx, checksum_idx) + 1 + for line in f: + parts = line.strip().split("\t") + if len(parts) >= min_cols: + fn = os.path.basename(parts[name_idx].strip()) + cs = parts[checksum_idx].strip().lower() + if fn and _is_md5_checksum(cs): + checksums[fn] = cs + + return checksums + + +def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: + """ + Compute an MD5 checksum for integrity validation, not for security use. + """ + try: + md5 = hashlib.md5(usedforsecurity=False) + except TypeError: + md5 = hashlib.md5() + with open(file_path, "rb") as file_handle: + while True: + chunk = file_handle.read(chunk_size) + if not chunk: + break + md5.update(chunk) + return md5.hexdigest() + + +def validate_download(file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, str]: + """ + Validate a local file exists, is non-empty, and checksum matches when provided. + """ + if not os.path.exists(file_path): + return False, "file does not exist" + if os.path.getsize(file_path) == 0: + return False, "file is empty" + if expected_checksum: + actual_checksum = compute_md5(file_path) + if actual_checksum.lower() != expected_checksum.lower(): + return False, ( + f"checksum mismatch (expected={expected_checksum.lower()}, actual={actual_checksum.lower()})" + ) + return True, "ok" + + +def _remove_if_exists(file_path: str) -> None: + """ + Remove a file if it already exists locally. + """ + if os.path.exists(file_path): + os.remove(file_path) diff --git a/pridepy/files/__init__.py b/pridepy/files/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pridepy/files/files.py b/pridepy/files/files.py deleted file mode 100644 index fbcf2f9..0000000 --- a/pridepy/files/files.py +++ /dev/null @@ -1,1951 +0,0 @@ -#!/usr/bin/env python -import ftplib -import hashlib -import importlib.resources -import logging -import os -import platform -import posixpath -import re -import subprocess -import urllib -import urllib.request -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from ftplib import FTP -from typing import Dict, List, Optional, Tuple -import socket -from urllib.parse import urlparse -import xml.etree.ElementTree as ET - -import boto3 -import botocore -import requests -from botocore.config import Config -from tqdm import tqdm - -from pridepy.authentication.authentication import Authentication -from pridepy.util.api_handling import Util - - -class Progress: - def __init__(self, total_size, file_name): - self.pbar = tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc="Downloading {}".format(file_name), - ) - - def __call__(self, bytes_amount): - self.pbar.update(bytes_amount) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.pbar.close() - - def close(self): - self.pbar.close() - - -class Files: - """ - This class handles PRIDE API files endpoint. - """ - - V3_API_BASE_URL = "https://www.ebi.ac.uk/pride/ws/archive/v3" - API_BASE_URL = "https://www.ebi.ac.uk/pride/ws/archive/v3" - API_PRIVATE_URL = "https://www.ebi.ac.uk/pride/private/ws/archive/v2" - PRIDE_ARCHIVE_FTP = "ftp.pride.ebi.ac.uk" - PRIDE_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.pride.ebi.ac.uk/" - PRIDE_ARCHIVE_HTTPS_URL_PREFIX = "https://ftp.pride.ebi.ac.uk/" - MASSIVE_ARCHIVE_FTP = "massive-ftp.ucsd.edu" - MASSIVE_ARCHIVE_FTP_URL_PREFIX = "ftp://massive-ftp.ucsd.edu/v01/" - S3_URL = "https://hh.fire.sdo.ebi.ac.uk" - S3_BUCKET = "pride-public" - PROTOCOL_ORDER = ["aspera", "s3", "ftp", "globus"] - MASSIVE_CATEGORY_MAP = { - "raw": "RAW", - "peak": "PEAK", - "ccms_peak": "PEAK", - "search": "SEARCH", - "result": "RESULT", - "ccms_result": "RESULT", - "quant": "RESULT", - "fasta": "FASTA", - "spectrum_library": "SPECTRUM_LIBRARY", - "library": "SPECTRUM_LIBRARY", - } - logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") - - def __init__(self): - pass - - @staticmethod - def _find_tsv_columns(header: str) -> Optional[Tuple[int, int]]: - """Return (name_idx, checksum_idx) from a TSV header, or None.""" - cols = [col.strip().lower() for col in header.split("\t")] - required_cols = {"file-name", "file-md5checksum", "file-size"} - if not required_cols.issubset(set(cols)): - return None - return cols.index("file-name"), cols.index("file-md5checksum") - - @staticmethod - def _is_md5_checksum(value: str) -> bool: - return len(value) == 32 and all(char in "0123456789abcdef" for char in value) - - @staticmethod - def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: - """ - Read PRIDE API checksum TSV and build {file_name: md5} map. - Expected format: File-Name\tFile-MD5Checksum\tFile-Size - """ - checksums: Dict[str, str] = {} - if not checksum_file_path or not os.path.exists(checksum_file_path): - return checksums - - with open(checksum_file_path, "r", encoding="utf-8") as f: - header = f.readline().strip() - if not header: - return checksums - - col_indices = Files._find_tsv_columns(header) - if col_indices is None: - logging.warning(f"Unrecognized checksum file format: {header}") - return checksums - - name_idx, checksum_idx = col_indices - min_cols = max(name_idx, checksum_idx) + 1 - for line in f: - parts = line.strip().split("\t") - if len(parts) >= min_cols: - fn = os.path.basename(parts[name_idx].strip()) - cs = parts[checksum_idx].strip().lower() - if fn and Files._is_md5_checksum(cs): - checksums[fn] = cs - - return checksums - - @staticmethod - def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: - """ - Compute an MD5 checksum for integrity validation, not for security use. - """ - try: - md5 = hashlib.md5(usedforsecurity=False) - except TypeError: - md5 = hashlib.md5() - with open(file_path, "rb") as file_handle: - while True: - chunk = file_handle.read(chunk_size) - if not chunk: - break - md5.update(chunk) - return md5.hexdigest() - - @staticmethod - def validate_download(file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, str]: - """ - Validate a local file exists, is non-empty, and checksum matches when provided. - """ - if not os.path.exists(file_path): - return False, "file does not exist" - if os.path.getsize(file_path) == 0: - return False, "file is empty" - if expected_checksum: - actual_checksum = Files.compute_md5(file_path) - if actual_checksum.lower() != expected_checksum.lower(): - return False, ( - f"checksum mismatch (expected={expected_checksum.lower()}, actual={actual_checksum.lower()})" - ) - return True, "ok" - - @staticmethod - def _remove_if_exists(file_path: str) -> None: - """ - Remove a file if it already exists locally. - """ - if os.path.exists(file_path): - os.remove(file_path) - - @staticmethod - def _get_download_url(file_record: Dict, protocol: str) -> str: - """ - Resolve the public download URL for a file and protocol. - - Raises ValueError when the requested protocol has no suitable location. - Aspera requires a dedicated "Aspera Protocol" entry; ftp/s3/globus - derive their URL from the "FTP Protocol" entry (falling back to an - arbitrary non-Aspera location would produce a URL the caller cannot - actually transfer with). - """ - locations = file_record.get("publicFileLocations", []) - if not locations: - raise ValueError("No public file locations present") - - aspera_url = None - ftp_url = None - for location in locations: - name = location.get("name") - if name == "Aspera Protocol": - aspera_url = location.get("value") - elif name == "FTP Protocol": - ftp_url = location.get("value") - - if protocol == "aspera": - if not aspera_url: - raise ValueError("Aspera URL not available") - return aspera_url - - if not ftp_url: - raise ValueError("FTP URL not available") - if protocol == "ftp": - return ftp_url - if protocol == "globus": - return ftp_url.replace( - Files.PRIDE_ARCHIVE_FTP_URL_PREFIX, - Files.PRIDE_ARCHIVE_HTTPS_URL_PREFIX, - 1, - ) - if protocol == "s3": - return ftp_url - raise ValueError(f"Unsupported protocol: {protocol}") - - @staticmethod - def _resolve_local_path(file_record: Dict, output_folder: str) -> str: - """ - Compute the canonical local path for a file regardless of transfer protocol. - """ - try: - canonical_url = Files._get_download_url(file_record, "ftp") - except ValueError: - canonical_url = "" - if canonical_url: - return Files.get_output_file_name(canonical_url, file_record, output_folder) - return os.path.join(output_folder, file_record["fileName"]) - - @staticmethod - def _protocol_sequence(protocol: str) -> List[str]: - """ - Build the ordered list of protocols to try for a requested download mode. - """ - if protocol not in Files.PROTOCOL_ORDER: - return [] - return [protocol] + [p for p in Files.PROTOCOL_ORDER if p != protocol] - - @staticmethod - def is_massive_accession(accession: str) -> bool: - """ - Return True when the accession looks like a MassIVE dataset accession. - """ - if not accession: - return False - return bool(re.fullmatch(r"R?MSV\d{9}", accession.upper())) - - @staticmethod - def _get_massive_public_root(accession: str) -> str: - normalized_accession = accession.upper() - return f"/v01/{normalized_accession}" - - @staticmethod - def _get_massive_public_ftp_url(accession: str, remote_path: str) -> str: - root_path = Files._get_massive_public_root(accession).rstrip("/") - relative_path = remote_path - if remote_path.startswith(root_path): - relative_path = remote_path[len(root_path) :].lstrip("/") - return f"{Files.MASSIVE_ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" - - @staticmethod - def _map_massive_collection_to_category(collection: str) -> str: - return Files.MASSIVE_CATEGORY_MAP.get(collection.lower(), "OTHER") - - @staticmethod - def _build_massive_file_record(accession: str, ftp_url: str) -> Dict: - parsed = urlparse(ftp_url) - root_prefix = f"/v01/{accession.upper()}/" - relative_path = parsed.path - if relative_path.startswith(root_prefix): - relative_path = relative_path[len(root_prefix) :] - relative_path = relative_path.lstrip("/") - collection = relative_path.split("/", 1)[0] if relative_path else "" - return { - "accession": accession.upper(), - "fileName": os.path.basename(parsed.path), - "fileCategory": {"value": Files._map_massive_collection_to_category(collection)}, - "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], - "relativePath": relative_path, - "collection": collection, - "source": "MassIVE", - } - - @staticmethod - def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: - """ - Recursively list files under a remote FTP directory. - """ - file_paths: List[str] = [] - try: - entries = list(ftp.mlsd(remote_dir)) - for name, facts in entries: - if name in {".", ".."}: - continue - child_path = posixpath.join(remote_dir.rstrip("/"), name) - if facts.get("type") == "dir": - file_paths.extend(Files._walk_ftp_tree(ftp, child_path)) - elif facts.get("type") == "file": - file_paths.append(child_path) - return file_paths - except (AttributeError, ftplib.error_perm): - pass - - current_dir = ftp.pwd() - listing: List[str] = [] - try: - ftp.cwd(remote_dir) - ftp.retrlines("LIST", listing.append) - for entry in listing: - parts = entry.split(maxsplit=8) - if len(parts) < 9: - continue - name = parts[8] - if name in {".", ".."}: - continue - child_path = posixpath.join(remote_dir.rstrip("/"), name) - if entry.startswith("d"): - file_paths.extend(Files._walk_ftp_tree(ftp, child_path)) - else: - file_paths.append(child_path) - finally: - ftp.cwd(current_dir) - return file_paths - - def _list_massive_public_files(self, accession: str) -> List[Dict]: - """ - Discover all public files for a MassIVE dataset from its anonymous FTP tree. - """ - normalized_accession = accession.upper() - remote_root = self._get_massive_public_root(normalized_accession) - ftp = FTP(self.MASSIVE_ARCHIVE_FTP, timeout=30) - try: - ftp.login() - ftp.set_pasv(True) - logging.info(f"Connected to FTP host: {self.MASSIVE_ARCHIVE_FTP}") - remote_files = self._walk_ftp_tree(ftp, remote_root) - except Exception as error: - raise RuntimeError( - f"Unable to list public files for MassIVE dataset {normalized_accession}: {error}" - ) from error - finally: - try: - ftp.quit() - except Exception: - ftp.close() - - return [ - self._build_massive_file_record( - normalized_accession, - self._get_massive_public_ftp_url(normalized_accession, remote_file), - ) - for remote_file in remote_files - ] - - def _download_massive_file_records( - self, - accession: str, - file_records: List[Dict], - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str, - ) -> None: - """ - Download public MassIVE files via anonymous FTP. - """ - if protocol != "ftp": - logging.warning( - "MassIVE direct downloads currently use ftp only. " - f"Ignoring requested protocol '{protocol}' for {accession}." - ) - - ftp_urls = [self._get_download_url(file_record, "ftp") for file_record in file_records] - if not ftp_urls: - logging.info(f"No files matched for MassIVE dataset {accession}") - return - - self.download_ftp_urls( - ftp_urls=ftp_urls, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - ) - - async def stream_all_files_metadata(self, output_file, accession=None): - """ - get stream all project files from PRIDE API in JSON format - """ - if accession is None: - request_url = f"{self.V3_API_BASE_URL}/files/all" - count_request_url = f"{self.V3_API_BASE_URL}/files/count" - else: - request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/all" - count_request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/count" - headers = {"Accept": "application/JSON"} - response = Util.get_api_call(count_request_url, headers) - total_records = response.json() - - regex_search_pattern = '"fileName"' - await Util.stream_response_to_file( - output_file, total_records, regex_search_pattern, request_url, headers - ) - - def stream_all_files_by_project(self, accession) -> List[Dict]: - """ - get stream all project files from PRIDE API in JSON format - """ - request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/all" - headers = {"Accept": "application/JSON"} - record_files = Util.read_json_stream(api_url=request_url, headers=headers) - return record_files - - def get_all_raw_file_list(self, project_accession): - """ - Get all raw file lists from PRIDE API for a given project_accession - :param project_accession: PRIDE accession - :return: raw file list in JSON format - """ - if self.is_massive_accession(project_accession): - record_files = self._list_massive_public_files(project_accession) - return [ - file for file in record_files if file["fileCategory"]["value"] == "RAW" - ] - - record_files = self.stream_all_files_by_project(project_accession) - - # Filter projects by fileCategory = RAW - raw_files = [file for file in record_files if file["fileCategory"]["value"] == "RAW"] - return raw_files - - def download_all_raw_files( - self, - accession, - output_folder, - skip_if_downloaded_already, - protocol, - aspera_maximum_bandwidth: str, - checksum_check: bool = False, - parallel_files: int = 1, - ): - """ - This method will download all the raw files from PRIDE PROJECT - :param output_folder: output directory where raw files will get saved - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - :param accession: PRIDE accession - :param protocol: ftp, aspera, globus - :param aspera_maximum_bandwidth: Aspera maximum bandwidth - :param checksum_check: Download checksum for a given project. - :return: None - """ - - if not (os.path.isdir(output_folder)): - os.mkdir(output_folder) - - raw_files = self.get_all_raw_file_list(accession) - - if self.is_massive_accession(accession): - self._download_massive_file_records( - accession=accession, - file_records=raw_files, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - ) - return - - self.download_files( - raw_files, - accession, - output_folder, - skip_if_downloaded_already, - protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, - parallel_files=parallel_files, - ) - - @staticmethod - def download_files_from_ftp( - file_list_json, - output_folder, - skip_if_downloaded_already, - max_connection_retries=3, - max_download_retries=3, - ): - """ - Download files using a single FTP connection with a retry mechanism and a progress bar for each file. - :param file_list_json: file list in JSON format - :param output_folder: folder to download the files - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - :param max_connection_retries: Number of attempts to reconnect to the FTP server if the connection is lost. - :param max_download_retries: Number of attempts to retry the download of a file in case of failure. - """ - - if not os.path.isdir(output_folder): - os.makedirs(output_folder) - - def connect_ftp(): - """Helper function to establish FTP connection.""" - ftp = FTP(Files.PRIDE_ARCHIVE_FTP, timeout=30) - ftp.login() # Anonymous login - ftp.set_pasv(True) # Enable passive mode - logging.info(f"Connected to FTP host: {Files.PRIDE_ARCHIVE_FTP}") - return ftp - - connection_attempt = 0 - while connection_attempt < max_connection_retries: - try: - ftp = connect_ftp() - for file in file_list_json: - try: - # Get FTP download URL - if file["publicFileLocations"][0]["name"] == "FTP Protocol": - download_url = file["publicFileLocations"][0]["value"] - else: - download_url = file["publicFileLocations"][1]["value"] - - logging.debug("ftp_filepath:" + download_url) - - # Get output file path - new_file_path = Files.get_output_file_name( - download_url, file, output_folder - ) - - if skip_if_downloaded_already and os.path.exists(new_file_path): - logging.info("Skipping download as file already exists") - continue - - # Extract file path from the download URL - parsed_url = urlparse(download_url) - ftp_file_path = urllib.parse.unquote(parsed_url.path.lstrip("/")) - - logging.info(f"Starting FTP download: {ftp_file_path}") - - # Retry download in case of failure - download_attempt = 0 - while download_attempt < max_download_retries: - try: - # Get file size for progress tracking - total_size = ftp.size(ftp_file_path) - logging.info(f"File size: {total_size} bytes") - - # Initialize progress bar - with open(new_file_path, "wb") as f: - with tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=new_file_path, - ) as pbar: - - def callback(data): - f.write(data) - pbar.update(len(data)) - - # Retrieve the file with progress callback - ftp.retrbinary(f"RETR {ftp_file_path}", callback) - - logging.info(f"Successfully downloaded {new_file_path}") - break # Exit download retry loop if successful - except ( - socket.timeout, - ftplib.error_temp, - ftplib.error_perm, - ) as e: - download_attempt += 1 - logging.error( - f"Download failed for {new_file_path} (attempt {download_attempt}): {str(e)}" - ) - if download_attempt >= max_download_retries: - logging.error( - f"Giving up on {new_file_path} after {max_download_retries} attempts." - ) - break # Give up on this file after max retries - except (KeyError, IndexError) as e: - logging.error(f"Failed to process file due to missing data: {str(e)}") - except Exception as e: - logging.error(f"Unexpected error while processing file: {str(e)}") - ftp.quit() # Close FTP connection after all files are downloaded - logging.info(f"Disconnected from FTP host: {Files.PRIDE_ARCHIVE_FTP}") - break # Exit connection retry loop if everything was successful - except ( - socket.timeout, - ftplib.error_temp, - ftplib.error_perm, - socket.error, - ) as e: - connection_attempt += 1 - logging.error(f"FTP connection failed (attempt {connection_attempt}): {str(e)}") - if connection_attempt < max_connection_retries: - logging.info("Retrying connection...") - time.sleep(5) # Optional delay before retrying - else: - logging.error( - f"Giving up after {max_connection_retries} failed connection attempts." - ) - break - - @staticmethod - def get_output_file_name(download_url, file, output_folder): - public_filepath_part = download_url.rsplit("/", 1) - accession = file.get("accession", "unknown-accession") - logging.debug(accession + " -> " + public_filepath_part[1]) - new_file_path = os.path.join(output_folder, f"{public_filepath_part[1]}") - return new_file_path - - @staticmethod - def download_files_from_aspera( - file_list_json: List[Dict], - output_folder: str, - skip_if_downloaded_already, - maximum_bandwidth: str = "100M", - ): - """ - Download files using aspera transfer url - :param file_list_json: file list in json format - :param output_folder: folder to download the files - :param maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - """ - ascp_path = Files.get_ascp_binary() - key_full_path = importlib.resources.files("pridepy").joinpath( - "aspera/key/asperaweb_id_dsa.openssh" - ) - key_path = os.path.abspath(key_full_path) - for file in file_list_json: - if file["publicFileLocations"][0]["name"] == "Aspera Protocol": - download_url = file["publicFileLocations"][0]["value"] - else: - download_url = file["publicFileLocations"][1]["value"] - - # Create a clean filename to save the downloaded file - logging.debug(f"Downloading via Aspera: {download_url}") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) - - if skip_if_downloaded_already == True and os.path.exists(new_file_path): - logging.info("Skipping download as file already exists") - continue - - try: - # Execute the ascp command using subprocess - subprocess.run( - [ - ascp_path, - "-QT", - "-P", - "33001", - "-l", - maximum_bandwidth, # Options for Aspera: adjust as necessary - "-i", - key_path, - download_url, - new_file_path, # Source and destination - ], - check=True, - ) - logging.info(f"Successfully downloaded {new_file_path} via Aspera") - except subprocess.CalledProcessError as e: - logging.error(f"Aspera download failed for {new_file_path}: {str(e)}") - - @staticmethod - def _download_range(url, file_path, start, end, pbar, max_retries=3): - """Download a byte range directly into the target file using seek.""" - for attempt in range(1, max_retries + 1): - try: - session = Util.create_session_with_retries() - headers = {"Range": f"bytes={start}-{end}"} - with session.get(url, headers=headers, stream=True, timeout=(15, 15)) as r: - r.raise_for_status() - if r.status_code != 206: - raise RuntimeError(f"Server did not honor Range request: {r.status_code}") - content_range = r.headers.get("Content-Range", "") - if not content_range.lower().startswith(f"bytes {start}-{end}/"): - raise RuntimeError(f"Unexpected Content-Range header: {content_range}") - with open(file_path, "r+b") as f: - f.seek(start) - for chunk in r.iter_content(chunk_size=8 * 1024 * 1024): - if chunk: - f.write(chunk) - pbar.update(len(chunk)) - return - except (requests.RequestException, RuntimeError, OSError) as exc: - logging.warning( - f"Range {start}-{end} attempt {attempt}/{max_retries} failed: {exc}" - ) - if attempt >= max_retries: - raise - time.sleep(2 * attempt) - - @staticmethod - def _parallel_download(url, file_path, position=0): - """Download a file via a single-connection HTTP stream with optional resume. - If a partial file exists and the server supports Range requests, resumes - from where it left off; otherwise restarts from scratch.""" - session = Util.create_session_with_retries() - try: - head = session.head(url, timeout=(30, 30)) - head.raise_for_status() - total_size = int(head.headers.get("content-length", 0)) - accept_ranges = head.headers.get("accept-ranges", "none").strip().lower() - except (requests.RequestException, ValueError) as exc: - logging.info(f"HEAD request failed, falling back to single connection: {exc}") - total_size = 0 - accept_ranges = "none" - - resume_size = 0 - if os.path.exists(file_path) and accept_ranges == "bytes" and total_size > 0: - resume_size = os.path.getsize(file_path) - if resume_size >= total_size: - logging.info(f"File already complete: {file_path}") - return - if resume_size > 0: - logging.info(f"Resuming download from {resume_size} bytes: {file_path}") - - headers = {"Range": f"bytes={resume_size}-"} if resume_size > 0 else {} - with session.get(url, headers=headers, stream=True, timeout=(30, 60)) as r: - r.raise_for_status() - if resume_size > 0 and r.status_code != 206: - logging.warning("Server did not honor Range request (status %s), restarting download", r.status_code) - resume_size = 0 - with tqdm(total=total_size, unit="B", unit_scale=True, desc=file_path, - initial=resume_size, position=position, leave=True) as pbar: - mode = "ab" if resume_size > 0 else "wb" - with open(file_path, mode, buffering=8 * 1024 * 1024) as f: - for chunk in r.iter_content(chunk_size=8 * 1024 * 1024): - if chunk: - f.write(chunk) - pbar.update(len(chunk)) - - @staticmethod - def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_retries=6, position=0): - """Download a single file via globus; used as a worker target.""" - download_url = Files._get_download_url(file, "globus") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) - - if skip_if_downloaded_already and os.path.exists(new_file_path): - logging.info(f"Skipping download as file already exists: {new_file_path}") - return - - for attempt in range(1, max_retries + 1): - try: - Files._parallel_download(download_url, new_file_path, position=position) - return - except Exception as e: - logging.warning(f"Attempt {attempt}/{max_retries} failed for {file.get('fileName', '?')}: {e}") - if attempt == max_retries: - raise - - @staticmethod - def download_files_from_globus( - file_list_json: List[Dict], output_folder, skip_if_downloaded_already, - parallel_files: int = 1, - checksum_map: Optional[Dict[str, str]] = None, - ): - """ - Download files using globus transfer url with progress bar for each file. - When skip_if_downloaded_already is True, files are pre-filtered so that - only missing or incomplete files are submitted to the worker pool, - ensuring the -w parallel_files parameter is fully utilised. - When checksum_map is provided, existing files are validated against - their expected checksum; corrupted files are re-downloaded. - :param file_list_json: file list in json format - :param output_folder: folder to download the files - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - :param parallel_files: number of files to download simultaneously - :param checksum_map: mapping of file name to expected MD5 checksum - """ - if checksum_map is None: - checksum_map = {} - - if not (os.path.isdir(output_folder)): - os.makedirs(output_folder, exist_ok=True) - - # --- Phase 0: pre-filter files that need downloading ----------------- - files_to_download: List[Dict] = [] - for file in file_list_json: - download_url = Files._get_download_url(file, "globus") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) - if skip_if_downloaded_already and os.path.exists(new_file_path): - expected_cs = checksum_map.get(file.get("fileName", "")) - if expected_cs: - valid, reason = Files.validate_download(new_file_path, expected_cs) - if not valid: - logging.warning(f"Corrupted file detected ({reason}), will re-download: {new_file_path}") - files_to_download.append(file) - continue - logging.info(f"Skipping download as file already exists: {new_file_path}") - continue - files_to_download.append(file) - - if not files_to_download: - logging.info("All files already downloaded, nothing to do.") - return - - logging.info( - f"{len(file_list_json) - len(files_to_download)} file(s) skipped, " - f"{len(files_to_download)} file(s) to download" - ) - - # --- Phase 1: download (skip check already done, pass False) --------- - parallel_files = min(parallel_files, 3, len(files_to_download)) - if parallel_files < 2: - for file in files_to_download: - try: - Files._globus_download_one( - file, output_folder, False - ) - new_file_path = Files.get_output_file_name( - Files._get_download_url(file, "globus"), file, output_folder - ) - logging.info(f"Successfully downloaded {new_file_path}") - except Exception as e: - logging.error(f"Download from Globus failed: {str(e)}") - else: - logging.info(f"Downloading {len(files_to_download)} file(s) with {parallel_files} parallel workers") - with ThreadPoolExecutor(max_workers=parallel_files) as executor: - futures = { - executor.submit( - Files._globus_download_one, - file, output_folder, False, - position=idx, - ): file - for idx, file in enumerate(files_to_download) - } - for future in as_completed(futures): - try: - future.result() - except Exception as e: - logging.error(f"Download from Globus failed: {str(e)}") - - @staticmethod - def download_files_from_s3( - file_list_json: List[Dict], output_folder: str, skip_if_downloaded_already - ): - """ - Download files using S3 transfer URL with a progress bar and retry logic. - :param file_list_json: file list in JSON format - :param output_folder: folder to download the files - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - """ - - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - # Retry and timeout config - retry_config = Config( - retries={"max_attempts": 5, "mode": "standard"}, - connect_timeout=120, # Increase timeout to 120 seconds - read_timeout=120, # Timeout for reading data - signature_version=botocore.UNSIGNED, # Unsigned requests for public data - ) - - s3_resource = boto3.resource( - "s3", - config=retry_config, - endpoint_url=Files.S3_URL, - ) - bucket = s3_resource.Bucket(Files.S3_BUCKET) - - for file in file_list_json: - try: - # Determine S3 or FTP path - download_url = ( - file["publicFileLocations"][0]["value"] - if file["publicFileLocations"][0]["name"] == "FTP Protocol" - else file["publicFileLocations"][1]["value"] - ) - - ftp_base_url = "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/" - s3_path = download_url.replace(ftp_base_url, "") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) - - if skip_if_downloaded_already == True and os.path.exists(new_file_path): - logging.info("Skipping download as file already exists") - continue - - logging.debug(f"Downloading From S3: {s3_path}") - - # Get file size for progress tracking - obj = bucket.Object(s3_path) - total_size = obj.content_length - - # Initialize progress bar - progress = Progress(total_size, new_file_path) - - # Download with progress bar and retry handling - for attempt in range(5): - try: - bucket.download_file(s3_path, new_file_path, Callback=progress) - progress.close() - logging.info(f"Successfully downloaded {new_file_path}") - break - except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] == "404": - logging.error("The object does not exist.") - break - else: - logging.error(f"Download failed: {e}") - if attempt < 4: - time.sleep(2**attempt) # Exponential backoff - logging.info(f"Retrying... ({attempt + 1}/5)") - else: - raise - except Exception as e: - logging.error(f"Failed to download {file['fileName']}: {e}") - - def get_submitted_file_path_prefix(self, accession): - """ - At pride repository, public data is disseminated according to a proper structure. - I.e. base/path/ + yyyy/mm/accession/ + submitted/ - This extracts the yyyy/mm/accession path fragment from the API by examine the file path - of a public file. - I.e. ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2018/10/PXD008644/7550GI_Y.raw - :param accession: PRIDE accession - :return: path fragment (eg: 2018/10/PXD008644) - """ - results = self.get_all_raw_file_list(accession) - first_file = results[0]["publicFileLocations"][0]["value"] - path_fragment = re.search(r"\d{4}/\d{2}/PXD\d*", first_file).group() - return path_fragment - - def download_file_by_name( - self, - accession, - file_name, - output_folder, - skip_if_downloaded_already, - protocol, - username, - password, - aspera_maximum_bandwidth, - checksum_check, - ): - """ - Download files from url - :param accession: PRIDE accession - :param file_name: file name to download - :param output_folder: folder to download the files - :param protocol: ftp, aspera, globus - :param username: Username for private datasets - :param password: Password for private datasets - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - :param aspera_maximum_bandwidth: Aspera maximum bandwidth - :param checksum_check: Download checksum for a given project. - """ - - if not (os.path.isdir(output_folder)): - os.mkdir(output_folder) - - ## Check type of project - if self.is_massive_accession(accession): - logging.info("Downloading file from public MassIVE dataset {}".format(accession)) - response = self.get_file_from_api(accession, file_name) - if not response: - raise Exception( - "File name {} not found in MassIVE dataset {}".format(file_name, accession) - ) - self._download_massive_file_records( - accession=accession, - file_records=response, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - ) - return - - public_project = False - project_status = Util.get_api_call(self.API_BASE_URL + "/status/{}".format(accession)) - - if project_status.status_code == 200: - if project_status.text == "PRIVATE": - public_project = False - elif project_status.text == "PUBLIC": - public_project = True - else: - raise Exception("Dataset {} is not present in PRIDE Archive".format(accession)) - - if public_project: - logging.info("Downloading file from public dataset {}".format(accession)) - response = self.get_file_from_api(accession, file_name) - self.download_files( - response, - accession, - output_folder, - skip_if_downloaded_already, - protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, - ) - elif not public_project and (username is not None and password is not None): - logging.info("Downloading file from private dataset {}".format(accession)) - self.download_private_file_name( - accession=accession, - file_name=file_name, - output_folder=output_folder, - username=username, - password=password, - ) - else: - logging.error( - "For a private dataset {} you must provide a username and password".format( - accession - ) - ) - raise Exception( - "For a private dataset {} you must provide a username and password".format( - accession - ) - ) - - def get_file_from_api(self, accession, file_name) -> List[Dict]: - """ - Fetches file from API - :param accession: PRIDE accession - :param file_name: file name - :return: file in json format - """ - - try: - if self.is_massive_accession(accession): - files = self._list_massive_public_files(accession) - return [f for f in files if f["fileName"] == file_name] - files = self.stream_all_files_by_project(accession) - file = [f for f in files if f["fileName"] == file_name] - return file - except Exception as e: - raise Exception("File not found " + str(e)) - - def download_private_file_name(self, accession, file_name, output_folder, username, password): - """ - Get the information for a given private file to be downloaded from the api. - :param accession: Project accession - :param file_name: The file name to be downloaded - :param username: Username with access to the dataset - :param password: Password for user with access to the dataset - """ - - auth = Authentication() - auth_token = auth.get_token(username, password) - validate_token = auth.validate_token(auth_token) - logging.info("Valid token after login: {}".format(validate_token)) - - url = self.API_PRIVATE_URL + "/projects/{}/files?search={}".format(accession, file_name) - content = requests.get(url, headers={"Authorization": "Bearer {}".format(auth_token)}) - if content.ok and content.status_code == 200: - json_file = content.json() - if ( - "_embedded" in json_file - and "files" in json_file["_embedded"] - and len(json_file["_embedded"]["files"]) == 1 - ): - download_url = json_file["_embedded"]["files"][0]["_links"]["download"]["href"] - logging.info(download_url) - - # Create a clean filename to save the downloaded file - new_file_path = os.path.join(output_folder, f"{file_name}") - - session = Util.create_session_with_retries() # Create session with retries - # Check if the file already exists - if os.path.exists(new_file_path): - resume_header = {"Range": f"bytes={os.path.getsize(new_file_path)}-"} - mode = "ab" # Append to file - resume_size = os.path.getsize(new_file_path) - else: - resume_header = {} - mode = "wb" # Write new file - resume_size = 0 - - with session.get( - download_url, stream=True, headers=resume_header, timeout=(10, 60) - ) as r: - r.raise_for_status() - total_size = int(r.headers.get("content-length", 0)) + resume_size - block_size = 1024 * 1024 # 1 MB chunks - - with tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=new_file_path, - initial=resume_size, - ) as pbar: - with open(new_file_path, mode) as f: - for chunk in r.iter_content(chunk_size=block_size): - if chunk: - f.write(chunk) - pbar.update(len(chunk)) - - logging.info(f"Successfully downloaded {new_file_path}") - - else: - logging.info( - "File name {} found more than once for the given project {}".format( - file_name, accession - ) - ) - else: - logging.info( - f"File name {file_name} now found in the project {accession}, or user don't have access" - ) - raise Exception( - f"File name {file_name} now found in the project {accession}, or user don't have access" - ) - - @staticmethod - def get_ascp_binary(): - """ - Detect the OS and architecture, and return the appropriate ascp binary path. - - Returns: - str: Path to the correct ascp binary. - """ - os_type = platform.system().lower() - arch, _ = platform.architecture() - aspera_dir = importlib.resources.files("pridepy").joinpath("aspera/") - - if os_type == "linux": - if arch == "32bit": - return os.path.join(aspera_dir, "linux-32", "ascp") - elif arch == "64bit": - return os.path.join(aspera_dir, "linux-64", "ascp") - elif os_type == "darwin": # macOS (intel-based) - return os.path.join(aspera_dir, "mac-intel", "ascp") - elif os_type == "windows": - if arch == "32bit": - return os.path.join(aspera_dir, "windows-32", "ascp.exe") - elif arch == "64bit": - return os.path.join(aspera_dir, "windows-64", "ascp.exe") - else: - raise OSError(f"Unsupported OS or architecture: {os_type}, {arch}") - - @staticmethod - def save_checksum_file(accession, output_folder): - """ - Download and persist the checksum manifest for a PRIDE accession. - """ - os.makedirs(output_folder, exist_ok=True) - url = f"{Files.V3_API_BASE_URL}/files/checksum/{accession}" - headers = {"accept": "text/plain"} - request = urllib.request.Request(url, headers=headers, method="GET") - logging.info(f"Fetching checksum file from {url}") - with urllib.request.urlopen(request) as response: - data = response.read().decode("utf-8") - # Save the data to a .tsv file - output_path = os.path.join(output_folder, f"{accession}-checksum.tsv") - with open(output_path, "w", encoding="utf-8") as file: - file.write(data) - return output_path - - @staticmethod - def _batch_download_by_protocol( - file_list: List[Dict], - output_folder: str, - protocol: str, - skip_if_downloaded_already: bool, - aspera_maximum_bandwidth: str, - parallel_files: int = 1, - checksum_map: Optional[Dict[str, str]] = None, - ) -> None: - """ - Transfer a batch of files with one protocol, reusing a single - connection where the underlying helper supports it (FTP, S3). - """ - if not file_list: - return - if protocol == "ftp": - Files.download_files_from_ftp( - file_list, - output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - ) - return - if protocol == "aspera": - Files.download_files_from_aspera( - file_list, - output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - maximum_bandwidth=aspera_maximum_bandwidth, - ) - return - if protocol == "globus": - Files.download_files_from_globus( - file_list, - output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - parallel_files=parallel_files, - checksum_map=checksum_map or {}, - ) - return - if protocol == "s3": - Files.download_files_from_s3( - file_list, - output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - ) - return - raise ValueError(f"Unsupported protocol: {protocol}") - - @staticmethod - def _download_with_fallback( - file_record: Dict, - output_folder: str, - protocol_sequence: List[str], - expected_checksum: Optional[str], - aspera_maximum_bandwidth: str, - max_protocol_retries: int = 2, - parallel_files: int = 1, - ) -> bool: - """ - Download one file by trying each protocol in sequence, validating - after every attempt. Intended as the per-file fallback path; batch - download of the primary protocol is handled separately. - """ - local_path = Files._resolve_local_path(file_record, output_folder) - - for protocol in protocol_sequence: - for attempt in range(1, max_protocol_retries + 1): - logging.info( - f"Downloading {file_record['fileName']} via {protocol} " - f"(attempt {attempt}/{max_protocol_retries})" - ) - try: - Files._remove_if_exists(local_path) - Files._batch_download_by_protocol( - [file_record], - output_folder, - protocol, - skip_if_downloaded_already=False, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - parallel_files=parallel_files, - ) - except Exception as error: - logging.error( - f"Protocol {protocol} failed for {file_record['fileName']}: {error}" - ) - - valid, reason = Files.validate_download(local_path, expected_checksum) - if valid: - logging.info( - f"File {file_record['fileName']} downloaded successfully via {protocol}" - ) - return True - - logging.warning( - f"Validation failed for {file_record['fileName']} via {protocol}: {reason}" - ) - Files._remove_if_exists(local_path) - - logging.warning( - f"Protocol {protocol} exhausted for {file_record['fileName']}, switching protocol." - ) - - logging.error(f"All protocol attempts failed for {file_record['fileName']}") - return False - - @staticmethod - def download_files( - file_list_json: List[Dict], - accession, - output_folder: str, - skip_if_downloaded_already, - protocol: str = "ftp", - aspera_maximum_bandwidth: str = "100M", # Aspera maximum bandwidth - checksum_check=False, - parallel_files: int = 1, - ): - """ - Download files using either FTP or Aspera transfer protocol. - :param file_list_json: File list in JSON format - :param accession: Project accession - :param output_folder: Folder to download the files - :param protocol: ftp, aspera, globus - :param aspera_maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - """ - protocols_supported = ["ftp", "aspera", "globus", "s3"] - if protocol not in protocols_supported: - logging.error("Protocol should be one of ftp, aspera, globus, s3") - return - - os.makedirs(output_folder, exist_ok=True) - - checksum_map: Dict[str, str] = {} - if checksum_check: - checksum_file_path = Files.save_checksum_file(accession, output_folder) - checksum_map = Files.read_checksum_file(checksum_file_path) - logging.info(f"Loaded checksums for {len(checksum_map)} files") - - if not file_list_json: - return - - protocol_sequence = Files._protocol_sequence(protocol) - primary_protocol = protocol_sequence[0] - # Retry with the primary protocol first, then fall back to others - fallback_sequence = protocol_sequence - - # Phase 1: batch download with the requested protocol. Reuses a single - # FTP/S3 connection for all files (the previous behaviour) instead of - # paying the per-file reconnect cost in the common happy path. - logging.info( - f"Downloading {len(file_list_json)} file(s) via {primary_protocol} (batch)" - ) - try: - Files._batch_download_by_protocol( - file_list_json, - output_folder, - primary_protocol, - skip_if_downloaded_already=skip_if_downloaded_already, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - parallel_files=parallel_files, - checksum_map=checksum_map, - ) - except Exception as exc: - logging.warning( - f"Batch {primary_protocol} run hit an error; will retry individual failures: {exc}" - ) - - # Phase 2: validate every file and fall back per-file for the ones - # that are missing or invalid. - logging.info("Phase 2: validating %d downloaded file(s)", len(file_list_json)) - failed_files: List[str] = [] - for i, file_record in enumerate(file_list_json, 1): - expected_checksum = checksum_map.get(file_record["fileName"]) - local_path = Files._resolve_local_path(file_record, output_folder) - logging.info("Validating [%d/%d] %s", i, len(file_list_json), file_record["fileName"]) - valid, reason = Files.validate_download(local_path, expected_checksum) - if valid: - continue - - logging.warning( - f"{file_record['fileName']} invalid after {primary_protocol} ({reason})" - ) - if "checksum mismatch" in reason: - Files._remove_if_exists(local_path) - - if not fallback_sequence: - failed_files.append(file_record.get("fileName", "")) - continue - - success = Files._download_with_fallback( - file_record=file_record, - output_folder=output_folder, - protocol_sequence=fallback_sequence, - expected_checksum=expected_checksum, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - parallel_files=parallel_files, - ) - if not success: - failed_files.append(file_record.get("fileName", "")) - - if failed_files: - failed_summary = ", ".join(failed_files) - logging.error(f"Failed to download {len(failed_files)} file(s): {failed_summary}") - raise RuntimeError(f"Failed to download {len(failed_files)} file(s): {failed_summary}") - - def download_files_by_list( - self, - accession: str, - file_names: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str = "ftp", - aspera_maximum_bandwidth: str = "100M", - checksum_check: bool = False, - parallel_files: int = 1, - ) -> None: - """Download a subset of project files identified by a filename list. - - Resolves each requested filename via the project metadata API and - delegates to :meth:`download_files` so the existing batch + protocol - fallback engine is reused. - - :param accession: PRIDE or MassIVE project accession (public) - :param file_names: filenames to download - :param output_folder: directory to write downloaded files into - :param skip_if_downloaded_already: skip files already present locally - :param protocol: preferred protocol; falls back across others on failure - :param aspera_maximum_bandwidth: aspera ascp bandwidth cap - :param checksum_check: download project checksums and validate - :param parallel_files: number of files to download simultaneously for globus - :raises ValueError: if ``file_names`` is empty or none match the project - """ - if not file_names: - raise ValueError("file_names must contain at least one filename") - - if self.is_massive_accession(accession): - all_files = self._list_massive_public_files(accession) - else: - all_files = self.stream_all_files_by_project(accession) - requested = set(file_names) - matched = [f for f in all_files if f.get("fileName") in requested] - missing = sorted(requested - {f.get("fileName") for f in matched}) - if missing: - logging.warning("Files not found in project %s: %s", accession, missing) - if not matched: - raise ValueError( - f"No matching files in project {accession} for: {sorted(requested)}" - ) - - if self.is_massive_accession(accession): - self._download_massive_file_records( - accession=accession, - file_records=matched, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - ) - return - - self.download_files( - matched, - accession, - output_folder, - skip_if_downloaded_already, - protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, - parallel_files=parallel_files, - ) - - @staticmethod - def _extract_pride_accession(url: str) -> Optional[str]: - """Extract a PRIDE accession (PXD/PRD followed by digits) from a URL path. - - PRIDE archive URLs follow the pattern - ``…/pride/data/archive/YYYY/MM//filename``. - Returns ``None`` when no accession can be identified. - """ - match = re.search(r"((?:PXD|PRD)\d{4,})", url) - return match.group(1) if match else None - - @staticmethod - def download_files_by_url( - urls: List[str], - output_folder: str, - skip_if_downloaded_already: bool = False, - protocol: str = "ftp", - parallel_files: int = 1, - checksum_check: bool = False, - ) -> None: - """Download files from a list of raw URLs, dispatched by URL scheme. - - Supported schemes: ``http``, ``https``, ``ftp``. Each URL is downloaded - independently; per-URL errors are logged, then aggregated and re-raised - as a single :class:`RuntimeError` so callers see a complete failure - summary. - - :param urls: fully-qualified URLs (each contains its scheme) - :param output_folder: directory to write downloaded files into - :param skip_if_downloaded_already: skip URLs whose target file exists - :param protocol: ``ftp`` (default) for single-connection per URL scheme; - ``globus`` for resume-capable http/https downloads (single-connection stream) - (no effect on ftp:// URLs which always use single-connection FTP) - :param checksum_check: validate downloads against PRIDE checksum API; - accessions are inferred from URL paths (only PRIDE URLs supported) - :raises ValueError: if ``urls`` is empty - :raises RuntimeError: if one or more URLs failed - """ - if not urls: - raise ValueError("urls must contain at least one URL") - - os.makedirs(output_folder, exist_ok=True) - - parallel_files = min(parallel_files, 3, len(urls)) - failures: List[Tuple[str, str]] = [] - if parallel_files < 2: - for url in urls: - try: - Files._download_single_url( - url, output_folder, skip_if_downloaded_already, protocol, - ) - except Exception as exc: # pylint: disable=broad-except - logging.error("Failed to download %s: %s", url, exc) - failures.append((url, str(exc))) - else: - logging.info( - "Downloading %d URL(s) with %d parallel workers", - len(urls), parallel_files, - ) - with ThreadPoolExecutor(max_workers=parallel_files) as executor: - futures = { - executor.submit( - Files._download_single_url, - url, output_folder, skip_if_downloaded_already, protocol, - position=idx, - ): url - for idx, url in enumerate(urls) - } - for future in as_completed(futures): - url = futures[future] - try: - future.result() - except Exception as exc: # pylint: disable=broad-except - logging.error("Failed to download %s: %s", url, exc) - failures.append((url, str(exc))) - - if failures: - summary = ", ".join(f"{u} ({e})" for u, e in failures) - raise RuntimeError( - f"Failed to download {len(failures)} URL(s): {summary}" - ) - - if checksum_check: - Files._validate_urls_checksums(urls, output_folder) - - @staticmethod - def _validate_urls_checksums(urls: List[str], output_folder: str) -> None: - """Validate downloaded files against PRIDE checksum API. - - Accessions are inferred from URL paths via - :meth:`_extract_pride_accession`. URLs that do not contain a - recognisable PRIDE accession are skipped with a warning. - - :raises RuntimeError: if one or more files fail validation - """ - accession_urls: Dict[str, List[str]] = {} - for url in urls: - acc = Files._extract_pride_accession(url) - if acc: - accession_urls.setdefault(acc, []).append(url) - else: - logging.warning( - "Cannot infer PRIDE accession from URL, skipping checksum: %s", url - ) - - validation_failures: List[str] = [] - for acc, acc_urls in accession_urls.items(): - checksum_file_path = Files.save_checksum_file(acc, output_folder) - checksum_map = Files.read_checksum_file(checksum_file_path) - logging.info( - "Loaded checksums for %d files (project %s)", - len(checksum_map), acc, - ) - for url in acc_urls: - file_name = os.path.basename(urlparse(url).path) - target = os.path.join(output_folder, file_name) - expected = checksum_map.get(file_name) - logging.info("Validating %s", file_name) - valid, reason = Files.validate_download(target, expected) - if not valid: - logging.error("Validation failed for %s: %s", file_name, reason) - validation_failures.append(f"{file_name} ({reason})") - else: - logging.info("Checksum OK: %s", file_name) - - if validation_failures: - raise RuntimeError( - f"Checksum validation failed for {len(validation_failures)} file(s): " - + ", ".join(validation_failures) - ) - - @staticmethod - def _download_single_url( - url: str, - output_folder: str, - skip_if_exists: bool = False, - protocol: str = "ftp", - position: int = 0, - ) -> str: - """Download one URL, dispatched by scheme; return the local file path.""" - parsed = urlparse(url) - if not (parsed.scheme or "").lower(): - raise ValueError(f"URL missing scheme: {url}") - - file_name = os.path.basename(parsed.path) - if not file_name: - raise ValueError(f"Cannot derive filename from URL: {url}") - - target = os.path.join(output_folder, file_name) - if skip_if_exists and os.path.isfile(target) and os.path.getsize(target) > 0: - logging.info("Skipping %s: already downloaded", file_name) - return target - - Files._dispatch_url_scheme(parsed, target, protocol, position=position) - - ok, reason = Files.validate_download(target) - if not ok: - Files._remove_if_exists(target) - raise RuntimeError(f"Download invalid: {reason} ({target})") - return target - - @staticmethod - def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: int = 0) -> None: - """Route a parsed URL to its protocol-specific downloader. - - ``protocol='globus'`` swaps the http/https single-connection streamer - for :meth:`_parallel_download` (single-connection with progress bar). - ftp:// URLs are unaffected. - """ - scheme = (parsed.scheme or "").lower() - if scheme in ("http", "https"): - if protocol == "globus": - Files._parallel_download(parsed.geturl(), target, position=position) - else: - Files._http_download_url(parsed.geturl(), target) - elif scheme == "ftp": - Files._ftp_download_url(parsed, target) - else: - raise ValueError(f"Unsupported URL scheme: {scheme}") - - @staticmethod - def _http_download_url(url: str, target: str) -> None: - """Stream an http/https URL into ``target`` with a progress bar.""" - session = Util.create_session_with_retries() - with session.get(url, stream=True, timeout=60) as response: - response.raise_for_status() - total = int(response.headers.get("Content-Length", 0)) - with open(target, "wb") as out, tqdm( - total=total, - unit="B", - unit_scale=True, - desc=os.path.basename(target), - ) as pbar: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - out.write(chunk) - pbar.update(len(chunk)) - - @staticmethod - def _ftp_download_url(parsed, target: str) -> None: - """Download a single file from an ftp:// URL with a progress bar.""" - host = parsed.hostname - if not host: - raise ValueError(f"FTP URL missing host: {parsed.geturl()}") - port = parsed.port or 21 - user = parsed.username or "anonymous" - pwd = parsed.password or "anonymous@" - remote_path = parsed.path - with FTP() as ftp: - ftp.connect(host, port, timeout=60) - ftp.login(user, pwd) - try: - total = ftp.size(remote_path) or 0 - except ftplib.error_perm: - total = 0 - with open(target, "wb") as out, tqdm( - total=total, - unit="B", - unit_scale=True, - desc=os.path.basename(target), - ) as pbar: - - def _callback(data: bytes) -> None: - out.write(data) - pbar.update(len(data)) - - ftp.retrbinary(f"RETR {remote_path}", _callback) - - def download_all_category_files( - self, - accession: str, - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str, - aspera_maximum_bandwidth: str, - checksum_check: bool, - categories: List[str] = None, - category: str = None, - parallel_files: int = 1, - ): - """ - Download all files of specified categories from a PRIDE project. - - :param accession: The PRIDE project accession identifier. - :param output_folder: The directory where the files will be downloaded. - :param skip_if_downloaded_already: If True, skips downloading files that already exist. - :param protocol: The transfer protocol to use (e.g., ftp, aspera, globus, s3). - :param aspera_maximum_bandwidth: Maximum bandwidth for Aspera transfers. - :param checksum_check: If True, downloads the checksum file for the project. - :param categories: List of file categories to download. - :param category: Single file category (deprecated, use categories instead). - """ - if categories is None: - categories = [category] if category else ["RAW"] - raw_files = self.get_all_category_file_list(accession, categories) - if self.is_massive_accession(accession): - self._download_massive_file_records( - accession=accession, - file_records=raw_files, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - ) - return - self.download_files( - raw_files, - accession, - output_folder, - skip_if_downloaded_already, - protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, - parallel_files=parallel_files, - ) - - def get_all_category_file_list( - self, accession: str, categories: "str | List[str]" - ) -> List[Dict]: - """ - Retrieve a list of files from a specific project that belong to given categories. - - :param accession: The PRIDE project accession identifier. - :param categories: A single category string or list of categories to filter by. - :return: A list of files matching the specified categories. - """ - if isinstance(categories, str): - categories = [categories] - category_set = {category.upper() for category in categories} - - if self.is_massive_accession(accession): - record_files = self._list_massive_public_files(accession) - else: - record_files = self.stream_all_files_by_project(accession) - - category_files = [ - file for file in record_files if file["fileCategory"]["value"] in category_set - ] - return category_files - - # ------------------------------- - # ProteomeXchange support - # ------------------------------- - - @staticmethod - def _normalize_px_xml_url(px_id_or_url: str) -> str: - """ - Build the ProteomeXchange XML endpoint from a dataset accession or a dataset web URL. - Examples accepted: - - PXD039236 - - https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236 - - https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236&anything - """ - if px_id_or_url.startswith("http://") or px_id_or_url.startswith("https://"): - parsed = urlparse(px_id_or_url) - # keep the ID param value if present; otherwise fallback to the path tail - query = parsed.query or "" - if "ID=" in query: - id_value = [q.split("=", 1)[1] for q in query.split("&") if q.startswith("ID=")] - if id_value: - return ( - f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID={id_value[0]}&outputMode=XML&test=no" - ) - # If the input URL already requests XML, just ensure flags - if parsed.path.endswith("/cgi/GetDataset"): - return ( - f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?{query}&outputMode=XML&test=no" - ) - # Assume it's a plain accession if not a URL - return ( - f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID={px_id_or_url}&outputMode=XML&test=no" - ) - - @staticmethod - def _parse_px_xml_for_raw_file_urls(px_xml_url: str) -> List[str]: - """ - Parse the PX XML and return a list of associated raw file URIs. - We extract cvParam with name "Associated raw file URI" under each DatasetFile. - """ - headers = {"Accept": "application/xml"} - response = Util.get_api_call(px_xml_url, headers) - response.raise_for_status() - root = ET.fromstring(response.content) - - urls: List[str] = [] - # The XML namespace is often absent in PX XML; access elements directly - for dataset_file in root.iter("DatasetFile"): - for cv in dataset_file.findall("cvParam"): - name = cv.attrib.get("name") - value = cv.attrib.get("value") - if name == "Associated raw file URI" and value: - urls.append(value) - return urls - - def download_px_raw_files( - self, - px_id_or_url: str, - output_folder: str, - skip_if_downloaded_already: bool = True, - ) -> None: - """ - Download all raw files referenced by a ProteomeXchange dataset. - Prefer FTP when the URL is ftp://, otherwise use HTTP(S). Supports resume and skip. - """ - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - px_xml_url = self._normalize_px_xml_url(px_id_or_url) - logging.info(f"Fetching PX XML: {px_xml_url}") - urls = self._parse_px_xml_for_raw_file_urls(px_xml_url) - if not urls: - logging.info("No Associated raw file URIs found in PX XML") - return - - ftp_urls = [u for u in urls if u.lower().startswith("ftp://")] - http_urls = [u for u in urls if u.lower().startswith("http://") or u.lower().startswith("https://")] - - if ftp_urls: - self.download_ftp_urls(ftp_urls, output_folder, skip_if_downloaded_already) - if http_urls: - self.download_http_urls(http_urls, output_folder, skip_if_downloaded_already) - - @staticmethod - def _local_path_for_url(download_url: str, output_folder: str) -> str: - filename = os.path.basename(urlparse(download_url).path) - return os.path.join(output_folder, filename) - - @staticmethod - def download_ftp_urls( - ftp_urls: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - max_connection_retries: int = 3, - max_download_retries: int = 3, - ) -> None: - """ - Download a list of FTP URLs using a single connection, with retries and progress bars. - """ - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - def connect_ftp(host: str): - ftp = FTP(host, timeout=30) - ftp.login() - ftp.set_pasv(True) - logging.info(f"Connected to FTP host: {host}") - return ftp - - # Group URLs by host to reuse connections efficiently - host_to_paths: Dict[str, List[str]] = {} - for url in ftp_urls: - parsed = urlparse(url) - host_to_paths.setdefault(parsed.hostname, []).append(parsed.path.lstrip("/")) - - for host, paths in host_to_paths.items(): - connection_attempt = 0 - while connection_attempt < max_connection_retries: - try: - ftp = connect_ftp(host) - for ftp_path in paths: - try: - local_path = os.path.join(output_folder, os.path.basename(ftp_path)) - if skip_if_downloaded_already and os.path.exists(local_path): - logging.info("Skipping download as file already exists") - continue - - logging.info(f"Starting FTP download: {host}/{ftp_path}") - download_attempt = 0 - while download_attempt < max_download_retries: - try: - total_size = ftp.size(ftp_path) - # Try to resume using REST if partial file exists - if os.path.exists(local_path): - current_size = os.path.getsize(local_path) - mode = "ab" - else: - current_size = 0 - mode = "wb" - - with open(local_path, mode) as f, tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=local_path, - initial=current_size, - ) as pbar: - def callback(data): - f.write(data) - pbar.update(len(data)) - - if current_size: - try: - ftp.sendcmd(f"REST {current_size}") - except Exception: - # If REST not supported, fall back to full download - current_size = 0 - f.seek(0) - f.truncate() - ftp.retrbinary(f"RETR {ftp_path}", callback) - logging.info(f"Successfully downloaded {local_path}") - break - except (socket.timeout, ftplib.error_temp, ftplib.error_perm) as e: - download_attempt += 1 - logging.error( - f"Download failed for {local_path} (attempt {download_attempt}): {str(e)}" - ) - if download_attempt >= max_download_retries: - logging.error( - f"Giving up on {local_path} after {max_download_retries} attempts." - ) - break - except Exception as e: - logging.error(f"Unexpected error while processing FTP path {ftp_path}: {str(e)}") - ftp.quit() - logging.info(f"Disconnected from FTP host: {host}") - break - except (socket.timeout, ftplib.error_temp, ftplib.error_perm, socket.error) as e: - connection_attempt += 1 - logging.error(f"FTP connection failed (attempt {connection_attempt}): {str(e)}") - if connection_attempt < max_connection_retries: - logging.info("Retrying connection...") - time.sleep(5) - else: - logging.error( - f"Giving up after {max_connection_retries} failed connection attempts to {host}." - ) - - @staticmethod - def download_http_urls( - http_urls: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - ) -> None: - """ - Download a list of HTTP(S) URLs with resume support and progress bars. - """ - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - session = Util.create_session_with_retries() - for url in http_urls: - try: - local_path = Files._local_path_for_url(url, output_folder) - if skip_if_downloaded_already and os.path.exists(local_path): - logging.info("Skipping download as file already exists") - continue - - if os.path.exists(local_path): - resume_size = os.path.getsize(local_path) - headers = {"Range": f"bytes={resume_size}-"} - mode = "ab" - else: - resume_size = 0 - headers = {} - mode = "wb" - - with session.get(url, stream=True, headers=headers, timeout=(10, 60)) as r: - r.raise_for_status() - total_size = int(r.headers.get("content-length", 0)) + resume_size - block_size = 1024 * 1024 - with tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=local_path, - initial=resume_size, - ) as pbar: - with open(local_path, mode) as f: - for chunk in r.iter_content(chunk_size=block_size): - if chunk: - f.write(chunk) - pbar.update(len(chunk)) - logging.info(f"Successfully downloaded {local_path}") - except Exception as e: - logging.error(f"HTTP download failed for {url}: {str(e)}") diff --git a/pridepy/pridepy.py b/pridepy/pridepy.py index 4929954..e307a61 100644 --- a/pridepy/pridepy.py +++ b/pridepy/pridepy.py @@ -2,7 +2,7 @@ import asyncio import logging import click -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.project.project import Project PROTOCOL_CHOICES = click.Choice(["ftp", "aspera", "globus", "s3"], case_sensitive=False) @@ -58,6 +58,13 @@ def main(): type=click.IntRange(1, 3), help="Number of files to download simultaneously (1-3). Primarily used by globus protocol. Default is 1.", ) +@click.option( + "--preserve-structure", + is_flag=True, + default=False, + help="Recreate the dataset's subdirectory layout under the output folder. " + "By default files are downloaded flat into the output folder.", +) def download_all_public_raw_files( accession, protocol, @@ -66,6 +73,7 @@ def download_all_public_raw_files( aspera_maximum_bandwidth: str = "50M", checksum_check: bool = False, parallel_files: int = 1, + preserve_structure: bool = False, ): """ Command to download all public raw files from a specified PRIDE or MassIVE dataset. @@ -95,6 +103,7 @@ def download_all_public_raw_files( aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, parallel_files=parallel_files, + flatten=not preserve_structure, ) @@ -149,6 +158,13 @@ def download_all_public_raw_files( type=click.IntRange(1, 3), help="Number of files to download simultaneously (1-3). Primarily used by globus protocol. Default is 1.", ) +@click.option( + "--preserve-structure", + is_flag=True, + default=False, + help="Recreate the dataset's subdirectory layout under the output folder. " + "By default files are downloaded flat into the output folder.", +) def download_all_public_category_files( accession: str, protocol: str, @@ -158,6 +174,7 @@ def download_all_public_category_files( checksum_check: bool = False, category: str = "RAW", parallel_files: int = 1, + preserve_structure: bool = False, ): """ Command to download all public files of a specified category from a given PRIDE or MassIVE dataset. @@ -198,6 +215,7 @@ def download_all_public_category_files( checksum_check=checksum_check, categories=categories, parallel_files=parallel_files, + flatten=not preserve_structure, ) @@ -258,10 +276,12 @@ def download_file_by_name( :param protocol: Protocol to use for download: ftp, aspera, globus, s3. Default is ftp. :param file_name: fileName to be downloaded :param output_folder: output folder to download or copy files - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. Default is False. + :param skip_if_downloaded_already: Boolean value to skip the download if the + file has already been downloaded. Default is False. :param username: PRIDE login username for private files :param password: PRIDE login password for private files - :param aspera_maximum_bandwidth: Aspera maximum bandwidth (e.g 50M, 100M, 200M), depending on the user's network bandwidth, default is 100M + :param aspera_maximum_bandwidth: Aspera maximum bandwidth (e.g 50M, 100M, + 200M), depending on the user's network bandwidth, default is 100M :param checksum_check: Download checksum file for project. """ @@ -309,11 +329,28 @@ def download_file_by_name( default=False, help="Skip the download if the file has already been downloaded.", ) -def download_px_raw_files(accession: str, output_folder: str, skip_if_downloaded_already: bool): +@click.option( + "--preserve-structure", + is_flag=True, + default=False, + help="Recreate the dataset's subdirectory layout under the output folder. " + "By default files are downloaded flat into the output folder.", +) +def download_px_raw_files( + accession: str, + output_folder: str, + skip_if_downloaded_already: bool, + preserve_structure: bool = False, +): """CLI wrapper to download raw files via ProteomeXchange XML.""" files = Files() logging.info(f"PX accession/URL: {accession}") - files.download_px_raw_files(accession, output_folder, skip_if_downloaded_already) + files.download_px_raw_files( + accession, + output_folder, + skip_if_downloaded_already, + flatten=not preserve_structure, + ) @main.command("list-private-files", help="List private files by project accession") @@ -416,10 +453,10 @@ def stream_files_metadata(accession, output_file): "-sf", "--sort-fields", required=False, - default=["submission_date"], + default=["submissionDate"], multiple=True, help="Field(s) for sorting the results on. Default for this " - "request is submission_date. More fields can be separated by " + "request is submissionDate. More fields can be separated by " "comma and passed. Example: submissionDate,accession", type=click.Choice( "accession,submissionDate,diseases,organismsPart,organisms,instruments,softwares," @@ -561,6 +598,13 @@ def _read_url_arguments(url_list_path, urls_csv=None): type=click.IntRange(1, 3), help="Number of files to download simultaneously (1-3). Primarily used by globus protocol. Default is 1.", ) +@click.option( + "--preserve-structure", + is_flag=True, + default=False, + help="Recreate the dataset's subdirectory layout under the output folder. " + "By default files are downloaded flat into the output folder.", +) def download_files_by_list( accession, protocol, @@ -571,6 +615,7 @@ def download_files_by_list( aspera_maximum_bandwidth, checksum_check, parallel_files, + preserve_structure: bool = False, ): """Download a named subset of files from a PRIDE project.""" file_names = _read_filename_arguments(file_list_path, files_csv) @@ -586,6 +631,7 @@ def download_files_by_list( aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, parallel_files=parallel_files, + flatten=not preserve_structure, ) @@ -640,7 +686,7 @@ def download_files_by_list( "--parallel-files", default=1, type=click.IntRange(1, 3), - help="Number of files to download simultaneously (1-3). Primarily used by globus protocol. Default is 1.", + help="Number of files to download simultaneously (1-3), for any URL scheme. Default is 1.", ) def download_files_by_url( url_list_path, diff --git a/pridepy/tests/_live_api.py b/pridepy/tests/_live_api.py new file mode 100644 index 0000000..e04240f --- /dev/null +++ b/pridepy/tests/_live_api.py @@ -0,0 +1,34 @@ +"""Helpers for the live PRIDE-API integration tests. + +A handful of tests hit ``www.ebi.ac.uk`` directly (no mocking) to validate +real behaviour. That endpoint is occasionally slow or unreachable from CI +runners, which used to fail the build on a transient read timeout. The +:func:`tolerate_api_outage` context manager turns an API outage into a clean +skip instead of a failure, keeping CI deterministic while still exercising the +real API whenever it is available. +""" +import unittest +from contextlib import contextmanager + +import requests + + +@contextmanager +def tolerate_api_outage(testcase: unittest.TestCase): + """Skip (don't fail) the wrapped block when the live PRIDE API is down. + + Wrap only the live API call(s) and their assertions. An API outage surfaces + as one of: + * ``requests.RequestException`` — ``Util.get_api_call`` lets connection + / read timeouts propagate; + * ``RuntimeError`` — ``Provider._list_files_checked`` raises when the API + helper returned ``None``; + * ``TypeError`` — a helper that returns ``None`` on failure is then + iterated / measured (e.g. ``len(None)``). + Genuine assertion failures raise ``AssertionError``, which is *not* caught, + so real regressions still fail the test. + """ + try: + yield + except (requests.RequestException, RuntimeError, TypeError) as exc: + testcase.skipTest(f"PRIDE API unavailable: {type(exc).__name__}: {exc}") diff --git a/pridepy/tests/test_authentication.py b/pridepy/tests/test_authentication.py index d7f5e75..ec0a4f9 100644 --- a/pridepy/tests/test_authentication.py +++ b/pridepy/tests/test_authentication.py @@ -3,7 +3,7 @@ import pytest from pridepy.authentication import authentication -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.project.project import Project diff --git a/pridepy/tests/test_cli_flatten.py b/pridepy/tests/test_cli_flatten.py new file mode 100644 index 0000000..b068a40 --- /dev/null +++ b/pridepy/tests/test_cli_flatten.py @@ -0,0 +1,88 @@ +"""CLI wiring for the --preserve-structure flag. + +By default the download commands flatten into the output folder (flatten=True); +--preserve-structure flips that to flatten=False. +""" +from unittest import TestCase +from unittest.mock import patch + +from click.testing import CliRunner + +from pridepy.pridepy import main + + +class TestCliPreserveStructure(TestCase): + def _invoke(self, args): + return CliRunner().invoke(main, args, catch_exceptions=False) + + def test_download_all_public_raw_files_flattens_by_default(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + ["download-all-public-raw-files", "-a", "MSV000012345", "-o", "/tmp/x"] + ) + kwargs = files_cls.return_value.download_all_raw_files.call_args.kwargs + assert kwargs["flatten"] is True + + def test_download_all_public_raw_files_preserve_structure(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-all-public-raw-files", + "-a", + "MSV000012345", + "-o", + "/tmp/x", + "--preserve-structure", + ] + ) + kwargs = files_cls.return_value.download_all_raw_files.call_args.kwargs + assert kwargs["flatten"] is False + + def test_download_all_public_category_files_preserve_structure(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-all-public-category-files", + "-a", + "MSV000012345", + "-o", + "/tmp/x", + "-c", + "RAW", + "--preserve-structure", + ] + ) + kwargs = files_cls.return_value.download_all_category_files.call_args.kwargs + assert kwargs["flatten"] is False + + def test_download_files_by_list_preserve_structure(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-files-by-list", + "-a", + "MSV000012345", + "-o", + "/tmp/x", + "-f", + "a.raw", + "--preserve-structure", + ] + ) + kwargs = files_cls.return_value.download_files_by_list.call_args.kwargs + assert kwargs["flatten"] is False + + def test_download_px_raw_files_preserve_structure(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-px-raw-files", + "-a", + "PXD000001", + "-o", + "/tmp/x", + "--preserve-structure", + ] + ) + kwargs = files_cls.return_value.download_px_raw_files.call_args.kwargs + assert kwargs["flatten"] is False diff --git a/pridepy/tests/test_download_by_list.py b/pridepy/tests/test_download_by_list.py index df81914..649017e 100644 --- a/pridepy/tests/test_download_by_list.py +++ b/pridepy/tests/test_download_by_list.py @@ -12,8 +12,9 @@ import click import pytest -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.pridepy import _read_filename_arguments +from pridepy.download.pride import PrideProvider class TestDownloadFilesByList(TestCase): @@ -36,8 +37,8 @@ def test_filters_metadata_and_delegates(self): {"fileName": "c.raw"}, ] with patch.object( - files_obj, "stream_all_files_by_project", return_value=api_response - ), patch.object(files_obj, "download_files") as mock_download: + PrideProvider, "list_files", return_value=api_response + ), patch.object(PrideProvider, "download_files") as mock_download: files_obj.download_files_by_list( accession="PXD001819", file_names=["a.raw", "c.raw"], @@ -46,16 +47,16 @@ def test_filters_metadata_and_delegates(self): protocol="ftp", ) - args, _ = mock_download.call_args - matched = args[0] + _, kwargs = mock_download.call_args + matched = kwargs["records"] assert {f["fileName"] for f in matched} == {"a.raw", "c.raw"} def test_warns_on_partial_match(self): files_obj = Files() api_response = [{"fileName": "a.raw"}] with patch.object( - files_obj, "stream_all_files_by_project", return_value=api_response - ), patch.object(files_obj, "download_files") as mock_download, self.assertLogs( + PrideProvider, "list_files", return_value=api_response + ), patch.object(PrideProvider, "download_files") as mock_download, self.assertLogs( level="WARNING" ) as log_ctx: files_obj.download_files_by_list( @@ -71,7 +72,7 @@ def test_warns_on_partial_match(self): def test_raises_when_no_files_match(self): files_obj = Files() with patch.object( - files_obj, "stream_all_files_by_project", return_value=[] + PrideProvider, "list_files", return_value=[] ): with pytest.raises(ValueError, match="No matching files"): files_obj.download_files_by_list( diff --git a/pridepy/tests/test_download_by_url.py b/pridepy/tests/test_download_by_url.py index fb34491..b8c66da 100644 --- a/pridepy/tests/test_download_by_url.py +++ b/pridepy/tests/test_download_by_url.py @@ -12,7 +12,8 @@ import click import pytest -from pridepy.files.files import Files +from pridepy.download import by_url +from pridepy.download.client import Client as Files from pridepy.pridepy import _read_url_arguments @@ -38,7 +39,7 @@ def fake_http(_url, target_path): _touch_valid(target_path) with patch.object( - Files, "_http_download_url", side_effect=fake_http + by_url, "_http_download_url", side_effect=fake_http ) as mock_http: Files.download_files_by_url( urls=["https://example.org/sample.raw"], @@ -56,7 +57,7 @@ def fake_ftp(_parsed, target_path): _touch_valid(target_path) with patch.object( - Files, "_ftp_download_url", side_effect=fake_ftp + by_url, "_ftp_download_url", side_effect=fake_ftp ) as mock_ftp: Files.download_files_by_url( urls=["ftp://ftp.pride.ebi.ac.uk/path/sample.raw"], @@ -86,7 +87,7 @@ def test_skip_if_exists_short_circuits(self): with tempfile.TemporaryDirectory() as tmp_dir: target = os.path.join(tmp_dir, "existing.raw") _touch_valid(target) - with patch.object(Files, "_http_download_url") as mock_http: + with patch.object(by_url, "_http_download_url") as mock_http: Files.download_files_by_url( urls=["https://example.org/existing.raw"], output_folder=tmp_dir, diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 21b1603..532f624 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -4,7 +4,13 @@ from unittest import TestCase from unittest.mock import Mock, patch -from pridepy.files.files import Files +from pridepy.download import by_url +from pridepy.download.client import Client as Files +from pridepy.download import transport +from pridepy.download import util as provider_util +from pridepy.download.massive import MassiveProvider +from pridepy.download.pride import PrideProvider +from pridepy.download.proteomexchange import ProteomeXchangeProvider class TestDownloadResilience(TestCase): @@ -40,7 +46,7 @@ def test_get_download_url_maps_globus_to_pride_archive_https(self): ] } - download_url = Files._get_download_url(file_record, "globus") + download_url = PrideProvider._get_download_url(file_record, "globus") assert download_url == "https://ftp.pride.ebi.ac.uk/path/file.raw" @@ -55,16 +61,17 @@ def test_parallel_download_streams_full_file(self): stream_response = Mock() stream_response.raise_for_status.return_value = None + stream_response.headers = {} stream_response.iter_content.return_value = [b"abc"] stream_response.__enter__ = Mock(return_value=stream_response) stream_response.__exit__ = Mock(return_value=None) session.get.return_value = stream_response with patch( - "pridepy.files.files.Util.create_session_with_retries", + "pridepy.download.transport.Util.create_session_with_retries", return_value=session, ): - Files._parallel_download( + transport._parallel_download( "https://example.org/file.raw", output_file, ) @@ -80,16 +87,17 @@ def test_parallel_download_falls_back_when_head_fails(self): fallback_response = Mock() fallback_response.raise_for_status.return_value = None + fallback_response.headers = {} fallback_response.iter_content.return_value = [b"abc"] fallback_response.__enter__ = Mock(return_value=fallback_response) fallback_response.__exit__ = Mock(return_value=None) session.get.return_value = fallback_response with patch( - "pridepy.files.files.Util.create_session_with_retries", + "pridepy.download.transport.Util.create_session_with_retries", return_value=session, ): - Files._parallel_download( + transport._parallel_download( "https://example.org/file.raw", output_file, ) @@ -108,16 +116,17 @@ def test_parallel_download_falls_back_without_accept_ranges(self): fallback_response = Mock() fallback_response.raise_for_status.return_value = None + fallback_response.headers = {} fallback_response.iter_content.return_value = [b"abc"] fallback_response.__enter__ = Mock(return_value=fallback_response) fallback_response.__exit__ = Mock(return_value=None) session.get.return_value = fallback_response with patch( - "pridepy.files.files.Util.create_session_with_retries", + "pridepy.download.transport.Util.create_session_with_retries", return_value=session, ): - Files._parallel_download( + transport._parallel_download( "https://example.org/file.raw", output_file, ) @@ -125,6 +134,252 @@ def test_parallel_download_falls_back_without_accept_ranges(self): with open(output_file, "rb") as handle: assert handle.read() == b"abc" + def test_parallel_download_raises_on_truncated_stream(self): + """A stream shorter than Content-Length must raise so the caller retries.""" + with tempfile.TemporaryDirectory() as tmp_dir: + output_file = os.path.join(tmp_dir, "file.raw") + session = Mock() + head = Mock() + head.headers = {"content-length": "5", "accept-ranges": "none"} + head.raise_for_status.return_value = None + session.head.return_value = head + + stream_response = Mock() + stream_response.raise_for_status.return_value = None + stream_response.headers = {} # no Content-Encoding -> size check active + stream_response.iter_content.return_value = [b"ab"] # only 2 of 5 bytes + stream_response.__enter__ = Mock(return_value=stream_response) + stream_response.__exit__ = Mock(return_value=None) + session.get.return_value = stream_response + + with patch( + "pridepy.download.transport.Util.create_session_with_retries", + return_value=session, + ): + with self.assertRaisesRegex(RuntimeError, "Incomplete download"): + transport._parallel_download( + "https://example.org/file.raw", + output_file, + ) + + def test_safe_join_preserves_subdirs_and_blocks_escape(self): + out = os.path.join("/tmp", "out") + # Nested dataset-relative path is preserved under output_folder. + assert transport._safe_join(out, "raw/sub/run.raw") == os.path.join( + out, "raw", "sub", "run.raw" + ) + # Traversal that escapes output_folder falls back to the basename. + assert transport._safe_join(out, "../../etc/passwd") == os.path.join( + out, "passwd" + ) + + def test_download_files_preserves_relative_paths_when_flatten_false(self): + """With flatten=False, base.Provider.download_files threads each + record's relativePath through to the transport layer so same-basename + files in different collections keep their subdirectory layout.""" + provider = MassiveProvider() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(transport, "download_ftp_urls") as ftp_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/out", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + flatten=False, + ) + kwargs = ftp_mock.call_args.kwargs + assert kwargs["relative_paths"] == ["raw/a/run.raw", "raw/b/run.raw"] + + def test_download_files_threads_relative_paths_for_http(self): + """With flatten=False, the HTTP partition also forwards relativePath to + download_http_urls.""" + + class _HttpProvider(MassiveProvider): + pass + + provider = _HttpProvider() + records = [ + { + "accession": "MSV000012345", + "fileName": "run.raw", + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "http://example.org/d1/run.raw"} + ], + "relativePath": "raw/d1/run.raw", + }, + ] + with patch.object(transport, "download_http_urls") as http_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/out", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + flatten=False, + ) + assert http_mock.call_args.kwargs["relative_paths"] == ["raw/d1/run.raw"] + + def test_download_http_urls_raises_when_a_file_fails(self): + """A failed HTTP transfer must surface as an exception, not be + swallowed into a false success.""" + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + transport, "_parallel_download", side_effect=RuntimeError("boom") + ): + with self.assertRaisesRegex(RuntimeError, "Failed to download"): + transport.download_http_urls( + http_urls=["https://example.org/a.raw"], + output_folder=tmp_dir, + skip_if_downloaded_already=False, + max_retries=1, + ) + + def test_download_ftp_urls_raises_when_a_file_fails(self): + """A failed FTP transfer must surface as an exception.""" + with tempfile.TemporaryDirectory() as tmp_dir: + fake_ftp = Mock() + with patch.object( + transport, "_open_ftp_connection", return_value=fake_ftp + ), patch.object( + transport, "_download_one_ftp_path", side_effect=RuntimeError("boom") + ): + with self.assertRaisesRegex(RuntimeError, "Failed to download"): + transport.download_ftp_urls( + ftp_urls=["ftp://ftp.example.org/p/a.raw"], + output_folder=tmp_dir, + skip_if_downloaded_already=False, + ) + + def test_by_url_http_download_raises_on_truncated_content(self): + """by_url's HTTP downloader must reject a stream shorter than + Content-Length instead of accepting a truncated file.""" + with tempfile.TemporaryDirectory() as tmp_dir: + target = os.path.join(tmp_dir, "a.raw") + session = Mock() + response = Mock() + response.raise_for_status.return_value = None + response.headers = {"Content-Length": "5"} + response.iter_content.return_value = [b"ab"] # only 2 of 5 bytes + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + session.get.return_value = response + with patch( + "pridepy.download.by_url.Util.create_session_with_retries", + return_value=session, + ): + with self.assertRaisesRegex(RuntimeError, "Incomplete download"): + by_url._http_download_url("https://example.org/a.raw", target) + + def test_by_url_http_download_skips_size_check_when_encoded(self): + """A gzip/deflate response is decompressed by requests, so the on-disk + size won't match Content-Length — the size check must be skipped to + avoid a false 'Incomplete download' on an intact file.""" + with tempfile.TemporaryDirectory() as tmp_dir: + target = os.path.join(tmp_dir, "a.txt") + session = Mock() + response = Mock() + response.raise_for_status.return_value = None + # Content-Length is the compressed size; decompressed payload is larger. + response.headers = {"Content-Length": "5", "Content-Encoding": "gzip"} + response.iter_content.return_value = [b"abcdefghij"] # 10 decompressed bytes + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + session.get.return_value = response + with patch( + "pridepy.download.by_url.Util.create_session_with_retries", + return_value=session, + ): + by_url._http_download_url("https://example.org/a.txt", target) + with open(target, "rb") as handle: + assert handle.read() == b"abcdefghij" + + def test_proteomexchange_relative_paths_handle_root_common_prefix(self): + """When raw URIs live in different top-level directories (common + prefix is '/'), the paths must still be disambiguated, not collapsed + to a colliding basename.""" + urls = [ + "ftp://ftp.example.org/run1/sample.raw", + "ftp://ftp.example.org/run2/sample.raw", + ] + with patch.object( + ProteomeXchangeProvider, "_normalize_px_xml_url", return_value="http://x" + ), patch.object( + ProteomeXchangeProvider, + "_parse_px_xml_for_raw_file_urls", + return_value=urls, + ): + records = ProteomeXchangeProvider().list_files("PXD1") + assert {r["relativePath"] for r in records} == { + "run1/sample.raw", + "run2/sample.raw", + } + + def test_download_files_propagates_transport_failure(self): + """Provider.download_files must propagate a transport failure so the + direct-download path doesn't report false success (parity with PRIDE).""" + provider = MassiveProvider() + record = MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a.raw", + ) + with patch.object( + transport, "download_ftp_urls", side_effect=RuntimeError("download failed") + ): + with self.assertRaises(RuntimeError): + provider.download_files( + accession="MSV000012345", + records=[record], + output_folder="/tmp/does-not-matter", + skip_if_downloaded_already=False, + protocol="ftp", + ) + + def test_proteomexchange_relative_paths_disambiguate_duplicate_basenames(self): + """download-px-raw-files must not flatten duplicate basenames from + different directories onto the same local file.""" + urls = [ + "ftp://ftp.pride.ebi.ac.uk/pride/PXD1/run1/sample.raw", + "ftp://ftp.pride.ebi.ac.uk/pride/PXD1/run2/sample.raw", + ] + with patch.object( + ProteomeXchangeProvider, "_normalize_px_xml_url", return_value="http://x" + ), patch.object( + ProteomeXchangeProvider, + "_parse_px_xml_for_raw_file_urls", + return_value=urls, + ): + records = ProteomeXchangeProvider().list_files("PXD1") + + assert {r["relativePath"] for r in records} == { + "run1/sample.raw", + "run2/sample.raw", + } + + def test_proteomexchange_single_file_relative_path_is_basename(self): + urls = ["ftp://ftp.pride.ebi.ac.uk/pride/PXD1/run1/sample.raw"] + with patch.object( + ProteomeXchangeProvider, "_normalize_px_xml_url", return_value="http://x" + ), patch.object( + ProteomeXchangeProvider, + "_parse_px_xml_for_raw_file_urls", + return_value=urls, + ): + records = ProteomeXchangeProvider().list_files("PXD1") + assert records[0]["relativePath"] == "sample.raw" + def test_validate_download_rejects_empty_and_bad_checksum(self): with tempfile.TemporaryDirectory() as tmp_dir: file_path = os.path.join(tmp_dir, "test.raw") @@ -142,8 +397,54 @@ def test_validate_download_rejects_empty_and_bad_checksum(self): assert "checksum mismatch" in reason def test_protocol_sequence_prefers_requested_then_fallback(self): - assert Files._protocol_sequence("ftp") == ["ftp", "aspera", "s3", "globus"] - assert Files._protocol_sequence("aspera") == ["aspera", "s3", "ftp", "globus"] + assert PrideProvider._protocol_sequence("ftp") == ["ftp", "aspera", "s3", "globus"] + assert PrideProvider._protocol_sequence("aspera") == ["aspera", "s3", "ftp", "globus"] + + def test_pride_ftp_batch_routes_through_shared_transport(self): + """PRIDE FTP batch downloads must use transport.download_ftp_urls + (per-file reconnect + REST resume + size checks) instead of the legacy + single-connection loop that cascades on one timeout (issue #107).""" + records = [ + { + "fileName": "a.raw", + "accession": "PXD000001", + "publicFileLocations": [ + { + "name": "FTP Protocol", + "value": "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2024/05/PXD000001/a.raw", + } + ], + }, + { + "fileName": "b.raw", + "accession": "PXD000001", + "publicFileLocations": [ + { + "name": "FTP Protocol", + "value": "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2024/05/PXD000001/b.raw", + } + ], + }, + ] + with patch.object(transport, "download_ftp_urls") as ftp_mock: + PrideProvider._batch_download_by_protocol( + records, + "/tmp/out", + "ftp", + skip_if_downloaded_already=False, + aspera_maximum_bandwidth="100M", + parallel_files=2, + ) + + ftp_mock.assert_called_once() + kwargs = ftp_mock.call_args.kwargs + assert kwargs["ftp_urls"] == [ + "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2024/05/PXD000001/a.raw", + "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2024/05/PXD000001/b.raw", + ] + assert kwargs["use_tls"] is False + assert kwargs["parallel_files"] == 2 + assert kwargs["skip_if_downloaded_already"] is False def test_download_with_fallback_switches_protocol_after_invalid_file(self): file_record = { @@ -168,8 +469,8 @@ def fake_batch(file_list, output_folder, protocol, skip_if_downloaded_already, with open(local_path, "wb") as handle: handle.write(b"abc") - with patch.object(Files, "_batch_download_by_protocol", side_effect=fake_batch): - success = Files._download_with_fallback( + with patch.object(PrideProvider, "_batch_download_by_protocol", side_effect=fake_batch): + success = PrideProvider._download_with_fallback( file_record=file_record, output_folder=tmp_dir, protocol_sequence=["aspera", "s3"], @@ -201,9 +502,9 @@ def fake_batch(file_list, output_folder, protocol, skip_if_downloaded_already, with open(local_path, "wb") as handle: handle.write(b"data") - with patch.object(Files, "_batch_download_by_protocol", side_effect=fake_batch) as batch_mock, \ - patch.object(Files, "_download_with_fallback") as fallback_mock: - Files.download_files( + with patch.object(PrideProvider, "_batch_download_by_protocol", side_effect=fake_batch) as batch_mock, \ + patch.object(PrideProvider, "_download_with_fallback") as fallback_mock: + PrideProvider._download_files_batch( file_list_json=[file_record], accession="PXD000000", output_folder=tmp_dir, @@ -229,8 +530,8 @@ def test_globus_parallel_workers_capped_to_file_count(self): ] with tempfile.TemporaryDirectory() as tmp_dir: - with patch.object(Files, "_globus_download_one") as mock_one: - Files.download_files_from_globus( + with patch.object(PrideProvider, "_globus_download_one") as mock_one: + PrideProvider.download_files_from_globus( file_list_json=file_records, output_folder=tmp_dir, skip_if_downloaded_already=False, @@ -243,7 +544,7 @@ def test_globus_parallel_workers_capped_to_file_count(self): def test_url_parallel_workers_capped_to_url_count(self): """download_files_by_url must cap workers to len(urls).""" with tempfile.TemporaryDirectory() as tmp_dir: - with patch.object(Files, "_download_single_url") as mock_single: + with patch.object(by_url, "_download_single_url") as mock_single: Files.download_files_by_url( urls=["https://example.org/a.raw"], output_folder=tmp_dir, @@ -258,13 +559,51 @@ def test_download_files_raises_when_any_file_fails(self): with tempfile.TemporaryDirectory() as tmp_dir: file_list = [{"fileName": "missing.raw"}] - with patch.object(Files, "_batch_download_by_protocol"), \ - patch.object(Files, "_download_with_fallback", return_value=False): + with patch.object(PrideProvider, "_batch_download_by_protocol"), \ + patch.object(PrideProvider, "_download_with_fallback", return_value=False): with self.assertRaisesRegex(RuntimeError, "missing.raw"): - Files.download_files( + PrideProvider._download_files_batch( file_list_json=file_list, accession="PXD000000", output_folder=tmp_dir, skip_if_downloaded_already=False, protocol="ftp", ) + + def test_facade_dispatches_pride_through_registry_to_fallback(self): + """Files().download_all_raw_files for a PXD accession must flow: + Files facade -> Registry.resolve -> PrideProvider.download_files + -> _batch_download_by_protocol (mocked). + + Patching PrideProvider._batch_download_by_protocol proves the patch + intercepts (i.e. PrideProvider owns the multi-protocol orchestrator + and no longer routes through Files). + """ + fake_records = [ + { + "accession": "PXD000001", + "fileName": "x.raw", + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "ftp://ftp.pride.ebi.ac.uk/.../x.raw"} + ], + }, + ] + + with tempfile.TemporaryDirectory() as tmp: + with patch.object(PrideProvider, "list_files", return_value=fake_records), \ + patch.object(PrideProvider, "_batch_download_by_protocol", return_value=[]) as batch_mock, \ + patch.object(provider_util, "validate_download", return_value=(True, "ok")), \ + patch.object(PrideProvider, "_download_with_fallback") as fallback_mock: + Files().download_all_raw_files( + accession="PXD000001", + output_folder=tmp, + skip_if_downloaded_already=False, + protocol="ftp", + aspera_maximum_bandwidth="100M", + ) + + batch_mock.assert_called_once() + # No fallback expected because all files passed validation after + # the primary-protocol batch run. + fallback_mock.assert_not_called() diff --git a/pridepy/tests/test_flatten_paths.py b/pridepy/tests/test_flatten_paths.py new file mode 100644 index 0000000..a92ba34 --- /dev/null +++ b/pridepy/tests/test_flatten_paths.py @@ -0,0 +1,39 @@ +"""Tests for flatten_relative_paths: collapsing dataset-relative paths to flat, +de-duplicated basenames for download into a single output folder.""" +from unittest import TestCase + +from pridepy.download.util import flatten_relative_paths + + +class TestFlattenRelativePaths(TestCase): + def test_distinct_basenames_are_kept_unchanged(self): + assert flatten_relative_paths( + ["raw/a.raw", "ccms_peak/b.mzML", "search/c.mzid"] + ) == ["a.raw", "b.mzML", "c.mzid"] + + def test_colliding_basenames_get_numeric_suffixes(self): + # raw/a/run.raw and raw/b/run.raw both collapse to run.raw. + result = flatten_relative_paths(["raw/a/run.raw", "raw/b/run.raw"]) + assert result == ["run.raw", "run_1.raw"] + + def test_suffix_assignment_is_deterministic_by_sorted_source_path(self): + # First by sorted source path keeps the bare name regardless of input + # order, so re-runs (any order) map a given source to the same name. + forward = flatten_relative_paths(["raw/b/run.raw", "raw/a/run.raw"]) + # Input order preserved in output; "raw/a/run.raw" (sorts first) -> run.raw + assert forward == ["run_1.raw", "run.raw"] + + def test_output_is_positionally_aligned_with_input(self): + names = flatten_relative_paths(["x/dup.txt", "y/uniq.txt", "z/dup.txt"]) + assert names == ["dup.txt", "uniq.txt", "dup_1.txt"] + + def test_multi_dot_extension_suffix_before_last_extension(self): + result = flatten_relative_paths(["a/data.tar.gz", "b/data.tar.gz"]) + assert result == ["data.tar.gz", "data.tar_1.gz"] + + def test_files_without_extension_get_plain_suffix(self): + result = flatten_relative_paths(["a/README", "b/README"]) + assert result == ["README", "README_1"] + + def test_leading_slash_is_stripped(self): + assert flatten_relative_paths(["/raw/a.raw"]) == ["a.raw"] diff --git a/pridepy/tests/test_ftp_download_validation.py b/pridepy/tests/test_ftp_download_validation.py new file mode 100644 index 0000000..7390045 --- /dev/null +++ b/pridepy/tests/test_ftp_download_validation.py @@ -0,0 +1,91 @@ +"""Coverage for the size-mismatch detection added to ``_download_one_ftp_path``. + +The FTP server's ``SIZE`` reply is the only integrity signal direct downloads +have (MassIVE/JPOST don't publish per-file MD5 manifests like PRIDE). After +``retrbinary`` returns, we re-check the local size against the server-reported +size and treat a mismatch as a retryable failure. +""" +import os +import tempfile +from unittest import TestCase +from unittest.mock import MagicMock + +import pytest + +from pridepy.download import transport + + +def _make_fake_ftp(expected_size, write_bytes_per_call): + """Return a MagicMock FTP that writes ``write_bytes_per_call`` bytes per call. + + ``retrbinary`` is invoked once per attempt; we record how many attempts + happened by counting calls and produce a different payload size for each. + """ + fake = MagicMock() + fake.size.return_value = expected_size + fake.sendcmd = MagicMock() + fake._call_count = 0 + + def retrbinary(cmd, callback): + idx = fake._call_count + fake._call_count += 1 + payload = b"x" * write_bytes_per_call[idx] + callback(payload) + + fake.retrbinary.side_effect = retrbinary + return fake + + +class TestSizeMismatchValidation(TestCase): + def test_size_mismatch_is_retried_then_succeeds(self): + """First attempt returns 50 bytes (expected 100) -> retry, second yields 50 more -> 100, OK.""" + with tempfile.TemporaryDirectory() as tmp: + local_path = os.path.join(tmp, "f.bin") + ftp = _make_fake_ftp(expected_size=100, write_bytes_per_call=[50, 50]) + + transport._download_one_ftp_path( + ftp=ftp, + ftp_path="/JPST000001/f.bin", + local_path=local_path, + skip_if_downloaded_already=False, + max_download_retries=3, + ) + + assert os.path.getsize(local_path) == 100 + assert ftp.retrbinary.call_count == 2 + # First attempt: file empty, no REST. Second: file has 50 bytes, REST 50 issued. + sendcmd_args = [call.args[0] for call in ftp.sendcmd.call_args_list] + assert sendcmd_args == ["REST 50"] + + def test_size_mismatch_after_retries_raises(self): + """Three attempts all undersize -> RuntimeError after giving up.""" + with tempfile.TemporaryDirectory() as tmp: + local_path = os.path.join(tmp, "f.bin") + ftp = _make_fake_ftp(expected_size=100, write_bytes_per_call=[10, 10, 10]) + + with pytest.raises(RuntimeError, match="Giving up"): + transport._download_one_ftp_path( + ftp=ftp, + ftp_path="/JPST000001/f.bin", + local_path=local_path, + skip_if_downloaded_already=False, + max_download_retries=3, + ) + + assert ftp.retrbinary.call_count == 3 + + def test_correct_size_returns_without_retry(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = os.path.join(tmp, "f.bin") + ftp = _make_fake_ftp(expected_size=50, write_bytes_per_call=[50]) + + transport._download_one_ftp_path( + ftp=ftp, + ftp_path="/JPST000001/f.bin", + local_path=local_path, + skip_if_downloaded_already=False, + max_download_retries=3, + ) + + assert os.path.getsize(local_path) == 50 + assert ftp.retrbinary.call_count == 1 diff --git a/pridepy/tests/test_iprox_files.py b/pridepy/tests/test_iprox_files.py new file mode 100644 index 0000000..3f9a487 --- /dev/null +++ b/pridepy/tests/test_iprox_files.py @@ -0,0 +1,150 @@ +"""iProX direct-download support. + +iProX publishes the ProteomeXchange XML for each dataset at a deterministic +path on its anonymous HTTP download server:: + + http://download.iprox.org//PX_.xml + +The referenced files are served from the same host over HTTP with byte-range +support, so resume and parallel downloads use the same plumbing as PRIDE +HTTP(S) transfers. +""" +import tempfile +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from pridepy.download.client import Client as Files +from pridepy.download import transport +from pridepy.download.iprox import IproxProvider +from pridepy.download.pride import PrideProvider + + +IPROX_XML_FIXTURE = """ + + + + + + + + + + + + + + + + + + + + + +""".encode("utf-8") + + +class TestIProXFiles(TestCase): + def test_is_iprox_accession_matches_ipx_format(self): + assert Files.is_iprox_accession("IPX0000123") + assert Files.is_iprox_accession("IPX0000123000") + assert Files.is_iprox_accession("ipx1234567") + assert not Files.is_iprox_accession("PXD000012") + assert not Files.is_iprox_accession("MSV000012345") + assert not Files.is_iprox_accession("JPST000001") + assert not Files.is_iprox_accession("IPX12") + assert not Files.is_iprox_accession("") + assert not Files.is_iprox_accession(None) + + def test_iprox_is_a_direct_download_accession(self): + assert Files.is_direct_download_accession("IPX0017413000") + + def test_build_iprox_file_record_maps_px_cv_to_category(self): + record = IproxProvider._build_file_record( + "IPX0017413000", + "http://download.iprox.org/IPX0017413000/IPX0017413001/sample.raw", + category_from_px="Associated raw file URI", + ) + assert record["fileName"] == "sample.raw" + assert record["fileCategory"]["value"] == "RAW" + assert record["source"] == "iProX" + # _download_direct_download_records dispatches by URL scheme, so the + # publicFileLocations URL must still be the HTTP download URL. + assert record["publicFileLocations"][0]["value"].startswith("http://") + + def test_list_iprox_public_files_parses_px_xml(self): + fake_response = MagicMock() + fake_response.content = IPROX_XML_FIXTURE + fake_response.raise_for_status = MagicMock() + with patch( + "pridepy.download.iprox.requests.get", return_value=fake_response + ) as req_mock: + records = IproxProvider().list_files("IPX0017413000") + + # The fetch hits the deterministic PX XML URL. + req_mock.assert_called_once() + called_url = req_mock.call_args[0][0] + assert called_url == ( + "http://download.iprox.org/IPX0017413000/PX_IPX0017413000.xml" + ) + + # 3 valid HTTP records; the ftp:// "Other URI" cvParam was filtered out. + assert len(records) == 3 + cats = {r["fileName"]: r["fileCategory"]["value"] for r in records} + assert cats == { + "sample1.raw": "RAW", + "sample2.raw": "RAW", + "results.tsv": "SEARCH", + } + for r in records: + assert r["source"] == "iProX" + assert r["publicFileLocations"][0]["value"].startswith("http://") + + def test_get_all_raw_file_list_filters_iprox_records(self): + files = Files() + fake_response = MagicMock() + fake_response.content = IPROX_XML_FIXTURE + fake_response.raise_for_status = MagicMock() + with patch( + "pridepy.download.iprox.requests.get", return_value=fake_response + ), patch.object(PrideProvider, "stream_all_files_by_project") as pride_mock: + raw_files = files.get_all_raw_file_list("IPX0017413000") + + pride_mock.assert_not_called() + assert {r["fileName"] for r in raw_files} == {"sample1.raw", "sample2.raw"} + + def test_download_file_by_name_routes_iprox_to_http_urls(self): + files = Files() + fake_response = MagicMock() + fake_response.content = IPROX_XML_FIXTURE + fake_response.raise_for_status = MagicMock() + with tempfile.TemporaryDirectory() as tmp_dir, patch( + "pridepy.download.iprox.requests.get", return_value=fake_response + ), patch.object(transport, "download_http_urls") as http_mock, patch.object( + transport, "download_ftp_urls" + ) as ftp_mock: + files.download_file_by_name( + accession="IPX0017413000", + file_name="results.tsv", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="ftp", + username=None, + password=None, + aspera_maximum_bandwidth="100M", + checksum_check=False, + ) + + # iProX is HTTP, not FTP — FTP path must not be called. + ftp_mock.assert_not_called() + http_mock.assert_called_once() + kwargs = http_mock.call_args.kwargs + assert kwargs["http_urls"] == [ + "http://download.iprox.org/IPX0017413000/IPX0017413001/results.tsv" + ] + assert kwargs["parallel_files"] == 1 + assert kwargs["skip_if_downloaded_already"] is False diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py new file mode 100644 index 0000000..e9546c0 --- /dev/null +++ b/pridepy/tests/test_jpost_files.py @@ -0,0 +1,148 @@ +import json +import tempfile +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from pridepy.download.client import Client as Files +from pridepy.download import transport +from pridepy.download.jpost import JpostProvider + + +class TestJPOSTFiles(TestCase): + def test_is_jpost_accession(self): + assert Files.is_jpost_accession("JPST000001") + assert Files.is_jpost_accession("jpst123456") + assert not Files.is_jpost_accession("PXD000012") + assert not Files.is_jpost_accession("MSV000012345") + assert not Files.is_jpost_accession("JPST12") + + def test_is_direct_download_accession_includes_jpost(self): + assert Files.is_direct_download_accession("JPST000001") + + def test_build_jpost_file_record_maps_collection_to_category(self): + record = JpostProvider._build_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/peak/sample.mzML", + ) + + assert record["fileName"] == "sample.mzML" + assert record["collection"] == "peak" + assert record["fileCategory"]["value"] == "PEAK" + assert record["source"] == "JPOST" + + def test_build_jpost_file_record_marks_raw_collection_as_raw(self): + record = JpostProvider._build_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/raw/run01.raw", + ) + + assert record["collection"] == "raw" + assert record["fileCategory"]["value"] == "RAW" + + def test_get_all_raw_file_list_filters_jpost_records(self): + files = Files() + jpost_records = [ + JpostProvider._build_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/raw/run1.raw", + ), + JpostProvider._build_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/result/results.tsv", + ), + ] + + with patch.object(JpostProvider, "list_files", return_value=jpost_records): + result = files.get_all_raw_file_list("JPST000001") + + assert len(result) == 1 + assert {file["fileName"] for file in result} == {"run1.raw"} + + def test_download_file_by_name_uses_jpost_ftp_listing(self): + files = Files() + file_record = JpostProvider._build_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/raw/folder/sample.raw", + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + JpostProvider, "list_files", return_value=[file_record] + ), patch.object(transport, "download_ftp_urls") as download_mock: + files.download_file_by_name( + accession="JPST000001", + file_name="sample.raw", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="ftp", + username=None, + password=None, + aspera_maximum_bandwidth="100M", + checksum_check=False, + ) + + download_mock.assert_called_once_with( + ftp_urls=["ftp://ftp.jpostdb.org/JPST000001/raw/folder/sample.raw"], + output_folder=tmp_dir, + skip_if_downloaded_already=False, + use_tls=False, + parallel_files=1, + relative_paths=["sample.raw"], + ) + + def test_proxi_listing_maps_cv_name_to_category(self): + proxi_response = { + "datasetFiles": [ + { + "accession": "PRIDE:0000404", + "name": "Associated raw file URI", + "value": "ftp://ftp.jpostdb.org/JPST002311/sample01.raw", + }, + { + "accession": "PRIDE:0000408", + "name": "Search engine output file URI", + "value": "ftp://ftp.jpostdb.org/JPST002311/sample01.sne", + }, + { + "accession": "PRIDE:0000999", + "name": "Some unknown CV", + "value": "ftp://ftp.jpostdb.org/JPST002311/misc/sample01.txt", + }, + { + "accession": "PRIDE:0000404", + "name": "Associated raw file URI", + "value": "https://example.org/not-ftp.raw", + }, + ] + } + fake_response = MagicMock() + fake_response.content = json.dumps(proxi_response).encode("utf-8") + fake_response.raise_for_status = MagicMock() + with patch("pridepy.download.jpost.requests.get", return_value=fake_response) as req_mock: + records = JpostProvider()._list_via_proxi("JPST002311") + + req_mock.assert_called_once() + call_url = req_mock.call_args[0][0] + assert call_url == "https://repository.jpostdb.org/proxi/datasets/JPST002311" + # Non-FTP URI ignored; three FTP entries kept. + assert len(records) == 3 + cats = {r["fileName"]: r["fileCategory"]["value"] for r in records} + assert cats["sample01.raw"] == "RAW" + assert cats["sample01.sne"] == "SEARCH" + # Unknown CV falls back to path-based heuristic (collection "misc" -> OTHER). + assert cats["sample01.txt"] == "OTHER" + + def test_proxi_falls_back_to_ftp_walk_on_error(self): + with patch.object( + JpostProvider, + "_list_via_proxi", + side_effect=RuntimeError("proxi down"), + ), patch.object( + transport, "_list_ftp_repo_files", return_value=["/JPST000001/raw/x.raw"] + ) as ftp_mock: + result = JpostProvider().list_files("JPST000001") + + ftp_mock.assert_called_once() + assert len(result) == 1 + assert result[0]["fileName"] == "x.raw" + assert result[0]["source"] == "JPOST" diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index f600b71..5ccc3d7 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -1,8 +1,11 @@ +import ftplib import tempfile from unittest import TestCase from unittest.mock import patch -from pridepy.files.files import Files +from pridepy.download.client import Client as Files +from pridepy.download import transport +from pridepy.download.massive import MassiveProvider class TestMassIVEFiles(TestCase): @@ -13,7 +16,7 @@ def test_is_massive_accession(self): assert not Files.is_massive_accession("MSV123") def test_build_massive_file_record_maps_collection_to_category(self): - record = Files._build_massive_file_record( + record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/ccms_peak/converted/sample.mzML", ) @@ -23,7 +26,7 @@ def test_build_massive_file_record_maps_collection_to_category(self): assert record["fileCategory"]["value"] == "PEAK" def test_build_massive_file_record_marks_raw_collection_as_raw(self): - record = Files._build_massive_file_record( + record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/run01.raw", ) @@ -32,7 +35,7 @@ def test_build_massive_file_record_marks_raw_collection_as_raw(self): assert record["fileCategory"]["value"] == "RAW" def test_build_massive_file_record_keeps_non_raw_collection_even_for_raw_like_file_names(self): - record = Files._build_massive_file_record( + record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/uploads/run01.raw", ) @@ -41,7 +44,7 @@ def test_build_massive_file_record_keeps_non_raw_collection_even_for_raw_like_fi assert record["fileCategory"]["value"] == "OTHER" def test_build_massive_file_record_marks_ab_sciex_scan_sidecar_as_raw_when_under_raw(self): - record = Files._build_massive_file_record( + record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/sample.wiff.scan", ) @@ -52,21 +55,21 @@ def test_build_massive_file_record_marks_ab_sciex_scan_sidecar_as_raw_when_under def test_get_all_raw_file_list_filters_massive_records(self): files = Files() massive_records = [ - Files._build_massive_file_record( + MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/run1.raw", ), - Files._build_massive_file_record( + MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/quant/results.tsv", ), - Files._build_massive_file_record( + MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/uploads/run2.mzML", ), ] - with patch.object(Files, "_list_massive_public_files", return_value=massive_records): + with patch.object(MassiveProvider, "list_files", return_value=massive_records): result = files.get_all_raw_file_list("MSV000012345") assert len(result) == 1 @@ -74,14 +77,14 @@ def test_get_all_raw_file_list_filters_massive_records(self): def test_download_file_by_name_uses_massive_ftp_listing(self): files = Files() - file_record = Files._build_massive_file_record( + file_record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/folder/sample.raw", ) with tempfile.TemporaryDirectory() as tmp_dir: - with patch.object(Files, "_list_massive_public_files", return_value=[file_record]), patch.object( - Files, "download_ftp_urls" + with patch.object(MassiveProvider, "list_files", return_value=[file_record]), patch.object( + transport, "download_ftp_urls" ) as download_mock: files.download_file_by_name( accession="MSV000012345", @@ -99,4 +102,343 @@ def test_download_file_by_name_uses_massive_ftp_listing(self): ftp_urls=["ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/folder/sample.raw"], output_folder=tmp_dir, skip_if_downloaded_already=False, + use_tls=True, + parallel_files=1, + relative_paths=["sample.raw"], ) + + def test_repo_uses_tls_true_for_massive_false_for_jpost(self): + assert Files._repo_uses_tls("MSV000012345") is True + assert Files._repo_uses_tls("JPST000001") is False + assert Files._repo_uses_tls("PXD000012") is False + + def test_download_all_raw_files_threads_parallel_files_for_massive(self): + files = Files() + massive_records = [ + MassiveProvider._build_file_record( + "MSV000012345", + f"ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/run{i}.raw", + ) + for i in range(3) + ] + + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + MassiveProvider, "list_files", return_value=massive_records + ), patch.object(transport, "download_ftp_urls") as download_mock: + files.download_all_raw_files( + accession="MSV000012345", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="ftp", + aspera_maximum_bandwidth="100M", + checksum_check=False, + parallel_files=3, + ) + + kwargs = download_mock.call_args.kwargs + assert kwargs["use_tls"] is True + assert kwargs["parallel_files"] == 3 + + def test_base_direct_download_provider_partitions_urls_by_scheme(self): + """Records mixing ftp:// and http(s):// route to the right transport.""" + from pridepy.download.massive import MassiveProvider + + provider = MassiveProvider() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a.raw", + ), + # Synthetic http record to verify partitioning (real MassIVE uses ftp). + { + "accession": "MSV000012345", + "fileName": "b.raw", + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "http://example.org/b.raw"} + ], + }, + ] + with patch.object(transport, "download_ftp_urls") as ftp_mock, \ + patch.object(transport, "download_http_urls") as http_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + ) + + ftp_mock.assert_called_once() + assert ftp_mock.call_args.kwargs["use_tls"] is True + assert ftp_mock.call_args.kwargs["ftp_urls"] == [ + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a.raw" + ] + http_mock.assert_called_once() + assert http_mock.call_args.kwargs["http_urls"] == ["http://example.org/b.raw"] + + def test_download_files_flattens_into_output_folder_by_default(self): + """By default, files land directly in the output folder (no tree), and + colliding basenames are de-duplicated.""" + provider = MassiveProvider() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(transport, "download_ftp_urls") as ftp_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + ) + + assert ftp_mock.call_args.kwargs["relative_paths"] == ["run.raw", "run_1.raw"] + + def test_download_files_preserves_structure_when_flatten_false(self): + provider = MassiveProvider() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(transport, "download_ftp_urls") as ftp_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + flatten=False, + ) + + assert ftp_mock.call_args.kwargs["relative_paths"] == [ + "raw/a/run.raw", + "raw/b/run.raw", + ] + + def test_client_download_all_raw_files_flattens_by_default(self): + files = Files() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(MassiveProvider, "list_files", return_value=records), patch.object( + transport, "download_ftp_urls" + ) as ftp_mock: + files.download_all_raw_files( + accession="MSV000012345", + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + aspera_maximum_bandwidth="100M", + checksum_check=False, + parallel_files=1, + ) + assert ftp_mock.call_args.kwargs["relative_paths"] == ["run.raw", "run_1.raw"] + + def test_client_download_all_raw_files_preserve_structure(self): + files = Files() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(MassiveProvider, "list_files", return_value=records), patch.object( + transport, "download_ftp_urls" + ) as ftp_mock: + files.download_all_raw_files( + accession="MSV000012345", + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + aspera_maximum_bandwidth="100M", + checksum_check=False, + parallel_files=1, + flatten=False, + ) + assert ftp_mock.call_args.kwargs["relative_paths"] == [ + "raw/a/run.raw", + "raw/b/run.raw", + ] + + def test_get_https_url_builds_proteosafe_endpoint(self): + url = MassiveProvider._get_https_url( + "MSV000012345", "raw/Raw spec/C 3.raw" + ) + # Path slashes/dots preserved, spaces percent-encoded. + assert url == ( + "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true" + "&file=f.MSV000012345/raw/Raw%20spec/C%203.raw" + ) + + def test_build_massive_file_record_handles_non_v01_version_root(self): + """Datasets live under v01..vNN; records must preserve the real root.""" + record = MassiveProvider._build_file_record( + "MSV000088302", + "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/ccms_peak/run.mzML", + ) + + assert record["relativePath"] == "ccms_peak/run.mzML" + assert record["collection"] == "ccms_peak" + assert record["fileName"] == "run.mzML" + assert record["fileCategory"]["value"] == "PEAK" + assert ( + record["publicFileLocations"][0]["value"] + == "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/ccms_peak/run.mzML" + ) + + def test_get_public_ftp_url_preserves_version_root(self): + url = MassiveProvider._get_public_ftp_url( + "MSV000088302", "/v04/MSV000088302/ccms_peak/run.mzML" + ) + assert url == "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/ccms_peak/run.mzML" + + def test_list_files_discovers_version_root_when_not_v01(self): + """list_files probes top-level roots and walks the one holding the + dataset, so a dataset under /v04 is listed with v04 download URLs.""" + def fake_nlst(command, callback): + assert command == "NLST /" + for name in ["v01", "v02", "v03", "v04", "v05"]: + callback(name) + + def fake_cwd(path): + # Only the real root accepts the CWD; others 550. + if path != "/v04/MSV000088302": + raise ftplib.error_perm("550 CD issue: file does not exist") + + with patch.object(transport, "_open_ftp_connection") as open_conn, patch.object( + transport, + "_walk_ftp_tree", + return_value=["/v04/MSV000088302/ccms_peak/run.mzML"], + ) as walk_mock: + fake_ftp = open_conn.return_value + fake_ftp.retrlines.side_effect = fake_nlst + fake_ftp.cwd.side_effect = fake_cwd + + records = MassiveProvider().list_files("MSV000088302") + + walk_mock.assert_called_once_with(fake_ftp, "/v04/MSV000088302") + assert len(records) == 1 + assert records[0]["relativePath"] == "ccms_peak/run.mzML" + assert ( + records[0]["publicFileLocations"][0]["value"] + == "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/ccms_peak/run.mzML" + ) + + def test_list_files_prefers_versioned_root_over_auxiliary_root(self): + """x01/z01 can hold a partial (peak-only) copy while the full dataset + lives under a vNN root, so the versioned root must win even when both + exist.""" + def fake_nlst(command, callback): + for name in ["x01", "z01", "v04"]: + callback(name) + + existing = {"/z01/MSV000088302", "/v04/MSV000088302"} + + def fake_cwd(path): + if path not in existing: + raise ftplib.error_perm("550 CD issue: file does not exist") + + with patch.object(transport, "_open_ftp_connection") as open_conn, patch.object( + transport, + "_walk_ftp_tree", + return_value=["/v04/MSV000088302/raw/run.raw"], + ) as walk_mock: + fake_ftp = open_conn.return_value + fake_ftp.retrlines.side_effect = fake_nlst + fake_ftp.cwd.side_effect = fake_cwd + + records = MassiveProvider().list_files("MSV000088302") + + walk_mock.assert_called_once_with(fake_ftp, "/v04/MSV000088302") + assert ( + records[0]["publicFileLocations"][0]["value"] + == "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/raw/run.raw" + ) + + def test_build_https_file_record_sets_relpath_category_and_https_location(self): + record = MassiveProvider._build_https_file_record( + "MSV000012345", "raw/sub/run.raw" + ) + assert record["relativePath"] == "raw/sub/run.raw" + assert record["fileName"] == "run.raw" + assert record["collection"] == "raw" + assert record["fileCategory"]["value"] == "RAW" + location = record["publicFileLocations"][0] + assert location["value"].startswith( + "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?" + ) + assert location["value"].endswith("file=f.MSV000012345/raw/sub/run.raw") + + def test_list_files_falls_back_to_https_when_ftps_blocked(self): + """When the FTPS tree walk raises (e.g. FTPS blocked), list_files must + fall back to the HTTPS file index and emit HTTPS-download records.""" + csv_text = ( + "usi,filepath\n" + "mzspec:MSV000012345:raw/a/run.raw,raw/a/run.raw\n" + "mzspec:MSV000012345:raw/b/run.raw,raw/b/run.raw\n" + "mzspec:MSV000012345:ccms_result/x.mzid,ccms_result/x.mzid\n" + ) + + class _FakeCSVResponse: + def raise_for_status(self): + return None + + def iter_lines(self): + for line in csv_text.splitlines(): + yield line.encode("utf-8") + + with patch.object( + transport, + "_resolve_and_walk_ftp_dataset", + side_effect=RuntimeError("FTPS blocked"), + ), patch( + "pridepy.download.massive.requests.get", return_value=_FakeCSVResponse() + ): + records = MassiveProvider().list_files("MSV000012345") + + assert {r["relativePath"] for r in records} == { + "raw/a/run.raw", + "raw/b/run.raw", + "ccms_result/x.mzid", + } + # Same-basename files in different collections are kept distinct. + run_records = [r for r in records if r["fileName"] == "run.raw"] + assert len(run_records) == 2 + for record in records: + assert record["publicFileLocations"][0]["value"].startswith("https://") + # Downstream RAW filtering still works on the HTTPS records. + raw_names = { + rec["fileName"] + for rec in records + if rec["fileCategory"]["value"] == "RAW" + } + assert raw_names == {"run.raw"} diff --git a/pridepy/tests/test_raw_files.py b/pridepy/tests/test_raw_files.py index 1ce2ca3..f969871 100644 --- a/pridepy/tests/test_raw_files.py +++ b/pridepy/tests/test_raw_files.py @@ -1,11 +1,16 @@ from unittest import TestCase -from pridepy.files.files import Files +from pridepy.download.client import Client as Files +from pridepy.tests._live_api import tolerate_api_outage class TestRawFiles(TestCase): """ A test class to test files related methods. + + These hit the live PRIDE API; each call is wrapped in + :func:`tolerate_api_outage` so a transient API outage skips rather than + fails the build. """ def test_get_all_raw_file_list(self): @@ -13,10 +18,10 @@ def test_get_all_raw_file_list(self): A test method to check if it is possible to fetch the list of raw files """ raw = Files() - - # This project has only two files - result = raw.get_all_raw_file_list("PXD008644") - assert len(result) == 2 + with tolerate_api_outage(self): + # This project has only two files + result = raw.get_all_raw_file_list("PXD008644") + assert len(result) == 2 def test_get_raw_file_path_prefix(self): """ @@ -27,16 +32,17 @@ def test_get_raw_file_path_prefix(self): I.e. ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2018/10/PXD008644/7550GI_Y.raw """ raw = Files() - assert raw.get_submitted_file_path_prefix("PXD008644") == "2018/10/PXD008644" + with tolerate_api_outage(self): + assert raw.get_submitted_file_path_prefix("PXD008644") == "2018/10/PXD008644" def test_get_all_category_file_list(self): - raw = Files() - result = raw.get_all_category_file_list("PXD008644", "RAW") - assert len(result) == 2 + with tolerate_api_outage(self): + result = raw.get_all_category_file_list("PXD008644", "RAW") + assert len(result) == 2 - result = raw.get_all_category_file_list("PXD008644", "SEARCH") - assert len(result) == 2 + result = raw.get_all_category_file_list("PXD008644", "SEARCH") + assert len(result) == 2 def test_get_all_category_file_list_multiple(self): """ @@ -44,9 +50,10 @@ def test_get_all_category_file_list_multiple(self): PXD008644 has 2 RAW + 2 SEARCH = 4 files combined. """ raw = Files() - result = raw.get_all_category_file_list("PXD008644", ["RAW", "SEARCH"]) - assert len(result) == 4 + with tolerate_api_outage(self): + result = raw.get_all_category_file_list("PXD008644", ["RAW", "SEARCH"]) + assert len(result) == 4 - # Verify both categories are present - categories = {file["fileCategory"]["value"] for file in result} - assert categories == {"RAW", "SEARCH"} + # Verify both categories are present + categories = {file["fileCategory"]["value"] for file in result} + assert categories == {"RAW", "SEARCH"} diff --git a/pridepy/tests/test_review_fixes.py b/pridepy/tests/test_review_fixes.py new file mode 100644 index 0000000..242941c --- /dev/null +++ b/pridepy/tests/test_review_fixes.py @@ -0,0 +1,166 @@ +"""Tests for hardening fixes from PR #106 code review. + +Covers: exception chaining, defensive guards for empty/partial listings, +FTP host validation, and protocol forwarding in the shared download path. +""" +import subprocess +import tempfile +from unittest import TestCase +from unittest.mock import Mock, patch + +import pytest + +from pridepy.download import registry, transport +from pridepy.download.client import Client +from pridepy.download.massive import MassiveProvider +from pridepy.download.pride import PrideProvider + + +def _pride_record(file_name="a.raw", accession="PXD000001", date="2018/10"): + return { + "fileName": file_name, + "accession": accession, + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + { + "name": "FTP Protocol", + "value": f"ftp://ftp.pride.ebi.ac.uk/pride/data/archive/{date}/{accession}/{file_name}", + } + ], + } + + +class TestReviewFixes(TestCase): + def test_get_file_from_api_chains_original_exception(self): + with patch.object(registry, "resolve", side_effect=KeyError("boom")): + with pytest.raises(Exception) as exc_info: + Client().get_file_from_api("PXD000001", "x.raw") + # The original cause must be preserved for debugging. + assert isinstance(exc_info.value.__cause__, KeyError) + + def test_get_submitted_prefix_raises_clear_error_when_no_raw_files(self): + provider = PrideProvider() + records = [ + { + "fileName": "results.tsv", + "fileCategory": {"value": "SEARCH"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "ftp://h/2018/10/PXD1/results.tsv"} + ], + } + ] + with patch.object(provider, "_list_files_checked", return_value=records): + with pytest.raises(ValueError): # not a bare IndexError + provider.get_submitted_file_path_prefix("PXD1") + + def test_get_submitted_prefix_raises_clear_error_when_path_has_no_prefix(self): + provider = PrideProvider() + records = [ + { + "fileName": "a.raw", + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "ftp://host/no-date-here/a.raw"} + ], + } + ] + with patch.object(provider, "_list_files_checked", return_value=records): + with pytest.raises(ValueError): # not a bare AttributeError on None.group() + provider.get_submitted_file_path_prefix("PXD1") + + def test_download_ftp_urls_rejects_url_without_host(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with pytest.raises(ValueError, match="host"): + transport.download_ftp_urls( + ftp_urls=["ftp:///pride/data/x.raw"], # no hostname + output_folder=tmp_dir, + skip_if_downloaded_already=False, + ) + + def test_get_raw_files_tolerates_records_missing_category(self): + records = [ + {"fileName": "a.raw"}, # no fileCategory at all + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b.raw", + ), + ] + with patch.object(MassiveProvider, "list_files", return_value=records): + result = MassiveProvider().get_raw_files("MSV000012345") + assert {r["fileName"] for r in result} == {"b.raw"} + + def test_get_submitted_prefix_supports_prd_accessions(self): + provider = PrideProvider() + records = [_pride_record("a.raw", accession="PRD000123", date="2012/03")] + with patch.object(provider, "_list_files_checked", return_value=records): + assert provider.get_submitted_file_path_prefix("PRD000123") == "2012/03/PRD000123" + + def test_aspera_batch_raises_when_a_file_fails(self): + records = [ + { + "fileName": "a.raw", + "accession": "PXD000001", + "publicFileLocations": [ + {"name": "Aspera Protocol", "value": "faspe://h/a.raw"} + ], + } + ] + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object(PrideProvider, "get_ascp_binary", return_value="/bin/false"), patch( + "pridepy.download.pride.subprocess.run", + side_effect=subprocess.CalledProcessError(1, "ascp"), + ): + with pytest.raises(RuntimeError, match="Aspera"): + PrideProvider.download_files_from_aspera( + records, tmp_dir, skip_if_downloaded_already=False + ) + + def test_globus_batch_raises_when_a_file_fails(self): + records = [_pride_record("a.raw")] + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + PrideProvider, "_globus_download_one", side_effect=RuntimeError("boom") + ): + with pytest.raises(RuntimeError, match="Globus"): + PrideProvider.download_files_from_globus( + records, tmp_dir, skip_if_downloaded_already=False + ) + + def test_s3_batch_raises_when_a_file_fails(self): + records = [_pride_record("a.raw")] + mock_obj = Mock() + mock_obj.content_length = 10 + mock_bucket = Mock() + mock_bucket.Object.return_value = mock_obj + mock_bucket.download_file.side_effect = Exception("boom") + mock_resource = Mock() + mock_resource.Bucket.return_value = mock_bucket + with tempfile.TemporaryDirectory() as tmp_dir: + with patch("pridepy.download.pride.boto3.resource", return_value=mock_resource): + with pytest.raises(RuntimeError, match="S3"): + PrideProvider.download_files_from_s3( + records, tmp_dir, skip_if_downloaded_already=False + ) + + def test_download_files_forwards_protocol_to_get_download_url(self): + seen = [] + + class _CapturingProvider(MassiveProvider): + def get_download_url(self, record, protocol="ftp"): + seen.append(protocol) + return record["publicFileLocations"][0]["value"] + + record = MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a.raw", + ) + with patch.object(transport, "download_ftp_urls"): + _CapturingProvider().download_files( + accession="MSV000012345", + records=[record], + output_folder="/tmp/x", + skip_if_downloaded_already=False, + protocol="aspera", + parallel_files=1, + ) + assert seen == ["aspera"] diff --git a/pridepy/tests/test_search.py b/pridepy/tests/test_search.py index a61e83c..2d1e6e3 100644 --- a/pridepy/tests/test_search.py +++ b/pridepy/tests/test_search.py @@ -1,7 +1,8 @@ from unittest import TestCase -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.project.project import Project +from pridepy.tests._live_api import tolerate_api_outage from pridepy.util.api_handling import Util import logging @@ -9,6 +10,10 @@ class TestSearch(TestCase): """ A test class to test files related methods. + + These hit the live PRIDE API; calls are wrapped in + :func:`tolerate_api_outage` so a transient API outage skips rather than + fails the build. """ def test_search_projects(self): @@ -17,46 +22,48 @@ def test_search_projects(self): """ project = Project() - result = project.search_by_keywords_and_filters( - keyword="PXD009476", - query_filter="", - page_size=100, - page=0, - sort_direction="DESC", - sort_fields="accession", - ) - assert len(result) > 0 # Search should return at least one result - assert any( - r["accession"] == "PXD009476" for r in result - ) # Search should return the queried project - - result = project.get_projects(77, 0, "ASC", "submission_date") - assert len(result) == 77 - - result = project.get_by_accession("PXD009476") - assert result["accession"] == "PXD009476" - - assert ( - len( - project.get_files_by_accession( - "PXD009476", + with tolerate_api_outage(self): + result = project.search_by_keywords_and_filters( + keyword="PXD009476", + query_filter="", + page_size=100, + page=0, + sort_direction="DESC", + sort_fields="accession", + ) + assert len(result) > 0 # Search should return at least one result + assert any( + r["accession"] == "PXD009476" for r in result + ) # Search should return the queried project + + result = project.get_projects(77, 0, "ASC", "submission_date") + assert len(result) == 77 + + result = project.get_by_accession("PXD009476") + assert result["accession"] == "PXD009476" + + assert ( + len( + project.get_files_by_accession( + "PXD009476", + ) ) + == 100 ) - == 100 - ) def test_status_dataset(self): files = Files() accession = "PXD044389" - project_status = Util.get_api_call(files.API_BASE_URL + "/status/{}".format(accession)) - public_project = False - if project_status.status_code == 200: - if project_status.text == "PRIVATE": - public_project = True - elif project_status.text == "PUBLIC": - public_project = False - else: - raise Exception("Dataset {} is not present in PRIDE Archive".format(accession)) - logging.debug(f"Public project: {public_project}") + with tolerate_api_outage(self): + project_status = Util.get_api_call(files.API_BASE_URL + "/status/{}".format(accession)) + public_project = False + if project_status.status_code == 200: + if project_status.text == "PRIVATE": + public_project = True + elif project_status.text == "PUBLIC": + public_project = False + else: + raise Exception("Dataset {} is not present in PRIDE Archive".format(accession)) + logging.debug(f"Public project: {public_project}") diff --git a/pridepy/util/api_handling.py b/pridepy/util/api_handling.py index cdf7ffc..e81622f 100644 --- a/pridepy/util/api_handling.py +++ b/pridepy/util/api_handling.py @@ -62,7 +62,7 @@ async def stream_response_to_file( pbar.update( 1 ) # Update progress bar by 1 for each detection - except PermissionError as e: + except PermissionError: print("[ERROR] No permissions to write to:", out_file) sys.exit(1) diff --git a/pyproject.toml b/pyproject.toml index 90f40ae..b94d076 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pridepy" -version = "0.0.15" +version = "0.0.16" description = "Python Client library for PRIDE Rest API" readme = "README.md" requires-python = ">=3.9" @@ -40,6 +40,7 @@ dependencies = [ "boto3>=1.34.61", "botocore>=1.34.74", "httpx>=0.27.0", + "defusedxml>=0.7.1", ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index 3bae141..613ad5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ boto3 botocore tqdm urllib3 -httpx \ No newline at end of file +httpx>=0.27.0 +defusedxml \ No newline at end of file