From d471e0b26729ac4f2d12d0e12f9b71b5251f0288 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 11:03:23 +0100 Subject: [PATCH 01/54] Add direct downloads for JPOST and iProX accessions; bump to 0.0.16 Extends the direct-download support introduced for MassIVE in PR #98 to two more proteomics repositories whose datasets are often standalone (no ProteomeXchange accession): - JPOST (Japan ProteOme STandard Repository): JPST\d{6} accessions, listed and downloaded from ftp.jpostdb.org. - iProX (Integrated Proteome resources): IPX\d{7,10} accessions, listed and downloaded from ftp.iprox.cn. Refactor: - Add is_direct_download_accession() unifying the MSV/JPST/IPX checks, plus _list_direct_download_files() and _download_direct_download_records() dispatchers. All call sites (get_all_raw_file_list, download_all_raw_files, download_all_category_files, get_file_from_api, download_file_by_name, download_files_by_list, get_all_category_file_list) now go through the unified entry points. - Extract _list_ftp_repo_files() helper so the FTP connection lifecycle (connect / login / passive / walk / quit) lives in one place. As part of that, fix the FTP-constructor-outside-try issue flagged in the PR #98 review: a connect failure no longer triggers NameError in finally. - Keep is_massive_accession, _list_massive_public_files, and _download_massive_file_records as thin backward-compatible wrappers so existing tests and external callers continue to work. Tests: add test_jpost_files.py and test_iprox_files.py mirroring the MassIVE coverage (regex match, record building, raw-only filtering, and the download_file_by_name happy path). All 19 direct-download tests pass. Version: 0.0.16. --- README.md | 26 +++- pridepy/files/files.py | 251 ++++++++++++++++++++++++++---- pridepy/tests/test_iprox_files.py | 89 +++++++++++ pridepy/tests/test_jpost_files.py | 88 +++++++++++ pyproject.toml | 2 +- 5 files changed, 416 insertions(+), 40 deletions(-) create mode 100644 pridepy/tests/test_iprox_files.py create mode 100644 pridepy/tests/test_jpost_files.py diff --git a/README.md b/README.md index 0715bc6..a661153 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ You can: - download public and private PRIDE files -- download public MassIVE datasets directly from `MSV...` accessions +- download public MassIVE (`MSV...`), JPOST (`JPST...`), and iProX (`IPX...`) datasets directly from their native FTP archives - download by category (`RAW`, `SEARCH`, `RESULT`, etc.) - stream project and file metadata - search projects by keyword and filters @@ -80,15 +80,26 @@ pridepy download-all-public-raw-files \ --checksum-check ``` -### 3) Download a public MassIVE dataset directly +### 3) Download a public MassIVE, JPOST, or iProX dataset directly ```bash +# MassIVE pridepy download-all-public-raw-files \ -a MSV000082297 \ -o ./downloads/MSV000082297 + +# JPOST +pridepy download-all-public-raw-files \ + -a JPST000123 \ + -o ./downloads/JPST000123 + +# iProX +pridepy download-all-public-raw-files \ + -a IPX0000123000 \ + -o ./downloads/IPX0000123000 ``` -For direct `MSV...` downloads, `pridepy` enumerates the dataset from MassIVE's public FTP tree. Raw downloads follow MassIVE's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. +For these direct downloads, `pridepy` enumerates the dataset from the repository's public FTP tree (MassIVE at `massive-ftp.ucsd.edu`, JPOST at `ftp.jpostdb.org`, iProX at `ftp.iprox.cn`). Raw downloads follow each repository's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. ### 4) Download only selected categories @@ -99,7 +110,7 @@ pridepy download-all-public-category-files \ -c RAW,SEARCH ``` -You can also request a specific MassIVE collection through the same category interface: +You can also request a specific MassIVE / JPOST / iProX collection through the same category interface: ```bash pridepy download-all-public-category-files \ @@ -244,14 +255,15 @@ print(f"RAW files: {len(raw_files)}") print(raw_files[0]["fileName"]) ``` -For MassIVE accessions, the same method returns the files found under the dataset's `raw/` collection: +For MassIVE / JPOST / iProX accessions, the same method returns the files found under the dataset's `raw/` collection: ```python from pridepy.files.files import Files files = Files() -raw_files = files.get_all_raw_file_list("MSV000082297") -print(f"MassIVE raw files: {len(raw_files)}") +for accession in ("MSV000082297", "JPST000123", "IPX0000123000"): + raw_files = files.get_all_raw_file_list(accession) + print(f"{accession} raw files: {len(raw_files)}") ``` ### Example: search projects diff --git a/pridepy/files/files.py b/pridepy/files/files.py index fbcf2f9..1e5ac75 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -63,6 +63,10 @@ class Files: PRIDE_ARCHIVE_HTTPS_URL_PREFIX = "https://ftp.pride.ebi.ac.uk/" MASSIVE_ARCHIVE_FTP = "massive-ftp.ucsd.edu" MASSIVE_ARCHIVE_FTP_URL_PREFIX = "ftp://massive-ftp.ucsd.edu/v01/" + JPOST_ARCHIVE_FTP = "ftp.jpostdb.org" + JPOST_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.jpostdb.org/" + IPROX_ARCHIVE_FTP = "ftp.iprox.cn" + IPROX_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.iprox.cn/" S3_URL = "https://hh.fire.sdo.ebi.ac.uk" S3_BUCKET = "pride-public" PROTOCOL_ORDER = ["aspera", "s3", "ftp", "globus"] @@ -280,6 +284,98 @@ def _build_massive_file_record(accession: str, ftp_url: str) -> Dict: "source": "MassIVE", } + @staticmethod + def is_jpost_accession(accession: str) -> bool: + """ + Return True when the accession looks like a JPOST dataset accession. + """ + if not accession: + return False + return bool(re.fullmatch(r"JPST\d{6}", accession.upper())) + + @staticmethod + def _get_jpost_public_root(accession: str) -> str: + return f"/{accession.upper()}" + + @staticmethod + def _get_jpost_public_ftp_url(accession: str, remote_path: str) -> str: + root_path = Files._get_jpost_public_root(accession).rstrip("/") + relative_path = remote_path + if remote_path.startswith(root_path): + relative_path = remote_path[len(root_path) :].lstrip("/") + return f"{Files.JPOST_ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" + + @staticmethod + def _build_jpost_file_record(accession: str, ftp_url: str) -> Dict: + parsed = urlparse(ftp_url) + root_prefix = f"/{accession.upper()}/" + relative_path = parsed.path + if relative_path.startswith(root_prefix): + relative_path = relative_path[len(root_prefix) :] + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + return { + "accession": accession.upper(), + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": Files._map_massive_collection_to_category(collection)}, + "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], + "relativePath": relative_path, + "collection": collection, + "source": "JPOST", + } + + @staticmethod + def is_iprox_accession(accession: str) -> bool: + """ + Return True when the accession looks like an iProX dataset accession. + """ + if not accession: + return False + return bool(re.fullmatch(r"IPX\d{7,10}", accession.upper())) + + @staticmethod + def _get_iprox_public_root(accession: str) -> str: + return f"/{accession.upper()}" + + @staticmethod + def _get_iprox_public_ftp_url(accession: str, remote_path: str) -> str: + root_path = Files._get_iprox_public_root(accession).rstrip("/") + relative_path = remote_path + if remote_path.startswith(root_path): + relative_path = remote_path[len(root_path) :].lstrip("/") + return f"{Files.IPROX_ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" + + @staticmethod + def _build_iprox_file_record(accession: str, ftp_url: str) -> Dict: + parsed = urlparse(ftp_url) + root_prefix = f"/{accession.upper()}/" + relative_path = parsed.path + if relative_path.startswith(root_prefix): + relative_path = relative_path[len(root_prefix) :] + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + return { + "accession": accession.upper(), + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": Files._map_massive_collection_to_category(collection)}, + "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], + "relativePath": relative_path, + "collection": collection, + "source": "iProX", + } + + @staticmethod + def is_direct_download_accession(accession: str) -> bool: + """ + Return True when the accession is served by a public FTP repository + that pridepy supports via direct downloads (no ProteomeXchange API). + """ + return ( + Files.is_massive_accession(accession) + or Files.is_jpost_accession(accession) + or Files.is_iprox_accession(accession) + ) + @staticmethod def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: """ @@ -321,28 +417,46 @@ def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: ftp.cwd(current_dir) return file_paths - def _list_massive_public_files(self, accession: str) -> List[Dict]: + def _list_ftp_repo_files( + self, host: str, remote_root: str, error_label: str + ) -> List[str]: """ - Discover all public files for a MassIVE dataset from its anonymous FTP tree. + Connect to an anonymous FTP host, walk a directory tree, and return file paths. + Centralizes connection lifecycle so the constructor failure case doesn't mask + the underlying error in ``finally`` (see PR #98 review). """ - normalized_accession = accession.upper() - remote_root = self._get_massive_public_root(normalized_accession) - ftp = FTP(self.MASSIVE_ARCHIVE_FTP, timeout=30) + ftp: Optional[FTP] = None try: + ftp = FTP(host, timeout=30) ftp.login() ftp.set_pasv(True) - logging.info(f"Connected to FTP host: {self.MASSIVE_ARCHIVE_FTP}") - remote_files = self._walk_ftp_tree(ftp, remote_root) + logging.info(f"Connected to FTP host: {host}") + return self._walk_ftp_tree(ftp, remote_root) except Exception as error: raise RuntimeError( - f"Unable to list public files for MassIVE dataset {normalized_accession}: {error}" + f"Unable to list public files for {error_label}: {error}" ) from error finally: - try: - ftp.quit() - except Exception: - ftp.close() + if ftp is not None: + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + def _list_massive_public_files(self, accession: str) -> List[Dict]: + """ + Discover all public files for a MassIVE dataset from its anonymous FTP tree. + """ + normalized_accession = accession.upper() + remote_root = self._get_massive_public_root(normalized_accession) + remote_files = self._list_ftp_repo_files( + host=self.MASSIVE_ARCHIVE_FTP, + remote_root=remote_root, + error_label=f"MassIVE dataset {normalized_accession}", + ) return [ self._build_massive_file_record( normalized_accession, @@ -362,15 +476,86 @@ def _download_massive_file_records( """ Download public MassIVE files via anonymous FTP. """ + self._download_direct_download_records( + accession=accession, + file_records=file_records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + ) + + def _list_jpost_public_files(self, accession: str) -> List[Dict]: + """ + Discover all public files for a JPOST dataset from its anonymous FTP tree. + """ + normalized_accession = accession.upper() + remote_root = self._get_jpost_public_root(normalized_accession) + remote_files = self._list_ftp_repo_files( + host=self.JPOST_ARCHIVE_FTP, + remote_root=remote_root, + error_label=f"JPOST dataset {normalized_accession}", + ) + return [ + self._build_jpost_file_record( + normalized_accession, + self._get_jpost_public_ftp_url(normalized_accession, remote_file), + ) + for remote_file in remote_files + ] + + def _list_iprox_public_files(self, accession: str) -> List[Dict]: + """ + Discover all public files for an iProX dataset from its anonymous FTP tree. + """ + normalized_accession = accession.upper() + remote_root = self._get_iprox_public_root(normalized_accession) + remote_files = self._list_ftp_repo_files( + host=self.IPROX_ARCHIVE_FTP, + remote_root=remote_root, + error_label=f"iProX dataset {normalized_accession}", + ) + return [ + self._build_iprox_file_record( + normalized_accession, + self._get_iprox_public_ftp_url(normalized_accession, remote_file), + ) + for remote_file in remote_files + ] + + def _list_direct_download_files(self, accession: str) -> List[Dict]: + """ + Dispatch to the right FTP-based listing for a direct-download repository. + """ + if self.is_massive_accession(accession): + return self._list_massive_public_files(accession) + if self.is_jpost_accession(accession): + return self._list_jpost_public_files(accession) + if self.is_iprox_accession(accession): + return self._list_iprox_public_files(accession) + raise ValueError( + f"Accession {accession} is not a direct-download repository accession" + ) + + def _download_direct_download_records( + self, + accession: str, + file_records: List[Dict], + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str, + ) -> None: + """ + Download files from a direct-download repository (MassIVE/JPOST/iProX) via anonymous FTP. + """ if protocol != "ftp": logging.warning( - "MassIVE direct downloads currently use ftp only. " + "Direct downloads currently use ftp only. " f"Ignoring requested protocol '{protocol}' for {accession}." ) ftp_urls = [self._get_download_url(file_record, "ftp") for file_record in file_records] if not ftp_urls: - logging.info(f"No files matched for MassIVE dataset {accession}") + logging.info(f"No files matched for direct-download dataset {accession}") return self.download_ftp_urls( @@ -413,8 +598,8 @@ def get_all_raw_file_list(self, project_accession): :param project_accession: PRIDE accession :return: raw file list in JSON format """ - if self.is_massive_accession(project_accession): - record_files = self._list_massive_public_files(project_accession) + if self.is_direct_download_accession(project_accession): + record_files = self._list_direct_download_files(project_accession) return [ file for file in record_files if file["fileCategory"]["value"] == "RAW" ] @@ -451,8 +636,8 @@ def download_all_raw_files( raw_files = self.get_all_raw_file_list(accession) - if self.is_massive_accession(accession): - self._download_massive_file_records( + if self.is_direct_download_accession(accession): + self._download_direct_download_records( accession=accession, file_records=raw_files, output_folder=output_folder, @@ -945,14 +1130,16 @@ def download_file_by_name( os.mkdir(output_folder) ## Check type of project - if self.is_massive_accession(accession): - logging.info("Downloading file from public MassIVE dataset {}".format(accession)) + if self.is_direct_download_accession(accession): + logging.info( + "Downloading file from public direct-download dataset {}".format(accession) + ) response = self.get_file_from_api(accession, file_name) if not response: raise Exception( - "File name {} not found in MassIVE dataset {}".format(file_name, accession) + "File name {} not found in dataset {}".format(file_name, accession) ) - self._download_massive_file_records( + self._download_direct_download_records( accession=accession, file_records=response, output_folder=output_folder, @@ -1014,8 +1201,8 @@ def get_file_from_api(self, accession, file_name) -> List[Dict]: """ try: - if self.is_massive_accession(accession): - files = self._list_massive_public_files(accession) + if self.is_direct_download_accession(accession): + files = self._list_direct_download_files(accession) return [f for f in files if f["fileName"] == file_name] files = self.stream_all_files_by_project(accession) file = [f for f in files if f["fileName"] == file_name] @@ -1380,8 +1567,8 @@ def download_files_by_list( if not file_names: raise ValueError("file_names must contain at least one filename") - if self.is_massive_accession(accession): - all_files = self._list_massive_public_files(accession) + if self.is_direct_download_accession(accession): + all_files = self._list_direct_download_files(accession) else: all_files = self.stream_all_files_by_project(accession) requested = set(file_names) @@ -1394,8 +1581,8 @@ def download_files_by_list( f"No matching files in project {accession} for: {sorted(requested)}" ) - if self.is_massive_accession(accession): - self._download_massive_file_records( + if self.is_direct_download_accession(accession): + self._download_direct_download_records( accession=accession, file_records=matched, output_folder=output_folder, @@ -1670,8 +1857,8 @@ def download_all_category_files( if categories is None: categories = [category] if category else ["RAW"] raw_files = self.get_all_category_file_list(accession, categories) - if self.is_massive_accession(accession): - self._download_massive_file_records( + if self.is_direct_download_accession(accession): + self._download_direct_download_records( accession=accession, file_records=raw_files, output_folder=output_folder, @@ -1704,8 +1891,8 @@ def get_all_category_file_list( categories = [categories] category_set = {category.upper() for category in categories} - if self.is_massive_accession(accession): - record_files = self._list_massive_public_files(accession) + if self.is_direct_download_accession(accession): + record_files = self._list_direct_download_files(accession) else: record_files = self.stream_all_files_by_project(accession) diff --git a/pridepy/tests/test_iprox_files.py b/pridepy/tests/test_iprox_files.py new file mode 100644 index 0000000..0af7194 --- /dev/null +++ b/pridepy/tests/test_iprox_files.py @@ -0,0 +1,89 @@ +import tempfile +from unittest import TestCase +from unittest.mock import patch + +from pridepy.files.files import Files + + +class TestIProXFiles(TestCase): + def test_is_iprox_accession(self): + assert Files.is_iprox_accession("IPX0000123") + assert Files.is_iprox_accession("IPX0000123000") + assert Files.is_iprox_accession("ipx1234567") + assert not Files.is_iprox_accession("PXD000012") + assert not Files.is_iprox_accession("MSV000012345") + assert not Files.is_iprox_accession("IPX12") + + def test_is_direct_download_accession_includes_iprox(self): + assert Files.is_direct_download_accession("IPX0000123000") + + def test_build_iprox_file_record_maps_collection_to_category(self): + record = Files._build_iprox_file_record( + "IPX0000123000", + "ftp://ftp.iprox.cn/IPX0000123000/peak/sample.mzML", + ) + + assert record["fileName"] == "sample.mzML" + assert record["collection"] == "peak" + assert record["fileCategory"]["value"] == "PEAK" + assert record["source"] == "iProX" + + def test_build_iprox_file_record_marks_raw_collection_as_raw(self): + record = Files._build_iprox_file_record( + "IPX0000123000", + "ftp://ftp.iprox.cn/IPX0000123000/raw/run01.raw", + ) + + assert record["collection"] == "raw" + assert record["fileCategory"]["value"] == "RAW" + + def test_get_all_raw_file_list_filters_iprox_records(self): + files = Files() + iprox_records = [ + Files._build_iprox_file_record( + "IPX0000123000", + "ftp://ftp.iprox.cn/IPX0000123000/raw/run1.raw", + ), + Files._build_iprox_file_record( + "IPX0000123000", + "ftp://ftp.iprox.cn/IPX0000123000/result/results.tsv", + ), + ] + + with patch.object(Files, "_list_iprox_public_files", return_value=iprox_records), patch.object( + Files, "stream_all_files_by_project" + ) as pride_mock: + result = files.get_all_raw_file_list("IPX0000123000") + + pride_mock.assert_not_called() + assert len(result) == 1 + assert {file["fileName"] for file in result} == {"run1.raw"} + + def test_download_file_by_name_uses_iprox_ftp_listing(self): + files = Files() + file_record = Files._build_iprox_file_record( + "IPX0000123000", + "ftp://ftp.iprox.cn/IPX0000123000/raw/folder/sample.raw", + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + Files, "_list_iprox_public_files", return_value=[file_record] + ), patch.object(Files, "download_ftp_urls") as download_mock: + files.download_file_by_name( + accession="IPX0000123000", + file_name="sample.raw", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="ftp", + username=None, + password=None, + aspera_maximum_bandwidth="100M", + checksum_check=False, + ) + + download_mock.assert_called_once_with( + ftp_urls=["ftp://ftp.iprox.cn/IPX0000123000/raw/folder/sample.raw"], + output_folder=tmp_dir, + skip_if_downloaded_already=False, + ) diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py new file mode 100644 index 0000000..d41cb4f --- /dev/null +++ b/pridepy/tests/test_jpost_files.py @@ -0,0 +1,88 @@ +import tempfile +from unittest import TestCase +from unittest.mock import patch + +from pridepy.files.files import Files + + +class TestJPOSTFiles(TestCase): + def test_is_jpost_accession(self): + assert Files.is_jpost_accession("JPST000001") + assert Files.is_jpost_accession("jpst123456") + assert not Files.is_jpost_accession("PXD000012") + assert not Files.is_jpost_accession("MSV000012345") + assert not Files.is_jpost_accession("JPST12") + + def test_is_direct_download_accession_includes_jpost(self): + assert Files.is_direct_download_accession("JPST000001") + + def test_build_jpost_file_record_maps_collection_to_category(self): + record = Files._build_jpost_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/peak/sample.mzML", + ) + + assert record["fileName"] == "sample.mzML" + assert record["collection"] == "peak" + assert record["fileCategory"]["value"] == "PEAK" + assert record["source"] == "JPOST" + + def test_build_jpost_file_record_marks_raw_collection_as_raw(self): + record = Files._build_jpost_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/raw/run01.raw", + ) + + assert record["collection"] == "raw" + assert record["fileCategory"]["value"] == "RAW" + + def test_get_all_raw_file_list_filters_jpost_records(self): + files = Files() + jpost_records = [ + Files._build_jpost_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/raw/run1.raw", + ), + Files._build_jpost_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/result/results.tsv", + ), + ] + + with patch.object(Files, "_list_jpost_public_files", return_value=jpost_records), patch.object( + Files, "stream_all_files_by_project" + ) as pride_mock: + result = files.get_all_raw_file_list("JPST000001") + + pride_mock.assert_not_called() + assert len(result) == 1 + assert {file["fileName"] for file in result} == {"run1.raw"} + + def test_download_file_by_name_uses_jpost_ftp_listing(self): + files = Files() + file_record = Files._build_jpost_file_record( + "JPST000001", + "ftp://ftp.jpostdb.org/JPST000001/raw/folder/sample.raw", + ) + + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + Files, "_list_jpost_public_files", return_value=[file_record] + ), patch.object(Files, "download_ftp_urls") as download_mock: + files.download_file_by_name( + accession="JPST000001", + file_name="sample.raw", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="ftp", + username=None, + password=None, + aspera_maximum_bandwidth="100M", + checksum_check=False, + ) + + download_mock.assert_called_once_with( + ftp_urls=["ftp://ftp.jpostdb.org/JPST000001/raw/folder/sample.raw"], + output_folder=tmp_dir, + skip_if_downloaded_already=False, + ) diff --git a/pyproject.toml b/pyproject.toml index 90f40ae..4a95f24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pridepy" -version = "0.0.15" +version = "0.0.16" description = "Python Client library for PRIDE Rest API" readme = "README.md" requires-python = ">=3.9" From 3f8ed3fb2c2a5484a3a3b955969d1efc6f90f351 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 11:29:11 +0100 Subject: [PATCH 02/54] Direct downloads: FTPS for MassIVE, parallelism, defer iProX Address feedback that direct downloads should match the PRIDE feature set (resume, parallel, retries) and that MassIVE's actual FTP server requires TLS. Live findings: - massive-ftp.ucsd.edu now rejects plain anonymous FTP with 421 TLS is required. The merged PR #98 code was effectively broken against the live server. Switch to FTP_TLS + PROT P. - ftp.jpostdb.org accepts plain anonymous FTP; keep as-is. - ftp.iprox.cn does not resolve (DNS fail) and no other iProX FTP host responds. iProX is HTTPS-only and needs a different transport (REST API). Defer iProX support; user to provide the endpoint. Implementation: - New static _open_ftp_connection(host, use_tls) opens FTP or FTP_TLS with the right TLS setup, and transparently falls back to FTPS if a plain FTP server replies 'TLS is required'. - _list_ftp_repo_files() and download_ftp_urls() both grow a use_tls flag. _repo_uses_tls(accession) wires this from the repo type (MassIVE = True, JPOST = False). - download_ftp_urls() grows parallel_files: when >1, a ThreadPoolExecutor runs that many FTP workers per host, each with its own connection. Existing serial single-connection-per-host path is preserved for parallel_files <= 1. - Extracted _download_one_ftp_path() with REST-based resume + per-file retries; _download_ftp_paths_serial() / _download_ftp_paths_parallel() pick the right scheduling. REST resume verified live (3 KB pre-stage -> 10 KB final, MD5 matches full file). - _download_direct_download_records() now accepts parallel_files and forwards it (along with use_tls derived from the accession) to download_ftp_urls. All call sites (download_all_raw_files, download_all_category_files, download_files_by_list) thread the user-supplied -w/--parallel-files through. Tests: - test_jpost_files / test_massive_files updated to assert the new kwargs (use_tls, parallel_files). - New test_repo_uses_tls_true_for_massive_false_for_jpost and test_download_all_raw_files_threads_parallel_files_for_massive. - test_iprox_files.py removed (iProX is deferred). - All 15 unit tests pass. Live testing: - MassIVE: listed MSV000080175 (44 files), single-file download of params.xml (10315 B, MD5 43d87368d705c3f380c1d030b14850c4), REST resume from a 3000 B partial, and 3-worker parallel download of files from MSV000080175 + MSV000078335 all succeeded. - JPOST: rate-limited from this IP ('421 too many connections'). Code path is structurally identical to MassIVE (same FTP helper), routing covered by unit tests; deferred to a follow-up live check. --- README.md | 17 +- pridepy/files/files.py | 428 ++++++++++++++++++---------- pridepy/tests/test_iprox_files.py | 89 ------ pridepy/tests/test_jpost_files.py | 2 + pridepy/tests/test_massive_files.py | 35 +++ 5 files changed, 317 insertions(+), 254 deletions(-) delete mode 100644 pridepy/tests/test_iprox_files.py diff --git a/README.md b/README.md index a661153..6139748 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ You can: - download public and private PRIDE files -- download public MassIVE (`MSV...`), JPOST (`JPST...`), and iProX (`IPX...`) datasets directly from their native FTP archives +- download public MassIVE (`MSV...`) and JPOST (`JPST...`) datasets directly from their native FTP archives - download by category (`RAW`, `SEARCH`, `RESULT`, etc.) - stream project and file metadata - search projects by keyword and filters @@ -80,7 +80,7 @@ pridepy download-all-public-raw-files \ --checksum-check ``` -### 3) Download a public MassIVE, JPOST, or iProX dataset directly +### 3) Download a public MassIVE or JPOST dataset directly ```bash # MassIVE @@ -92,14 +92,9 @@ pridepy download-all-public-raw-files \ pridepy download-all-public-raw-files \ -a JPST000123 \ -o ./downloads/JPST000123 - -# iProX -pridepy download-all-public-raw-files \ - -a IPX0000123000 \ - -o ./downloads/IPX0000123000 ``` -For these direct downloads, `pridepy` enumerates the dataset from the repository's public FTP tree (MassIVE at `massive-ftp.ucsd.edu`, JPOST at `ftp.jpostdb.org`, iProX at `ftp.iprox.cn`). Raw downloads follow each repository's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. +For these direct downloads, `pridepy` enumerates the dataset from the repository's public FTP tree (MassIVE at `massive-ftp.ucsd.edu` over FTPS, JPOST at `ftp.jpostdb.org` over plain FTP). Raw downloads follow each repository's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. ### 4) Download only selected categories @@ -110,7 +105,7 @@ pridepy download-all-public-category-files \ -c RAW,SEARCH ``` -You can also request a specific MassIVE / JPOST / iProX collection through the same category interface: +You can also request a specific MassIVE / JPOST collection through the same category interface: ```bash pridepy download-all-public-category-files \ @@ -255,13 +250,13 @@ print(f"RAW files: {len(raw_files)}") print(raw_files[0]["fileName"]) ``` -For MassIVE / JPOST / iProX accessions, the same method returns the files found under the dataset's `raw/` collection: +For MassIVE / JPOST accessions, the same method returns the files found under the dataset's `raw/` collection: ```python from pridepy.files.files import Files files = Files() -for accession in ("MSV000082297", "JPST000123", "IPX0000123000"): +for accession in ("MSV000082297", "JPST000123"): raw_files = files.get_all_raw_file_list(accession) print(f"{accession} raw files: {len(raw_files)}") ``` diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 1e5ac75..464e761 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -65,8 +65,6 @@ class Files: MASSIVE_ARCHIVE_FTP_URL_PREFIX = "ftp://massive-ftp.ucsd.edu/v01/" JPOST_ARCHIVE_FTP = "ftp.jpostdb.org" JPOST_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.jpostdb.org/" - IPROX_ARCHIVE_FTP = "ftp.iprox.cn" - IPROX_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.iprox.cn/" S3_URL = "https://hh.fire.sdo.ebi.ac.uk" S3_BUCKET = "pride-public" PROTOCOL_ORDER = ["aspera", "s3", "ftp", "globus"] @@ -324,46 +322,6 @@ def _build_jpost_file_record(accession: str, ftp_url: str) -> Dict: "source": "JPOST", } - @staticmethod - def is_iprox_accession(accession: str) -> bool: - """ - Return True when the accession looks like an iProX dataset accession. - """ - if not accession: - return False - return bool(re.fullmatch(r"IPX\d{7,10}", accession.upper())) - - @staticmethod - def _get_iprox_public_root(accession: str) -> str: - return f"/{accession.upper()}" - - @staticmethod - def _get_iprox_public_ftp_url(accession: str, remote_path: str) -> str: - root_path = Files._get_iprox_public_root(accession).rstrip("/") - relative_path = remote_path - if remote_path.startswith(root_path): - relative_path = remote_path[len(root_path) :].lstrip("/") - return f"{Files.IPROX_ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" - - @staticmethod - def _build_iprox_file_record(accession: str, ftp_url: str) -> Dict: - parsed = urlparse(ftp_url) - root_prefix = f"/{accession.upper()}/" - relative_path = parsed.path - if relative_path.startswith(root_prefix): - relative_path = relative_path[len(root_prefix) :] - relative_path = relative_path.lstrip("/") - collection = relative_path.split("/", 1)[0] if relative_path else "" - return { - "accession": accession.upper(), - "fileName": os.path.basename(parsed.path), - "fileCategory": {"value": Files._map_massive_collection_to_category(collection)}, - "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], - "relativePath": relative_path, - "collection": collection, - "source": "iProX", - } - @staticmethod def is_direct_download_accession(accession: str) -> bool: """ @@ -373,9 +331,17 @@ def is_direct_download_accession(accession: str) -> bool: return ( Files.is_massive_accession(accession) or Files.is_jpost_accession(accession) - or Files.is_iprox_accession(accession) ) + @staticmethod + def _repo_uses_tls(accession: str) -> bool: + """ + Whether the public FTP server for ``accession`` requires FTP over TLS. + MassIVE rejects plain anonymous FTP (``421 TLS is required``); JPOST + accepts plain FTP. + """ + return Files.is_massive_accession(accession) + @staticmethod def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: """ @@ -417,20 +383,55 @@ def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: ftp.cwd(current_dir) return file_paths + @staticmethod + def _open_ftp_connection(host: str, use_tls: bool, timeout: int = 30) -> FTP: + """ + Open an anonymous FTP connection, transparently using FTPS when the + server requires TLS (e.g., MassIVE). When ``use_tls`` is False but the + server replies ``421 TLS is required`` to ``login``, transparently + retry with FTPS so callers don't need to know the policy in advance. + """ + if use_tls: + ftp: FTP = ftplib.FTP_TLS(host, timeout=timeout) + ftp.login() + ftp.prot_p() + else: + ftp = FTP(host, timeout=timeout) + try: + ftp.login() + except ftplib.error_temp as e: + if "TLS" in str(e).upper(): + try: + ftp.close() + except Exception: + pass + ftp = ftplib.FTP_TLS(host, timeout=timeout) + ftp.login() + ftp.prot_p() + else: + raise + ftp.set_pasv(True) + return ftp + def _list_ftp_repo_files( - self, host: str, remote_root: str, error_label: str + self, + host: str, + remote_root: str, + error_label: str, + use_tls: bool = False, ) -> List[str]: """ - Connect to an anonymous FTP host, walk a directory tree, and return file paths. - Centralizes connection lifecycle so the constructor failure case doesn't mask - the underlying error in ``finally`` (see PR #98 review). + Connect to an anonymous FTP host (FTP or FTPS), walk a directory tree, + and return file paths. + + ``use_tls`` should be True for servers that reject plain FTP (e.g. + MassIVE). Centralizes connection lifecycle so a constructor failure + doesn't mask the underlying error in ``finally`` (PR #98 review). """ ftp: Optional[FTP] = None try: - ftp = FTP(host, timeout=30) - ftp.login() - ftp.set_pasv(True) - logging.info(f"Connected to FTP host: {host}") + ftp = self._open_ftp_connection(host, use_tls=use_tls) + logging.info(f"Connected to FTP host: {host} (tls={use_tls})") return self._walk_ftp_tree(ftp, remote_root) except Exception as error: raise RuntimeError( @@ -456,6 +457,7 @@ def _list_massive_public_files(self, accession: str) -> List[Dict]: host=self.MASSIVE_ARCHIVE_FTP, remote_root=remote_root, error_label=f"MassIVE dataset {normalized_accession}", + use_tls=True, ) return [ self._build_massive_file_record( @@ -472,9 +474,11 @@ def _download_massive_file_records( output_folder: str, skip_if_downloaded_already: bool, protocol: str, + parallel_files: int = 1, ) -> None: """ - Download public MassIVE files via anonymous FTP. + Download public MassIVE files via anonymous FTP (now FTPS). + Backward-compat wrapper around :meth:`_download_direct_download_records`. """ self._download_direct_download_records( accession=accession, @@ -482,6 +486,7 @@ def _download_massive_file_records( output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, + parallel_files=parallel_files, ) def _list_jpost_public_files(self, accession: str) -> List[Dict]: @@ -503,25 +508,6 @@ def _list_jpost_public_files(self, accession: str) -> List[Dict]: for remote_file in remote_files ] - def _list_iprox_public_files(self, accession: str) -> List[Dict]: - """ - Discover all public files for an iProX dataset from its anonymous FTP tree. - """ - normalized_accession = accession.upper() - remote_root = self._get_iprox_public_root(normalized_accession) - remote_files = self._list_ftp_repo_files( - host=self.IPROX_ARCHIVE_FTP, - remote_root=remote_root, - error_label=f"iProX dataset {normalized_accession}", - ) - return [ - self._build_iprox_file_record( - normalized_accession, - self._get_iprox_public_ftp_url(normalized_accession, remote_file), - ) - for remote_file in remote_files - ] - def _list_direct_download_files(self, accession: str) -> List[Dict]: """ Dispatch to the right FTP-based listing for a direct-download repository. @@ -530,8 +516,6 @@ def _list_direct_download_files(self, accession: str) -> List[Dict]: return self._list_massive_public_files(accession) if self.is_jpost_accession(accession): return self._list_jpost_public_files(accession) - if self.is_iprox_accession(accession): - return self._list_iprox_public_files(accession) raise ValueError( f"Accession {accession} is not a direct-download repository accession" ) @@ -543,9 +527,12 @@ def _download_direct_download_records( output_folder: str, skip_if_downloaded_already: bool, protocol: str, + parallel_files: int = 1, ) -> None: """ - Download files from a direct-download repository (MassIVE/JPOST/iProX) via anonymous FTP. + Download files from a direct-download repository (MassIVE/JPOST) via + anonymous FTP. Supports REST-based resume, per-file retries, and + parallel workers (one connection per worker, capped at file count). """ if protocol != "ftp": logging.warning( @@ -562,6 +549,8 @@ def _download_direct_download_records( ftp_urls=ftp_urls, output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=self._repo_uses_tls(accession), + parallel_files=parallel_files, ) async def stream_all_files_metadata(self, output_file, accession=None): @@ -643,6 +632,7 @@ def download_all_raw_files( output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, + parallel_files=parallel_files, ) return @@ -1588,6 +1578,7 @@ def download_files_by_list( output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, + parallel_files=parallel_files, ) return @@ -1864,6 +1855,7 @@ def download_all_category_files( output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, + parallel_files=parallel_files, ) return self.download_files( @@ -1988,6 +1980,181 @@ def _local_path_for_url(download_url: str, output_folder: str) -> str: filename = os.path.basename(urlparse(download_url).path) return os.path.join(output_folder, filename) + @staticmethod + def _download_one_ftp_path( + ftp: FTP, + ftp_path: str, + local_path: str, + skip_if_downloaded_already: bool, + max_download_retries: int, + position: int = 0, + ) -> None: + """ + Download a single FTP path over an existing connection, with REST resume + and per-file retry. Raises on giving up so the caller can decide what to do. + """ + if skip_if_downloaded_already and os.path.exists(local_path): + logging.info(f"Skipping download as file already exists: {local_path}") + return + + attempt = 0 + last_error: Optional[Exception] = None + while attempt < max_download_retries: + try: + total_size = ftp.size(ftp_path) + if os.path.exists(local_path): + current_size = os.path.getsize(local_path) + mode = "ab" + else: + current_size = 0 + mode = "wb" + + with open(local_path, mode) as f, tqdm( + total=total_size, + unit="B", + unit_scale=True, + desc=local_path, + initial=current_size, + position=position, + leave=True, + ) as pbar: + def callback(data): + f.write(data) + pbar.update(len(data)) + + if current_size: + try: + ftp.sendcmd(f"REST {current_size}") + except Exception: + current_size = 0 + f.seek(0) + f.truncate() + ftp.retrbinary(f"RETR {ftp_path}", callback) + logging.info(f"Successfully downloaded {local_path}") + return + except (socket.timeout, ftplib.error_temp, ftplib.error_perm) as e: + attempt += 1 + last_error = e + logging.error( + f"Download failed for {local_path} (attempt {attempt}): {e}" + ) + raise RuntimeError( + f"Giving up on {local_path} after {max_download_retries} attempts" + ) from last_error + + @staticmethod + def _download_ftp_paths_serial( + host: str, + paths: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + use_tls: bool, + max_connection_retries: int, + max_download_retries: int, + ) -> None: + """Download all paths from one host over a single (reused) connection.""" + connection_attempt = 0 + while connection_attempt < max_connection_retries: + try: + ftp = Files._open_ftp_connection(host, use_tls=use_tls) + logging.info(f"Connected to FTP host: {host} (tls={use_tls})") + for ftp_path in paths: + local_path = os.path.join(output_folder, os.path.basename(ftp_path)) + try: + Files._download_one_ftp_path( + ftp=ftp, + ftp_path=ftp_path, + local_path=local_path, + skip_if_downloaded_already=skip_if_downloaded_already, + max_download_retries=max_download_retries, + ) + except Exception as e: + logging.error( + f"Failed to download {ftp_path} from {host}: {e}" + ) + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + logging.info(f"Disconnected from FTP host: {host}") + return + except (socket.timeout, ftplib.error_temp, ftplib.error_perm, OSError) as e: + connection_attempt += 1 + logging.error( + f"FTP connection failed (attempt {connection_attempt}): {e}" + ) + if connection_attempt < max_connection_retries: + logging.info("Retrying connection...") + time.sleep(5) + else: + logging.error( + f"Giving up after {max_connection_retries} failed connection attempts to {host}." + ) + + @staticmethod + def _download_ftp_paths_parallel( + host: str, + paths: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + use_tls: bool, + max_connection_retries: int, + max_download_retries: int, + parallel_files: int, + ) -> None: + """ + Download paths concurrently using ``parallel_files`` workers; each + worker opens its own FTP connection so transfers don't serialize. + """ + def worker(ftp_path: str, position: int) -> None: + local_path = os.path.join(output_folder, os.path.basename(ftp_path)) + if skip_if_downloaded_already and os.path.exists(local_path): + logging.info(f"Skipping download as file already exists: {local_path}") + return + connection_attempt = 0 + while connection_attempt < max_connection_retries: + try: + ftp = Files._open_ftp_connection(host, use_tls=use_tls) + try: + Files._download_one_ftp_path( + ftp=ftp, + ftp_path=ftp_path, + local_path=local_path, + skip_if_downloaded_already=False, + max_download_retries=max_download_retries, + position=position, + ) + return + finally: + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + except (socket.timeout, ftplib.error_temp, ftplib.error_perm, OSError) as e: + connection_attempt += 1 + logging.error( + f"FTP connection failed for {ftp_path} (attempt {connection_attempt}): {e}" + ) + if connection_attempt < max_connection_retries: + time.sleep(5) + logging.error(f"Giving up on {ftp_path} from {host}") + + with ThreadPoolExecutor(max_workers=parallel_files) as executor: + futures = [ + executor.submit(worker, path, idx) for idx, path in enumerate(paths) + ] + for future in as_completed(futures): + try: + future.result() + except Exception as e: + logging.error(f"Parallel FTP download error: {e}") + @staticmethod def download_ftp_urls( ftp_urls: List[str], @@ -1995,98 +2162,51 @@ def download_ftp_urls( skip_if_downloaded_already: bool, max_connection_retries: int = 3, max_download_retries: int = 3, + use_tls: bool = False, + parallel_files: int = 1, ) -> None: """ - Download a list of FTP URLs using a single connection, with retries and progress bars. + Download a list of FTP URLs with retries, REST-based resume, and + optional parallel workers. + + :param use_tls: Open the FTP connection with TLS (FTP_TLS / PROT P). + Required for hosts that reject plain anonymous FTP (e.g. MassIVE). + When False but the server replies ``421 TLS is required``, the + connection is transparently retried over TLS. + :param parallel_files: When >1, downloads run concurrently with that + many worker connections per host (capped at the number of files). """ if not os.path.isdir(output_folder): os.makedirs(output_folder, exist_ok=True) - def connect_ftp(host: str): - ftp = FTP(host, timeout=30) - ftp.login() - ftp.set_pasv(True) - logging.info(f"Connected to FTP host: {host}") - return ftp - - # Group URLs by host to reuse connections efficiently host_to_paths: Dict[str, List[str]] = {} for url in ftp_urls: parsed = urlparse(url) host_to_paths.setdefault(parsed.hostname, []).append(parsed.path.lstrip("/")) for host, paths in host_to_paths.items(): - connection_attempt = 0 - while connection_attempt < max_connection_retries: - try: - ftp = connect_ftp(host) - for ftp_path in paths: - try: - local_path = os.path.join(output_folder, os.path.basename(ftp_path)) - if skip_if_downloaded_already and os.path.exists(local_path): - logging.info("Skipping download as file already exists") - continue - - logging.info(f"Starting FTP download: {host}/{ftp_path}") - download_attempt = 0 - while download_attempt < max_download_retries: - try: - total_size = ftp.size(ftp_path) - # Try to resume using REST if partial file exists - if os.path.exists(local_path): - current_size = os.path.getsize(local_path) - mode = "ab" - else: - current_size = 0 - mode = "wb" - - with open(local_path, mode) as f, tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=local_path, - initial=current_size, - ) as pbar: - def callback(data): - f.write(data) - pbar.update(len(data)) - - if current_size: - try: - ftp.sendcmd(f"REST {current_size}") - except Exception: - # If REST not supported, fall back to full download - current_size = 0 - f.seek(0) - f.truncate() - ftp.retrbinary(f"RETR {ftp_path}", callback) - logging.info(f"Successfully downloaded {local_path}") - break - except (socket.timeout, ftplib.error_temp, ftplib.error_perm) as e: - download_attempt += 1 - logging.error( - f"Download failed for {local_path} (attempt {download_attempt}): {str(e)}" - ) - if download_attempt >= max_download_retries: - logging.error( - f"Giving up on {local_path} after {max_download_retries} attempts." - ) - break - except Exception as e: - logging.error(f"Unexpected error while processing FTP path {ftp_path}: {str(e)}") - ftp.quit() - logging.info(f"Disconnected from FTP host: {host}") - break - except (socket.timeout, ftplib.error_temp, ftplib.error_perm, socket.error) as e: - connection_attempt += 1 - logging.error(f"FTP connection failed (attempt {connection_attempt}): {str(e)}") - if connection_attempt < max_connection_retries: - logging.info("Retrying connection...") - time.sleep(5) - else: - logging.error( - f"Giving up after {max_connection_retries} failed connection attempts to {host}." - ) + workers = max(1, min(parallel_files, len(paths))) + if workers > 1: + Files._download_ftp_paths_parallel( + host=host, + paths=paths, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=use_tls, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + parallel_files=workers, + ) + else: + Files._download_ftp_paths_serial( + host=host, + paths=paths, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=use_tls, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + ) @staticmethod def download_http_urls( diff --git a/pridepy/tests/test_iprox_files.py b/pridepy/tests/test_iprox_files.py deleted file mode 100644 index 0af7194..0000000 --- a/pridepy/tests/test_iprox_files.py +++ /dev/null @@ -1,89 +0,0 @@ -import tempfile -from unittest import TestCase -from unittest.mock import patch - -from pridepy.files.files import Files - - -class TestIProXFiles(TestCase): - def test_is_iprox_accession(self): - assert Files.is_iprox_accession("IPX0000123") - assert Files.is_iprox_accession("IPX0000123000") - assert Files.is_iprox_accession("ipx1234567") - assert not Files.is_iprox_accession("PXD000012") - assert not Files.is_iprox_accession("MSV000012345") - assert not Files.is_iprox_accession("IPX12") - - def test_is_direct_download_accession_includes_iprox(self): - assert Files.is_direct_download_accession("IPX0000123000") - - def test_build_iprox_file_record_maps_collection_to_category(self): - record = Files._build_iprox_file_record( - "IPX0000123000", - "ftp://ftp.iprox.cn/IPX0000123000/peak/sample.mzML", - ) - - assert record["fileName"] == "sample.mzML" - assert record["collection"] == "peak" - assert record["fileCategory"]["value"] == "PEAK" - assert record["source"] == "iProX" - - def test_build_iprox_file_record_marks_raw_collection_as_raw(self): - record = Files._build_iprox_file_record( - "IPX0000123000", - "ftp://ftp.iprox.cn/IPX0000123000/raw/run01.raw", - ) - - assert record["collection"] == "raw" - assert record["fileCategory"]["value"] == "RAW" - - def test_get_all_raw_file_list_filters_iprox_records(self): - files = Files() - iprox_records = [ - Files._build_iprox_file_record( - "IPX0000123000", - "ftp://ftp.iprox.cn/IPX0000123000/raw/run1.raw", - ), - Files._build_iprox_file_record( - "IPX0000123000", - "ftp://ftp.iprox.cn/IPX0000123000/result/results.tsv", - ), - ] - - with patch.object(Files, "_list_iprox_public_files", return_value=iprox_records), patch.object( - Files, "stream_all_files_by_project" - ) as pride_mock: - result = files.get_all_raw_file_list("IPX0000123000") - - pride_mock.assert_not_called() - assert len(result) == 1 - assert {file["fileName"] for file in result} == {"run1.raw"} - - def test_download_file_by_name_uses_iprox_ftp_listing(self): - files = Files() - file_record = Files._build_iprox_file_record( - "IPX0000123000", - "ftp://ftp.iprox.cn/IPX0000123000/raw/folder/sample.raw", - ) - - with tempfile.TemporaryDirectory() as tmp_dir: - with patch.object( - Files, "_list_iprox_public_files", return_value=[file_record] - ), patch.object(Files, "download_ftp_urls") as download_mock: - files.download_file_by_name( - accession="IPX0000123000", - file_name="sample.raw", - output_folder=tmp_dir, - skip_if_downloaded_already=False, - protocol="ftp", - username=None, - password=None, - aspera_maximum_bandwidth="100M", - checksum_check=False, - ) - - download_mock.assert_called_once_with( - ftp_urls=["ftp://ftp.iprox.cn/IPX0000123000/raw/folder/sample.raw"], - output_folder=tmp_dir, - skip_if_downloaded_already=False, - ) diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py index d41cb4f..021401e 100644 --- a/pridepy/tests/test_jpost_files.py +++ b/pridepy/tests/test_jpost_files.py @@ -85,4 +85,6 @@ def test_download_file_by_name_uses_jpost_ftp_listing(self): ftp_urls=["ftp://ftp.jpostdb.org/JPST000001/raw/folder/sample.raw"], output_folder=tmp_dir, skip_if_downloaded_already=False, + use_tls=False, + parallel_files=1, ) diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index f600b71..a958309 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -99,4 +99,39 @@ def test_download_file_by_name_uses_massive_ftp_listing(self): ftp_urls=["ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/folder/sample.raw"], output_folder=tmp_dir, skip_if_downloaded_already=False, + use_tls=True, + parallel_files=1, ) + + def test_repo_uses_tls_true_for_massive_false_for_jpost(self): + assert Files._repo_uses_tls("MSV000012345") is True + assert Files._repo_uses_tls("JPST000001") is False + assert Files._repo_uses_tls("PXD000012") is False + + def test_download_all_raw_files_threads_parallel_files_for_massive(self): + files = Files() + massive_records = [ + Files._build_massive_file_record( + "MSV000012345", + f"ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/run{i}.raw", + ) + for i in range(3) + ] + + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + Files, "_list_massive_public_files", return_value=massive_records + ), patch.object(Files, "download_ftp_urls") as download_mock: + files.download_all_raw_files( + accession="MSV000012345", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="ftp", + aspera_maximum_bandwidth="100M", + checksum_check=False, + parallel_files=3, + ) + + kwargs = download_mock.call_args.kwargs + assert kwargs["use_tls"] is True + assert kwargs["parallel_files"] == 3 From a3603785826a5ea2b27b5e825f356db3571bbc46 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 11:51:06 +0100 Subject: [PATCH 03/54] JPOST PROXI listing, post-transfer size check, iProX accession guard Address deferred items from PR review: 1. JPOST PROXI listing ftp.jpostdb.org rate-limits aggressively per source IP (sticky 421 on any retry within ~10 min). For listing this is fatal because every pridepy invocation needs a fresh tree walk. JPOST publishes a JSON PROXI endpoint at https://repository.jpostdb.org/proxi/datasets/ that returns datasetFiles[*].value as ftp:// URIs alongside CV labels (Associated raw file URI, Search engine output file URI, Result file URI, Peak list file URI, ...). _list_jpost_public_files now hits PROXI first, builds file records with categories derived from the CV name (mapped via JPOST_PROXI_CATEGORY_MAP), and falls back to the FTP tree walk only if PROXI is unreachable or returns no records. Live-tested against JPST002311 -> 160 files (88 RAW, 72 SEARCH). 2. Post-transfer size check Neither MassIVE nor JPOST publishes per-file checksum manifests in a standard location, so md5 verification isn't an option for these datasets. As a lighter-weight integrity signal, _download_one_ftp_path now compares the local file size against ftp.size() after retrbinary returns and treats a mismatch as a retryable failure (next attempt resumes via REST from the current partial). This catches half-finished transfers where the data channel was closed early without retrbinary raising. 3. iProX accession guard Probing showed iProX REST endpoints (PMD009Controller/findByProjectId.jsonp, findFilesBySubProjectID.jsonp) all redirect unauthenticated callers to a CAS login page, and downloads use faspe:// URLs with per-session tokens. Native support is therefore blocked until iProX exposes an anonymous JSON API or pridepy carries iProX credentials. Until then, add is_iprox_accession() and _raise_if_iprox() so every public entry point (get_all_raw_file_list, download_all_raw_files, download_all_category_files, download_file_by_name, download_files_by_list, get_file_from_api) emits a clear NotImplementedError instead of silently falling through to the PRIDE API and 404-ing. Tests - test_jpost_files: PROXI listing maps CV names to PRIDE categories; PROXI failure falls back to FTP walk. - test_iprox_guard: regex coverage; assert each entry point raises NotImplementedError for IPX accessions; assert is_direct_download_accession returns False for IPX. - test_ftp_download_validation: size mismatch retries until success; repeated mismatch raises after max_download_retries; correct size skips retries. - 25 tests total (10 MassIVE + 8 JPOST + 5 iProX guard + 3 size check). Live verification (same MassIVE FTPS path as before) - MSV000080175 listing + params.xml download still produces 10315 B, MD5 43d87368d705c3f380c1d030b14850c4. - JPST002311 PROXI listing returns 160 files with correct categories. Actual file transfer is still blocked from this IP by JPOST's 421-too-many-connections rate limit; the code path is shared with MassIVE (same _download_one_ftp_path / download_ftp_urls), so a fresh IP should succeed. --- README.md | 10 +- pridepy/files/files.py | 150 ++++++++++++++++-- pridepy/tests/test_ftp_download_validation.py | 91 +++++++++++ pridepy/tests/test_iprox_guard.py | 62 ++++++++ pridepy/tests/test_jpost_files.py | 65 +++++++- 5 files changed, 361 insertions(+), 17 deletions(-) create mode 100644 pridepy/tests/test_ftp_download_validation.py create mode 100644 pridepy/tests/test_iprox_guard.py diff --git a/README.md b/README.md index 6139748..9609f26 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ You can: - download public and private PRIDE files -- download public MassIVE (`MSV...`) and JPOST (`JPST...`) datasets directly from their native FTP archives +- download public MassIVE (`MSV...`) and JPOST (`JPST...`) datasets directly. MassIVE goes through FTPS at `massive-ftp.ucsd.edu`; JPOST uses the JSON PROXI endpoint at `repository.jpostdb.org` for listings and `ftp.jpostdb.org` for transfers - download by category (`RAW`, `SEARCH`, `RESULT`, etc.) - stream project and file metadata - search projects by keyword and filters @@ -94,7 +94,13 @@ pridepy download-all-public-raw-files \ -o ./downloads/JPST000123 ``` -For these direct downloads, `pridepy` enumerates the dataset from the repository's public FTP tree (MassIVE at `massive-ftp.ucsd.edu` over FTPS, JPOST at `ftp.jpostdb.org` over plain FTP). Raw downloads follow each repository's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. +For these direct downloads, `pridepy` enumerates the dataset from the repository: +- **MassIVE** lists files by walking the FTPS tree at `massive-ftp.ucsd.edu` (TLS is required by the server). +- **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads them from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. + +Raw downloads follow each repository's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. Direct downloads support REST-based resume, per-file retries, parallel workers (`-w N` up to 3), and post-transfer size verification against the server-reported size. + +iProX accessions (`IPX...`) are recognised so the CLI gives you a clear "not supported yet" error rather than treating them as unknown PRIDE accessions. Native iProX download support is blocked on their REST API requiring CAS authentication and downloads going through Aspera with per-session tokens; track that work upstream. ### 4) Download only selected categories diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 464e761..ecacd16 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -65,6 +65,16 @@ class Files: MASSIVE_ARCHIVE_FTP_URL_PREFIX = "ftp://massive-ftp.ucsd.edu/v01/" JPOST_ARCHIVE_FTP = "ftp.jpostdb.org" JPOST_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.jpostdb.org/" + JPOST_PROXI_BASE_URL = "https://repository.jpostdb.org/proxi/datasets/" + JPOST_PROXI_CATEGORY_MAP = { + "Associated raw file URI": "RAW", + "Result file URI": "RESULT", + "Search engine output file URI": "SEARCH", + "Peak list file URI": "PEAK", + "Spectrum library file URI": "SPECTRUM_LIBRARY", + "Sequence database URI": "FASTA", + "Quantification file URI": "RESULT", + } S3_URL = "https://hh.fire.sdo.ebi.ac.uk" S3_BUCKET = "pride-public" PROTOCOL_ORDER = ["aspera", "s3", "ftp", "globus"] @@ -304,7 +314,17 @@ def _get_jpost_public_ftp_url(accession: str, remote_path: str) -> str: return f"{Files.JPOST_ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" @staticmethod - def _build_jpost_file_record(accession: str, ftp_url: str) -> Dict: + def _build_jpost_file_record( + accession: str, ftp_url: str, category_from_proxi: Optional[str] = None + ) -> Dict: + """ + Build a pridepy file record for a JPOST file. + + When ``category_from_proxi`` is provided (e.g. ``"Associated raw file URI"``), + the PROXI CV name takes precedence over the heuristic collection-from-path + mapping. Falls back to the same path-segment heuristic used for MassIVE + when the category isn't known. + """ parsed = urlparse(ftp_url) root_prefix = f"/{accession.upper()}/" relative_path = parsed.path @@ -312,10 +332,14 @@ def _build_jpost_file_record(accession: str, ftp_url: str) -> Dict: relative_path = relative_path[len(root_prefix) :] relative_path = relative_path.lstrip("/") collection = relative_path.split("/", 1)[0] if relative_path else "" + if category_from_proxi and category_from_proxi in Files.JPOST_PROXI_CATEGORY_MAP: + category = Files.JPOST_PROXI_CATEGORY_MAP[category_from_proxi] + else: + category = Files._map_massive_collection_to_category(collection) return { "accession": accession.upper(), "fileName": os.path.basename(parsed.path), - "fileCategory": {"value": Files._map_massive_collection_to_category(collection)}, + "fileCategory": {"value": category}, "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], "relativePath": relative_path, "collection": collection, @@ -333,6 +357,34 @@ def is_direct_download_accession(accession: str) -> bool: or Files.is_jpost_accession(accession) ) + @staticmethod + def is_iprox_accession(accession: str) -> bool: + """ + Return True when the accession looks like an iProX dataset accession + (``IPX`` followed by 7-10 digits). iProX is recognised so the CLI can + emit a clear error rather than treating IPX as an unknown PRIDE + accession; direct downloads from iProX are not yet supported because + their listing API requires CAS authentication and downloads go through + Aspera with per-session tokens. + """ + if not accession: + return False + return bool(re.fullmatch(r"IPX\d{7,10}", accession.upper())) + + @staticmethod + def _raise_if_iprox(accession: str) -> None: + """ + Raise a clear ``NotImplementedError`` when a user passes an iProX + accession. iProX downloads need CAS authentication and Aspera-tokenised + ``faspe://`` URLs which pridepy does not handle yet. + """ + if Files.is_iprox_accession(accession): + raise NotImplementedError( + f"iProX accession {accession} is recognised but not yet supported. " + "iProX requires CAS authentication and Aspera-tokenised downloads; " + "track this in pridepy or use the iProX web interface for now." + ) + @staticmethod def _repo_uses_tls(accession: str) -> bool: """ @@ -491,22 +543,71 @@ def _download_massive_file_records( def _list_jpost_public_files(self, accession: str) -> List[Dict]: """ - Discover all public files for a JPOST dataset from its anonymous FTP tree. + Discover all public files for a JPOST dataset. + + Prefers the JPOST PROXI JSON endpoint at + ``https://repository.jpostdb.org/proxi/datasets/`` since it + returns file URLs with category labels and avoids the anonymous-FTP + rate limit that ``ftp.jpostdb.org`` applies per source IP. Falls back + to walking the FTP tree if PROXI is unreachable or returns no files. """ normalized_accession = accession.upper() - remote_root = self._get_jpost_public_root(normalized_accession) - remote_files = self._list_ftp_repo_files( - host=self.JPOST_ARCHIVE_FTP, - remote_root=remote_root, - error_label=f"JPOST dataset {normalized_accession}", + try: + return self._list_jpost_public_files_via_proxi(normalized_accession) + except Exception as proxi_error: + logging.warning( + f"JPOST PROXI listing failed for {normalized_accession} " + f"({proxi_error}); falling back to FTP tree walk." + ) + remote_root = self._get_jpost_public_root(normalized_accession) + remote_files = self._list_ftp_repo_files( + host=self.JPOST_ARCHIVE_FTP, + remote_root=remote_root, + error_label=f"JPOST dataset {normalized_accession}", + ) + return [ + self._build_jpost_file_record( + normalized_accession, + self._get_jpost_public_ftp_url(normalized_accession, remote_file), + ) + for remote_file in remote_files + ] + + def _list_jpost_public_files_via_proxi(self, accession: str) -> List[Dict]: + """ + Fetch the JPOST PROXI dataset metadata and turn each ``datasetFiles`` + entry into a pridepy file record. The PROXI ``name`` field is mapped to + a PRIDE-style category so existing RAW/SEARCH/RESULT filtering works. + """ + import json as _json + + proxi_url = f"{self.JPOST_PROXI_BASE_URL}{accession}" + logging.info(f"Fetching JPOST PROXI metadata: {proxi_url}") + response = requests.get( + proxi_url, + headers={"Accept": "application/json"}, + timeout=30, ) - return [ - self._build_jpost_file_record( - normalized_accession, - self._get_jpost_public_ftp_url(normalized_accession, remote_file), + response.raise_for_status() + data = _json.loads(response.content) + dataset_files = data.get("datasetFiles") or [] + records: List[Dict] = [] + for entry in dataset_files: + value = (entry or {}).get("value") + if not value or not value.startswith("ftp://"): + continue + records.append( + self._build_jpost_file_record( + accession, + value, + category_from_proxi=(entry or {}).get("name"), + ) ) - for remote_file in remote_files - ] + if not records: + raise RuntimeError( + f"JPOST PROXI returned no FTP file URIs for {accession}" + ) + return records def _list_direct_download_files(self, accession: str) -> List[Dict]: """ @@ -587,6 +688,7 @@ def get_all_raw_file_list(self, project_accession): :param project_accession: PRIDE accession :return: raw file list in JSON format """ + self._raise_if_iprox(project_accession) if self.is_direct_download_accession(project_accession): record_files = self._list_direct_download_files(project_accession) return [ @@ -619,6 +721,7 @@ def download_all_raw_files( :param checksum_check: Download checksum for a given project. :return: None """ + self._raise_if_iprox(accession) if not (os.path.isdir(output_folder)): os.mkdir(output_folder) @@ -1115,6 +1218,7 @@ def download_file_by_name( :param aspera_maximum_bandwidth: Aspera maximum bandwidth :param checksum_check: Download checksum for a given project. """ + self._raise_if_iprox(accession) if not (os.path.isdir(output_folder)): os.mkdir(output_folder) @@ -1189,6 +1293,7 @@ def get_file_from_api(self, accession, file_name) -> List[Dict]: :param file_name: file name :return: file in json format """ + self._raise_if_iprox(accession) try: if self.is_direct_download_accession(accession): @@ -1556,6 +1661,7 @@ def download_files_by_list( """ if not file_names: raise ValueError("file_names must contain at least one filename") + self._raise_if_iprox(accession) if self.is_direct_download_accession(accession): all_files = self._list_direct_download_files(accession) @@ -1845,6 +1951,7 @@ def download_all_category_files( :param categories: List of file categories to download. :param category: Single file category (deprecated, use categories instead). """ + self._raise_if_iprox(accession) if categories is None: categories = [category] if category else ["RAW"] raw_files = self.get_all_category_file_list(accession, categories) @@ -2030,6 +2137,21 @@ def callback(data): f.seek(0) f.truncate() ftp.retrbinary(f"RETR {ftp_path}", callback) + + # Post-transfer integrity check: server-reported size must match + # the local size. Catches half-finished transfers that retrbinary + # didn't raise on (e.g. server closed the data channel early). + # The next iteration will REST-resume from where we left off. + if total_size: + final_size = os.path.getsize(local_path) + if final_size != total_size: + attempt += 1 + logging.error( + f"Size mismatch for {local_path}: " + f"got {final_size} bytes, expected {total_size} " + f"(attempt {attempt})" + ) + continue logging.info(f"Successfully downloaded {local_path}") return except (socket.timeout, ftplib.error_temp, ftplib.error_perm) as e: diff --git a/pridepy/tests/test_ftp_download_validation.py b/pridepy/tests/test_ftp_download_validation.py new file mode 100644 index 0000000..10bbfb5 --- /dev/null +++ b/pridepy/tests/test_ftp_download_validation.py @@ -0,0 +1,91 @@ +"""Coverage for the size-mismatch detection added to ``_download_one_ftp_path``. + +The FTP server's ``SIZE`` reply is the only integrity signal direct downloads +have (MassIVE/JPOST don't publish per-file MD5 manifests like PRIDE). After +``retrbinary`` returns, we re-check the local size against the server-reported +size and treat a mismatch as a retryable failure. +""" +import os +import tempfile +from unittest import TestCase +from unittest.mock import MagicMock + +import pytest + +from pridepy.files.files import Files + + +def _make_fake_ftp(expected_size, write_bytes_per_call): + """Return a MagicMock FTP that writes ``write_bytes_per_call`` bytes per call. + + ``retrbinary`` is invoked once per attempt; we record how many attempts + happened by counting calls and produce a different payload size for each. + """ + fake = MagicMock() + fake.size.return_value = expected_size + fake.sendcmd = MagicMock() + fake._call_count = 0 + + def retrbinary(cmd, callback): + idx = fake._call_count + fake._call_count += 1 + payload = b"x" * write_bytes_per_call[idx] + callback(payload) + + fake.retrbinary.side_effect = retrbinary + return fake + + +class TestSizeMismatchValidation(TestCase): + def test_size_mismatch_is_retried_then_succeeds(self): + """First attempt returns 50 bytes (expected 100) -> retry, second yields 50 more -> 100, OK.""" + with tempfile.TemporaryDirectory() as tmp: + local_path = os.path.join(tmp, "f.bin") + ftp = _make_fake_ftp(expected_size=100, write_bytes_per_call=[50, 50]) + + Files._download_one_ftp_path( + ftp=ftp, + ftp_path="/JPST000001/f.bin", + local_path=local_path, + skip_if_downloaded_already=False, + max_download_retries=3, + ) + + assert os.path.getsize(local_path) == 100 + assert ftp.retrbinary.call_count == 2 + # First attempt: file empty, no REST. Second: file has 50 bytes, REST 50 issued. + sendcmd_args = [call.args[0] for call in ftp.sendcmd.call_args_list] + assert sendcmd_args == ["REST 50"] + + def test_size_mismatch_after_retries_raises(self): + """Three attempts all undersize -> RuntimeError after giving up.""" + with tempfile.TemporaryDirectory() as tmp: + local_path = os.path.join(tmp, "f.bin") + ftp = _make_fake_ftp(expected_size=100, write_bytes_per_call=[10, 10, 10]) + + with pytest.raises(RuntimeError, match="Giving up"): + Files._download_one_ftp_path( + ftp=ftp, + ftp_path="/JPST000001/f.bin", + local_path=local_path, + skip_if_downloaded_already=False, + max_download_retries=3, + ) + + assert ftp.retrbinary.call_count == 3 + + def test_correct_size_returns_without_retry(self): + with tempfile.TemporaryDirectory() as tmp: + local_path = os.path.join(tmp, "f.bin") + ftp = _make_fake_ftp(expected_size=50, write_bytes_per_call=[50]) + + Files._download_one_ftp_path( + ftp=ftp, + ftp_path="/JPST000001/f.bin", + local_path=local_path, + skip_if_downloaded_already=False, + max_download_retries=3, + ) + + assert os.path.getsize(local_path) == 50 + assert ftp.retrbinary.call_count == 1 diff --git a/pridepy/tests/test_iprox_guard.py b/pridepy/tests/test_iprox_guard.py new file mode 100644 index 0000000..f28f490 --- /dev/null +++ b/pridepy/tests/test_iprox_guard.py @@ -0,0 +1,62 @@ +"""iProX accession recognition and unsupported-accession guard. + +iProX direct downloads are not implemented (the iProX REST API gates listing +behind CAS authentication and files are served over Aspera with per-session +tokens). pridepy still recognises the accession format so the user gets a +clear ``NotImplementedError`` instead of a confusing PRIDE-API 404. +""" +import tempfile +from unittest import TestCase + +import pytest + +from pridepy.files.files import Files + + +class TestIProXGuard(TestCase): + def test_is_iprox_accession_matches_ipx_format(self): + assert Files.is_iprox_accession("IPX0000123") + assert Files.is_iprox_accession("IPX0000123000") + assert Files.is_iprox_accession("ipx1234567") + assert not Files.is_iprox_accession("PXD000012") + assert not Files.is_iprox_accession("MSV000012345") + assert not Files.is_iprox_accession("JPST000001") + assert not Files.is_iprox_accession("IPX12") + assert not Files.is_iprox_accession("") + assert not Files.is_iprox_accession(None) + + def test_iprox_is_not_a_direct_download_accession(self): + assert Files.is_direct_download_accession("IPX0000123000") is False + + def test_get_all_raw_file_list_raises_for_iprox(self): + files = Files() + with pytest.raises(NotImplementedError, match="iProX"): + files.get_all_raw_file_list("IPX0006033000") + + def test_download_file_by_name_raises_for_iprox(self): + files = Files() + with tempfile.TemporaryDirectory() as tmp_dir: + with pytest.raises(NotImplementedError, match="iProX"): + files.download_file_by_name( + accession="IPX0006033000", + file_name="foo.raw", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="ftp", + username=None, + password=None, + aspera_maximum_bandwidth="100M", + checksum_check=False, + ) + + def test_download_all_raw_files_raises_for_iprox(self): + files = Files() + with tempfile.TemporaryDirectory() as tmp_dir: + with pytest.raises(NotImplementedError, match="iProX"): + files.download_all_raw_files( + accession="IPX0006033000", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="ftp", + aspera_maximum_bandwidth="100M", + ) diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py index 021401e..678adda 100644 --- a/pridepy/tests/test_jpost_files.py +++ b/pridepy/tests/test_jpost_files.py @@ -1,6 +1,7 @@ +import json import tempfile from unittest import TestCase -from unittest.mock import patch +from unittest.mock import MagicMock, patch from pridepy.files.files import Files @@ -88,3 +89,65 @@ def test_download_file_by_name_uses_jpost_ftp_listing(self): use_tls=False, parallel_files=1, ) + + def test_proxi_listing_maps_cv_name_to_category(self): + files = Files() + proxi_response = { + "datasetFiles": [ + { + "accession": "PRIDE:0000404", + "name": "Associated raw file URI", + "value": "ftp://ftp.jpostdb.org/JPST002311/sample01.raw", + }, + { + "accession": "PRIDE:0000408", + "name": "Search engine output file URI", + "value": "ftp://ftp.jpostdb.org/JPST002311/sample01.sne", + }, + { + "accession": "PRIDE:0000999", + "name": "Some unknown CV", + "value": "ftp://ftp.jpostdb.org/JPST002311/misc/sample01.txt", + }, + { + "accession": "PRIDE:0000404", + "name": "Associated raw file URI", + "value": "https://example.org/not-ftp.raw", + }, + ] + } + fake_response = MagicMock() + fake_response.content = json.dumps(proxi_response).encode("utf-8") + fake_response.raise_for_status = MagicMock() + with patch("pridepy.files.files.requests.get", return_value=fake_response) as req_mock: + records = files._list_jpost_public_files_via_proxi("JPST002311") + + req_mock.assert_called_once() + call_url = req_mock.call_args[0][0] + assert call_url == "https://repository.jpostdb.org/proxi/datasets/JPST002311" + # Non-FTP URI ignored; three FTP entries kept. + assert len(records) == 3 + cats = {r["fileName"]: r["fileCategory"]["value"] for r in records} + assert cats["sample01.raw"] == "RAW" + assert cats["sample01.sne"] == "SEARCH" + # Unknown CV falls back to path-based heuristic (collection "misc" -> OTHER). + assert cats["sample01.txt"] == "OTHER" + + def test_proxi_falls_back_to_ftp_walk_on_error(self): + files = Files() + ftp_record = Files._build_jpost_file_record( + "JPST000001", "ftp://ftp.jpostdb.org/JPST000001/raw/x.raw" + ) + with patch.object( + Files, + "_list_jpost_public_files_via_proxi", + side_effect=RuntimeError("proxi down"), + ), patch.object( + Files, "_list_ftp_repo_files", return_value=["/JPST000001/raw/x.raw"] + ) as ftp_mock: + result = files._list_jpost_public_files("JPST000001") + + ftp_mock.assert_called_once() + assert len(result) == 1 + assert result[0]["fileName"] == "x.raw" + assert result[0]["source"] == "JPOST" From df165ab64ee1ec3e5779064e47cd32c255a4b0b0 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 12:23:40 +0100 Subject: [PATCH 04/54] iProX direct downloads via PX XML + anonymous HTTPS at download.iprox.org User pointed out the iProX dataset XML endpoint: https://www.iprox.cn/FAF016Controller/readXml.jsonp?fileId=file__xml Probing showed something simpler works: iProX publishes the ProteomeXchange XML for every public dataset at a deterministic, anonymous-accessible path on download.iprox.org. No CAS auth, no Aspera tokens, no fileId discovery needed: http://download.iprox.org//PX_.xml The XML embeds Associated raw file URI / Search engine output file URI cvParams pointing at HTTPS file URLs on the same host (Accept-Ranges: bytes, so we get resume for free). The earlier 'iProX is auth-gated' finding was specifically about the iProX UI's CAS-protected JSON endpoints (PMD009Controller/findBySubProjectId.jsonp etc.); the public download server is anonymous. Changes ------- - IPROX_DOWNLOAD_BASE_URL, IPROX_PX_XML_URL_TEMPLATE, IPROX_PX_CATEGORY_MAP added (the latter is the same CV map JPOST PROXI uses). - _build_iprox_file_record() and _list_iprox_public_files() added, mirroring the JPOST helpers but using the PX XML schema. - is_direct_download_accession() now returns True for IPX accessions. - _raise_if_iprox() removed entirely. All entry points that previously called it no longer do, so IPX accessions flow through the unified direct-download dispatcher. - _list_direct_download_files() dispatches IPX -> _list_iprox_public_files. - _download_direct_download_records() now partitions URLs by scheme: ftp:// records go through download_ftp_urls (MassIVE/JPOST), http(s):// records through download_http_urls (iProX). A dataset whose records somehow contain both flows through both paths correctly. - download_http_urls() grew parallel_files + max_retries kwargs and a ThreadPoolExecutor path. The per-file worker _http_download_one() wraps _parallel_download() (reused from the globus codepath) with retry, so iProX gets the same HEAD-then-Range resume and restart-on- non-206 behaviour we already use for PRIDE HTTPS. Tests (26 pass) --------------- - test_iprox_guard.py renamed to test_iprox_files.py and rewritten as positive-path tests: regex coverage, IPX is a direct-download accession, PX XML parsing maps cvParam name -> PRIDE category, RAW filtering ignores non-FTP/HTTPS URIs, download_file_by_name routes iProX URLs to download_http_urls (not download_ftp_urls) with parallel_files=1. - All previously-added MassIVE / JPOST / size-validation tests still pass (20 + 6 iProX = 26 total). Live verification (this branch, against download.iprox.org) ----------------------------------------------------------- - IPX0017413000 listing: 7 files, correctly categorised (RAW + SEARCH). - download_file_by_name(IPX0017413000, protein_annotation_profile.xlsx): 3,253,230 B downloaded, MD5 c17baf230ffde1e2837ec4eb32dcea68, valid XLSX (PK header). - Range-based resume: pre-staged 1,000,000 B partial, completed to 3,253,230 B with the same MD5. - Parallel HTTPS: 3 worker downloads from IPX0017413000 (xlsx + 2 RAW) ran concurrently; cancelled mid-flight after observing all 3 files growing in parallel (xlsx finished at 3.25 MB, Tumor_NK1.raw was at 142 MB and Control_NK3.raw at 419 MB before cancellation). --- README.md | 24 +-- pridepy/files/files.py | 290 ++++++++++++++++++++++-------- pridepy/tests/test_iprox_files.py | 148 +++++++++++++++ pridepy/tests/test_iprox_guard.py | 62 ------- 4 files changed, 374 insertions(+), 150 deletions(-) create mode 100644 pridepy/tests/test_iprox_files.py delete mode 100644 pridepy/tests/test_iprox_guard.py diff --git a/README.md b/README.md index 9609f26..27bab43 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ You can: - download public and private PRIDE files -- download public MassIVE (`MSV...`) and JPOST (`JPST...`) datasets directly. MassIVE goes through FTPS at `massive-ftp.ucsd.edu`; JPOST uses the JSON PROXI endpoint at `repository.jpostdb.org` for listings and `ftp.jpostdb.org` for transfers +- download public MassIVE (`MSV...`), JPOST (`JPST...`), and iProX (`IPX...`) datasets directly. MassIVE goes through FTPS at `massive-ftp.ucsd.edu`; JPOST uses the JSON PROXI endpoint at `repository.jpostdb.org` for listings and `ftp.jpostdb.org` for transfers; iProX fetches the dataset's ProteomeXchange XML from `download.iprox.org` and downloads files over anonymous HTTPS - download by category (`RAW`, `SEARCH`, `RESULT`, etc.) - stream project and file metadata - search projects by keyword and filters @@ -80,7 +80,7 @@ pridepy download-all-public-raw-files \ --checksum-check ``` -### 3) Download a public MassIVE or JPOST dataset directly +### 3) Download a public MassIVE, JPOST, or iProX dataset directly ```bash # MassIVE @@ -90,17 +90,21 @@ pridepy download-all-public-raw-files \ # JPOST pridepy download-all-public-raw-files \ - -a JPST000123 \ - -o ./downloads/JPST000123 + -a JPST002311 \ + -o ./downloads/JPST002311 + +# iProX +pridepy download-all-public-raw-files \ + -a IPX0017413000 \ + -o ./downloads/IPX0017413000 ``` For these direct downloads, `pridepy` enumerates the dataset from the repository: - **MassIVE** lists files by walking the FTPS tree at `massive-ftp.ucsd.edu` (TLS is required by the server). - **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads them from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. +- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTPS. iProX exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTPS endpoint instead so no iProX credentials are required. -Raw downloads follow each repository's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. Direct downloads support REST-based resume, per-file retries, parallel workers (`-w N` up to 3), and post-transfer size verification against the server-reported size. - -iProX accessions (`IPX...`) are recognised so the CLI gives you a clear "not supported yet" error rather than treating them as unknown PRIDE accessions. Native iProX download support is blocked on their REST API requiring CAS authentication and downloads going through Aspera with per-session tokens; track that work upstream. +Raw downloads follow each repository's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. Direct downloads support resume (REST for FTP, byte-Range for HTTPS), per-file retries, parallel workers (`-w N` up to 3), and post-transfer size verification against the server-reported size. ### 4) Download only selected categories @@ -111,7 +115,7 @@ pridepy download-all-public-category-files \ -c RAW,SEARCH ``` -You can also request a specific MassIVE / JPOST collection through the same category interface: +You can also request a specific MassIVE / JPOST / iProX collection through the same category interface: ```bash pridepy download-all-public-category-files \ @@ -256,13 +260,13 @@ print(f"RAW files: {len(raw_files)}") print(raw_files[0]["fileName"]) ``` -For MassIVE / JPOST accessions, the same method returns the files found under the dataset's `raw/` collection: +For MassIVE / JPOST / iProX accessions, the same method returns the files found under the dataset's `raw/` collection: ```python from pridepy.files.files import Files files = Files() -for accession in ("MSV000082297", "JPST000123"): +for accession in ("MSV000082297", "JPST002311", "IPX0017413000"): raw_files = files.get_all_raw_file_list(accession) print(f"{accession} raw files: {len(raw_files)}") ``` diff --git a/pridepy/files/files.py b/pridepy/files/files.py index ecacd16..0f8e029 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -75,6 +75,14 @@ class Files: "Sequence database URI": "FASTA", "Quantification file URI": "RESULT", } + IPROX_DOWNLOAD_BASE_URL = "http://download.iprox.org/" + IPROX_PX_XML_URL_TEMPLATE = ( + "http://download.iprox.org/{accession}/PX_{accession}.xml" + ) + # iProX PX XML uses the same PSI-MS cvParam "name" values as JPOST, so the + # JPOST PROXI category map applies. PX XML cvParam "Associated raw file URI" + # is the canonical raw-file label per the PSI-MS CV (MS:1002846). + IPROX_PX_CATEGORY_MAP = JPOST_PROXI_CATEGORY_MAP S3_URL = "https://hh.fire.sdo.ebi.ac.uk" S3_BUCKET = "pride-public" PROTOCOL_ORDER = ["aspera", "s3", "ftp", "globus"] @@ -346,45 +354,69 @@ def _build_jpost_file_record( "source": "JPOST", } + @staticmethod + def _build_iprox_file_record( + accession: str, https_url: str, category_from_px: Optional[str] = None + ) -> Dict: + """ + Build a pridepy file record for an iProX file. iProX exposes files + over anonymous HTTPS at + ``http://download.iprox.org///``; + ``category_from_px`` is the ``cvParam`` ``name`` from the dataset's + ProteomeXchange XML (e.g. ``"Associated raw file URI"``). + """ + parsed = urlparse(https_url) + root_prefix = f"/{accession.upper()}/" + relative_path = parsed.path + if relative_path.startswith(root_prefix): + relative_path = relative_path[len(root_prefix) :] + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + if category_from_px and category_from_px in Files.IPROX_PX_CATEGORY_MAP: + category = Files.IPROX_PX_CATEGORY_MAP[category_from_px] + else: + category = Files._map_massive_collection_to_category(collection) + return { + "accession": accession.upper(), + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": category}, + # ``FTP Protocol`` is the existing label the download dispatcher + # uses to locate a file URL; here it actually points at HTTPS. + # ``_download_direct_download_records`` routes by URL scheme. + "publicFileLocations": [{"name": "FTP Protocol", "value": https_url}], + "relativePath": relative_path, + "collection": collection, + "source": "iProX", + } + @staticmethod def is_direct_download_accession(accession: str) -> bool: """ - Return True when the accession is served by a public FTP repository - that pridepy supports via direct downloads (no ProteomeXchange API). + Return True when the accession is served by a public repository that + pridepy supports via direct downloads (no ProteomeXchange API). + MassIVE and JPOST use FTP(S); iProX uses anonymous HTTPS via + ``download.iprox.org``. """ return ( Files.is_massive_accession(accession) or Files.is_jpost_accession(accession) + or Files.is_iprox_accession(accession) ) @staticmethod def is_iprox_accession(accession: str) -> bool: """ Return True when the accession looks like an iProX dataset accession - (``IPX`` followed by 7-10 digits). iProX is recognised so the CLI can - emit a clear error rather than treating IPX as an unknown PRIDE - accession; direct downloads from iProX are not yet supported because - their listing API requires CAS authentication and downloads go through - Aspera with per-session tokens. + (``IPX`` followed by 7-10 digits). iProX exposes the dataset + ProteomeXchange XML at + ``http://download.iprox.org//PX_.xml`` and the + referenced files are downloadable from ``download.iprox.org`` over + anonymous HTTPS with byte-range support. """ if not accession: return False return bool(re.fullmatch(r"IPX\d{7,10}", accession.upper())) - @staticmethod - def _raise_if_iprox(accession: str) -> None: - """ - Raise a clear ``NotImplementedError`` when a user passes an iProX - accession. iProX downloads need CAS authentication and Aspera-tokenised - ``faspe://`` URLs which pridepy does not handle yet. - """ - if Files.is_iprox_accession(accession): - raise NotImplementedError( - f"iProX accession {accession} is recognised but not yet supported. " - "iProX requires CAS authentication and Aspera-tokenised downloads; " - "track this in pridepy or use the iProX web interface for now." - ) - @staticmethod def _repo_uses_tls(accession: str) -> bool: """ @@ -609,14 +641,67 @@ def _list_jpost_public_files_via_proxi(self, accession: str) -> List[Dict]: ) return records + def _list_iprox_public_files(self, accession: str) -> List[Dict]: + """ + Discover all public files for an iProX dataset. + + iProX publishes the ProteomeXchange XML for every public dataset at a + deterministic path on its anonymous HTTPS download server:: + + http://download.iprox.org//PX_.xml + + We fetch that XML, walk every ````'s ``cvParam`` entries, + and turn each ``Associated raw file URI`` (and sibling URIs for + search-engine output, result files, etc.) into a pridepy file record. + File downloads themselves go through plain HTTPS on the same host, + which supports ``Range`` requests for resume. + """ + normalized_accession = accession.upper() + xml_url = self.IPROX_PX_XML_URL_TEMPLATE.format(accession=normalized_accession) + logging.info(f"Fetching iProX PX XML: {xml_url}") + response = requests.get(xml_url, timeout=30) + response.raise_for_status() + try: + root = ET.fromstring(response.content) + except ET.ParseError as parse_error: + raise RuntimeError( + f"Unable to parse iProX PX XML for {normalized_accession}: {parse_error}" + ) from parse_error + + records: List[Dict] = [] + for dataset_file in root.iter("DatasetFile"): + for cv in dataset_file.findall("cvParam"): + name = cv.attrib.get("name") + value = cv.attrib.get("value") + if not value or not name or not name.endswith("URI"): + continue + if not value.lower().startswith(("http://", "https://")): + continue + records.append( + self._build_iprox_file_record( + normalized_accession, + value, + category_from_px=name, + ) + ) + if not records: + raise RuntimeError( + f"iProX PX XML for {normalized_accession} contained no downloadable HTTPS URIs" + ) + return records + def _list_direct_download_files(self, accession: str) -> List[Dict]: """ - Dispatch to the right FTP-based listing for a direct-download repository. + Dispatch to the right listing transport for a direct-download + repository: MassIVE walks FTPS, JPOST uses PROXI JSON over HTTPS with + an FTP fallback, iProX uses the dataset's PX XML over HTTPS. """ if self.is_massive_accession(accession): return self._list_massive_public_files(accession) if self.is_jpost_accession(accession): return self._list_jpost_public_files(accession) + if self.is_iprox_accession(accession): + return self._list_iprox_public_files(accession) raise ValueError( f"Accession {accession} is not a direct-download repository accession" ) @@ -631,28 +716,42 @@ def _download_direct_download_records( parallel_files: int = 1, ) -> None: """ - Download files from a direct-download repository (MassIVE/JPOST) via - anonymous FTP. Supports REST-based resume, per-file retries, and - parallel workers (one connection per worker, capped at file count). + Download files from a direct-download repository. + + MassIVE and JPOST use anonymous FTP(S) with REST-based resume and + per-host parallel workers. iProX uses anonymous HTTPS via + ``download.iprox.org`` with ``Range``-based resume and per-file + parallel workers. URLs are partitioned by scheme so a mixed batch + (e.g. a JPOST PX XML that ever pointed at HTTPS) routes correctly. """ - if protocol != "ftp": + if protocol not in ("ftp", "https", "http"): logging.warning( - "Direct downloads currently use ftp only. " + "Direct downloads currently use ftp / https only. " f"Ignoring requested protocol '{protocol}' for {accession}." ) - ftp_urls = [self._get_download_url(file_record, "ftp") for file_record in file_records] - if not ftp_urls: + all_urls = [self._get_download_url(record, "ftp") for record in file_records] + ftp_urls = [u for u in all_urls if u.lower().startswith("ftp://")] + http_urls = [u for u in all_urls if u.lower().startswith(("http://", "https://"))] + if not ftp_urls and not http_urls: logging.info(f"No files matched for direct-download dataset {accession}") return - self.download_ftp_urls( - ftp_urls=ftp_urls, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - use_tls=self._repo_uses_tls(accession), - parallel_files=parallel_files, - ) + if ftp_urls: + self.download_ftp_urls( + ftp_urls=ftp_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=self._repo_uses_tls(accession), + parallel_files=parallel_files, + ) + if http_urls: + self.download_http_urls( + http_urls=http_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + parallel_files=parallel_files, + ) async def stream_all_files_metadata(self, output_file, accession=None): """ @@ -688,7 +787,6 @@ def get_all_raw_file_list(self, project_accession): :param project_accession: PRIDE accession :return: raw file list in JSON format """ - self._raise_if_iprox(project_accession) if self.is_direct_download_accession(project_accession): record_files = self._list_direct_download_files(project_accession) return [ @@ -721,7 +819,6 @@ def download_all_raw_files( :param checksum_check: Download checksum for a given project. :return: None """ - self._raise_if_iprox(accession) if not (os.path.isdir(output_folder)): os.mkdir(output_folder) @@ -1218,7 +1315,6 @@ def download_file_by_name( :param aspera_maximum_bandwidth: Aspera maximum bandwidth :param checksum_check: Download checksum for a given project. """ - self._raise_if_iprox(accession) if not (os.path.isdir(output_folder)): os.mkdir(output_folder) @@ -1293,7 +1389,6 @@ def get_file_from_api(self, accession, file_name) -> List[Dict]: :param file_name: file name :return: file in json format """ - self._raise_if_iprox(accession) try: if self.is_direct_download_accession(accession): @@ -1661,7 +1756,6 @@ def download_files_by_list( """ if not file_names: raise ValueError("file_names must contain at least one filename") - self._raise_if_iprox(accession) if self.is_direct_download_accession(accession): all_files = self._list_direct_download_files(accession) @@ -1951,7 +2045,6 @@ def download_all_category_files( :param categories: List of file categories to download. :param category: Single file category (deprecated, use categories instead). """ - self._raise_if_iprox(accession) if categories is None: categories = [category] if category else ["RAW"] raw_files = self.get_all_category_file_list(accession, categories) @@ -2330,51 +2423,92 @@ def download_ftp_urls( max_download_retries=max_download_retries, ) + @staticmethod + def _http_download_one( + url: str, + output_folder: str, + skip_if_downloaded_already: bool, + max_retries: int = 3, + position: int = 0, + ) -> None: + """ + Download a single HTTP(S) URL with HEAD-then-Range resume and retry. + Used as the worker target for both the serial loop and the parallel + ThreadPoolExecutor path. Reuses :meth:`_parallel_download` so the same + resume / restart-on-non-206 behaviour is shared with globus downloads. + """ + local_path = Files._local_path_for_url(url, output_folder) + if skip_if_downloaded_already and os.path.exists(local_path): + logging.info(f"Skipping download as file already exists: {local_path}") + return + last_error: Optional[Exception] = None + for attempt in range(1, max_retries + 1): + try: + Files._parallel_download(url, local_path, position=position) + logging.info(f"Successfully downloaded {local_path}") + return + except Exception as e: + last_error = e + logging.warning( + f"HTTP download attempt {attempt}/{max_retries} failed for {url}: {e}" + ) + raise RuntimeError( + f"Giving up on {local_path} after {max_retries} HTTP attempts" + ) from last_error + @staticmethod def download_http_urls( http_urls: List[str], output_folder: str, skip_if_downloaded_already: bool, + parallel_files: int = 1, + max_retries: int = 3, ) -> None: """ - Download a list of HTTP(S) URLs with resume support and progress bars. + Download a list of HTTP(S) URLs with HEAD-then-Range resume, per-file + retries, and an optional ``parallel_files`` worker pool. + + When ``parallel_files`` > 1, downloads run concurrently using a + :class:`ThreadPoolExecutor`. Each worker manages its own file (a new + ``requests`` session is opened inside ``_parallel_download``) so the + only shared resource is the output directory. """ if not os.path.isdir(output_folder): os.makedirs(output_folder, exist_ok=True) - session = Util.create_session_with_retries() - for url in http_urls: - try: - local_path = Files._local_path_for_url(url, output_folder) - if skip_if_downloaded_already and os.path.exists(local_path): - logging.info("Skipping download as file already exists") - continue - - if os.path.exists(local_path): - resume_size = os.path.getsize(local_path) - headers = {"Range": f"bytes={resume_size}-"} - mode = "ab" - else: - resume_size = 0 - headers = {} - mode = "wb" + if not http_urls: + return - with session.get(url, stream=True, headers=headers, timeout=(10, 60)) as r: - r.raise_for_status() - total_size = int(r.headers.get("content-length", 0)) + resume_size - block_size = 1024 * 1024 - with tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=local_path, - initial=resume_size, - ) as pbar: - with open(local_path, mode) as f: - for chunk in r.iter_content(chunk_size=block_size): - if chunk: - f.write(chunk) - pbar.update(len(chunk)) - logging.info(f"Successfully downloaded {local_path}") - except Exception as e: - logging.error(f"HTTP download failed for {url}: {str(e)}") + workers = max(1, min(parallel_files, len(http_urls))) + if workers > 1: + logging.info( + f"Downloading {len(http_urls)} HTTP(S) file(s) with {workers} parallel workers" + ) + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [ + executor.submit( + Files._http_download_one, + url, + output_folder, + skip_if_downloaded_already, + max_retries, + idx, + ) + for idx, url in enumerate(http_urls) + ] + for future in as_completed(futures): + try: + future.result() + except Exception as e: + logging.error(f"Parallel HTTP download error: {e}") + else: + for url in http_urls: + try: + Files._http_download_one( + url, + output_folder, + skip_if_downloaded_already, + max_retries, + ) + except Exception as e: + logging.error(f"HTTP download failed for {url}: {e}") diff --git a/pridepy/tests/test_iprox_files.py b/pridepy/tests/test_iprox_files.py new file mode 100644 index 0000000..dfdcd21 --- /dev/null +++ b/pridepy/tests/test_iprox_files.py @@ -0,0 +1,148 @@ +"""iProX direct-download support. + +iProX publishes the ProteomeXchange XML for each dataset at a deterministic +path on its anonymous HTTPS download server:: + + http://download.iprox.org//PX_.xml + +The referenced files are served from the same host over HTTPS with byte-range +support, so resume and parallel downloads use the same plumbing as PRIDE +HTTP(S) transfers. +""" +import tempfile +from unittest import TestCase +from unittest.mock import MagicMock, patch + +from pridepy.files.files import Files + + +IPROX_XML_FIXTURE = """ + + + + + + + + + + + + + + + + + + + + + +""".encode("utf-8") + + +class TestIProXFiles(TestCase): + def test_is_iprox_accession_matches_ipx_format(self): + assert Files.is_iprox_accession("IPX0000123") + assert Files.is_iprox_accession("IPX0000123000") + assert Files.is_iprox_accession("ipx1234567") + assert not Files.is_iprox_accession("PXD000012") + assert not Files.is_iprox_accession("MSV000012345") + assert not Files.is_iprox_accession("JPST000001") + assert not Files.is_iprox_accession("IPX12") + assert not Files.is_iprox_accession("") + assert not Files.is_iprox_accession(None) + + def test_iprox_is_a_direct_download_accession(self): + assert Files.is_direct_download_accession("IPX0017413000") + + def test_build_iprox_file_record_maps_px_cv_to_category(self): + record = Files._build_iprox_file_record( + "IPX0017413000", + "http://download.iprox.org/IPX0017413000/IPX0017413001/sample.raw", + category_from_px="Associated raw file URI", + ) + assert record["fileName"] == "sample.raw" + assert record["fileCategory"]["value"] == "RAW" + assert record["source"] == "iProX" + # _download_direct_download_records dispatches by URL scheme, so the + # publicFileLocations URL must still be the HTTPS download URL. + assert record["publicFileLocations"][0]["value"].startswith("http://") + + def test_list_iprox_public_files_parses_px_xml(self): + files = Files() + fake_response = MagicMock() + fake_response.content = IPROX_XML_FIXTURE + fake_response.raise_for_status = MagicMock() + with patch( + "pridepy.files.files.requests.get", return_value=fake_response + ) as req_mock: + records = files._list_iprox_public_files("IPX0017413000") + + # The fetch hits the deterministic PX XML URL. + req_mock.assert_called_once() + called_url = req_mock.call_args[0][0] + assert called_url == ( + "http://download.iprox.org/IPX0017413000/PX_IPX0017413000.xml" + ) + + # 3 valid HTTPS records; the ftp:// "Other URI" cvParam was filtered out. + assert len(records) == 3 + cats = {r["fileName"]: r["fileCategory"]["value"] for r in records} + assert cats == { + "sample1.raw": "RAW", + "sample2.raw": "RAW", + "results.tsv": "SEARCH", + } + for r in records: + assert r["source"] == "iProX" + assert r["publicFileLocations"][0]["value"].startswith("http://") + + def test_get_all_raw_file_list_filters_iprox_records(self): + files = Files() + fake_response = MagicMock() + fake_response.content = IPROX_XML_FIXTURE + fake_response.raise_for_status = MagicMock() + with patch( + "pridepy.files.files.requests.get", return_value=fake_response + ), patch.object(Files, "stream_all_files_by_project") as pride_mock: + raw_files = files.get_all_raw_file_list("IPX0017413000") + + pride_mock.assert_not_called() + assert {r["fileName"] for r in raw_files} == {"sample1.raw", "sample2.raw"} + + def test_download_file_by_name_routes_iprox_to_http_urls(self): + files = Files() + fake_response = MagicMock() + fake_response.content = IPROX_XML_FIXTURE + fake_response.raise_for_status = MagicMock() + with tempfile.TemporaryDirectory() as tmp_dir, patch( + "pridepy.files.files.requests.get", return_value=fake_response + ), patch.object(Files, "download_http_urls") as http_mock, patch.object( + Files, "download_ftp_urls" + ) as ftp_mock: + files.download_file_by_name( + accession="IPX0017413000", + file_name="results.tsv", + output_folder=tmp_dir, + skip_if_downloaded_already=False, + protocol="ftp", + username=None, + password=None, + aspera_maximum_bandwidth="100M", + checksum_check=False, + ) + + # iProX is HTTPS, not FTP — FTP path must not be called. + ftp_mock.assert_not_called() + http_mock.assert_called_once() + kwargs = http_mock.call_args.kwargs + assert kwargs["http_urls"] == [ + "http://download.iprox.org/IPX0017413000/IPX0017413001/results.tsv" + ] + assert kwargs["parallel_files"] == 1 + assert kwargs["skip_if_downloaded_already"] is False diff --git a/pridepy/tests/test_iprox_guard.py b/pridepy/tests/test_iprox_guard.py deleted file mode 100644 index f28f490..0000000 --- a/pridepy/tests/test_iprox_guard.py +++ /dev/null @@ -1,62 +0,0 @@ -"""iProX accession recognition and unsupported-accession guard. - -iProX direct downloads are not implemented (the iProX REST API gates listing -behind CAS authentication and files are served over Aspera with per-session -tokens). pridepy still recognises the accession format so the user gets a -clear ``NotImplementedError`` instead of a confusing PRIDE-API 404. -""" -import tempfile -from unittest import TestCase - -import pytest - -from pridepy.files.files import Files - - -class TestIProXGuard(TestCase): - def test_is_iprox_accession_matches_ipx_format(self): - assert Files.is_iprox_accession("IPX0000123") - assert Files.is_iprox_accession("IPX0000123000") - assert Files.is_iprox_accession("ipx1234567") - assert not Files.is_iprox_accession("PXD000012") - assert not Files.is_iprox_accession("MSV000012345") - assert not Files.is_iprox_accession("JPST000001") - assert not Files.is_iprox_accession("IPX12") - assert not Files.is_iprox_accession("") - assert not Files.is_iprox_accession(None) - - def test_iprox_is_not_a_direct_download_accession(self): - assert Files.is_direct_download_accession("IPX0000123000") is False - - def test_get_all_raw_file_list_raises_for_iprox(self): - files = Files() - with pytest.raises(NotImplementedError, match="iProX"): - files.get_all_raw_file_list("IPX0006033000") - - def test_download_file_by_name_raises_for_iprox(self): - files = Files() - with tempfile.TemporaryDirectory() as tmp_dir: - with pytest.raises(NotImplementedError, match="iProX"): - files.download_file_by_name( - accession="IPX0006033000", - file_name="foo.raw", - output_folder=tmp_dir, - skip_if_downloaded_already=False, - protocol="ftp", - username=None, - password=None, - aspera_maximum_bandwidth="100M", - checksum_check=False, - ) - - def test_download_all_raw_files_raises_for_iprox(self): - files = Files() - with tempfile.TemporaryDirectory() as tmp_dir: - with pytest.raises(NotImplementedError, match="iProX"): - files.download_all_raw_files( - accession="IPX0006033000", - output_folder=tmp_dir, - skip_if_downloaded_already=False, - protocol="ftp", - aspera_maximum_bandwidth="100M", - ) From eedbef1b46dce227afdcb69bbc5061dd59eba608 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 15:36:25 +0100 Subject: [PATCH 05/54] refactor(providers): scaffold providers/ package with Provider ABC + Registry Empty scaffolding for the per-provider refactor (spec: docs/specs/2026-05-27-files-py-provider-refactor-design.md). Introduces the Provider abstract base class, BaseDirectDownloadProvider with a shared download_files() that partitions URLs by scheme and routes through Files.download_ftp_urls / Files.download_http_urls (preserving test patches), and a Registry for accession -> provider resolution. No behaviour change. No providers registered yet. --- pridepy/providers/__init__.py | 7 +++ pridepy/providers/base.py | 107 ++++++++++++++++++++++++++++++++++ pridepy/providers/registry.py | 38 ++++++++++++ 3 files changed, 152 insertions(+) create mode 100644 pridepy/providers/__init__.py create mode 100644 pridepy/providers/base.py create mode 100644 pridepy/providers/registry.py diff --git a/pridepy/providers/__init__.py b/pridepy/providers/__init__.py new file mode 100644 index 0000000..ee1de17 --- /dev/null +++ b/pridepy/providers/__init__.py @@ -0,0 +1,7 @@ +"""Per-repository provider classes used by :class:`pridepy.files.files.Files`. + +Each module under this package owns the listing, transport choice, and +record-construction logic for one repository: PRIDE, MassIVE, JPOST, iProX. +The :mod:`registry` module maps an accession to the right provider; the +:mod:`transport` module hosts the shared FTP/FTPS/HTTPS download plumbing. +""" diff --git a/pridepy/providers/base.py b/pridepy/providers/base.py new file mode 100644 index 0000000..f9fa8bc --- /dev/null +++ b/pridepy/providers/base.py @@ -0,0 +1,107 @@ +"""Abstract base classes for pridepy providers.""" +from abc import ABC, abstractmethod +from typing import ClassVar, Dict, List, Optional + + +class Provider(ABC): + """Abstract base for every repository pridepy can list and download from.""" + + name: ClassVar[str] # "pride", "massive", "jpost", "iprox" + + @staticmethod + @abstractmethod + def matches(accession: str) -> bool: + """Return True if this provider should handle ``accession``.""" + + @abstractmethod + def list_files(self, accession: str) -> List[Dict]: + """Return pridepy file records for the dataset. + + Each record is a dict shaped like the PRIDE V3 API file response, + with at minimum: ``accession``, ``fileName``, ``fileCategory`` + (with nested ``value``), ``publicFileLocations`` (list of + ``{"name": ..., "value": }``). + """ + + @abstractmethod + def download_files( + self, + accession: str, + records: List[Dict], + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str, + parallel_files: int = 1, + checksum_check: bool = False, + aspera_maximum_bandwidth: str = "100M", + username: Optional[str] = None, + password: Optional[str] = None, + ) -> None: + """Download the given records into ``output_folder``.""" + + +class BaseDirectDownloadProvider(Provider): + """Shared ``download_files`` for MassIVE / JPOST / iProX. + + Subclasses set the ``use_tls`` class var (True for MassIVE FTPS, False for + JPOST plain FTP) and override :meth:`list_files`. The shared + ``download_files`` implementation partitions record URLs by scheme: + ``ftp://`` URLs are handed to :meth:`Files.download_ftp_urls`; ``http(s)://`` + URLs go to :meth:`Files.download_http_urls`. It calls **back** into + ``Files`` so that test patches on ``Files.download_ftp_urls`` / + ``Files.download_http_urls`` continue to intercept the calls. + """ + + use_tls: ClassVar[bool] = False + + def download_files( + self, + accession: str, + records: List[Dict], + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str, + parallel_files: int = 1, + checksum_check: bool = False, + aspera_maximum_bandwidth: str = "100M", + username: Optional[str] = None, + password: Optional[str] = None, + ) -> None: + # Lazy import: providers know about Files (the facade) only via the + # public attributes that tests may patch; avoid module-load cycle. + from pridepy.files.files import Files + + if protocol not in ("ftp", "https", "http"): + import logging + logging.warning( + "Direct downloads currently use ftp / https only. " + f"Ignoring requested protocol '{protocol}' for {accession}." + ) + + all_urls = [Files._get_download_url(record, "ftp") for record in records] + ftp_urls = [u for u in all_urls if u.lower().startswith("ftp://")] + http_urls = [ + u for u in all_urls if u.lower().startswith(("http://", "https://")) + ] + if not ftp_urls and not http_urls: + import logging + logging.info( + f"No files matched for direct-download dataset {accession}" + ) + return + + if ftp_urls: + Files.download_ftp_urls( + ftp_urls=ftp_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=self.use_tls, + parallel_files=parallel_files, + ) + if http_urls: + Files.download_http_urls( + http_urls=http_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + parallel_files=parallel_files, + ) diff --git a/pridepy/providers/registry.py b/pridepy/providers/registry.py new file mode 100644 index 0000000..7d2c20d --- /dev/null +++ b/pridepy/providers/registry.py @@ -0,0 +1,38 @@ +"""Accession-to-provider resolution. + +Providers are tried in priority order; direct-download repositories +(MassIVE / JPOST / iProX) are tried first because their accession patterns +are unambiguous. PRIDE is tried last and acts as the catch-all for +``PXD\\d+`` / ``PRD\\d+`` accessions. +""" +from typing import List, Type + +from pridepy.providers.base import Provider + +_PROVIDERS: List[Type[Provider]] = [] # populated by individual provider modules + + +def register(provider_cls: Type[Provider]) -> Type[Provider]: + """Register a provider class. Usable as a decorator.""" + if provider_cls not in _PROVIDERS: + _PROVIDERS.append(provider_cls) + return provider_cls + + +def resolve(accession: str) -> Provider: + """Return a provider instance that matches ``accession``. + + :raises ValueError: when no registered provider matches. + """ + for cls in _PROVIDERS: + if cls.matches(accession): + return cls() + raise ValueError(f"No provider registered for accession {accession!r}") + + +def is_known(accession: str) -> bool: + """Return True if any registered provider matches ``accession``.""" + for cls in _PROVIDERS: + if cls.matches(accession): + return True + return False From 912287d9db0f0f8867de5cc665aba827aca7225f Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 15:46:41 +0100 Subject: [PATCH 06/54] refactor(providers): move FTP/HTTPS transport into providers/transport.py Pulled download_ftp_urls, download_http_urls, and their helpers (_open_ftp_connection, _walk_ftp_tree, _list_ftp_repo_files, _download_one_ftp_path, _download_ftp_paths_serial/_parallel, _http_download_one, _parallel_download, _local_path_for_url) out of the Files class verbatim into providers/transport.py. Files keeps a shim staticmethod for each function that does a lazy import and delegates, so existing test patches like patch.object(Files, 'download_ftp_urls') keep intercepting calls. No behaviour change. Test suite green. --- pridepy/files/files.py | 487 ++++++------------------------- pridepy/providers/transport.py | 504 +++++++++++++++++++++++++++++++++ 2 files changed, 583 insertions(+), 408 deletions(-) create mode 100644 pridepy/providers/transport.py diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 0f8e029..0de339a 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -428,108 +428,21 @@ def _repo_uses_tls(accession: str) -> bool: @staticmethod def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: - """ - Recursively list files under a remote FTP directory. - """ - file_paths: List[str] = [] - try: - entries = list(ftp.mlsd(remote_dir)) - for name, facts in entries: - if name in {".", ".."}: - continue - child_path = posixpath.join(remote_dir.rstrip("/"), name) - if facts.get("type") == "dir": - file_paths.extend(Files._walk_ftp_tree(ftp, child_path)) - elif facts.get("type") == "file": - file_paths.append(child_path) - return file_paths - except (AttributeError, ftplib.error_perm): - pass - - current_dir = ftp.pwd() - listing: List[str] = [] - try: - ftp.cwd(remote_dir) - ftp.retrlines("LIST", listing.append) - for entry in listing: - parts = entry.split(maxsplit=8) - if len(parts) < 9: - continue - name = parts[8] - if name in {".", ".."}: - continue - child_path = posixpath.join(remote_dir.rstrip("/"), name) - if entry.startswith("d"): - file_paths.extend(Files._walk_ftp_tree(ftp, child_path)) - else: - file_paths.append(child_path) - finally: - ftp.cwd(current_dir) - return file_paths + """Shim — see :func:`pridepy.providers.transport._walk_ftp_tree`.""" + from pridepy.providers import transport + return transport._walk_ftp_tree(ftp=ftp, remote_dir=remote_dir) @staticmethod def _open_ftp_connection(host: str, use_tls: bool, timeout: int = 30) -> FTP: - """ - Open an anonymous FTP connection, transparently using FTPS when the - server requires TLS (e.g., MassIVE). When ``use_tls`` is False but the - server replies ``421 TLS is required`` to ``login``, transparently - retry with FTPS so callers don't need to know the policy in advance. - """ - if use_tls: - ftp: FTP = ftplib.FTP_TLS(host, timeout=timeout) - ftp.login() - ftp.prot_p() - else: - ftp = FTP(host, timeout=timeout) - try: - ftp.login() - except ftplib.error_temp as e: - if "TLS" in str(e).upper(): - try: - ftp.close() - except Exception: - pass - ftp = ftplib.FTP_TLS(host, timeout=timeout) - ftp.login() - ftp.prot_p() - else: - raise - ftp.set_pasv(True) - return ftp + """Shim — see :func:`pridepy.providers.transport._open_ftp_connection`.""" + from pridepy.providers import transport + return transport._open_ftp_connection(host=host, use_tls=use_tls, timeout=timeout) - def _list_ftp_repo_files( - self, - host: str, - remote_root: str, - error_label: str, - use_tls: bool = False, - ) -> List[str]: - """ - Connect to an anonymous FTP host (FTP or FTPS), walk a directory tree, - and return file paths. - - ``use_tls`` should be True for servers that reject plain FTP (e.g. - MassIVE). Centralizes connection lifecycle so a constructor failure - doesn't mask the underlying error in ``finally`` (PR #98 review). - """ - ftp: Optional[FTP] = None - try: - ftp = self._open_ftp_connection(host, use_tls=use_tls) - logging.info(f"Connected to FTP host: {host} (tls={use_tls})") - return self._walk_ftp_tree(ftp, remote_root) - except Exception as error: - raise RuntimeError( - f"Unable to list public files for {error_label}: {error}" - ) from error - finally: - if ftp is not None: - try: - ftp.quit() - except Exception: - try: - ftp.close() - except Exception: - pass + @staticmethod + def _list_ftp_repo_files(host, remote_root, error_label, use_tls=False): + """Shim — see :func:`pridepy.providers.transport._list_ftp_repo_files`.""" + from pridepy.providers import transport + return transport._list_ftp_repo_files(host=host, remote_root=remote_root, error_label=error_label, use_tls=use_tls) def _list_massive_public_files(self, accession: str) -> List[Dict]: """ @@ -1061,43 +974,9 @@ def _download_range(url, file_path, start, end, pbar, max_retries=3): @staticmethod def _parallel_download(url, file_path, position=0): - """Download a file via a single-connection HTTP stream with optional resume. - If a partial file exists and the server supports Range requests, resumes - from where it left off; otherwise restarts from scratch.""" - session = Util.create_session_with_retries() - try: - head = session.head(url, timeout=(30, 30)) - head.raise_for_status() - total_size = int(head.headers.get("content-length", 0)) - accept_ranges = head.headers.get("accept-ranges", "none").strip().lower() - except (requests.RequestException, ValueError) as exc: - logging.info(f"HEAD request failed, falling back to single connection: {exc}") - total_size = 0 - accept_ranges = "none" - - resume_size = 0 - if os.path.exists(file_path) and accept_ranges == "bytes" and total_size > 0: - resume_size = os.path.getsize(file_path) - if resume_size >= total_size: - logging.info(f"File already complete: {file_path}") - return - if resume_size > 0: - logging.info(f"Resuming download from {resume_size} bytes: {file_path}") - - headers = {"Range": f"bytes={resume_size}-"} if resume_size > 0 else {} - with session.get(url, headers=headers, stream=True, timeout=(30, 60)) as r: - r.raise_for_status() - if resume_size > 0 and r.status_code != 206: - logging.warning("Server did not honor Range request (status %s), restarting download", r.status_code) - resume_size = 0 - with tqdm(total=total_size, unit="B", unit_scale=True, desc=file_path, - initial=resume_size, position=position, leave=True) as pbar: - mode = "ab" if resume_size > 0 else "wb" - with open(file_path, mode, buffering=8 * 1024 * 1024) as f: - for chunk in r.iter_content(chunk_size=8 * 1024 * 1024): - if chunk: - f.write(chunk) - pbar.update(len(chunk)) + """Shim — see :func:`pridepy.providers.transport._parallel_download`.""" + from pridepy.providers import transport + return transport._parallel_download(url=url, file_path=file_path, position=position) @staticmethod def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_retries=6, position=0): @@ -2177,8 +2056,9 @@ def download_px_raw_files( @staticmethod def _local_path_for_url(download_url: str, output_folder: str) -> str: - filename = os.path.basename(urlparse(download_url).path) - return os.path.join(output_folder, filename) + """Shim — see :func:`pridepy.providers.transport._local_path_for_url`.""" + from pridepy.providers import transport + return transport._local_path_for_url(download_url=download_url, output_folder=output_folder) @staticmethod def _download_one_ftp_path( @@ -2189,73 +2069,16 @@ def _download_one_ftp_path( max_download_retries: int, position: int = 0, ) -> None: - """ - Download a single FTP path over an existing connection, with REST resume - and per-file retry. Raises on giving up so the caller can decide what to do. - """ - if skip_if_downloaded_already and os.path.exists(local_path): - logging.info(f"Skipping download as file already exists: {local_path}") - return - - attempt = 0 - last_error: Optional[Exception] = None - while attempt < max_download_retries: - try: - total_size = ftp.size(ftp_path) - if os.path.exists(local_path): - current_size = os.path.getsize(local_path) - mode = "ab" - else: - current_size = 0 - mode = "wb" - - with open(local_path, mode) as f, tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=local_path, - initial=current_size, - position=position, - leave=True, - ) as pbar: - def callback(data): - f.write(data) - pbar.update(len(data)) - - if current_size: - try: - ftp.sendcmd(f"REST {current_size}") - except Exception: - current_size = 0 - f.seek(0) - f.truncate() - ftp.retrbinary(f"RETR {ftp_path}", callback) - - # Post-transfer integrity check: server-reported size must match - # the local size. Catches half-finished transfers that retrbinary - # didn't raise on (e.g. server closed the data channel early). - # The next iteration will REST-resume from where we left off. - if total_size: - final_size = os.path.getsize(local_path) - if final_size != total_size: - attempt += 1 - logging.error( - f"Size mismatch for {local_path}: " - f"got {final_size} bytes, expected {total_size} " - f"(attempt {attempt})" - ) - continue - logging.info(f"Successfully downloaded {local_path}") - return - except (socket.timeout, ftplib.error_temp, ftplib.error_perm) as e: - attempt += 1 - last_error = e - logging.error( - f"Download failed for {local_path} (attempt {attempt}): {e}" - ) - raise RuntimeError( - f"Giving up on {local_path} after {max_download_retries} attempts" - ) from last_error + """Shim — see :func:`pridepy.providers.transport._download_one_ftp_path`.""" + from pridepy.providers import transport + return transport._download_one_ftp_path( + ftp=ftp, + ftp_path=ftp_path, + local_path=local_path, + skip_if_downloaded_already=skip_if_downloaded_already, + max_download_retries=max_download_retries, + position=position, + ) @staticmethod def _download_ftp_paths_serial( @@ -2267,47 +2090,17 @@ def _download_ftp_paths_serial( max_connection_retries: int, max_download_retries: int, ) -> None: - """Download all paths from one host over a single (reused) connection.""" - connection_attempt = 0 - while connection_attempt < max_connection_retries: - try: - ftp = Files._open_ftp_connection(host, use_tls=use_tls) - logging.info(f"Connected to FTP host: {host} (tls={use_tls})") - for ftp_path in paths: - local_path = os.path.join(output_folder, os.path.basename(ftp_path)) - try: - Files._download_one_ftp_path( - ftp=ftp, - ftp_path=ftp_path, - local_path=local_path, - skip_if_downloaded_already=skip_if_downloaded_already, - max_download_retries=max_download_retries, - ) - except Exception as e: - logging.error( - f"Failed to download {ftp_path} from {host}: {e}" - ) - try: - ftp.quit() - except Exception: - try: - ftp.close() - except Exception: - pass - logging.info(f"Disconnected from FTP host: {host}") - return - except (socket.timeout, ftplib.error_temp, ftplib.error_perm, OSError) as e: - connection_attempt += 1 - logging.error( - f"FTP connection failed (attempt {connection_attempt}): {e}" - ) - if connection_attempt < max_connection_retries: - logging.info("Retrying connection...") - time.sleep(5) - else: - logging.error( - f"Giving up after {max_connection_retries} failed connection attempts to {host}." - ) + """Shim — see :func:`pridepy.providers.transport._download_ftp_paths_serial`.""" + from pridepy.providers import transport + return transport._download_ftp_paths_serial( + host=host, + paths=paths, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=use_tls, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + ) @staticmethod def _download_ftp_paths_parallel( @@ -2320,55 +2113,18 @@ def _download_ftp_paths_parallel( max_download_retries: int, parallel_files: int, ) -> None: - """ - Download paths concurrently using ``parallel_files`` workers; each - worker opens its own FTP connection so transfers don't serialize. - """ - def worker(ftp_path: str, position: int) -> None: - local_path = os.path.join(output_folder, os.path.basename(ftp_path)) - if skip_if_downloaded_already and os.path.exists(local_path): - logging.info(f"Skipping download as file already exists: {local_path}") - return - connection_attempt = 0 - while connection_attempt < max_connection_retries: - try: - ftp = Files._open_ftp_connection(host, use_tls=use_tls) - try: - Files._download_one_ftp_path( - ftp=ftp, - ftp_path=ftp_path, - local_path=local_path, - skip_if_downloaded_already=False, - max_download_retries=max_download_retries, - position=position, - ) - return - finally: - try: - ftp.quit() - except Exception: - try: - ftp.close() - except Exception: - pass - except (socket.timeout, ftplib.error_temp, ftplib.error_perm, OSError) as e: - connection_attempt += 1 - logging.error( - f"FTP connection failed for {ftp_path} (attempt {connection_attempt}): {e}" - ) - if connection_attempt < max_connection_retries: - time.sleep(5) - logging.error(f"Giving up on {ftp_path} from {host}") - - with ThreadPoolExecutor(max_workers=parallel_files) as executor: - futures = [ - executor.submit(worker, path, idx) for idx, path in enumerate(paths) - ] - for future in as_completed(futures): - try: - future.result() - except Exception as e: - logging.error(f"Parallel FTP download error: {e}") + """Shim — see :func:`pridepy.providers.transport._download_ftp_paths_parallel`.""" + from pridepy.providers import transport + return transport._download_ftp_paths_parallel( + host=host, + paths=paths, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=use_tls, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + parallel_files=parallel_files, + ) @staticmethod def download_ftp_urls( @@ -2380,48 +2136,17 @@ def download_ftp_urls( use_tls: bool = False, parallel_files: int = 1, ) -> None: - """ - Download a list of FTP URLs with retries, REST-based resume, and - optional parallel workers. - - :param use_tls: Open the FTP connection with TLS (FTP_TLS / PROT P). - Required for hosts that reject plain anonymous FTP (e.g. MassIVE). - When False but the server replies ``421 TLS is required``, the - connection is transparently retried over TLS. - :param parallel_files: When >1, downloads run concurrently with that - many worker connections per host (capped at the number of files). - """ - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - host_to_paths: Dict[str, List[str]] = {} - for url in ftp_urls: - parsed = urlparse(url) - host_to_paths.setdefault(parsed.hostname, []).append(parsed.path.lstrip("/")) - - for host, paths in host_to_paths.items(): - workers = max(1, min(parallel_files, len(paths))) - if workers > 1: - Files._download_ftp_paths_parallel( - host=host, - paths=paths, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - use_tls=use_tls, - max_connection_retries=max_connection_retries, - max_download_retries=max_download_retries, - parallel_files=workers, - ) - else: - Files._download_ftp_paths_serial( - host=host, - paths=paths, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - use_tls=use_tls, - max_connection_retries=max_connection_retries, - max_download_retries=max_download_retries, - ) + """Shim — see :func:`pridepy.providers.transport.download_ftp_urls`.""" + from pridepy.providers import transport + return transport.download_ftp_urls( + ftp_urls=ftp_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + use_tls=use_tls, + parallel_files=parallel_files, + ) @staticmethod def _http_download_one( @@ -2431,30 +2156,15 @@ def _http_download_one( max_retries: int = 3, position: int = 0, ) -> None: - """ - Download a single HTTP(S) URL with HEAD-then-Range resume and retry. - Used as the worker target for both the serial loop and the parallel - ThreadPoolExecutor path. Reuses :meth:`_parallel_download` so the same - resume / restart-on-non-206 behaviour is shared with globus downloads. - """ - local_path = Files._local_path_for_url(url, output_folder) - if skip_if_downloaded_already and os.path.exists(local_path): - logging.info(f"Skipping download as file already exists: {local_path}") - return - last_error: Optional[Exception] = None - for attempt in range(1, max_retries + 1): - try: - Files._parallel_download(url, local_path, position=position) - logging.info(f"Successfully downloaded {local_path}") - return - except Exception as e: - last_error = e - logging.warning( - f"HTTP download attempt {attempt}/{max_retries} failed for {url}: {e}" - ) - raise RuntimeError( - f"Giving up on {local_path} after {max_retries} HTTP attempts" - ) from last_error + """Shim — see :func:`pridepy.providers.transport._http_download_one`.""" + from pridepy.providers import transport + return transport._http_download_one( + url=url, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + max_retries=max_retries, + position=position, + ) @staticmethod def download_http_urls( @@ -2464,51 +2174,12 @@ def download_http_urls( parallel_files: int = 1, max_retries: int = 3, ) -> None: - """ - Download a list of HTTP(S) URLs with HEAD-then-Range resume, per-file - retries, and an optional ``parallel_files`` worker pool. - - When ``parallel_files`` > 1, downloads run concurrently using a - :class:`ThreadPoolExecutor`. Each worker manages its own file (a new - ``requests`` session is opened inside ``_parallel_download``) so the - only shared resource is the output directory. - """ - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - if not http_urls: - return - - workers = max(1, min(parallel_files, len(http_urls))) - if workers > 1: - logging.info( - f"Downloading {len(http_urls)} HTTP(S) file(s) with {workers} parallel workers" - ) - with ThreadPoolExecutor(max_workers=workers) as executor: - futures = [ - executor.submit( - Files._http_download_one, - url, - output_folder, - skip_if_downloaded_already, - max_retries, - idx, - ) - for idx, url in enumerate(http_urls) - ] - for future in as_completed(futures): - try: - future.result() - except Exception as e: - logging.error(f"Parallel HTTP download error: {e}") - else: - for url in http_urls: - try: - Files._http_download_one( - url, - output_folder, - skip_if_downloaded_already, - max_retries, - ) - except Exception as e: - logging.error(f"HTTP download failed for {url}: {e}") + """Shim — see :func:`pridepy.providers.transport.download_http_urls`.""" + from pridepy.providers import transport + return transport.download_http_urls( + http_urls=http_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + parallel_files=parallel_files, + max_retries=max_retries, + ) diff --git a/pridepy/providers/transport.py b/pridepy/providers/transport.py new file mode 100644 index 0000000..6649657 --- /dev/null +++ b/pridepy/providers/transport.py @@ -0,0 +1,504 @@ +"""Shared FTP / FTPS / HTTPS download transport. + +Stateless helpers used by the per-repository providers (and re-exported on +:class:`pridepy.files.files.Files` for backward compatibility with tests that +patch ``Files.download_ftp_urls`` etc.). +""" +import ftplib +import logging +import os +import socket +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from ftplib import FTP +from typing import Dict, List, Optional +from urllib.parse import urlparse + +import requests +from tqdm import tqdm + +from pridepy.util.api_handling import Util + + +def _local_path_for_url(download_url: str, output_folder: str) -> str: + filename = os.path.basename(urlparse(download_url).path) + return os.path.join(output_folder, filename) + + +def _open_ftp_connection(host: str, use_tls: bool, timeout: int = 30) -> FTP: + """ + Open an anonymous FTP connection, transparently using FTPS when the + server requires TLS (e.g., MassIVE). When ``use_tls`` is False but the + server replies ``421 TLS is required`` to ``login``, transparently + retry with FTPS so callers don't need to know the policy in advance. + """ + if use_tls: + ftp: FTP = ftplib.FTP_TLS(host, timeout=timeout) + ftp.login() + ftp.prot_p() + else: + ftp = FTP(host, timeout=timeout) + try: + ftp.login() + except ftplib.error_temp as e: + if "TLS" in str(e).upper(): + try: + ftp.close() + except Exception: + pass + ftp = ftplib.FTP_TLS(host, timeout=timeout) + ftp.login() + ftp.prot_p() + else: + raise + ftp.set_pasv(True) + return ftp + + +def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: + """ + Recursively list files under a remote FTP directory. + """ + import posixpath + file_paths: List[str] = [] + try: + entries = list(ftp.mlsd(remote_dir)) + for name, facts in entries: + if name in {".", ".."}: + continue + child_path = posixpath.join(remote_dir.rstrip("/"), name) + if facts.get("type") == "dir": + file_paths.extend(_walk_ftp_tree(ftp, child_path)) + elif facts.get("type") == "file": + file_paths.append(child_path) + return file_paths + except (AttributeError, ftplib.error_perm): + pass + + current_dir = ftp.pwd() + listing: List[str] = [] + try: + ftp.cwd(remote_dir) + ftp.retrlines("LIST", listing.append) + for entry in listing: + parts = entry.split(maxsplit=8) + if len(parts) < 9: + continue + name = parts[8] + if name in {".", ".."}: + continue + child_path = posixpath.join(remote_dir.rstrip("/"), name) + if entry.startswith("d"): + file_paths.extend(_walk_ftp_tree(ftp, child_path)) + else: + file_paths.append(child_path) + finally: + ftp.cwd(current_dir) + return file_paths + + +def _list_ftp_repo_files( + host: str, + remote_root: str, + error_label: str, + use_tls: bool = False, +) -> List[str]: + """ + Connect to an anonymous FTP host (FTP or FTPS), walk a directory tree, + and return file paths. + + ``use_tls`` should be True for servers that reject plain FTP (e.g. + MassIVE). Centralizes connection lifecycle so a constructor failure + doesn't mask the underlying error in ``finally`` (PR #98 review). + """ + ftp: Optional[FTP] = None + try: + ftp = _open_ftp_connection(host, use_tls=use_tls) + logging.info(f"Connected to FTP host: {host} (tls={use_tls})") + return _walk_ftp_tree(ftp, remote_root) + except Exception as error: + raise RuntimeError( + f"Unable to list public files for {error_label}: {error}" + ) from error + finally: + if ftp is not None: + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + + +def _download_one_ftp_path( + ftp: FTP, + ftp_path: str, + local_path: str, + skip_if_downloaded_already: bool, + max_download_retries: int, + position: int = 0, +) -> None: + """ + Download a single FTP path over an existing connection, with REST resume + and per-file retry. Raises on giving up so the caller can decide what to do. + """ + if skip_if_downloaded_already and os.path.exists(local_path): + logging.info(f"Skipping download as file already exists: {local_path}") + return + + attempt = 0 + last_error: Optional[Exception] = None + while attempt < max_download_retries: + try: + total_size = ftp.size(ftp_path) + if os.path.exists(local_path): + current_size = os.path.getsize(local_path) + mode = "ab" + else: + current_size = 0 + mode = "wb" + + with open(local_path, mode) as f, tqdm( + total=total_size, + unit="B", + unit_scale=True, + desc=local_path, + initial=current_size, + position=position, + leave=True, + ) as pbar: + def callback(data): + f.write(data) + pbar.update(len(data)) + + if current_size: + try: + ftp.sendcmd(f"REST {current_size}") + except Exception: + current_size = 0 + f.seek(0) + f.truncate() + ftp.retrbinary(f"RETR {ftp_path}", callback) + + # Post-transfer integrity check: server-reported size must match + # the local size. Catches half-finished transfers that retrbinary + # didn't raise on (e.g. server closed the data channel early). + # The next iteration will REST-resume from where we left off. + if total_size: + final_size = os.path.getsize(local_path) + if final_size != total_size: + attempt += 1 + logging.error( + f"Size mismatch for {local_path}: " + f"got {final_size} bytes, expected {total_size} " + f"(attempt {attempt})" + ) + continue + logging.info(f"Successfully downloaded {local_path}") + return + except (socket.timeout, ftplib.error_temp, ftplib.error_perm) as e: + attempt += 1 + last_error = e + logging.error( + f"Download failed for {local_path} (attempt {attempt}): {e}" + ) + raise RuntimeError( + f"Giving up on {local_path} after {max_download_retries} attempts" + ) from last_error + + +def _download_ftp_paths_serial( + host: str, + paths: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + use_tls: bool, + max_connection_retries: int, + max_download_retries: int, +) -> None: + """Download all paths from one host over a single (reused) connection.""" + connection_attempt = 0 + while connection_attempt < max_connection_retries: + try: + ftp = _open_ftp_connection(host, use_tls=use_tls) + logging.info(f"Connected to FTP host: {host} (tls={use_tls})") + for ftp_path in paths: + local_path = os.path.join(output_folder, os.path.basename(ftp_path)) + try: + _download_one_ftp_path( + ftp=ftp, + ftp_path=ftp_path, + local_path=local_path, + skip_if_downloaded_already=skip_if_downloaded_already, + max_download_retries=max_download_retries, + ) + except Exception as e: + logging.error( + f"Failed to download {ftp_path} from {host}: {e}" + ) + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + logging.info(f"Disconnected from FTP host: {host}") + return + except (socket.timeout, ftplib.error_temp, ftplib.error_perm, OSError) as e: + connection_attempt += 1 + logging.error( + f"FTP connection failed (attempt {connection_attempt}): {e}" + ) + if connection_attempt < max_connection_retries: + logging.info("Retrying connection...") + time.sleep(5) + else: + logging.error( + f"Giving up after {max_connection_retries} failed connection attempts to {host}." + ) + + +def _download_ftp_paths_parallel( + host: str, + paths: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + use_tls: bool, + max_connection_retries: int, + max_download_retries: int, + parallel_files: int, +) -> None: + """ + Download paths concurrently using ``parallel_files`` workers; each + worker opens its own FTP connection so transfers don't serialize. + """ + def worker(ftp_path: str, position: int) -> None: + local_path = os.path.join(output_folder, os.path.basename(ftp_path)) + if skip_if_downloaded_already and os.path.exists(local_path): + logging.info(f"Skipping download as file already exists: {local_path}") + return + connection_attempt = 0 + while connection_attempt < max_connection_retries: + try: + ftp = _open_ftp_connection(host, use_tls=use_tls) + try: + _download_one_ftp_path( + ftp=ftp, + ftp_path=ftp_path, + local_path=local_path, + skip_if_downloaded_already=False, + max_download_retries=max_download_retries, + position=position, + ) + return + finally: + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + except (socket.timeout, ftplib.error_temp, ftplib.error_perm, OSError) as e: + connection_attempt += 1 + logging.error( + f"FTP connection failed for {ftp_path} (attempt {connection_attempt}): {e}" + ) + if connection_attempt < max_connection_retries: + time.sleep(5) + logging.error(f"Giving up on {ftp_path} from {host}") + + with ThreadPoolExecutor(max_workers=parallel_files) as executor: + futures = [ + executor.submit(worker, path, idx) for idx, path in enumerate(paths) + ] + for future in as_completed(futures): + try: + future.result() + except Exception as e: + logging.error(f"Parallel FTP download error: {e}") + + +def download_ftp_urls( + ftp_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + max_connection_retries: int = 3, + max_download_retries: int = 3, + use_tls: bool = False, + parallel_files: int = 1, +) -> None: + """ + Download a list of FTP URLs with retries, REST-based resume, and + optional parallel workers. + + :param use_tls: Open the FTP connection with TLS (FTP_TLS / PROT P). + Required for hosts that reject plain anonymous FTP (e.g. MassIVE). + When False but the server replies ``421 TLS is required``, the + connection is transparently retried over TLS. + :param parallel_files: When >1, downloads run concurrently with that + many worker connections per host (capped at the number of files). + """ + if not os.path.isdir(output_folder): + os.makedirs(output_folder, exist_ok=True) + + host_to_paths: Dict[str, List[str]] = {} + for url in ftp_urls: + parsed = urlparse(url) + host_to_paths.setdefault(parsed.hostname, []).append(parsed.path.lstrip("/")) + + for host, paths in host_to_paths.items(): + workers = max(1, min(parallel_files, len(paths))) + if workers > 1: + _download_ftp_paths_parallel( + host=host, + paths=paths, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=use_tls, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + parallel_files=workers, + ) + else: + _download_ftp_paths_serial( + host=host, + paths=paths, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=use_tls, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + ) + + +def _parallel_download(url, file_path, position=0): + """Download a file via a single-connection HTTP stream with optional resume. + If a partial file exists and the server supports Range requests, resumes + from where it left off; otherwise restarts from scratch.""" + session = Util.create_session_with_retries() + try: + head = session.head(url, timeout=(30, 30)) + head.raise_for_status() + total_size = int(head.headers.get("content-length", 0)) + accept_ranges = head.headers.get("accept-ranges", "none").strip().lower() + except (requests.RequestException, ValueError) as exc: + logging.info(f"HEAD request failed, falling back to single connection: {exc}") + total_size = 0 + accept_ranges = "none" + + resume_size = 0 + if os.path.exists(file_path) and accept_ranges == "bytes" and total_size > 0: + resume_size = os.path.getsize(file_path) + if resume_size >= total_size: + logging.info(f"File already complete: {file_path}") + return + if resume_size > 0: + logging.info(f"Resuming download from {resume_size} bytes: {file_path}") + + headers = {"Range": f"bytes={resume_size}-"} if resume_size > 0 else {} + with session.get(url, headers=headers, stream=True, timeout=(30, 60)) as r: + r.raise_for_status() + if resume_size > 0 and r.status_code != 206: + logging.warning("Server did not honor Range request (status %s), restarting download", r.status_code) + resume_size = 0 + with tqdm(total=total_size, unit="B", unit_scale=True, desc=file_path, + initial=resume_size, position=position, leave=True) as pbar: + mode = "ab" if resume_size > 0 else "wb" + with open(file_path, mode, buffering=8 * 1024 * 1024) as f: + for chunk in r.iter_content(chunk_size=8 * 1024 * 1024): + if chunk: + f.write(chunk) + pbar.update(len(chunk)) + + +def _http_download_one( + url: str, + output_folder: str, + skip_if_downloaded_already: bool, + max_retries: int = 3, + position: int = 0, +) -> None: + """ + Download a single HTTP(S) URL with HEAD-then-Range resume and retry. + Used as the worker target for both the serial loop and the parallel + ThreadPoolExecutor path. Reuses :meth:`_parallel_download` so the same + resume / restart-on-non-206 behaviour is shared with globus downloads. + """ + local_path = _local_path_for_url(url, output_folder) + if skip_if_downloaded_already and os.path.exists(local_path): + logging.info(f"Skipping download as file already exists: {local_path}") + return + last_error: Optional[Exception] = None + for attempt in range(1, max_retries + 1): + try: + _parallel_download(url, local_path, position=position) + logging.info(f"Successfully downloaded {local_path}") + return + except Exception as e: + last_error = e + logging.warning( + f"HTTP download attempt {attempt}/{max_retries} failed for {url}: {e}" + ) + raise RuntimeError( + f"Giving up on {local_path} after {max_retries} HTTP attempts" + ) from last_error + + +def download_http_urls( + http_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + parallel_files: int = 1, + max_retries: int = 3, +) -> None: + """ + Download a list of HTTP(S) URLs with HEAD-then-Range resume, per-file + retries, and an optional ``parallel_files`` worker pool. + + When ``parallel_files`` > 1, downloads run concurrently using a + :class:`ThreadPoolExecutor`. Each worker manages its own file (a new + ``requests`` session is opened inside ``_parallel_download``) so the + only shared resource is the output directory. + """ + if not os.path.isdir(output_folder): + os.makedirs(output_folder, exist_ok=True) + + if not http_urls: + return + + workers = max(1, min(parallel_files, len(http_urls))) + if workers > 1: + logging.info( + f"Downloading {len(http_urls)} HTTP(S) file(s) with {workers} parallel workers" + ) + with ThreadPoolExecutor(max_workers=workers) as executor: + futures = [ + executor.submit( + _http_download_one, + url, + output_folder, + skip_if_downloaded_already, + max_retries, + idx, + ) + for idx, url in enumerate(http_urls) + ] + for future in as_completed(futures): + try: + future.result() + except Exception as e: + logging.error(f"Parallel HTTP download error: {e}") + else: + for url in http_urls: + try: + _http_download_one( + url, + output_folder, + skip_if_downloaded_already, + max_retries, + ) + except Exception as e: + logging.error(f"HTTP download failed for {url}: {e}") From e16c8709ede0bc9d9c7d846e21c32c5c73743ae5 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 15:53:10 +0100 Subject: [PATCH 07/54] refactor(providers): move cross-cutting utilities into providers/util.py Moved Progress, _find_tsv_columns, _is_md5_checksum, read_checksum_file, compute_md5, validate_download, _remove_if_exists, _get_download_url, _resolve_local_path from Files into providers/util.py. Files keeps shim re-exports so existing references keep working. No behaviour change. Test suite green. --- pridepy/files/files.py | 166 ++++++---------------------------- pridepy/providers/util.py | 183 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 210 insertions(+), 139 deletions(-) create mode 100644 pridepy/providers/util.py diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 0de339a..e919915 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -28,26 +28,9 @@ from pridepy.util.api_handling import Util -class Progress: - def __init__(self, total_size, file_name): - self.pbar = tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc="Downloading {}".format(file_name), - ) - - def __call__(self, bytes_amount): - self.pbar.update(bytes_amount) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.pbar.close() - - def close(self): - self.pbar.close() +# Re-export from providers.util so external `from pridepy.files.files import Progress` +# still works. +from pridepy.providers.util import Progress # noqa: F401 class Files: @@ -105,146 +88,51 @@ def __init__(self): @staticmethod def _find_tsv_columns(header: str) -> Optional[Tuple[int, int]]: - """Return (name_idx, checksum_idx) from a TSV header, or None.""" - cols = [col.strip().lower() for col in header.split("\t")] - required_cols = {"file-name", "file-md5checksum", "file-size"} - if not required_cols.issubset(set(cols)): - return None - return cols.index("file-name"), cols.index("file-md5checksum") + """Shim — see :func:`pridepy.providers.util._find_tsv_columns`.""" + from pridepy.providers import util + return util._find_tsv_columns(header) @staticmethod def _is_md5_checksum(value: str) -> bool: - return len(value) == 32 and all(char in "0123456789abcdef" for char in value) + """Shim — see :func:`pridepy.providers.util._is_md5_checksum`.""" + from pridepy.providers import util + return util._is_md5_checksum(value) @staticmethod def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: - """ - Read PRIDE API checksum TSV and build {file_name: md5} map. - Expected format: File-Name\tFile-MD5Checksum\tFile-Size - """ - checksums: Dict[str, str] = {} - if not checksum_file_path or not os.path.exists(checksum_file_path): - return checksums - - with open(checksum_file_path, "r", encoding="utf-8") as f: - header = f.readline().strip() - if not header: - return checksums - - col_indices = Files._find_tsv_columns(header) - if col_indices is None: - logging.warning(f"Unrecognized checksum file format: {header}") - return checksums - - name_idx, checksum_idx = col_indices - min_cols = max(name_idx, checksum_idx) + 1 - for line in f: - parts = line.strip().split("\t") - if len(parts) >= min_cols: - fn = os.path.basename(parts[name_idx].strip()) - cs = parts[checksum_idx].strip().lower() - if fn and Files._is_md5_checksum(cs): - checksums[fn] = cs - - return checksums + """Shim — see :func:`pridepy.providers.util.read_checksum_file`.""" + from pridepy.providers import util + return util.read_checksum_file(checksum_file_path) @staticmethod def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: - """ - Compute an MD5 checksum for integrity validation, not for security use. - """ - try: - md5 = hashlib.md5(usedforsecurity=False) - except TypeError: - md5 = hashlib.md5() - with open(file_path, "rb") as file_handle: - while True: - chunk = file_handle.read(chunk_size) - if not chunk: - break - md5.update(chunk) - return md5.hexdigest() + """Shim — see :func:`pridepy.providers.util.compute_md5`.""" + from pridepy.providers import util + return util.compute_md5(file_path, chunk_size) @staticmethod def validate_download(file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, str]: - """ - Validate a local file exists, is non-empty, and checksum matches when provided. - """ - if not os.path.exists(file_path): - return False, "file does not exist" - if os.path.getsize(file_path) == 0: - return False, "file is empty" - if expected_checksum: - actual_checksum = Files.compute_md5(file_path) - if actual_checksum.lower() != expected_checksum.lower(): - return False, ( - f"checksum mismatch (expected={expected_checksum.lower()}, actual={actual_checksum.lower()})" - ) - return True, "ok" + """Shim — see :func:`pridepy.providers.util.validate_download`.""" + from pridepy.providers import util + return util.validate_download(file_path, expected_checksum) @staticmethod def _remove_if_exists(file_path: str) -> None: - """ - Remove a file if it already exists locally. - """ - if os.path.exists(file_path): - os.remove(file_path) + """Shim — see :func:`pridepy.providers.util._remove_if_exists`.""" + from pridepy.providers import util + return util._remove_if_exists(file_path) @staticmethod def _get_download_url(file_record: Dict, protocol: str) -> str: - """ - Resolve the public download URL for a file and protocol. - - Raises ValueError when the requested protocol has no suitable location. - Aspera requires a dedicated "Aspera Protocol" entry; ftp/s3/globus - derive their URL from the "FTP Protocol" entry (falling back to an - arbitrary non-Aspera location would produce a URL the caller cannot - actually transfer with). - """ - locations = file_record.get("publicFileLocations", []) - if not locations: - raise ValueError("No public file locations present") - - aspera_url = None - ftp_url = None - for location in locations: - name = location.get("name") - if name == "Aspera Protocol": - aspera_url = location.get("value") - elif name == "FTP Protocol": - ftp_url = location.get("value") - - if protocol == "aspera": - if not aspera_url: - raise ValueError("Aspera URL not available") - return aspera_url - - if not ftp_url: - raise ValueError("FTP URL not available") - if protocol == "ftp": - return ftp_url - if protocol == "globus": - return ftp_url.replace( - Files.PRIDE_ARCHIVE_FTP_URL_PREFIX, - Files.PRIDE_ARCHIVE_HTTPS_URL_PREFIX, - 1, - ) - if protocol == "s3": - return ftp_url - raise ValueError(f"Unsupported protocol: {protocol}") + """Shim — see :func:`pridepy.providers.util._get_download_url`.""" + from pridepy.providers import util + return util._get_download_url(file_record, protocol) @staticmethod def _resolve_local_path(file_record: Dict, output_folder: str) -> str: - """ - Compute the canonical local path for a file regardless of transfer protocol. - """ - try: - canonical_url = Files._get_download_url(file_record, "ftp") - except ValueError: - canonical_url = "" - if canonical_url: - return Files.get_output_file_name(canonical_url, file_record, output_folder) - return os.path.join(output_folder, file_record["fileName"]) + """Shim — see :func:`pridepy.providers.util._resolve_local_path`.""" + from pridepy.providers import util + return util._resolve_local_path(file_record, output_folder) @staticmethod def _protocol_sequence(protocol: str) -> List[str]: diff --git a/pridepy/providers/util.py b/pridepy/providers/util.py new file mode 100644 index 0000000..0fc5791 --- /dev/null +++ b/pridepy/providers/util.py @@ -0,0 +1,183 @@ +"""Cross-cutting utilities used by providers and the Files facade. + +Pure functions (and one tiny Progress class) for checksums, record-shape +helpers, and download progress. Originally on ``Files`` as @staticmethods; +moved here so providers can use them without depending on Files at import +time, and Files keeps shim re-exports for backward compatibility with +existing test patches. +""" +import hashlib +import logging +import os +from typing import Dict, List, Optional, Tuple + +from tqdm import tqdm + + +class Progress: + def __init__(self, total_size, file_name): + self.pbar = tqdm( + total=total_size, + unit="B", + unit_scale=True, + desc="Downloading {}".format(file_name), + ) + + def __call__(self, bytes_amount): + self.pbar.update(bytes_amount) + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.pbar.close() + + def close(self): + self.pbar.close() + + +def _find_tsv_columns(header: str) -> Optional[Tuple[int, int]]: + """Return (name_idx, checksum_idx) from a TSV header, or None.""" + cols = [col.strip().lower() for col in header.split("\t")] + required_cols = {"file-name", "file-md5checksum", "file-size"} + if not required_cols.issubset(set(cols)): + return None + return cols.index("file-name"), cols.index("file-md5checksum") + + +def _is_md5_checksum(value: str) -> bool: + return len(value) == 32 and all(char in "0123456789abcdef" for char in value) + + +def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: + """ + Read PRIDE API checksum TSV and build {file_name: md5} map. + Expected format: File-Name\tFile-MD5Checksum\tFile-Size + """ + checksums: Dict[str, str] = {} + if not checksum_file_path or not os.path.exists(checksum_file_path): + return checksums + + with open(checksum_file_path, "r", encoding="utf-8") as f: + header = f.readline().strip() + if not header: + return checksums + + col_indices = _find_tsv_columns(header) + if col_indices is None: + logging.warning(f"Unrecognized checksum file format: {header}") + return checksums + + name_idx, checksum_idx = col_indices + min_cols = max(name_idx, checksum_idx) + 1 + for line in f: + parts = line.strip().split("\t") + if len(parts) >= min_cols: + fn = os.path.basename(parts[name_idx].strip()) + cs = parts[checksum_idx].strip().lower() + if fn and _is_md5_checksum(cs): + checksums[fn] = cs + + return checksums + + +def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: + """ + Compute an MD5 checksum for integrity validation, not for security use. + """ + try: + md5 = hashlib.md5(usedforsecurity=False) + except TypeError: + md5 = hashlib.md5() + with open(file_path, "rb") as file_handle: + while True: + chunk = file_handle.read(chunk_size) + if not chunk: + break + md5.update(chunk) + return md5.hexdigest() + + +def validate_download(file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, str]: + """ + Validate a local file exists, is non-empty, and checksum matches when provided. + """ + if not os.path.exists(file_path): + return False, "file does not exist" + if os.path.getsize(file_path) == 0: + return False, "file is empty" + if expected_checksum: + actual_checksum = compute_md5(file_path) + if actual_checksum.lower() != expected_checksum.lower(): + return False, ( + f"checksum mismatch (expected={expected_checksum.lower()}, actual={actual_checksum.lower()})" + ) + return True, "ok" + + +def _remove_if_exists(file_path: str) -> None: + """ + Remove a file if it already exists locally. + """ + if os.path.exists(file_path): + os.remove(file_path) + + +def _get_download_url(file_record: Dict, protocol: str) -> str: + """ + Resolve the public download URL for a file and protocol. + + Raises ValueError when the requested protocol has no suitable location. + Aspera requires a dedicated "Aspera Protocol" entry; ftp/s3/globus + derive their URL from the "FTP Protocol" entry (falling back to an + arbitrary non-Aspera location would produce a URL the caller cannot + actually transfer with). + """ + from pridepy.files.files import Files + + locations = file_record.get("publicFileLocations", []) + if not locations: + raise ValueError("No public file locations present") + + aspera_url = None + ftp_url = None + for location in locations: + name = location.get("name") + if name == "Aspera Protocol": + aspera_url = location.get("value") + elif name == "FTP Protocol": + ftp_url = location.get("value") + + if protocol == "aspera": + if not aspera_url: + raise ValueError("Aspera URL not available") + return aspera_url + + if not ftp_url: + raise ValueError("FTP URL not available") + if protocol == "ftp": + return ftp_url + if protocol == "globus": + return ftp_url.replace( + Files.PRIDE_ARCHIVE_FTP_URL_PREFIX, + Files.PRIDE_ARCHIVE_HTTPS_URL_PREFIX, + 1, + ) + if protocol == "s3": + return ftp_url + raise ValueError(f"Unsupported protocol: {protocol}") + + +def _resolve_local_path(file_record: Dict, output_folder: str) -> str: + """ + Compute the canonical local path for a file regardless of transfer protocol. + """ + from pridepy.files.files import Files + + try: + canonical_url = _get_download_url(file_record, "ftp") + except ValueError: + canonical_url = "" + if canonical_url: + return Files.get_output_file_name(canonical_url, file_record, output_folder) + return os.path.join(output_folder, file_record["fileName"]) From 41539e7ab898b06d96605eec159bbec4a6fa13cc Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 15:59:01 +0100 Subject: [PATCH 08/54] refactor(providers): extract MassiveProvider Moved MassIVE listing + record building from Files into providers/massive.py as MassiveProvider(BaseDirectDownloadProvider). Provider is registered with the Registry. Files keeps shim methods (is_massive_accession, _list_massive_public_files, _build_massive_file_record, _get_massive_public_root, _get_massive_public_ftp_url, _map_massive_collection_to_category) that delegate to the provider. MASSIVE_CATEGORY_MAP / MASSIVE_ARCHIVE_FTP constants remain on Files as class-attribute re-exports. All 10 MassIVE tests pass without modification. Full suite green. --- pridepy/files/files.py | 85 +++++++++---------------------- pridepy/providers/massive.py | 97 ++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+), 62 deletions(-) create mode 100644 pridepy/providers/massive.py diff --git a/pridepy/files/files.py b/pridepy/files/files.py index e919915..328150e 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -44,8 +44,15 @@ class Files: PRIDE_ARCHIVE_FTP = "ftp.pride.ebi.ac.uk" PRIDE_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.pride.ebi.ac.uk/" PRIDE_ARCHIVE_HTTPS_URL_PREFIX = "https://ftp.pride.ebi.ac.uk/" - MASSIVE_ARCHIVE_FTP = "massive-ftp.ucsd.edu" - MASSIVE_ARCHIVE_FTP_URL_PREFIX = "ftp://massive-ftp.ucsd.edu/v01/" + # Re-exported from providers/massive.py — kept here for back-compat. + from pridepy.providers.massive import ( # noqa: E402 + MASSIVE_CATEGORY_MAP as _MASSIVE_CATEGORY_MAP, + MassiveProvider as _MassiveProvider, + ) + MASSIVE_CATEGORY_MAP = _MASSIVE_CATEGORY_MAP + MASSIVE_ARCHIVE_FTP = _MassiveProvider.ARCHIVE_FTP + MASSIVE_ARCHIVE_FTP_URL_PREFIX = _MassiveProvider.ARCHIVE_FTP_URL_PREFIX + del _MASSIVE_CATEGORY_MAP, _MassiveProvider JPOST_ARCHIVE_FTP = "ftp.jpostdb.org" JPOST_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.jpostdb.org/" JPOST_PROXI_BASE_URL = "https://repository.jpostdb.org/proxi/datasets/" @@ -69,18 +76,6 @@ class Files: S3_URL = "https://hh.fire.sdo.ebi.ac.uk" S3_BUCKET = "pride-public" PROTOCOL_ORDER = ["aspera", "s3", "ftp", "globus"] - MASSIVE_CATEGORY_MAP = { - "raw": "RAW", - "peak": "PEAK", - "ccms_peak": "PEAK", - "search": "SEARCH", - "result": "RESULT", - "ccms_result": "RESULT", - "quant": "RESULT", - "fasta": "FASTA", - "spectrum_library": "SPECTRUM_LIBRARY", - "library": "SPECTRUM_LIBRARY", - } logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") def __init__(self): @@ -145,48 +140,29 @@ def _protocol_sequence(protocol: str) -> List[str]: @staticmethod def is_massive_accession(accession: str) -> bool: - """ - Return True when the accession looks like a MassIVE dataset accession. - """ - if not accession: - return False - return bool(re.fullmatch(r"R?MSV\d{9}", accession.upper())) + """Shim — see :meth:`pridepy.providers.massive.MassiveProvider.matches`.""" + from pridepy.providers.massive import MassiveProvider + return MassiveProvider.matches(accession) @staticmethod def _get_massive_public_root(accession: str) -> str: - normalized_accession = accession.upper() - return f"/v01/{normalized_accession}" + from pridepy.providers.massive import MassiveProvider + return MassiveProvider._get_public_root(accession) @staticmethod def _get_massive_public_ftp_url(accession: str, remote_path: str) -> str: - root_path = Files._get_massive_public_root(accession).rstrip("/") - relative_path = remote_path - if remote_path.startswith(root_path): - relative_path = remote_path[len(root_path) :].lstrip("/") - return f"{Files.MASSIVE_ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" + from pridepy.providers.massive import MassiveProvider + return MassiveProvider._get_public_ftp_url(accession, remote_path) @staticmethod def _map_massive_collection_to_category(collection: str) -> str: - return Files.MASSIVE_CATEGORY_MAP.get(collection.lower(), "OTHER") + from pridepy.providers.massive import MassiveProvider + return MassiveProvider._map_collection_to_category(collection) @staticmethod def _build_massive_file_record(accession: str, ftp_url: str) -> Dict: - parsed = urlparse(ftp_url) - root_prefix = f"/v01/{accession.upper()}/" - relative_path = parsed.path - if relative_path.startswith(root_prefix): - relative_path = relative_path[len(root_prefix) :] - relative_path = relative_path.lstrip("/") - collection = relative_path.split("/", 1)[0] if relative_path else "" - return { - "accession": accession.upper(), - "fileName": os.path.basename(parsed.path), - "fileCategory": {"value": Files._map_massive_collection_to_category(collection)}, - "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], - "relativePath": relative_path, - "collection": collection, - "source": "MassIVE", - } + from pridepy.providers.massive import MassiveProvider + return MassiveProvider._build_file_record(accession, ftp_url) @staticmethod def is_jpost_accession(accession: str) -> bool: @@ -333,24 +309,9 @@ def _list_ftp_repo_files(host, remote_root, error_label, use_tls=False): return transport._list_ftp_repo_files(host=host, remote_root=remote_root, error_label=error_label, use_tls=use_tls) def _list_massive_public_files(self, accession: str) -> List[Dict]: - """ - Discover all public files for a MassIVE dataset from its anonymous FTP tree. - """ - normalized_accession = accession.upper() - remote_root = self._get_massive_public_root(normalized_accession) - remote_files = self._list_ftp_repo_files( - host=self.MASSIVE_ARCHIVE_FTP, - remote_root=remote_root, - error_label=f"MassIVE dataset {normalized_accession}", - use_tls=True, - ) - return [ - self._build_massive_file_record( - normalized_accession, - self._get_massive_public_ftp_url(normalized_accession, remote_file), - ) - for remote_file in remote_files - ] + """Shim — see :meth:`pridepy.providers.massive.MassiveProvider.list_files`.""" + from pridepy.providers.massive import MassiveProvider + return MassiveProvider().list_files(accession) def _download_massive_file_records( self, diff --git a/pridepy/providers/massive.py b/pridepy/providers/massive.py new file mode 100644 index 0000000..cdc466b --- /dev/null +++ b/pridepy/providers/massive.py @@ -0,0 +1,97 @@ +"""MassIVE direct-download provider. + +Lists files by walking the FTPS tree at massive-ftp.ucsd.edu (TLS is +required by the server). Downloads files via the shared transport layer +with ``use_tls=True``. +""" +import os +import re +from typing import ClassVar, Dict, List +from urllib.parse import urlparse + +from pridepy.providers import registry +from pridepy.providers.base import BaseDirectDownloadProvider + + +MASSIVE_CATEGORY_MAP = { + "raw": "RAW", + "peak": "PEAK", + "ccms_peak": "PEAK", + "search": "SEARCH", + "result": "RESULT", + "ccms_result": "RESULT", + "quant": "RESULT", + "fasta": "FASTA", + "spectrum_library": "SPECTRUM_LIBRARY", + "library": "SPECTRUM_LIBRARY", +} + + +@registry.register +class MassiveProvider(BaseDirectDownloadProvider): + name: ClassVar[str] = "massive" + use_tls: ClassVar[bool] = True + + ARCHIVE_FTP: ClassVar[str] = "massive-ftp.ucsd.edu" + ARCHIVE_FTP_URL_PREFIX: ClassVar[str] = "ftp://massive-ftp.ucsd.edu/v01/" + + @staticmethod + def matches(accession: str) -> bool: + """Return True when ``accession`` is a MassIVE dataset accession.""" + if not accession: + return False + return bool(re.fullmatch(r"R?MSV\d{9}", accession.upper())) + + @staticmethod + def _get_public_root(accession: str) -> str: + return f"/v01/{accession.upper()}" + + @classmethod + def _get_public_ftp_url(cls, accession: str, remote_path: str) -> str: + root_path = cls._get_public_root(accession).rstrip("/") + relative_path = remote_path + if remote_path.startswith(root_path): + relative_path = remote_path[len(root_path):].lstrip("/") + return f"{cls.ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" + + @staticmethod + def _map_collection_to_category(collection: str) -> str: + return MASSIVE_CATEGORY_MAP.get(collection.lower(), "OTHER") + + @classmethod + def _build_file_record(cls, accession: str, ftp_url: str) -> Dict: + """Build a pridepy file record from an FTP URL inside the dataset.""" + parsed = urlparse(ftp_url) + root_prefix = f"/v01/{accession.upper()}/" + relative_path = parsed.path + if relative_path.startswith(root_prefix): + relative_path = relative_path[len(root_prefix):] + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + return { + "accession": accession.upper(), + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": cls._map_collection_to_category(collection)}, + "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], + "relativePath": relative_path, + "collection": collection, + "source": "MassIVE", + } + + def list_files(self, accession: str) -> List[Dict]: + from pridepy.providers import transport + normalized = accession.upper() + remote_root = self._get_public_root(normalized) + remote_files = transport._list_ftp_repo_files( + host=self.ARCHIVE_FTP, + remote_root=remote_root, + error_label=f"MassIVE dataset {normalized}", + use_tls=True, + ) + return [ + self._build_file_record( + normalized, + self._get_public_ftp_url(normalized, remote_file), + ) + for remote_file in remote_files + ] From 2010f6375083227ae637b4b84e0d0987c67b0af4 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 16:06:04 +0100 Subject: [PATCH 09/54] refactor(providers): extract JpostProvider with PROXI + FTP listing Moved JPOST listing (PROXI JSON primary + FTP tree walk fallback) and record building from Files into providers/jpost.py as JpostProvider(BaseDirectDownloadProvider). Provider registered with the Registry. Files keeps shim methods (is_jpost_accession, _list_jpost_public_files, _list_jpost_public_files_via_proxi, _build_jpost_file_record, _get_jpost_public_root, _get_jpost_public_ftp_url) that delegate to the provider. JPOST_ARCHIVE_FTP / JPOST_PROXI_BASE_URL / JPOST_PROXI_CATEGORY_MAP constants remain on Files as class-attribute re-exports. All JPOST tests pass without modification. Full suite green. --- pridepy/files/files.py | 123 +++++++----------------------- pridepy/providers/jpost.py | 150 +++++++++++++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 97 deletions(-) create mode 100644 pridepy/providers/jpost.py diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 328150e..69d477e 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -53,18 +53,12 @@ class Files: MASSIVE_ARCHIVE_FTP = _MassiveProvider.ARCHIVE_FTP MASSIVE_ARCHIVE_FTP_URL_PREFIX = _MassiveProvider.ARCHIVE_FTP_URL_PREFIX del _MASSIVE_CATEGORY_MAP, _MassiveProvider - JPOST_ARCHIVE_FTP = "ftp.jpostdb.org" - JPOST_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.jpostdb.org/" - JPOST_PROXI_BASE_URL = "https://repository.jpostdb.org/proxi/datasets/" - JPOST_PROXI_CATEGORY_MAP = { - "Associated raw file URI": "RAW", - "Result file URI": "RESULT", - "Search engine output file URI": "SEARCH", - "Peak list file URI": "PEAK", - "Spectrum library file URI": "SPECTRUM_LIBRARY", - "Sequence database URI": "FASTA", - "Quantification file URI": "RESULT", - } + from pridepy.providers.jpost import JpostProvider as _JpostProvider + JPOST_ARCHIVE_FTP = _JpostProvider.ARCHIVE_FTP + JPOST_ARCHIVE_FTP_URL_PREFIX = _JpostProvider.ARCHIVE_FTP_URL_PREFIX + JPOST_PROXI_BASE_URL = _JpostProvider.PROXI_BASE_URL + JPOST_PROXI_CATEGORY_MAP = _JpostProvider.PROXI_CATEGORY_MAP + del _JpostProvider IPROX_DOWNLOAD_BASE_URL = "http://download.iprox.org/" IPROX_PX_XML_URL_TEMPLATE = ( "http://download.iprox.org/{accession}/PX_{accession}.xml" @@ -166,57 +160,24 @@ def _build_massive_file_record(accession: str, ftp_url: str) -> Dict: @staticmethod def is_jpost_accession(accession: str) -> bool: - """ - Return True when the accession looks like a JPOST dataset accession. - """ - if not accession: - return False - return bool(re.fullmatch(r"JPST\d{6}", accession.upper())) + """Shim — see :meth:`pridepy.providers.jpost.JpostProvider.matches`.""" + from pridepy.providers.jpost import JpostProvider + return JpostProvider.matches(accession) @staticmethod def _get_jpost_public_root(accession: str) -> str: - return f"/{accession.upper()}" + from pridepy.providers.jpost import JpostProvider + return JpostProvider._get_public_root(accession) @staticmethod def _get_jpost_public_ftp_url(accession: str, remote_path: str) -> str: - root_path = Files._get_jpost_public_root(accession).rstrip("/") - relative_path = remote_path - if remote_path.startswith(root_path): - relative_path = remote_path[len(root_path) :].lstrip("/") - return f"{Files.JPOST_ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" + from pridepy.providers.jpost import JpostProvider + return JpostProvider._get_public_ftp_url(accession, remote_path) @staticmethod - def _build_jpost_file_record( - accession: str, ftp_url: str, category_from_proxi: Optional[str] = None - ) -> Dict: - """ - Build a pridepy file record for a JPOST file. - - When ``category_from_proxi`` is provided (e.g. ``"Associated raw file URI"``), - the PROXI CV name takes precedence over the heuristic collection-from-path - mapping. Falls back to the same path-segment heuristic used for MassIVE - when the category isn't known. - """ - parsed = urlparse(ftp_url) - root_prefix = f"/{accession.upper()}/" - relative_path = parsed.path - if relative_path.startswith(root_prefix): - relative_path = relative_path[len(root_prefix) :] - relative_path = relative_path.lstrip("/") - collection = relative_path.split("/", 1)[0] if relative_path else "" - if category_from_proxi and category_from_proxi in Files.JPOST_PROXI_CATEGORY_MAP: - category = Files.JPOST_PROXI_CATEGORY_MAP[category_from_proxi] - else: - category = Files._map_massive_collection_to_category(collection) - return { - "accession": accession.upper(), - "fileName": os.path.basename(parsed.path), - "fileCategory": {"value": category}, - "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], - "relativePath": relative_path, - "collection": collection, - "source": "JPOST", - } + def _build_jpost_file_record(accession, ftp_url, category_from_proxi=None): + from pridepy.providers.jpost import JpostProvider + return JpostProvider._build_file_record(accession, ftp_url, category_from_proxi) @staticmethod def _build_iprox_file_record( @@ -339,12 +300,11 @@ def _list_jpost_public_files(self, accession: str) -> List[Dict]: """ Discover all public files for a JPOST dataset. - Prefers the JPOST PROXI JSON endpoint at - ``https://repository.jpostdb.org/proxi/datasets/`` since it - returns file URLs with category labels and avoids the anonymous-FTP - rate limit that ``ftp.jpostdb.org`` applies per source IP. Falls back - to walking the FTP tree if PROXI is unreachable or returns no files. + Delegates to JpostProvider but routes via the shim methods so that + test patches on ``_list_jpost_public_files_via_proxi`` and + ``_list_ftp_repo_files`` continue to intercept. """ + from pridepy.providers.jpost import JpostProvider normalized_accession = accession.upper() try: return self._list_jpost_public_files_via_proxi(normalized_accession) @@ -353,55 +313,24 @@ def _list_jpost_public_files(self, accession: str) -> List[Dict]: f"JPOST PROXI listing failed for {normalized_accession} " f"({proxi_error}); falling back to FTP tree walk." ) - remote_root = self._get_jpost_public_root(normalized_accession) + remote_root = JpostProvider._get_public_root(normalized_accession) remote_files = self._list_ftp_repo_files( - host=self.JPOST_ARCHIVE_FTP, + host=JpostProvider.ARCHIVE_FTP, remote_root=remote_root, error_label=f"JPOST dataset {normalized_accession}", ) return [ self._build_jpost_file_record( normalized_accession, - self._get_jpost_public_ftp_url(normalized_accession, remote_file), + JpostProvider._get_public_ftp_url(normalized_accession, remote_file), ) for remote_file in remote_files ] def _list_jpost_public_files_via_proxi(self, accession: str) -> List[Dict]: - """ - Fetch the JPOST PROXI dataset metadata and turn each ``datasetFiles`` - entry into a pridepy file record. The PROXI ``name`` field is mapped to - a PRIDE-style category so existing RAW/SEARCH/RESULT filtering works. - """ - import json as _json - - proxi_url = f"{self.JPOST_PROXI_BASE_URL}{accession}" - logging.info(f"Fetching JPOST PROXI metadata: {proxi_url}") - response = requests.get( - proxi_url, - headers={"Accept": "application/json"}, - timeout=30, - ) - response.raise_for_status() - data = _json.loads(response.content) - dataset_files = data.get("datasetFiles") or [] - records: List[Dict] = [] - for entry in dataset_files: - value = (entry or {}).get("value") - if not value or not value.startswith("ftp://"): - continue - records.append( - self._build_jpost_file_record( - accession, - value, - category_from_proxi=(entry or {}).get("name"), - ) - ) - if not records: - raise RuntimeError( - f"JPOST PROXI returned no FTP file URIs for {accession}" - ) - return records + """Shim — see :meth:`pridepy.providers.jpost.JpostProvider._list_via_proxi`.""" + from pridepy.providers.jpost import JpostProvider + return JpostProvider()._list_via_proxi(accession) def _list_iprox_public_files(self, accession: str) -> List[Dict]: """ diff --git a/pridepy/providers/jpost.py b/pridepy/providers/jpost.py new file mode 100644 index 0000000..a9ab23e --- /dev/null +++ b/pridepy/providers/jpost.py @@ -0,0 +1,150 @@ +"""JPOST direct-download provider. + +PRIMARY listing: PROXI JSON at repository.jpostdb.org. The PROXI endpoint +returns ``datasetFiles[*].value`` as ``ftp://`` URLs alongside CV labels +(Associated raw file URI, Search engine output file URI, etc.) which map +cleanly to PRIDE file categories. + +FALLBACK listing: when PROXI fails, walk the FTP tree at ftp.jpostdb.org. +This is needed because JPOST's FTP server rate-limits aggressively per +source IP (sticky 421-too-many-connections); the PROXI path lets us avoid +walking the FTP tree just for a listing. +""" +import logging +import os +import re +from typing import ClassVar, Dict, List, Optional +from urllib.parse import urlparse + +import requests + +from pridepy.providers import registry +from pridepy.providers.base import BaseDirectDownloadProvider + + +@registry.register +class JpostProvider(BaseDirectDownloadProvider): + name: ClassVar[str] = "jpost" + use_tls: ClassVar[bool] = False + + ARCHIVE_FTP: ClassVar[str] = "ftp.jpostdb.org" + ARCHIVE_FTP_URL_PREFIX: ClassVar[str] = "ftp://ftp.jpostdb.org/" + PROXI_BASE_URL: ClassVar[str] = "https://repository.jpostdb.org/proxi/datasets/" + + PROXI_CATEGORY_MAP: ClassVar[Dict[str, str]] = { + "Associated raw file URI": "RAW", + "Result file URI": "RESULT", + "Search engine output file URI": "SEARCH", + "Peak list file URI": "PEAK", + "Spectrum library file URI": "SPECTRUM_LIBRARY", + "Sequence database URI": "FASTA", + "Quantification file URI": "RESULT", + } + + @staticmethod + def matches(accession: str) -> bool: + if not accession: + return False + return bool(re.fullmatch(r"JPST\d{6}", accession.upper())) + + @staticmethod + def _get_public_root(accession: str) -> str: + return f"/{accession.upper()}" + + @classmethod + def _get_public_ftp_url(cls, accession: str, remote_path: str) -> str: + root_path = cls._get_public_root(accession).rstrip("/") + relative_path = remote_path + if remote_path.startswith(root_path): + relative_path = remote_path[len(root_path):].lstrip("/") + return f"{cls.ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" + + @classmethod + def _build_file_record( + cls, accession: str, ftp_url: str, category_from_proxi: Optional[str] = None + ) -> Dict: + """Build a pridepy file record from an FTP URL. + + When ``category_from_proxi`` is provided (e.g. ``"Associated raw file URI"``), + the PROXI CV name takes precedence over the heuristic collection-from-path + mapping. Falls back to the same path-segment heuristic used for MassIVE + when the category isn't known. + """ + # Import the MassIVE collection->category map for the fallback heuristic. + from pridepy.providers.massive import MassiveProvider + parsed = urlparse(ftp_url) + root_prefix = f"/{accession.upper()}/" + relative_path = parsed.path + if relative_path.startswith(root_prefix): + relative_path = relative_path[len(root_prefix):] + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + if category_from_proxi and category_from_proxi in cls.PROXI_CATEGORY_MAP: + category = cls.PROXI_CATEGORY_MAP[category_from_proxi] + else: + category = MassiveProvider._map_collection_to_category(collection) + return { + "accession": accession.upper(), + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": category}, + "publicFileLocations": [{"name": "FTP Protocol", "value": ftp_url}], + "relativePath": relative_path, + "collection": collection, + "source": "JPOST", + } + + def list_files(self, accession: str) -> List[Dict]: + """PRIMARY: PROXI JSON. FALLBACK: FTP tree walk.""" + normalized = accession.upper() + try: + return self._list_via_proxi(normalized) + except Exception as proxi_error: + logging.warning( + f"JPOST PROXI listing failed for {normalized} " + f"({proxi_error}); falling back to FTP tree walk." + ) + from pridepy.providers import transport + remote_root = self._get_public_root(normalized) + remote_files = transport._list_ftp_repo_files( + host=self.ARCHIVE_FTP, + remote_root=remote_root, + error_label=f"JPOST dataset {normalized}", + ) + return [ + self._build_file_record( + normalized, + self._get_public_ftp_url(normalized, remote_file), + ) + for remote_file in remote_files + ] + + def _list_via_proxi(self, accession: str) -> List[Dict]: + """Fetch JPOST PROXI dataset metadata and turn each datasetFiles entry into a file record.""" + import json as _json + proxi_url = f"{self.PROXI_BASE_URL}{accession}" + logging.info(f"Fetching JPOST PROXI metadata: {proxi_url}") + response = requests.get( + proxi_url, + headers={"Accept": "application/json"}, + timeout=30, + ) + response.raise_for_status() + data = _json.loads(response.content) + dataset_files = data.get("datasetFiles") or [] + records: List[Dict] = [] + for entry in dataset_files: + value = (entry or {}).get("value") + if not value or not value.startswith("ftp://"): + continue + records.append( + self._build_file_record( + accession, + value, + category_from_proxi=(entry or {}).get("name"), + ) + ) + if not records: + raise RuntimeError( + f"JPOST PROXI returned no FTP file URIs for {accession}" + ) + return records From e128a16d8754977aad843a40f565944975098e7c Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 16:19:49 +0100 Subject: [PATCH 10/54] refactor(providers): extract IproxProvider with PX XML listing Moved iProX listing (PX XML from download.iprox.org) and record building from Files into providers/iprox.py as IproxProvider(BaseDirectDownloadProvider). Provider registered with the Registry. Files keeps shim methods (is_iprox_accession, _list_iprox_public_files, _build_iprox_file_record, _get_iprox_public_root, _get_iprox_public_ftp_url) that delegate to the provider. IPROX_DOWNLOAD_BASE_URL / IPROX_PX_XML_URL_TEMPLATE / IPROX_PX_CATEGORY_MAP constants remain on Files as class-attribute re-exports. _download_direct_download_records on Files now dispatches via the registry instead of branching manually. _list_direct_download_files keeps shim dispatching so existing test patches on _list_massive_public_files and _list_jpost_public_files continue to intercept the calls. All iProX tests pass without modification. Full suite green. --- pridepy/files/files.py | 176 ++++++++----------------------------- pridepy/providers/iprox.py | 129 +++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 138 deletions(-) create mode 100644 pridepy/providers/iprox.py diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 69d477e..afe6067 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -59,14 +59,11 @@ class Files: JPOST_PROXI_BASE_URL = _JpostProvider.PROXI_BASE_URL JPOST_PROXI_CATEGORY_MAP = _JpostProvider.PROXI_CATEGORY_MAP del _JpostProvider - IPROX_DOWNLOAD_BASE_URL = "http://download.iprox.org/" - IPROX_PX_XML_URL_TEMPLATE = ( - "http://download.iprox.org/{accession}/PX_{accession}.xml" - ) - # iProX PX XML uses the same PSI-MS cvParam "name" values as JPOST, so the - # JPOST PROXI category map applies. PX XML cvParam "Associated raw file URI" - # is the canonical raw-file label per the PSI-MS CV (MS:1002846). - IPROX_PX_CATEGORY_MAP = JPOST_PROXI_CATEGORY_MAP + from pridepy.providers.iprox import IproxProvider as _IproxProvider + IPROX_DOWNLOAD_BASE_URL = _IproxProvider.DOWNLOAD_BASE_URL + IPROX_PX_XML_URL_TEMPLATE = _IproxProvider.PX_XML_URL_TEMPLATE + IPROX_PX_CATEGORY_MAP = _IproxProvider.PX_CATEGORY_MAP + del _IproxProvider S3_URL = "https://hh.fire.sdo.ebi.ac.uk" S3_BUCKET = "pride-public" PROTOCOL_ORDER = ["aspera", "s3", "ftp", "globus"] @@ -180,67 +177,32 @@ def _build_jpost_file_record(accession, ftp_url, category_from_proxi=None): return JpostProvider._build_file_record(accession, ftp_url, category_from_proxi) @staticmethod - def _build_iprox_file_record( - accession: str, https_url: str, category_from_px: Optional[str] = None - ) -> Dict: - """ - Build a pridepy file record for an iProX file. iProX exposes files - over anonymous HTTPS at - ``http://download.iprox.org///``; - ``category_from_px`` is the ``cvParam`` ``name`` from the dataset's - ProteomeXchange XML (e.g. ``"Associated raw file URI"``). - """ - parsed = urlparse(https_url) - root_prefix = f"/{accession.upper()}/" - relative_path = parsed.path - if relative_path.startswith(root_prefix): - relative_path = relative_path[len(root_prefix) :] - relative_path = relative_path.lstrip("/") - collection = relative_path.split("/", 1)[0] if relative_path else "" - if category_from_px and category_from_px in Files.IPROX_PX_CATEGORY_MAP: - category = Files.IPROX_PX_CATEGORY_MAP[category_from_px] - else: - category = Files._map_massive_collection_to_category(collection) - return { - "accession": accession.upper(), - "fileName": os.path.basename(parsed.path), - "fileCategory": {"value": category}, - # ``FTP Protocol`` is the existing label the download dispatcher - # uses to locate a file URL; here it actually points at HTTPS. - # ``_download_direct_download_records`` routes by URL scheme. - "publicFileLocations": [{"name": "FTP Protocol", "value": https_url}], - "relativePath": relative_path, - "collection": collection, - "source": "iProX", - } + def _build_iprox_file_record(accession, https_url, category_from_px=None): + """Shim — see :meth:`pridepy.providers.iprox.IproxProvider._build_file_record`.""" + from pridepy.providers.iprox import IproxProvider + return IproxProvider._build_file_record(accession, https_url, category_from_px) + + @staticmethod + def _get_iprox_public_root(accession: str) -> str: + from pridepy.providers.iprox import IproxProvider + return IproxProvider._get_public_root(accession) + + @staticmethod + def _get_iprox_public_ftp_url(accession: str, remote_path: str) -> str: + from pridepy.providers.iprox import IproxProvider + return IproxProvider._get_public_ftp_url(accession, remote_path) @staticmethod def is_direct_download_accession(accession: str) -> bool: - """ - Return True when the accession is served by a public repository that - pridepy supports via direct downloads (no ProteomeXchange API). - MassIVE and JPOST use FTP(S); iProX uses anonymous HTTPS via - ``download.iprox.org``. - """ - return ( - Files.is_massive_accession(accession) - or Files.is_jpost_accession(accession) - or Files.is_iprox_accession(accession) - ) + """Shim — True for any registered direct-download provider (MSV/JPST/IPX).""" + from pridepy.providers import registry + return registry.is_known(accession) @staticmethod def is_iprox_accession(accession: str) -> bool: - """ - Return True when the accession looks like an iProX dataset accession - (``IPX`` followed by 7-10 digits). iProX exposes the dataset - ProteomeXchange XML at - ``http://download.iprox.org//PX_.xml`` and the - referenced files are downloadable from ``download.iprox.org`` over - anonymous HTTPS with byte-range support. - """ - if not accession: - return False - return bool(re.fullmatch(r"IPX\d{7,10}", accession.upper())) + """Shim — see :meth:`pridepy.providers.iprox.IproxProvider.matches`.""" + from pridepy.providers.iprox import IproxProvider + return IproxProvider.matches(accession) @staticmethod def _repo_uses_tls(accession: str) -> bool: @@ -333,53 +295,9 @@ def _list_jpost_public_files_via_proxi(self, accession: str) -> List[Dict]: return JpostProvider()._list_via_proxi(accession) def _list_iprox_public_files(self, accession: str) -> List[Dict]: - """ - Discover all public files for an iProX dataset. - - iProX publishes the ProteomeXchange XML for every public dataset at a - deterministic path on its anonymous HTTPS download server:: - - http://download.iprox.org//PX_.xml - - We fetch that XML, walk every ````'s ``cvParam`` entries, - and turn each ``Associated raw file URI`` (and sibling URIs for - search-engine output, result files, etc.) into a pridepy file record. - File downloads themselves go through plain HTTPS on the same host, - which supports ``Range`` requests for resume. - """ - normalized_accession = accession.upper() - xml_url = self.IPROX_PX_XML_URL_TEMPLATE.format(accession=normalized_accession) - logging.info(f"Fetching iProX PX XML: {xml_url}") - response = requests.get(xml_url, timeout=30) - response.raise_for_status() - try: - root = ET.fromstring(response.content) - except ET.ParseError as parse_error: - raise RuntimeError( - f"Unable to parse iProX PX XML for {normalized_accession}: {parse_error}" - ) from parse_error - - records: List[Dict] = [] - for dataset_file in root.iter("DatasetFile"): - for cv in dataset_file.findall("cvParam"): - name = cv.attrib.get("name") - value = cv.attrib.get("value") - if not value or not name or not name.endswith("URI"): - continue - if not value.lower().startswith(("http://", "https://")): - continue - records.append( - self._build_iprox_file_record( - normalized_accession, - value, - category_from_px=name, - ) - ) - if not records: - raise RuntimeError( - f"iProX PX XML for {normalized_accession} contained no downloadable HTTPS URIs" - ) - return records + """Shim — see :meth:`pridepy.providers.iprox.IproxProvider.list_files`.""" + from pridepy.providers.iprox import IproxProvider + return IproxProvider().list_files(accession) def _list_direct_download_files(self, accession: str) -> List[Dict]: """ @@ -414,35 +332,17 @@ def _download_direct_download_records( ``download.iprox.org`` with ``Range``-based resume and per-file parallel workers. URLs are partitioned by scheme so a mixed batch (e.g. a JPOST PX XML that ever pointed at HTTPS) routes correctly. + Dispatches via the provider registry. """ - if protocol not in ("ftp", "https", "http"): - logging.warning( - "Direct downloads currently use ftp / https only. " - f"Ignoring requested protocol '{protocol}' for {accession}." - ) - - all_urls = [self._get_download_url(record, "ftp") for record in file_records] - ftp_urls = [u for u in all_urls if u.lower().startswith("ftp://")] - http_urls = [u for u in all_urls if u.lower().startswith(("http://", "https://"))] - if not ftp_urls and not http_urls: - logging.info(f"No files matched for direct-download dataset {accession}") - return - - if ftp_urls: - self.download_ftp_urls( - ftp_urls=ftp_urls, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - use_tls=self._repo_uses_tls(accession), - parallel_files=parallel_files, - ) - if http_urls: - self.download_http_urls( - http_urls=http_urls, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - parallel_files=parallel_files, - ) + from pridepy.providers import registry + return registry.resolve(accession).download_files( + accession=accession, + records=file_records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + ) async def stream_all_files_metadata(self, output_file, accession=None): """ diff --git a/pridepy/providers/iprox.py b/pridepy/providers/iprox.py new file mode 100644 index 0000000..292307c --- /dev/null +++ b/pridepy/providers/iprox.py @@ -0,0 +1,129 @@ +"""iProX direct-download provider. + +iProX publishes the ProteomeXchange XML for each dataset at a +deterministic path on its anonymous HTTPS download server:: + + http://download.iprox.org//PX_.xml + +We fetch the XML, walk every ````'s ``cvParam`` entries, and +turn each ``Associated raw file URI`` (and sibling URIs for search-engine +output, result files, etc.) into a pridepy file record. File downloads +themselves go through plain HTTPS on the same host, which supports +``Range`` requests for resume. +""" +import logging +import os +import re +import xml.etree.ElementTree as ET +from typing import ClassVar, Dict, List, Optional +from urllib.parse import urlparse + +import requests + +from pridepy.providers import registry +from pridepy.providers.base import BaseDirectDownloadProvider +from pridepy.providers.jpost import JpostProvider + + +@registry.register +class IproxProvider(BaseDirectDownloadProvider): + name: ClassVar[str] = "iprox" + use_tls: ClassVar[bool] = False # download.iprox.org serves over plain HTTP + + DOWNLOAD_BASE_URL: ClassVar[str] = "http://download.iprox.org/" + PX_XML_URL_TEMPLATE: ClassVar[str] = ( + "http://download.iprox.org/{accession}/PX_{accession}.xml" + ) + # iProX PX XML uses the same PSI-MS cvParam "name" values as JPOST PROXI, + # so we reuse JpostProvider's category map. + PX_CATEGORY_MAP: ClassVar[Dict[str, str]] = JpostProvider.PROXI_CATEGORY_MAP + + @staticmethod + def matches(accession: str) -> bool: + """Return True when ``accession`` looks like an iProX dataset accession.""" + if not accession: + return False + return bool(re.fullmatch(r"IPX\d{7,10}", accession.upper())) + + @staticmethod + def _get_public_root(accession: str) -> str: + return f"/{accession.upper()}" + + @classmethod + def _get_public_ftp_url(cls, accession: str, remote_path: str) -> str: + # NOTE: name kept as `_get_public_ftp_url` for parity with other providers, + # but iProX URLs are http(s) not ftp. The dispatcher routes by scheme. + root_path = cls._get_public_root(accession).rstrip("/") + relative_path = remote_path + if remote_path.startswith(root_path): + relative_path = remote_path[len(root_path):].lstrip("/") + return f"{cls.DOWNLOAD_BASE_URL}{accession.upper()}/{relative_path}" + + @classmethod + def _build_file_record( + cls, accession: str, https_url: str, category_from_px: Optional[str] = None + ) -> Dict: + """Build a pridepy file record for an iProX file. + + ``category_from_px`` is the ``cvParam`` ``name`` from the dataset's + ProteomeXchange XML (e.g. ``"Associated raw file URI"``). + """ + from pridepy.providers.massive import MassiveProvider + parsed = urlparse(https_url) + root_prefix = f"/{accession.upper()}/" + relative_path = parsed.path + if relative_path.startswith(root_prefix): + relative_path = relative_path[len(root_prefix):] + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + if category_from_px and category_from_px in cls.PX_CATEGORY_MAP: + category = cls.PX_CATEGORY_MAP[category_from_px] + else: + category = MassiveProvider._map_collection_to_category(collection) + return { + "accession": accession.upper(), + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": category}, + # "FTP Protocol" is the existing label the download dispatcher uses + # to locate a file URL; here it actually points at HTTPS. + # BaseDirectDownloadProvider.download_files routes by URL scheme. + "publicFileLocations": [{"name": "FTP Protocol", "value": https_url}], + "relativePath": relative_path, + "collection": collection, + "source": "iProX", + } + + def list_files(self, accession: str) -> List[Dict]: + normalized = accession.upper() + xml_url = self.PX_XML_URL_TEMPLATE.format(accession=normalized) + logging.info(f"Fetching iProX PX XML: {xml_url}") + response = requests.get(xml_url, timeout=30) + response.raise_for_status() + try: + root = ET.fromstring(response.content) + except ET.ParseError as parse_error: + raise RuntimeError( + f"Unable to parse iProX PX XML for {normalized}: {parse_error}" + ) from parse_error + + records: List[Dict] = [] + for dataset_file in root.iter("DatasetFile"): + for cv in dataset_file.findall("cvParam"): + name = cv.attrib.get("name") + value = cv.attrib.get("value") + if not value or not name or not name.endswith("URI"): + continue + if not value.lower().startswith(("http://", "https://")): + continue + records.append( + self._build_file_record( + normalized, + value, + category_from_px=name, + ) + ) + if not records: + raise RuntimeError( + f"iProX PX XML for {normalized} contained no downloadable HTTPS URIs" + ) + return records From 835d77b998958ece069508228b797a60eb21a8cc Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 16:24:14 +0100 Subject: [PATCH 11/54] test(providers): verify BaseDirectDownloadProvider URL-scheme partitioning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mixed ftp:// + http(s):// records partition correctly: ftp URLs go to Files.download_ftp_urls with use_tls=True (the MassIVE setting), http URLs go to Files.download_http_urls. Both calls intercepted by patch.object(Files, ...) — proving the provider routes back through Files rather than calling transport directly (preserves test patches). --- pridepy/tests/test_massive_files.py | 39 +++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index a958309..fd6a4ac 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -135,3 +135,42 @@ def test_download_all_raw_files_threads_parallel_files_for_massive(self): kwargs = download_mock.call_args.kwargs assert kwargs["use_tls"] is True assert kwargs["parallel_files"] == 3 + + def test_base_direct_download_provider_partitions_urls_by_scheme(self): + """Records mixing ftp:// and http(s):// route to the right transport.""" + from pridepy.providers.massive import MassiveProvider + + provider = MassiveProvider() + records = [ + Files._build_massive_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a.raw", + ), + # Synthetic http record to verify partitioning (real MassIVE uses ftp). + { + "accession": "MSV000012345", + "fileName": "b.raw", + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "http://example.org/b.raw"} + ], + }, + ] + with patch.object(Files, "download_ftp_urls") as ftp_mock, \ + patch.object(Files, "download_http_urls") as http_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + ) + + ftp_mock.assert_called_once() + assert ftp_mock.call_args.kwargs["use_tls"] is True + assert ftp_mock.call_args.kwargs["ftp_urls"] == [ + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a.raw" + ] + http_mock.assert_called_once() + assert http_mock.call_args.kwargs["http_urls"] == ["http://example.org/b.raw"] From 3e358ae64304d11d314ed14fcafb15f90f3a3454 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 16:34:57 +0100 Subject: [PATCH 12/54] refactor(providers): extract PrideProvider with multi-protocol fallback Moved PRIDE-specific logic (V3 API listing, multi-protocol batch downloader with aspera/s3/ftp/globus fallback, private-dataset path, submitter helpers, Globus/S3 per-protocol downloaders, legacy single-connection FTP) from Files into providers/pride.py as PrideProvider(Provider). ~510 LOC moved out of files.py. Files keeps shim methods for every patched helper (_batch_download_by_protocol, _download_with_fallback, _globus_download_one, download_files_from_globus, download_files_from_s3, download_files_from_ftp, download_private_file_name, get_ascp_binary, save_checksum_file, stream_all_files_by_project, stream_all_files_metadata, get_submitted_file_path_prefix, _protocol_sequence, download_files). PRIDE class constants (V3_API_BASE_URL, API_BASE_URL, API_PRIVATE_URL, PRIDE_ARCHIVE_FTP, *_URL_PREFIX, S3_URL, S3_BUCKET, PROTOCOL_ORDER) remain on Files as re-exports. PrideProvider's internal calls to patch-sensitive helpers go through Files.X (lazy import) so existing test patches in test_download_resilience.py keep working without modification. is_direct_download_accession updated to exclude PRIDE (returns True only for MSV/JPST/IPX) now that PrideProvider is registered too. All 8 patch-sensitive tests in test_download_resilience.py pass. Full suite green at 67 passed, 4 skipped. --- pridepy/files/files.py | 729 +++++----------------------------- pridepy/providers/pride.py | 790 +++++++++++++++++++++++++++++++++++++ 2 files changed, 899 insertions(+), 620 deletions(-) create mode 100644 pridepy/providers/pride.py diff --git a/pridepy/files/files.py b/pridepy/files/files.py index afe6067..b4cc3b9 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -1,30 +1,19 @@ #!/usr/bin/env python import ftplib -import hashlib -import importlib.resources import logging import os -import platform -import posixpath import re -import subprocess import urllib import urllib.request -import time from concurrent.futures import ThreadPoolExecutor, as_completed from ftplib import FTP from typing import Dict, List, Optional, Tuple -import socket from urllib.parse import urlparse import xml.etree.ElementTree as ET -import boto3 -import botocore import requests -from botocore.config import Config from tqdm import tqdm -from pridepy.authentication.authentication import Authentication from pridepy.util.api_handling import Util @@ -38,12 +27,18 @@ class Files: This class handles PRIDE API files endpoint. """ - V3_API_BASE_URL = "https://www.ebi.ac.uk/pride/ws/archive/v3" - API_BASE_URL = "https://www.ebi.ac.uk/pride/ws/archive/v3" - API_PRIVATE_URL = "https://www.ebi.ac.uk/pride/private/ws/archive/v2" - PRIDE_ARCHIVE_FTP = "ftp.pride.ebi.ac.uk" - PRIDE_ARCHIVE_FTP_URL_PREFIX = "ftp://ftp.pride.ebi.ac.uk/" - PRIDE_ARCHIVE_HTTPS_URL_PREFIX = "https://ftp.pride.ebi.ac.uk/" + # Re-exported from providers/pride.py — kept here for back-compat. + from pridepy.providers.pride import PrideProvider as _PrideProvider + V3_API_BASE_URL = _PrideProvider.V3_API_BASE_URL + API_BASE_URL = _PrideProvider.API_BASE_URL + API_PRIVATE_URL = _PrideProvider.API_PRIVATE_URL + PRIDE_ARCHIVE_FTP = _PrideProvider.ARCHIVE_FTP + PRIDE_ARCHIVE_FTP_URL_PREFIX = _PrideProvider.ARCHIVE_FTP_URL_PREFIX + PRIDE_ARCHIVE_HTTPS_URL_PREFIX = _PrideProvider.ARCHIVE_HTTPS_URL_PREFIX + S3_URL = _PrideProvider.S3_URL + S3_BUCKET = _PrideProvider.S3_BUCKET + PROTOCOL_ORDER = _PrideProvider.PROTOCOL_ORDER + del _PrideProvider # Re-exported from providers/massive.py — kept here for back-compat. from pridepy.providers.massive import ( # noqa: E402 MASSIVE_CATEGORY_MAP as _MASSIVE_CATEGORY_MAP, @@ -64,9 +59,6 @@ class Files: IPROX_PX_XML_URL_TEMPLATE = _IproxProvider.PX_XML_URL_TEMPLATE IPROX_PX_CATEGORY_MAP = _IproxProvider.PX_CATEGORY_MAP del _IproxProvider - S3_URL = "https://hh.fire.sdo.ebi.ac.uk" - S3_BUCKET = "pride-public" - PROTOCOL_ORDER = ["aspera", "s3", "ftp", "globus"] logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") def __init__(self): @@ -122,12 +114,9 @@ def _resolve_local_path(file_record: Dict, output_folder: str) -> str: @staticmethod def _protocol_sequence(protocol: str) -> List[str]: - """ - Build the ordered list of protocols to try for a requested download mode. - """ - if protocol not in Files.PROTOCOL_ORDER: - return [] - return [protocol] + [p for p in Files.PROTOCOL_ORDER if p != protocol] + """Shim — see :meth:`pridepy.providers.pride.PrideProvider._protocol_sequence`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider._protocol_sequence(protocol) @staticmethod def is_massive_accession(accession: str) -> bool: @@ -194,9 +183,19 @@ def _get_iprox_public_ftp_url(accession: str, remote_path: str) -> str: @staticmethod def is_direct_download_accession(accession: str) -> bool: - """Shim — True for any registered direct-download provider (MSV/JPST/IPX).""" + """Shim — True for MassIVE/JPOST/iProX (explicitly excludes PRIDE). + + PRIDE is also a registered provider but PRIDE downloads go through + the multi-protocol orchestrator (FTP/Aspera/S3/Globus with checksum + validation and fallback), not the direct-download partitioned-by-URL- + scheme path. So we filter PRIDE out here. + """ from pridepy.providers import registry - return registry.is_known(accession) + try: + provider = registry.resolve(accession) + except ValueError: + return False + return provider.name != "pride" @staticmethod def is_iprox_accession(accession: str) -> bool: @@ -345,32 +344,14 @@ def _download_direct_download_records( ) async def stream_all_files_metadata(self, output_file, accession=None): - """ - get stream all project files from PRIDE API in JSON format - """ - if accession is None: - request_url = f"{self.V3_API_BASE_URL}/files/all" - count_request_url = f"{self.V3_API_BASE_URL}/files/count" - else: - request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/all" - count_request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/count" - headers = {"Accept": "application/JSON"} - response = Util.get_api_call(count_request_url, headers) - total_records = response.json() - - regex_search_pattern = '"fileName"' - await Util.stream_response_to_file( - output_file, total_records, regex_search_pattern, request_url, headers - ) + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.stream_all_files_metadata`.""" + from pridepy.providers.pride import PrideProvider + return await PrideProvider().stream_all_files_metadata(output_file, accession) def stream_all_files_by_project(self, accession) -> List[Dict]: - """ - get stream all project files from PRIDE API in JSON format - """ - request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/all" - headers = {"Accept": "application/JSON"} - record_files = Util.read_json_stream(api_url=request_url, headers=headers) - return record_files + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.stream_all_files_by_project`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider().stream_all_files_by_project(accession) def get_all_raw_file_list(self, project_accession): """ @@ -446,118 +427,15 @@ def download_files_from_ftp( max_connection_retries=3, max_download_retries=3, ): - """ - Download files using a single FTP connection with a retry mechanism and a progress bar for each file. - :param file_list_json: file list in JSON format - :param output_folder: folder to download the files - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - :param max_connection_retries: Number of attempts to reconnect to the FTP server if the connection is lost. - :param max_download_retries: Number of attempts to retry the download of a file in case of failure. - """ - - if not os.path.isdir(output_folder): - os.makedirs(output_folder) - - def connect_ftp(): - """Helper function to establish FTP connection.""" - ftp = FTP(Files.PRIDE_ARCHIVE_FTP, timeout=30) - ftp.login() # Anonymous login - ftp.set_pasv(True) # Enable passive mode - logging.info(f"Connected to FTP host: {Files.PRIDE_ARCHIVE_FTP}") - return ftp - - connection_attempt = 0 - while connection_attempt < max_connection_retries: - try: - ftp = connect_ftp() - for file in file_list_json: - try: - # Get FTP download URL - if file["publicFileLocations"][0]["name"] == "FTP Protocol": - download_url = file["publicFileLocations"][0]["value"] - else: - download_url = file["publicFileLocations"][1]["value"] - - logging.debug("ftp_filepath:" + download_url) - - # Get output file path - new_file_path = Files.get_output_file_name( - download_url, file, output_folder - ) - - if skip_if_downloaded_already and os.path.exists(new_file_path): - logging.info("Skipping download as file already exists") - continue - - # Extract file path from the download URL - parsed_url = urlparse(download_url) - ftp_file_path = urllib.parse.unquote(parsed_url.path.lstrip("/")) - - logging.info(f"Starting FTP download: {ftp_file_path}") - - # Retry download in case of failure - download_attempt = 0 - while download_attempt < max_download_retries: - try: - # Get file size for progress tracking - total_size = ftp.size(ftp_file_path) - logging.info(f"File size: {total_size} bytes") - - # Initialize progress bar - with open(new_file_path, "wb") as f: - with tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=new_file_path, - ) as pbar: - - def callback(data): - f.write(data) - pbar.update(len(data)) - - # Retrieve the file with progress callback - ftp.retrbinary(f"RETR {ftp_file_path}", callback) - - logging.info(f"Successfully downloaded {new_file_path}") - break # Exit download retry loop if successful - except ( - socket.timeout, - ftplib.error_temp, - ftplib.error_perm, - ) as e: - download_attempt += 1 - logging.error( - f"Download failed for {new_file_path} (attempt {download_attempt}): {str(e)}" - ) - if download_attempt >= max_download_retries: - logging.error( - f"Giving up on {new_file_path} after {max_download_retries} attempts." - ) - break # Give up on this file after max retries - except (KeyError, IndexError) as e: - logging.error(f"Failed to process file due to missing data: {str(e)}") - except Exception as e: - logging.error(f"Unexpected error while processing file: {str(e)}") - ftp.quit() # Close FTP connection after all files are downloaded - logging.info(f"Disconnected from FTP host: {Files.PRIDE_ARCHIVE_FTP}") - break # Exit connection retry loop if everything was successful - except ( - socket.timeout, - ftplib.error_temp, - ftplib.error_perm, - socket.error, - ) as e: - connection_attempt += 1 - logging.error(f"FTP connection failed (attempt {connection_attempt}): {str(e)}") - if connection_attempt < max_connection_retries: - logging.info("Retrying connection...") - time.sleep(5) # Optional delay before retrying - else: - logging.error( - f"Giving up after {max_connection_retries} failed connection attempts." - ) - break + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files_from_ftp`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider.download_files_from_ftp( + file_list_json, + output_folder, + skip_if_downloaded_already, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + ) @staticmethod def get_output_file_name(download_url, file, output_folder): @@ -658,22 +536,12 @@ def _parallel_download(url, file_path, position=0): @staticmethod def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_retries=6, position=0): - """Download a single file via globus; used as a worker target.""" - download_url = Files._get_download_url(file, "globus") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) - - if skip_if_downloaded_already and os.path.exists(new_file_path): - logging.info(f"Skipping download as file already exists: {new_file_path}") - return - - for attempt in range(1, max_retries + 1): - try: - Files._parallel_download(download_url, new_file_path, position=position) - return - except Exception as e: - logging.warning(f"Attempt {attempt}/{max_retries} failed for {file.get('fileName', '?')}: {e}") - if attempt == max_retries: - raise + """Shim — see :meth:`pridepy.providers.pride.PrideProvider._globus_download_one`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider._globus_download_one( + file, output_folder, skip_if_downloaded_already, + max_retries=max_retries, position=position, + ) @staticmethod def download_files_from_globus( @@ -681,172 +549,28 @@ def download_files_from_globus( parallel_files: int = 1, checksum_map: Optional[Dict[str, str]] = None, ): - """ - Download files using globus transfer url with progress bar for each file. - When skip_if_downloaded_already is True, files are pre-filtered so that - only missing or incomplete files are submitted to the worker pool, - ensuring the -w parallel_files parameter is fully utilised. - When checksum_map is provided, existing files are validated against - their expected checksum; corrupted files are re-downloaded. - :param file_list_json: file list in json format - :param output_folder: folder to download the files - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - :param parallel_files: number of files to download simultaneously - :param checksum_map: mapping of file name to expected MD5 checksum - """ - if checksum_map is None: - checksum_map = {} - - if not (os.path.isdir(output_folder)): - os.makedirs(output_folder, exist_ok=True) - - # --- Phase 0: pre-filter files that need downloading ----------------- - files_to_download: List[Dict] = [] - for file in file_list_json: - download_url = Files._get_download_url(file, "globus") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) - if skip_if_downloaded_already and os.path.exists(new_file_path): - expected_cs = checksum_map.get(file.get("fileName", "")) - if expected_cs: - valid, reason = Files.validate_download(new_file_path, expected_cs) - if not valid: - logging.warning(f"Corrupted file detected ({reason}), will re-download: {new_file_path}") - files_to_download.append(file) - continue - logging.info(f"Skipping download as file already exists: {new_file_path}") - continue - files_to_download.append(file) - - if not files_to_download: - logging.info("All files already downloaded, nothing to do.") - return - - logging.info( - f"{len(file_list_json) - len(files_to_download)} file(s) skipped, " - f"{len(files_to_download)} file(s) to download" + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files_from_globus`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider.download_files_from_globus( + file_list_json, output_folder, skip_if_downloaded_already, + parallel_files=parallel_files, + checksum_map=checksum_map, ) - # --- Phase 1: download (skip check already done, pass False) --------- - parallel_files = min(parallel_files, 3, len(files_to_download)) - if parallel_files < 2: - for file in files_to_download: - try: - Files._globus_download_one( - file, output_folder, False - ) - new_file_path = Files.get_output_file_name( - Files._get_download_url(file, "globus"), file, output_folder - ) - logging.info(f"Successfully downloaded {new_file_path}") - except Exception as e: - logging.error(f"Download from Globus failed: {str(e)}") - else: - logging.info(f"Downloading {len(files_to_download)} file(s) with {parallel_files} parallel workers") - with ThreadPoolExecutor(max_workers=parallel_files) as executor: - futures = { - executor.submit( - Files._globus_download_one, - file, output_folder, False, - position=idx, - ): file - for idx, file in enumerate(files_to_download) - } - for future in as_completed(futures): - try: - future.result() - except Exception as e: - logging.error(f"Download from Globus failed: {str(e)}") - @staticmethod def download_files_from_s3( file_list_json: List[Dict], output_folder: str, skip_if_downloaded_already ): - """ - Download files using S3 transfer URL with a progress bar and retry logic. - :param file_list_json: file list in JSON format - :param output_folder: folder to download the files - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - """ - - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - # Retry and timeout config - retry_config = Config( - retries={"max_attempts": 5, "mode": "standard"}, - connect_timeout=120, # Increase timeout to 120 seconds - read_timeout=120, # Timeout for reading data - signature_version=botocore.UNSIGNED, # Unsigned requests for public data + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files_from_s3`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider.download_files_from_s3( + file_list_json, output_folder, skip_if_downloaded_already, ) - s3_resource = boto3.resource( - "s3", - config=retry_config, - endpoint_url=Files.S3_URL, - ) - bucket = s3_resource.Bucket(Files.S3_BUCKET) - - for file in file_list_json: - try: - # Determine S3 or FTP path - download_url = ( - file["publicFileLocations"][0]["value"] - if file["publicFileLocations"][0]["name"] == "FTP Protocol" - else file["publicFileLocations"][1]["value"] - ) - - ftp_base_url = "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/" - s3_path = download_url.replace(ftp_base_url, "") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) - - if skip_if_downloaded_already == True and os.path.exists(new_file_path): - logging.info("Skipping download as file already exists") - continue - - logging.debug(f"Downloading From S3: {s3_path}") - - # Get file size for progress tracking - obj = bucket.Object(s3_path) - total_size = obj.content_length - - # Initialize progress bar - progress = Progress(total_size, new_file_path) - - # Download with progress bar and retry handling - for attempt in range(5): - try: - bucket.download_file(s3_path, new_file_path, Callback=progress) - progress.close() - logging.info(f"Successfully downloaded {new_file_path}") - break - except botocore.exceptions.ClientError as e: - if e.response["Error"]["Code"] == "404": - logging.error("The object does not exist.") - break - else: - logging.error(f"Download failed: {e}") - if attempt < 4: - time.sleep(2**attempt) # Exponential backoff - logging.info(f"Retrying... ({attempt + 1}/5)") - else: - raise - except Exception as e: - logging.error(f"Failed to download {file['fileName']}: {e}") - def get_submitted_file_path_prefix(self, accession): - """ - At pride repository, public data is disseminated according to a proper structure. - I.e. base/path/ + yyyy/mm/accession/ + submitted/ - This extracts the yyyy/mm/accession path fragment from the API by examine the file path - of a public file. - I.e. ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2018/10/PXD008644/7550GI_Y.raw - :param accession: PRIDE accession - :return: path fragment (eg: 2018/10/PXD008644) - """ - results = self.get_all_raw_file_list(accession) - first_file = results[0]["publicFileLocations"][0]["value"] - path_fragment = re.search(r"\d{4}/\d{2}/PXD\d*", first_file).group() - return path_fragment + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.get_submitted_file_path_prefix`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider().get_submitted_file_path_prefix(accession) def download_file_by_name( self, @@ -958,125 +682,23 @@ def get_file_from_api(self, accession, file_name) -> List[Dict]: raise Exception("File not found " + str(e)) def download_private_file_name(self, accession, file_name, output_folder, username, password): - """ - Get the information for a given private file to be downloaded from the api. - :param accession: Project accession - :param file_name: The file name to be downloaded - :param username: Username with access to the dataset - :param password: Password for user with access to the dataset - """ - - auth = Authentication() - auth_token = auth.get_token(username, password) - validate_token = auth.validate_token(auth_token) - logging.info("Valid token after login: {}".format(validate_token)) - - url = self.API_PRIVATE_URL + "/projects/{}/files?search={}".format(accession, file_name) - content = requests.get(url, headers={"Authorization": "Bearer {}".format(auth_token)}) - if content.ok and content.status_code == 200: - json_file = content.json() - if ( - "_embedded" in json_file - and "files" in json_file["_embedded"] - and len(json_file["_embedded"]["files"]) == 1 - ): - download_url = json_file["_embedded"]["files"][0]["_links"]["download"]["href"] - logging.info(download_url) - - # Create a clean filename to save the downloaded file - new_file_path = os.path.join(output_folder, f"{file_name}") - - session = Util.create_session_with_retries() # Create session with retries - # Check if the file already exists - if os.path.exists(new_file_path): - resume_header = {"Range": f"bytes={os.path.getsize(new_file_path)}-"} - mode = "ab" # Append to file - resume_size = os.path.getsize(new_file_path) - else: - resume_header = {} - mode = "wb" # Write new file - resume_size = 0 - - with session.get( - download_url, stream=True, headers=resume_header, timeout=(10, 60) - ) as r: - r.raise_for_status() - total_size = int(r.headers.get("content-length", 0)) + resume_size - block_size = 1024 * 1024 # 1 MB chunks - - with tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=new_file_path, - initial=resume_size, - ) as pbar: - with open(new_file_path, mode) as f: - for chunk in r.iter_content(chunk_size=block_size): - if chunk: - f.write(chunk) - pbar.update(len(chunk)) - - logging.info(f"Successfully downloaded {new_file_path}") - - else: - logging.info( - "File name {} found more than once for the given project {}".format( - file_name, accession - ) - ) - else: - logging.info( - f"File name {file_name} now found in the project {accession}, or user don't have access" - ) - raise Exception( - f"File name {file_name} now found in the project {accession}, or user don't have access" - ) + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_private_file_name`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider().download_private_file_name( + accession, file_name, output_folder, username, password, + ) @staticmethod def get_ascp_binary(): - """ - Detect the OS and architecture, and return the appropriate ascp binary path. - - Returns: - str: Path to the correct ascp binary. - """ - os_type = platform.system().lower() - arch, _ = platform.architecture() - aspera_dir = importlib.resources.files("pridepy").joinpath("aspera/") - - if os_type == "linux": - if arch == "32bit": - return os.path.join(aspera_dir, "linux-32", "ascp") - elif arch == "64bit": - return os.path.join(aspera_dir, "linux-64", "ascp") - elif os_type == "darwin": # macOS (intel-based) - return os.path.join(aspera_dir, "mac-intel", "ascp") - elif os_type == "windows": - if arch == "32bit": - return os.path.join(aspera_dir, "windows-32", "ascp.exe") - elif arch == "64bit": - return os.path.join(aspera_dir, "windows-64", "ascp.exe") - else: - raise OSError(f"Unsupported OS or architecture: {os_type}, {arch}") + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.get_ascp_binary`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider.get_ascp_binary() @staticmethod def save_checksum_file(accession, output_folder): - """ - Download and persist the checksum manifest for a PRIDE accession. - """ - os.makedirs(output_folder, exist_ok=True) - url = f"{Files.V3_API_BASE_URL}/files/checksum/{accession}" - headers = {"accept": "text/plain"} - request = urllib.request.Request(url, headers=headers, method="GET") - logging.info(f"Fetching checksum file from {url}") - with urllib.request.urlopen(request) as response: - data = response.read().decode("utf-8") - # Save the data to a .tsv file - output_path = os.path.join(output_folder, f"{accession}-checksum.tsv") - with open(output_path, "w", encoding="utf-8") as file: - file.write(data) - return output_path + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.save_checksum_file`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider.save_checksum_file(accession, output_folder) @staticmethod def _batch_download_by_protocol( @@ -1088,44 +710,22 @@ def _batch_download_by_protocol( parallel_files: int = 1, checksum_map: Optional[Dict[str, str]] = None, ) -> None: + """Shim — see :meth:`pridepy.providers.pride.PrideProvider._batch_download_by_protocol`. + + Tests patch this method via ``patch.object(Files, "_batch_download_by_protocol")``; + :class:`PrideProvider` calls back through ``Files.X`` so those patches + keep intercepting. """ - Transfer a batch of files with one protocol, reusing a single - connection where the underlying helper supports it (FTP, S3). - """ - if not file_list: - return - if protocol == "ftp": - Files.download_files_from_ftp( - file_list, - output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - ) - return - if protocol == "aspera": - Files.download_files_from_aspera( - file_list, - output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - maximum_bandwidth=aspera_maximum_bandwidth, - ) - return - if protocol == "globus": - Files.download_files_from_globus( - file_list, - output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - parallel_files=parallel_files, - checksum_map=checksum_map or {}, - ) - return - if protocol == "s3": - Files.download_files_from_s3( - file_list, - output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - ) - return - raise ValueError(f"Unsupported protocol: {protocol}") + from pridepy.providers.pride import PrideProvider + return PrideProvider._batch_download_by_protocol( + file_list, + output_folder, + protocol, + skip_if_downloaded_already, + aspera_maximum_bandwidth, + parallel_files=parallel_files, + checksum_map=checksum_map, + ) @staticmethod def _download_with_fallback( @@ -1137,52 +737,17 @@ def _download_with_fallback( max_protocol_retries: int = 2, parallel_files: int = 1, ) -> bool: - """ - Download one file by trying each protocol in sequence, validating - after every attempt. Intended as the per-file fallback path; batch - download of the primary protocol is handled separately. - """ - local_path = Files._resolve_local_path(file_record, output_folder) - - for protocol in protocol_sequence: - for attempt in range(1, max_protocol_retries + 1): - logging.info( - f"Downloading {file_record['fileName']} via {protocol} " - f"(attempt {attempt}/{max_protocol_retries})" - ) - try: - Files._remove_if_exists(local_path) - Files._batch_download_by_protocol( - [file_record], - output_folder, - protocol, - skip_if_downloaded_already=False, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - parallel_files=parallel_files, - ) - except Exception as error: - logging.error( - f"Protocol {protocol} failed for {file_record['fileName']}: {error}" - ) - - valid, reason = Files.validate_download(local_path, expected_checksum) - if valid: - logging.info( - f"File {file_record['fileName']} downloaded successfully via {protocol}" - ) - return True - - logging.warning( - f"Validation failed for {file_record['fileName']} via {protocol}: {reason}" - ) - Files._remove_if_exists(local_path) - - logging.warning( - f"Protocol {protocol} exhausted for {file_record['fileName']}, switching protocol." - ) - - logging.error(f"All protocol attempts failed for {file_record['fileName']}") - return False + """Shim — see :meth:`pridepy.providers.pride.PrideProvider._download_with_fallback`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider._download_with_fallback( + file_record, + output_folder, + protocol_sequence, + expected_checksum, + aspera_maximum_bandwidth, + max_protocol_retries=max_protocol_retries, + parallel_files=parallel_files, + ) @staticmethod def download_files( @@ -1195,94 +760,18 @@ def download_files( checksum_check=False, parallel_files: int = 1, ): - """ - Download files using either FTP or Aspera transfer protocol. - :param file_list_json: File list in JSON format - :param accession: Project accession - :param output_folder: Folder to download the files - :param protocol: ftp, aspera, globus - :param aspera_maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - """ - protocols_supported = ["ftp", "aspera", "globus", "s3"] - if protocol not in protocols_supported: - logging.error("Protocol should be one of ftp, aspera, globus, s3") - return - - os.makedirs(output_folder, exist_ok=True) - - checksum_map: Dict[str, str] = {} - if checksum_check: - checksum_file_path = Files.save_checksum_file(accession, output_folder) - checksum_map = Files.read_checksum_file(checksum_file_path) - logging.info(f"Loaded checksums for {len(checksum_map)} files") - - if not file_list_json: - return - - protocol_sequence = Files._protocol_sequence(protocol) - primary_protocol = protocol_sequence[0] - # Retry with the primary protocol first, then fall back to others - fallback_sequence = protocol_sequence - - # Phase 1: batch download with the requested protocol. Reuses a single - # FTP/S3 connection for all files (the previous behaviour) instead of - # paying the per-file reconnect cost in the common happy path. - logging.info( - f"Downloading {len(file_list_json)} file(s) via {primary_protocol} (batch)" + """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files`.""" + from pridepy.providers.pride import PrideProvider + return PrideProvider.download_files( + file_list_json, + accession, + output_folder, + skip_if_downloaded_already, + protocol=protocol, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, ) - try: - Files._batch_download_by_protocol( - file_list_json, - output_folder, - primary_protocol, - skip_if_downloaded_already=skip_if_downloaded_already, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - parallel_files=parallel_files, - checksum_map=checksum_map, - ) - except Exception as exc: - logging.warning( - f"Batch {primary_protocol} run hit an error; will retry individual failures: {exc}" - ) - - # Phase 2: validate every file and fall back per-file for the ones - # that are missing or invalid. - logging.info("Phase 2: validating %d downloaded file(s)", len(file_list_json)) - failed_files: List[str] = [] - for i, file_record in enumerate(file_list_json, 1): - expected_checksum = checksum_map.get(file_record["fileName"]) - local_path = Files._resolve_local_path(file_record, output_folder) - logging.info("Validating [%d/%d] %s", i, len(file_list_json), file_record["fileName"]) - valid, reason = Files.validate_download(local_path, expected_checksum) - if valid: - continue - - logging.warning( - f"{file_record['fileName']} invalid after {primary_protocol} ({reason})" - ) - if "checksum mismatch" in reason: - Files._remove_if_exists(local_path) - - if not fallback_sequence: - failed_files.append(file_record.get("fileName", "")) - continue - - success = Files._download_with_fallback( - file_record=file_record, - output_folder=output_folder, - protocol_sequence=fallback_sequence, - expected_checksum=expected_checksum, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - parallel_files=parallel_files, - ) - if not success: - failed_files.append(file_record.get("fileName", "")) - - if failed_files: - failed_summary = ", ".join(failed_files) - logging.error(f"Failed to download {len(failed_files)} file(s): {failed_summary}") - raise RuntimeError(f"Failed to download {len(failed_files)} file(s): {failed_summary}") def download_files_by_list( self, diff --git a/pridepy/providers/pride.py b/pridepy/providers/pride.py new file mode 100644 index 0000000..ea60a5d --- /dev/null +++ b/pridepy/providers/pride.py @@ -0,0 +1,790 @@ +"""PRIDE Archive provider. + +PRIDE has the richest behaviour of all providers: multi-protocol batch +download with aspera/s3/ftp/globus fallback, private-dataset path with +username/password auth, checksum TSV validation, and submitter-path +helpers. This module hosts all of those; the :class:`Files` facade +delegates via lightweight shim methods. + +Implementation note: PRIDE-specific helpers that the existing test suite +patches via ``patch.object(Files, "X")`` are called from inside this +provider via ``Files.X(...)`` (lazy import) — never ``self.X`` — so the +patches keep intercepting. This is a deliberate backward-compat choice +documented in the refactor plan (Task 8). +""" +import ftplib +import importlib.resources +import logging +import os +import platform +import re +import socket +import subprocess +import time +import urllib +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from ftplib import FTP +from typing import ClassVar, Dict, List, Optional +from urllib.parse import urlparse + +import boto3 +import botocore +import requests +from botocore.config import Config +from tqdm import tqdm + +from pridepy.authentication.authentication import Authentication +from pridepy.providers import registry +from pridepy.providers.base import Provider +from pridepy.providers.util import Progress +from pridepy.util.api_handling import Util + + +@registry.register +class PrideProvider(Provider): + """PRIDE Archive provider with multi-protocol fallback orchestration.""" + + name: ClassVar[str] = "pride" + + V3_API_BASE_URL: ClassVar[str] = "https://www.ebi.ac.uk/pride/ws/archive/v3" + API_BASE_URL: ClassVar[str] = "https://www.ebi.ac.uk/pride/ws/archive/v3" + API_PRIVATE_URL: ClassVar[str] = "https://www.ebi.ac.uk/pride/private/ws/archive/v2" + ARCHIVE_FTP: ClassVar[str] = "ftp.pride.ebi.ac.uk" + ARCHIVE_FTP_URL_PREFIX: ClassVar[str] = "ftp://ftp.pride.ebi.ac.uk/" + ARCHIVE_HTTPS_URL_PREFIX: ClassVar[str] = "https://ftp.pride.ebi.ac.uk/" + S3_URL: ClassVar[str] = "https://hh.fire.sdo.ebi.ac.uk" + S3_BUCKET: ClassVar[str] = "pride-public" + PROTOCOL_ORDER: ClassVar[List[str]] = ["aspera", "s3", "ftp", "globus"] + + @staticmethod + def matches(accession: str) -> bool: + """Return True when ``accession`` is a PRIDE dataset accession.""" + if not accession: + return False + return bool(re.fullmatch(r"(?:PXD|PRD)\d+", accession.upper())) + + # ------------------------------------------------------------------ + # Listing + # ------------------------------------------------------------------ + + async def stream_all_files_metadata(self, output_file, accession=None): + """ + get stream all project files from PRIDE API in JSON format + """ + if accession is None: + request_url = f"{self.V3_API_BASE_URL}/files/all" + count_request_url = f"{self.V3_API_BASE_URL}/files/count" + else: + request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/all" + count_request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/count" + headers = {"Accept": "application/JSON"} + response = Util.get_api_call(count_request_url, headers) + total_records = response.json() + + regex_search_pattern = '"fileName"' + await Util.stream_response_to_file( + output_file, total_records, regex_search_pattern, request_url, headers + ) + + def stream_all_files_by_project(self, accession) -> List[Dict]: + """ + get stream all project files from PRIDE API in JSON format + """ + request_url = f"{self.V3_API_BASE_URL}/projects/{accession}/files/all" + headers = {"Accept": "application/JSON"} + record_files = Util.read_json_stream(api_url=request_url, headers=headers) + return record_files + + def list_files(self, accession: str) -> List[Dict]: + """Return PRIDE file records for the dataset.""" + return self.stream_all_files_by_project(accession) + + def get_submitted_file_path_prefix(self, accession): + """ + At pride repository, public data is disseminated according to a proper structure. + I.e. base/path/ + yyyy/mm/accession/ + submitted/ + This extracts the yyyy/mm/accession path fragment from the API by examine the file path + of a public file. + I.e. ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2018/10/PXD008644/7550GI_Y.raw + :param accession: PRIDE accession + :return: path fragment (eg: 2018/10/PXD008644) + """ + # Use Files facade so test patches on get_all_raw_file_list keep working. + from pridepy.files.files import Files + results = Files().get_all_raw_file_list(accession) + first_file = results[0]["publicFileLocations"][0]["value"] + path_fragment = re.search(r"\d{4}/\d{2}/PXD\d*", first_file).group() + return path_fragment + + # ------------------------------------------------------------------ + # Static utilities + # ------------------------------------------------------------------ + + @staticmethod + def _protocol_sequence(protocol: str) -> List[str]: + """ + Build the ordered list of protocols to try for a requested download mode. + """ + if protocol not in PrideProvider.PROTOCOL_ORDER: + return [] + return [protocol] + [p for p in PrideProvider.PROTOCOL_ORDER if p != protocol] + + @staticmethod + def get_ascp_binary(): + """ + Detect the OS and architecture, and return the appropriate ascp binary path. + + Returns: + str: Path to the correct ascp binary. + """ + os_type = platform.system().lower() + arch, _ = platform.architecture() + aspera_dir = importlib.resources.files("pridepy").joinpath("aspera/") + + if os_type == "linux": + if arch == "32bit": + return os.path.join(aspera_dir, "linux-32", "ascp") + elif arch == "64bit": + return os.path.join(aspera_dir, "linux-64", "ascp") + elif os_type == "darwin": # macOS (intel-based) + return os.path.join(aspera_dir, "mac-intel", "ascp") + elif os_type == "windows": + if arch == "32bit": + return os.path.join(aspera_dir, "windows-32", "ascp.exe") + elif arch == "64bit": + return os.path.join(aspera_dir, "windows-64", "ascp.exe") + else: + raise OSError(f"Unsupported OS or architecture: {os_type}, {arch}") + + @staticmethod + def save_checksum_file(accession, output_folder): + """ + Download and persist the checksum manifest for a PRIDE accession. + """ + os.makedirs(output_folder, exist_ok=True) + url = f"{PrideProvider.V3_API_BASE_URL}/files/checksum/{accession}" + headers = {"accept": "text/plain"} + request = urllib.request.Request(url, headers=headers, method="GET") + logging.info(f"Fetching checksum file from {url}") + with urllib.request.urlopen(request) as response: + data = response.read().decode("utf-8") + # Save the data to a .tsv file + output_path = os.path.join(output_folder, f"{accession}-checksum.tsv") + with open(output_path, "w", encoding="utf-8") as file: + file.write(data) + return output_path + + # ------------------------------------------------------------------ + # Per-protocol single-file workers + # ------------------------------------------------------------------ + + @staticmethod + def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_retries=6, position=0): + """Download a single file via globus; used as a worker target.""" + # Use Files facade so test patches on Files helpers keep working. + from pridepy.files.files import Files + + download_url = Files._get_download_url(file, "globus") + new_file_path = Files.get_output_file_name(download_url, file, output_folder) + + if skip_if_downloaded_already and os.path.exists(new_file_path): + logging.info(f"Skipping download as file already exists: {new_file_path}") + return + + for attempt in range(1, max_retries + 1): + try: + Files._parallel_download(download_url, new_file_path, position=position) + return + except Exception as e: + logging.warning(f"Attempt {attempt}/{max_retries} failed for {file.get('fileName', '?')}: {e}") + if attempt == max_retries: + raise + + # ------------------------------------------------------------------ + # Per-protocol batch helpers + # ------------------------------------------------------------------ + + @staticmethod + def download_files_from_ftp( + file_list_json, + output_folder, + skip_if_downloaded_already, + max_connection_retries=3, + max_download_retries=3, + ): + """ + Download files using a single FTP connection with a retry mechanism and a progress bar for each file. + :param file_list_json: file list in JSON format + :param output_folder: folder to download the files + :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + :param max_connection_retries: Number of attempts to reconnect to the FTP server if the connection is lost. + :param max_download_retries: Number of attempts to retry the download of a file in case of failure. + """ + from pridepy.files.files import Files + + if not os.path.isdir(output_folder): + os.makedirs(output_folder) + + def connect_ftp(): + """Helper function to establish FTP connection.""" + ftp = FTP(PrideProvider.ARCHIVE_FTP, timeout=30) + ftp.login() # Anonymous login + ftp.set_pasv(True) # Enable passive mode + logging.info(f"Connected to FTP host: {PrideProvider.ARCHIVE_FTP}") + return ftp + + connection_attempt = 0 + while connection_attempt < max_connection_retries: + try: + ftp = connect_ftp() + for file in file_list_json: + try: + # Get FTP download URL + if file["publicFileLocations"][0]["name"] == "FTP Protocol": + download_url = file["publicFileLocations"][0]["value"] + else: + download_url = file["publicFileLocations"][1]["value"] + + logging.debug("ftp_filepath:" + download_url) + + # Get output file path + new_file_path = Files.get_output_file_name( + download_url, file, output_folder + ) + + if skip_if_downloaded_already and os.path.exists(new_file_path): + logging.info("Skipping download as file already exists") + continue + + # Extract file path from the download URL + parsed_url = urlparse(download_url) + ftp_file_path = urllib.parse.unquote(parsed_url.path.lstrip("/")) + + logging.info(f"Starting FTP download: {ftp_file_path}") + + # Retry download in case of failure + download_attempt = 0 + while download_attempt < max_download_retries: + try: + # Get file size for progress tracking + total_size = ftp.size(ftp_file_path) + logging.info(f"File size: {total_size} bytes") + + # Initialize progress bar + with open(new_file_path, "wb") as f: + with tqdm( + total=total_size, + unit="B", + unit_scale=True, + desc=new_file_path, + ) as pbar: + + def callback(data): + f.write(data) + pbar.update(len(data)) + + # Retrieve the file with progress callback + ftp.retrbinary(f"RETR {ftp_file_path}", callback) + + logging.info(f"Successfully downloaded {new_file_path}") + break # Exit download retry loop if successful + except ( + socket.timeout, + ftplib.error_temp, + ftplib.error_perm, + ) as e: + download_attempt += 1 + logging.error( + f"Download failed for {new_file_path} (attempt {download_attempt}): {str(e)}" + ) + if download_attempt >= max_download_retries: + logging.error( + f"Giving up on {new_file_path} after {max_download_retries} attempts." + ) + break # Give up on this file after max retries + except (KeyError, IndexError) as e: + logging.error(f"Failed to process file due to missing data: {str(e)}") + except Exception as e: + logging.error(f"Unexpected error while processing file: {str(e)}") + ftp.quit() # Close FTP connection after all files are downloaded + logging.info(f"Disconnected from FTP host: {PrideProvider.ARCHIVE_FTP}") + break # Exit connection retry loop if everything was successful + except ( + socket.timeout, + ftplib.error_temp, + ftplib.error_perm, + socket.error, + ) as e: + connection_attempt += 1 + logging.error(f"FTP connection failed (attempt {connection_attempt}): {str(e)}") + if connection_attempt < max_connection_retries: + logging.info("Retrying connection...") + time.sleep(5) # Optional delay before retrying + else: + logging.error( + f"Giving up after {max_connection_retries} failed connection attempts." + ) + break + + @staticmethod + def download_files_from_globus( + file_list_json: List[Dict], output_folder, skip_if_downloaded_already, + parallel_files: int = 1, + checksum_map: Optional[Dict[str, str]] = None, + ): + """ + Download files using globus transfer url with progress bar for each file. + When skip_if_downloaded_already is True, files are pre-filtered so that + only missing or incomplete files are submitted to the worker pool, + ensuring the -w parallel_files parameter is fully utilised. + When checksum_map is provided, existing files are validated against + their expected checksum; corrupted files are re-downloaded. + :param file_list_json: file list in json format + :param output_folder: folder to download the files + :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + :param parallel_files: number of files to download simultaneously + :param checksum_map: mapping of file name to expected MD5 checksum + """ + # Use Files facade so test patches on Files._globus_download_one etc. keep working. + from pridepy.files.files import Files + + if checksum_map is None: + checksum_map = {} + + if not (os.path.isdir(output_folder)): + os.makedirs(output_folder, exist_ok=True) + + # --- Phase 0: pre-filter files that need downloading ----------------- + files_to_download: List[Dict] = [] + for file in file_list_json: + download_url = Files._get_download_url(file, "globus") + new_file_path = Files.get_output_file_name(download_url, file, output_folder) + if skip_if_downloaded_already and os.path.exists(new_file_path): + expected_cs = checksum_map.get(file.get("fileName", "")) + if expected_cs: + valid, reason = Files.validate_download(new_file_path, expected_cs) + if not valid: + logging.warning(f"Corrupted file detected ({reason}), will re-download: {new_file_path}") + files_to_download.append(file) + continue + logging.info(f"Skipping download as file already exists: {new_file_path}") + continue + files_to_download.append(file) + + if not files_to_download: + logging.info("All files already downloaded, nothing to do.") + return + + logging.info( + f"{len(file_list_json) - len(files_to_download)} file(s) skipped, " + f"{len(files_to_download)} file(s) to download" + ) + + # --- Phase 1: download (skip check already done, pass False) --------- + parallel_files = min(parallel_files, 3, len(files_to_download)) + if parallel_files < 2: + for file in files_to_download: + try: + Files._globus_download_one( + file, output_folder, False + ) + new_file_path = Files.get_output_file_name( + Files._get_download_url(file, "globus"), file, output_folder + ) + logging.info(f"Successfully downloaded {new_file_path}") + except Exception as e: + logging.error(f"Download from Globus failed: {str(e)}") + else: + logging.info(f"Downloading {len(files_to_download)} file(s) with {parallel_files} parallel workers") + with ThreadPoolExecutor(max_workers=parallel_files) as executor: + futures = { + executor.submit( + Files._globus_download_one, + file, output_folder, False, + position=idx, + ): file + for idx, file in enumerate(files_to_download) + } + for future in as_completed(futures): + try: + future.result() + except Exception as e: + logging.error(f"Download from Globus failed: {str(e)}") + + @staticmethod + def download_files_from_s3( + file_list_json: List[Dict], output_folder: str, skip_if_downloaded_already + ): + """ + Download files using S3 transfer URL with a progress bar and retry logic. + :param file_list_json: file list in JSON format + :param output_folder: folder to download the files + :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + """ + from pridepy.files.files import Files + + if not os.path.isdir(output_folder): + os.makedirs(output_folder, exist_ok=True) + + # Retry and timeout config + retry_config = Config( + retries={"max_attempts": 5, "mode": "standard"}, + connect_timeout=120, # Increase timeout to 120 seconds + read_timeout=120, # Timeout for reading data + signature_version=botocore.UNSIGNED, # Unsigned requests for public data + ) + + s3_resource = boto3.resource( + "s3", + config=retry_config, + endpoint_url=PrideProvider.S3_URL, + ) + bucket = s3_resource.Bucket(PrideProvider.S3_BUCKET) + + for file in file_list_json: + try: + # Determine S3 or FTP path + download_url = ( + file["publicFileLocations"][0]["value"] + if file["publicFileLocations"][0]["name"] == "FTP Protocol" + else file["publicFileLocations"][1]["value"] + ) + + ftp_base_url = "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/" + s3_path = download_url.replace(ftp_base_url, "") + new_file_path = Files.get_output_file_name(download_url, file, output_folder) + + if skip_if_downloaded_already == True and os.path.exists(new_file_path): + logging.info("Skipping download as file already exists") + continue + + logging.debug(f"Downloading From S3: {s3_path}") + + # Get file size for progress tracking + obj = bucket.Object(s3_path) + total_size = obj.content_length + + # Initialize progress bar + progress = Progress(total_size, new_file_path) + + # Download with progress bar and retry handling + for attempt in range(5): + try: + bucket.download_file(s3_path, new_file_path, Callback=progress) + progress.close() + logging.info(f"Successfully downloaded {new_file_path}") + break + except botocore.exceptions.ClientError as e: + if e.response["Error"]["Code"] == "404": + logging.error("The object does not exist.") + break + else: + logging.error(f"Download failed: {e}") + if attempt < 4: + time.sleep(2**attempt) # Exponential backoff + logging.info(f"Retrying... ({attempt + 1}/5)") + else: + raise + except Exception as e: + logging.error(f"Failed to download {file['fileName']}: {e}") + + # ------------------------------------------------------------------ + # Private dataset download + # ------------------------------------------------------------------ + + def download_private_file_name(self, accession, file_name, output_folder, username, password): + """ + Get the information for a given private file to be downloaded from the api. + :param accession: Project accession + :param file_name: The file name to be downloaded + :param username: Username with access to the dataset + :param password: Password for user with access to the dataset + """ + + auth = Authentication() + auth_token = auth.get_token(username, password) + validate_token = auth.validate_token(auth_token) + logging.info("Valid token after login: {}".format(validate_token)) + + url = self.API_PRIVATE_URL + "/projects/{}/files?search={}".format(accession, file_name) + content = requests.get(url, headers={"Authorization": "Bearer {}".format(auth_token)}) + if content.ok and content.status_code == 200: + json_file = content.json() + if ( + "_embedded" in json_file + and "files" in json_file["_embedded"] + and len(json_file["_embedded"]["files"]) == 1 + ): + download_url = json_file["_embedded"]["files"][0]["_links"]["download"]["href"] + logging.info(download_url) + + # Create a clean filename to save the downloaded file + new_file_path = os.path.join(output_folder, f"{file_name}") + + session = Util.create_session_with_retries() # Create session with retries + # Check if the file already exists + if os.path.exists(new_file_path): + resume_header = {"Range": f"bytes={os.path.getsize(new_file_path)}-"} + mode = "ab" # Append to file + resume_size = os.path.getsize(new_file_path) + else: + resume_header = {} + mode = "wb" # Write new file + resume_size = 0 + + with session.get( + download_url, stream=True, headers=resume_header, timeout=(10, 60) + ) as r: + r.raise_for_status() + total_size = int(r.headers.get("content-length", 0)) + resume_size + block_size = 1024 * 1024 # 1 MB chunks + + with tqdm( + total=total_size, + unit="B", + unit_scale=True, + desc=new_file_path, + initial=resume_size, + ) as pbar: + with open(new_file_path, mode) as f: + for chunk in r.iter_content(chunk_size=block_size): + if chunk: + f.write(chunk) + pbar.update(len(chunk)) + + logging.info(f"Successfully downloaded {new_file_path}") + + else: + logging.info( + "File name {} found more than once for the given project {}".format( + file_name, accession + ) + ) + else: + logging.info( + f"File name {file_name} now found in the project {accession}, or user don't have access" + ) + raise Exception( + f"File name {file_name} now found in the project {accession}, or user don't have access" + ) + + # ------------------------------------------------------------------ + # Multi-protocol orchestrator + # ------------------------------------------------------------------ + + @staticmethod + def _batch_download_by_protocol( + file_list: List[Dict], + output_folder: str, + protocol: str, + skip_if_downloaded_already: bool, + aspera_maximum_bandwidth: str, + parallel_files: int = 1, + checksum_map: Optional[Dict[str, str]] = None, + ) -> None: + """ + Transfer a batch of files with one protocol, reusing a single + connection where the underlying helper supports it (FTP, S3). + """ + # Use Files facade so test patches on each per-protocol helper keep working. + from pridepy.files.files import Files + + if not file_list: + return + if protocol == "ftp": + Files.download_files_from_ftp( + file_list, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + ) + return + if protocol == "aspera": + Files.download_files_from_aspera( + file_list, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + maximum_bandwidth=aspera_maximum_bandwidth, + ) + return + if protocol == "globus": + Files.download_files_from_globus( + file_list, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + parallel_files=parallel_files, + checksum_map=checksum_map or {}, + ) + return + if protocol == "s3": + Files.download_files_from_s3( + file_list, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + ) + return + raise ValueError(f"Unsupported protocol: {protocol}") + + @staticmethod + def _download_with_fallback( + file_record: Dict, + output_folder: str, + protocol_sequence: List[str], + expected_checksum: Optional[str], + aspera_maximum_bandwidth: str, + max_protocol_retries: int = 2, + parallel_files: int = 1, + ) -> bool: + """ + Download one file by trying each protocol in sequence, validating + after every attempt. Intended as the per-file fallback path; batch + download of the primary protocol is handled separately. + """ + # Patch-sensitive: call through Files so test patches intercept. + from pridepy.files.files import Files + + local_path = Files._resolve_local_path(file_record, output_folder) + + for protocol in protocol_sequence: + for attempt in range(1, max_protocol_retries + 1): + logging.info( + f"Downloading {file_record['fileName']} via {protocol} " + f"(attempt {attempt}/{max_protocol_retries})" + ) + try: + Files._remove_if_exists(local_path) + Files._batch_download_by_protocol( + [file_record], + output_folder, + protocol, + skip_if_downloaded_already=False, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + parallel_files=parallel_files, + ) + except Exception as error: + logging.error( + f"Protocol {protocol} failed for {file_record['fileName']}: {error}" + ) + + valid, reason = Files.validate_download(local_path, expected_checksum) + if valid: + logging.info( + f"File {file_record['fileName']} downloaded successfully via {protocol}" + ) + return True + + logging.warning( + f"Validation failed for {file_record['fileName']} via {protocol}: {reason}" + ) + Files._remove_if_exists(local_path) + + logging.warning( + f"Protocol {protocol} exhausted for {file_record['fileName']}, switching protocol." + ) + + logging.error(f"All protocol attempts failed for {file_record['fileName']}") + return False + + @staticmethod + def download_files( + file_list_json: List[Dict], + accession, + output_folder: str, + skip_if_downloaded_already, + protocol: str = "ftp", + aspera_maximum_bandwidth: str = "100M", # Aspera maximum bandwidth + checksum_check=False, + parallel_files: int = 1, + ): + """ + Download files using either FTP or Aspera transfer protocol. + :param file_list_json: File list in JSON format + :param accession: Project accession + :param output_folder: Folder to download the files + :param protocol: ftp, aspera, globus + :param aspera_maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. + :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + """ + # Patch-sensitive: call _batch_download_by_protocol and + # _download_with_fallback through Files so test patches intercept. + from pridepy.files.files import Files + + protocols_supported = ["ftp", "aspera", "globus", "s3"] + if protocol not in protocols_supported: + logging.error("Protocol should be one of ftp, aspera, globus, s3") + return + + os.makedirs(output_folder, exist_ok=True) + + checksum_map: Dict[str, str] = {} + if checksum_check: + checksum_file_path = Files.save_checksum_file(accession, output_folder) + checksum_map = Files.read_checksum_file(checksum_file_path) + logging.info(f"Loaded checksums for {len(checksum_map)} files") + + if not file_list_json: + return + + protocol_sequence = Files._protocol_sequence(protocol) + primary_protocol = protocol_sequence[0] + # Retry with the primary protocol first, then fall back to others + fallback_sequence = protocol_sequence + + # Phase 1: batch download with the requested protocol. Reuses a single + # FTP/S3 connection for all files (the previous behaviour) instead of + # paying the per-file reconnect cost in the common happy path. + logging.info( + f"Downloading {len(file_list_json)} file(s) via {primary_protocol} (batch)" + ) + try: + Files._batch_download_by_protocol( + file_list_json, + output_folder, + primary_protocol, + skip_if_downloaded_already=skip_if_downloaded_already, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + parallel_files=parallel_files, + checksum_map=checksum_map, + ) + except Exception as exc: + logging.warning( + f"Batch {primary_protocol} run hit an error; will retry individual failures: {exc}" + ) + + # Phase 2: validate every file and fall back per-file for the ones + # that are missing or invalid. + logging.info("Phase 2: validating %d downloaded file(s)", len(file_list_json)) + failed_files: List[str] = [] + for i, file_record in enumerate(file_list_json, 1): + expected_checksum = checksum_map.get(file_record["fileName"]) + local_path = Files._resolve_local_path(file_record, output_folder) + logging.info("Validating [%d/%d] %s", i, len(file_list_json), file_record["fileName"]) + valid, reason = Files.validate_download(local_path, expected_checksum) + if valid: + continue + + logging.warning( + f"{file_record['fileName']} invalid after {primary_protocol} ({reason})" + ) + if "checksum mismatch" in reason: + Files._remove_if_exists(local_path) + + if not fallback_sequence: + failed_files.append(file_record.get("fileName", "")) + continue + + success = Files._download_with_fallback( + file_record=file_record, + output_folder=output_folder, + protocol_sequence=fallback_sequence, + expected_checksum=expected_checksum, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + parallel_files=parallel_files, + ) + if not success: + failed_files.append(file_record.get("fileName", "")) + + if failed_files: + failed_summary = ", ".join(failed_files) + logging.error(f"Failed to download {len(failed_files)} file(s): {failed_summary}") + raise RuntimeError(f"Failed to download {len(failed_files)} file(s): {failed_summary}") From 6fd743dd5bcc0334b3c802d4db3f57a2ab8bfa0d Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 16:49:27 +0100 Subject: [PATCH 13/54] refactor(providers): rewire Files facade through Registry Files public methods (get_all_raw_file_list, download_all_raw_files, download_all_category_files, get_all_category_file_list, download_file_by_name, get_file_from_api, download_files_by_list) now dispatch via registry.resolve(accession).{list,download}_files(...). Removed dead helpers: _list_direct_download_files, _download_direct_download_records. Kept _repo_uses_tls as a registry shim. Fixed _download_massive_file_records to use registry. PrideProvider.download_files refactored: the old static method is now _download_files_batch; a proper Provider-interface instance method download_files(self, accession, records, ...) wraps it so PRIDE routes uniformly through the registry like other providers. Tests updated: patches on Files._list_massive_public_files, Files._list_jpost_public_files, files_obj.stream_all_files_by_project, and files_obj.download_files now target the provider classes (MassiveProvider, JpostProvider, PrideProvider) directly. Full suite green. files.py size: 1254 LOC (was 1352 before Task 9). --- pridepy/files/files.py | 238 ++++++++----------------- pridepy/providers/pride.py | 27 ++- pridepy/tests/test_download_by_list.py | 15 +- pridepy/tests/test_jpost_files.py | 8 +- pridepy/tests/test_massive_files.py | 7 +- 5 files changed, 111 insertions(+), 184 deletions(-) diff --git a/pridepy/files/files.py b/pridepy/files/files.py index b4cc3b9..e9cde55 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -205,12 +205,13 @@ def is_iprox_accession(accession: str) -> bool: @staticmethod def _repo_uses_tls(accession: str) -> bool: - """ - Whether the public FTP server for ``accession`` requires FTP over TLS. - MassIVE rejects plain anonymous FTP (``421 TLS is required``); JPOST - accepts plain FTP. - """ - return Files.is_massive_accession(accession) + """Shim — returns the resolved provider's use_tls flag (False if unknown).""" + from pridepy.providers import registry + try: + provider = registry.resolve(accession) + except ValueError: + return False + return getattr(provider, "use_tls", False) @staticmethod def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: @@ -246,11 +247,12 @@ def _download_massive_file_records( ) -> None: """ Download public MassIVE files via anonymous FTP (now FTPS). - Backward-compat wrapper around :meth:`_download_direct_download_records`. + Backward-compat shim — dispatches via the provider registry. """ - self._download_direct_download_records( + from pridepy.providers import registry + registry.resolve(accession).download_files( accession=accession, - file_records=file_records, + records=file_records, output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, @@ -298,50 +300,6 @@ def _list_iprox_public_files(self, accession: str) -> List[Dict]: from pridepy.providers.iprox import IproxProvider return IproxProvider().list_files(accession) - def _list_direct_download_files(self, accession: str) -> List[Dict]: - """ - Dispatch to the right listing transport for a direct-download - repository: MassIVE walks FTPS, JPOST uses PROXI JSON over HTTPS with - an FTP fallback, iProX uses the dataset's PX XML over HTTPS. - """ - if self.is_massive_accession(accession): - return self._list_massive_public_files(accession) - if self.is_jpost_accession(accession): - return self._list_jpost_public_files(accession) - if self.is_iprox_accession(accession): - return self._list_iprox_public_files(accession) - raise ValueError( - f"Accession {accession} is not a direct-download repository accession" - ) - - def _download_direct_download_records( - self, - accession: str, - file_records: List[Dict], - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str, - parallel_files: int = 1, - ) -> None: - """ - Download files from a direct-download repository. - - MassIVE and JPOST use anonymous FTP(S) with REST-based resume and - per-host parallel workers. iProX uses anonymous HTTPS via - ``download.iprox.org`` with ``Range``-based resume and per-file - parallel workers. URLs are partitioned by scheme so a mixed batch - (e.g. a JPOST PX XML that ever pointed at HTTPS) routes correctly. - Dispatches via the provider registry. - """ - from pridepy.providers import registry - return registry.resolve(accession).download_files( - accession=accession, - records=file_records, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - ) async def stream_all_files_metadata(self, output_file, accession=None): """Shim — see :meth:`pridepy.providers.pride.PrideProvider.stream_all_files_metadata`.""" @@ -354,22 +312,14 @@ def stream_all_files_by_project(self, accession) -> List[Dict]: return PrideProvider().stream_all_files_by_project(accession) def get_all_raw_file_list(self, project_accession): - """ - Get all raw file lists from PRIDE API for a given project_accession - :param project_accession: PRIDE accession - :return: raw file list in JSON format - """ - if self.is_direct_download_accession(project_accession): - record_files = self._list_direct_download_files(project_accession) - return [ - file for file in record_files if file["fileCategory"]["value"] == "RAW" - ] - - record_files = self.stream_all_files_by_project(project_accession) + """Get raw file list for any registered provider. - # Filter projects by fileCategory = RAW - raw_files = [file for file in record_files if file["fileCategory"]["value"] == "RAW"] - return raw_files + Returns the dataset's file records filtered to fileCategory == "RAW". + """ + from pridepy.providers import registry + provider = registry.resolve(project_accession) + records = provider.list_files(project_accession) + return [r for r in records if r["fileCategory"]["value"] == "RAW"] def download_all_raw_files( self, @@ -381,42 +331,21 @@ def download_all_raw_files( checksum_check: bool = False, parallel_files: int = 1, ): - """ - This method will download all the raw files from PRIDE PROJECT - :param output_folder: output directory where raw files will get saved - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - :param accession: PRIDE accession - :param protocol: ftp, aspera, globus - :param aspera_maximum_bandwidth: Aspera maximum bandwidth - :param checksum_check: Download checksum for a given project. - :return: None - """ - - if not (os.path.isdir(output_folder)): + """Download all RAW files for any registered provider.""" + if not os.path.isdir(output_folder): os.mkdir(output_folder) - - raw_files = self.get_all_raw_file_list(accession) - - if self.is_direct_download_accession(accession): - self._download_direct_download_records( - accession=accession, - file_records=raw_files, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - ) - return - - self.download_files( - raw_files, - accession, - output_folder, - skip_if_downloaded_already, - protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, + from pridepy.providers import registry + provider = registry.resolve(accession) + records = self.get_all_raw_file_list(accession) + provider.download_files( + accession=accession, + records=records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, ) @staticmethod @@ -597,11 +526,14 @@ def download_file_by_name( :param checksum_check: Download checksum for a given project. """ - if not (os.path.isdir(output_folder)): + if not os.path.isdir(output_folder): os.mkdir(output_folder) + from pridepy.providers import registry + provider = registry.resolve(accession) + ## Check type of project - if self.is_direct_download_accession(accession): + if provider.name in ("massive", "jpost", "iprox"): logging.info( "Downloading file from public direct-download dataset {}".format(accession) ) @@ -610,9 +542,9 @@ def download_file_by_name( raise Exception( "File name {} not found in dataset {}".format(file_name, accession) ) - self._download_direct_download_records( + provider.download_files( accession=accession, - file_records=response, + records=response, output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, @@ -670,14 +602,10 @@ def get_file_from_api(self, accession, file_name) -> List[Dict]: :param file_name: file name :return: file in json format """ - + from pridepy.providers import registry try: - if self.is_direct_download_accession(accession): - files = self._list_direct_download_files(accession) - return [f for f in files if f["fileName"] == file_name] - files = self.stream_all_files_by_project(accession) - file = [f for f in files if f["fileName"] == file_name] - return file + records = registry.resolve(accession).list_files(accession) + return [r for r in records if r["fileName"] == file_name] except Exception as e: raise Exception("File not found " + str(e)) @@ -760,9 +688,9 @@ def download_files( checksum_check=False, parallel_files: int = 1, ): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files`.""" + """Shim — see :meth:`pridepy.providers.pride.PrideProvider._download_files_batch`.""" from pridepy.providers.pride import PrideProvider - return PrideProvider.download_files( + return PrideProvider._download_files_batch( file_list_json, accession, output_folder, @@ -803,10 +731,10 @@ def download_files_by_list( if not file_names: raise ValueError("file_names must contain at least one filename") - if self.is_direct_download_accession(accession): - all_files = self._list_direct_download_files(accession) - else: - all_files = self.stream_all_files_by_project(accession) + from pridepy.providers import registry + provider = registry.resolve(accession) + all_files = provider.list_files(accession) + requested = set(file_names) matched = [f for f in all_files if f.get("fileName") in requested] missing = sorted(requested - {f.get("fileName") for f in matched}) @@ -817,26 +745,15 @@ def download_files_by_list( f"No matching files in project {accession} for: {sorted(requested)}" ) - if self.is_direct_download_accession(accession): - self._download_direct_download_records( - accession=accession, - file_records=matched, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - ) - return - - self.download_files( - matched, - accession, - output_folder, - skip_if_downloaded_already, - protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, + provider.download_files( + accession=accession, + records=matched, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, ) @staticmethod @@ -1093,26 +1010,18 @@ def download_all_category_files( """ if categories is None: categories = [category] if category else ["RAW"] - raw_files = self.get_all_category_file_list(accession, categories) - if self.is_direct_download_accession(accession): - self._download_direct_download_records( - accession=accession, - file_records=raw_files, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - ) - return - self.download_files( - raw_files, - accession, - output_folder, - skip_if_downloaded_already, - protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, + records = self.get_all_category_file_list(accession, categories) + from pridepy.providers import registry + provider = registry.resolve(accession) + provider.download_files( + accession=accession, + records=records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, ) def get_all_category_file_list( @@ -1127,17 +1036,10 @@ def get_all_category_file_list( """ if isinstance(categories, str): categories = [categories] - category_set = {category.upper() for category in categories} - - if self.is_direct_download_accession(accession): - record_files = self._list_direct_download_files(accession) - else: - record_files = self.stream_all_files_by_project(accession) - - category_files = [ - file for file in record_files if file["fileCategory"]["value"] in category_set - ] - return category_files + category_set = {c.upper() for c in categories} + from pridepy.providers import registry + records = registry.resolve(accession).list_files(accession) + return [r for r in records if r["fileCategory"]["value"] in category_set] # ------------------------------- # ProteomeXchange support diff --git a/pridepy/providers/pride.py b/pridepy/providers/pride.py index ea60a5d..c8c40ca 100644 --- a/pridepy/providers/pride.py +++ b/pridepy/providers/pride.py @@ -685,8 +685,33 @@ def _download_with_fallback( logging.error(f"All protocol attempts failed for {file_record['fileName']}") return False - @staticmethod def download_files( + self, + accession, + records: List[Dict], + output_folder: str, + skip_if_downloaded_already, + protocol: str = "ftp", + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + parallel_files: int = 1, + username: Optional[str] = None, + password: Optional[str] = None, + ): + """Implement Provider.download_files — maps to the legacy static batch downloader.""" + PrideProvider._download_files_batch( + file_list_json=records, + accession=accession, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, + ) + + @staticmethod + def _download_files_batch( file_list_json: List[Dict], accession, output_folder: str, diff --git a/pridepy/tests/test_download_by_list.py b/pridepy/tests/test_download_by_list.py index df81914..5115b4e 100644 --- a/pridepy/tests/test_download_by_list.py +++ b/pridepy/tests/test_download_by_list.py @@ -14,6 +14,7 @@ from pridepy.files.files import Files from pridepy.pridepy import _read_filename_arguments +from pridepy.providers.pride import PrideProvider class TestDownloadFilesByList(TestCase): @@ -36,8 +37,8 @@ def test_filters_metadata_and_delegates(self): {"fileName": "c.raw"}, ] with patch.object( - files_obj, "stream_all_files_by_project", return_value=api_response - ), patch.object(files_obj, "download_files") as mock_download: + PrideProvider, "list_files", return_value=api_response + ), patch.object(PrideProvider, "download_files") as mock_download: files_obj.download_files_by_list( accession="PXD001819", file_names=["a.raw", "c.raw"], @@ -46,16 +47,16 @@ def test_filters_metadata_and_delegates(self): protocol="ftp", ) - args, _ = mock_download.call_args - matched = args[0] + _, kwargs = mock_download.call_args + matched = kwargs["records"] assert {f["fileName"] for f in matched} == {"a.raw", "c.raw"} def test_warns_on_partial_match(self): files_obj = Files() api_response = [{"fileName": "a.raw"}] with patch.object( - files_obj, "stream_all_files_by_project", return_value=api_response - ), patch.object(files_obj, "download_files") as mock_download, self.assertLogs( + PrideProvider, "list_files", return_value=api_response + ), patch.object(PrideProvider, "download_files") as mock_download, self.assertLogs( level="WARNING" ) as log_ctx: files_obj.download_files_by_list( @@ -71,7 +72,7 @@ def test_warns_on_partial_match(self): def test_raises_when_no_files_match(self): files_obj = Files() with patch.object( - files_obj, "stream_all_files_by_project", return_value=[] + PrideProvider, "list_files", return_value=[] ): with pytest.raises(ValueError, match="No matching files"): files_obj.download_files_by_list( diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py index 678adda..1e4c652 100644 --- a/pridepy/tests/test_jpost_files.py +++ b/pridepy/tests/test_jpost_files.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock, patch from pridepy.files.files import Files +from pridepy.providers.jpost import JpostProvider class TestJPOSTFiles(TestCase): @@ -50,12 +51,9 @@ def test_get_all_raw_file_list_filters_jpost_records(self): ), ] - with patch.object(Files, "_list_jpost_public_files", return_value=jpost_records), patch.object( - Files, "stream_all_files_by_project" - ) as pride_mock: + with patch.object(JpostProvider, "list_files", return_value=jpost_records): result = files.get_all_raw_file_list("JPST000001") - pride_mock.assert_not_called() assert len(result) == 1 assert {file["fileName"] for file in result} == {"run1.raw"} @@ -68,7 +66,7 @@ def test_download_file_by_name_uses_jpost_ftp_listing(self): with tempfile.TemporaryDirectory() as tmp_dir: with patch.object( - Files, "_list_jpost_public_files", return_value=[file_record] + JpostProvider, "list_files", return_value=[file_record] ), patch.object(Files, "download_ftp_urls") as download_mock: files.download_file_by_name( accession="JPST000001", diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index fd6a4ac..a4e9278 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -3,6 +3,7 @@ from unittest.mock import patch from pridepy.files.files import Files +from pridepy.providers.massive import MassiveProvider class TestMassIVEFiles(TestCase): @@ -66,7 +67,7 @@ def test_get_all_raw_file_list_filters_massive_records(self): ), ] - with patch.object(Files, "_list_massive_public_files", return_value=massive_records): + with patch.object(MassiveProvider, "list_files", return_value=massive_records): result = files.get_all_raw_file_list("MSV000012345") assert len(result) == 1 @@ -80,7 +81,7 @@ def test_download_file_by_name_uses_massive_ftp_listing(self): ) with tempfile.TemporaryDirectory() as tmp_dir: - with patch.object(Files, "_list_massive_public_files", return_value=[file_record]), patch.object( + with patch.object(MassiveProvider, "list_files", return_value=[file_record]), patch.object( Files, "download_ftp_urls" ) as download_mock: files.download_file_by_name( @@ -120,7 +121,7 @@ def test_download_all_raw_files_threads_parallel_files_for_massive(self): with tempfile.TemporaryDirectory() as tmp_dir: with patch.object( - Files, "_list_massive_public_files", return_value=massive_records + MassiveProvider, "list_files", return_value=massive_records ), patch.object(Files, "download_ftp_urls") as download_mock: files.download_all_raw_files( accession="MSV000012345", From 6d5637bf40f22d2f84134105a86d5493275d6585 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 16:54:56 +0100 Subject: [PATCH 14/54] test: integration test for PRIDE multi-protocol fallback through facade Verifies that Files().download_all_raw_files for a PXD accession flows through Registry.resolve -> PrideProvider.download_files -> _batch_download_by_protocol (patched via Files), and that _download_with_fallback is only called when batch returns failed files. Also mocks validate_download to return success so the happy-path test correctly asserts that fallback is not invoked when all files pass validation after the primary-protocol batch run. Spec acceptance criterion #5. --- pridepy/tests/test_download_resilience.py | 40 +++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 21b1603..0f86013 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -268,3 +268,43 @@ def test_download_files_raises_when_any_file_fails(self): skip_if_downloaded_already=False, protocol="ftp", ) + + def test_facade_dispatches_pride_through_registry_to_fallback(self): + """Files().download_all_raw_files for a PXD accession must flow: + Files facade -> Registry.resolve -> PrideProvider.download_files + -> _batch_download_by_protocol (mocked). + + Patching Files._batch_download_by_protocol proves the patch intercepts + (i.e. PrideProvider calls *back* through Files, preserving the test + contract for the multi-protocol orchestrator). + """ + from pridepy.providers.pride import PrideProvider + + fake_records = [ + { + "accession": "PXD000001", + "fileName": "x.raw", + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "ftp://ftp.pride.ebi.ac.uk/.../x.raw"} + ], + }, + ] + + with tempfile.TemporaryDirectory() as tmp: + with patch.object(PrideProvider, "list_files", return_value=fake_records), \ + patch.object(Files, "_batch_download_by_protocol", return_value=[]) as batch_mock, \ + patch.object(Files, "validate_download", return_value=(True, "ok")), \ + patch.object(Files, "_download_with_fallback") as fallback_mock: + Files().download_all_raw_files( + accession="PXD000001", + output_folder=tmp, + skip_if_downloaded_already=False, + protocol="ftp", + aspera_maximum_bandwidth="100M", + ) + + batch_mock.assert_called_once() + # No fallback expected because all files passed validation after + # the primary-protocol batch run. + fallback_mock.assert_not_called() From f60a69c367b0d7f697728d0e886aace5843c06c9 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 16:55:50 +0100 Subject: [PATCH 15/54] chore(release): bump version to 0.0.17 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4a95f24..f5b74ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pridepy" -version = "0.0.16" +version = "0.0.17" description = "Python Client library for PRIDE Rest API" readme = "README.md" requires-python = ">=3.9" From 02de4017482e305af47723f9910b919e86aaf8bc Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 18:03:53 +0100 Subject: [PATCH 16/54] refactor(commands): scaffold commands/ package Empty scaffold for the follow-up refactor that extracts cross-cutting commands (download_files_by_url, download_files_by_list, download_px_raw_files) from Files into their own modules. No code moved yet. No behaviour change. Test suite green at 68 passed, 4 skipped. --- pridepy/commands/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 pridepy/commands/__init__.py diff --git a/pridepy/commands/__init__.py b/pridepy/commands/__init__.py new file mode 100644 index 0000000..c94f89e --- /dev/null +++ b/pridepy/commands/__init__.py @@ -0,0 +1,12 @@ +"""Cross-cutting download commands. + +Each module under this package owns one user-facing command that doesn't +fit any single provider: + +- ``by_url``: download a list of explicit URLs (ftp/http/https) +- ``by_list``: download a subset of a project's files by filename +- ``proteomexchange``: download raw files from a ProteomeXchange XML + +The ``pridepy.files.files.Files`` facade keeps shim methods that +delegate here, so existing test patches on ``Files.X`` keep working. +""" From c165345783ece06fbdba98d43a94b416bd055f65 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 18:06:03 +0100 Subject: [PATCH 17/54] refactor(commands): move ProteomeXchange XML download into commands/proteomexchange.py Moved download_px_raw_files, _normalize_px_xml_url, _parse_px_xml_for_raw_file_urls from Files into commands/proteomexchange.py. Files keeps shim re-exports. Also removed now-unused xml.etree.ElementTree import from files.py. No behaviour change. Test suite green. --- pridepy/commands/proteomexchange.py | 94 +++++++++++++++++++++++++++++ pridepy/files/files.py | 77 +++-------------------- 2 files changed, 104 insertions(+), 67 deletions(-) create mode 100644 pridepy/commands/proteomexchange.py diff --git a/pridepy/commands/proteomexchange.py b/pridepy/commands/proteomexchange.py new file mode 100644 index 0000000..d86cd24 --- /dev/null +++ b/pridepy/commands/proteomexchange.py @@ -0,0 +1,94 @@ +"""ProteomeXchange XML download command. + +Given a PXD accession or a ProteomeXchange XML URL, parse the XML for +``Associated raw file URI`` cvParams and download each one over its +native scheme (ftp:// via FTP, http(s):// via HTTPS). +""" +import logging +import os +import xml.etree.ElementTree as ET +from typing import List +from urllib.parse import urlparse + +from pridepy.util.api_handling import Util + + +def _normalize_px_xml_url(px_id_or_url: str) -> str: + """ + Build the ProteomeXchange XML endpoint from a dataset accession or a dataset web URL. + Examples accepted: + - PXD039236 + - https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236 + - https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236&anything + """ + if px_id_or_url.startswith("http://") or px_id_or_url.startswith("https://"): + parsed = urlparse(px_id_or_url) + # keep the ID param value if present; otherwise fallback to the path tail + query = parsed.query or "" + if "ID=" in query: + id_value = [q.split("=", 1)[1] for q in query.split("&") if q.startswith("ID=")] + if id_value: + return ( + f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID={id_value[0]}&outputMode=XML&test=no" + ) + # If the input URL already requests XML, just ensure flags + if parsed.path.endswith("/cgi/GetDataset"): + return ( + f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?{query}&outputMode=XML&test=no" + ) + # Assume it's a plain accession if not a URL + return ( + f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID={px_id_or_url}&outputMode=XML&test=no" + ) + + +def _parse_px_xml_for_raw_file_urls(px_xml_url: str) -> List[str]: + """ + Parse the PX XML and return a list of associated raw file URIs. + We extract cvParam with name "Associated raw file URI" under each DatasetFile. + """ + headers = {"Accept": "application/xml"} + response = Util.get_api_call(px_xml_url, headers) + response.raise_for_status() + root = ET.fromstring(response.content) + + urls: List[str] = [] + # The XML namespace is often absent in PX XML; access elements directly + for dataset_file in root.iter("DatasetFile"): + for cv in dataset_file.findall("cvParam"): + name = cv.attrib.get("name") + value = cv.attrib.get("value") + if name == "Associated raw file URI" and value: + urls.append(value) + return urls + + +def download_px_raw_files( + px_id_or_url: str, + output_folder: str, + skip_if_downloaded_already: bool = True, +) -> None: + """Download all raw files referenced by a ProteomeXchange dataset. + + Prefers FTP when the URL is ftp://, otherwise uses HTTP(S). Supports + resume and skip. + """ + from pridepy.files.files import Files # lazy: avoid module-load cycle + + if not os.path.isdir(output_folder): + os.makedirs(output_folder, exist_ok=True) + + px_xml_url = _normalize_px_xml_url(px_id_or_url) + logging.info(f"Fetching PX XML: {px_xml_url}") + urls = _parse_px_xml_for_raw_file_urls(px_xml_url) + if not urls: + logging.info("No Associated raw file URIs found in PX XML") + return + + ftp_urls = [u for u in urls if u.lower().startswith("ftp://")] + http_urls = [u for u in urls if u.lower().startswith(("http://", "https://"))] + + if ftp_urls: + Files.download_ftp_urls(ftp_urls, output_folder, skip_if_downloaded_already) + if http_urls: + Files.download_http_urls(http_urls, output_folder, skip_if_downloaded_already) diff --git a/pridepy/files/files.py b/pridepy/files/files.py index e9cde55..612152b 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -9,7 +9,6 @@ from ftplib import FTP from typing import Dict, List, Optional, Tuple from urllib.parse import urlparse -import xml.etree.ElementTree as ET import requests from tqdm import tqdm @@ -1047,53 +1046,15 @@ def get_all_category_file_list( @staticmethod def _normalize_px_xml_url(px_id_or_url: str) -> str: - """ - Build the ProteomeXchange XML endpoint from a dataset accession or a dataset web URL. - Examples accepted: - - PXD039236 - - https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236 - - https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236&anything - """ - if px_id_or_url.startswith("http://") or px_id_or_url.startswith("https://"): - parsed = urlparse(px_id_or_url) - # keep the ID param value if present; otherwise fallback to the path tail - query = parsed.query or "" - if "ID=" in query: - id_value = [q.split("=", 1)[1] for q in query.split("&") if q.startswith("ID=")] - if id_value: - return ( - f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID={id_value[0]}&outputMode=XML&test=no" - ) - # If the input URL already requests XML, just ensure flags - if parsed.path.endswith("/cgi/GetDataset"): - return ( - f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?{query}&outputMode=XML&test=no" - ) - # Assume it's a plain accession if not a URL - return ( - f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID={px_id_or_url}&outputMode=XML&test=no" - ) + """Shim — see :func:`pridepy.commands.proteomexchange._normalize_px_xml_url`.""" + from pridepy.commands import proteomexchange + return proteomexchange._normalize_px_xml_url(px_id_or_url) @staticmethod - def _parse_px_xml_for_raw_file_urls(px_xml_url: str) -> List[str]: - """ - Parse the PX XML and return a list of associated raw file URIs. - We extract cvParam with name "Associated raw file URI" under each DatasetFile. - """ - headers = {"Accept": "application/xml"} - response = Util.get_api_call(px_xml_url, headers) - response.raise_for_status() - root = ET.fromstring(response.content) - - urls: List[str] = [] - # The XML namespace is often absent in PX XML; access elements directly - for dataset_file in root.iter("DatasetFile"): - for cv in dataset_file.findall("cvParam"): - name = cv.attrib.get("name") - value = cv.attrib.get("value") - if name == "Associated raw file URI" and value: - urls.append(value) - return urls + def _parse_px_xml_for_raw_file_urls(px_xml_url: str): + """Shim — see :func:`pridepy.commands.proteomexchange._parse_px_xml_for_raw_file_urls`.""" + from pridepy.commands import proteomexchange + return proteomexchange._parse_px_xml_for_raw_file_urls(px_xml_url) def download_px_raw_files( self, @@ -1101,27 +1062,9 @@ def download_px_raw_files( output_folder: str, skip_if_downloaded_already: bool = True, ) -> None: - """ - Download all raw files referenced by a ProteomeXchange dataset. - Prefer FTP when the URL is ftp://, otherwise use HTTP(S). Supports resume and skip. - """ - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - px_xml_url = self._normalize_px_xml_url(px_id_or_url) - logging.info(f"Fetching PX XML: {px_xml_url}") - urls = self._parse_px_xml_for_raw_file_urls(px_xml_url) - if not urls: - logging.info("No Associated raw file URIs found in PX XML") - return - - ftp_urls = [u for u in urls if u.lower().startswith("ftp://")] - http_urls = [u for u in urls if u.lower().startswith("http://") or u.lower().startswith("https://")] - - if ftp_urls: - self.download_ftp_urls(ftp_urls, output_folder, skip_if_downloaded_already) - if http_urls: - self.download_http_urls(http_urls, output_folder, skip_if_downloaded_already) + """Shim — see :func:`pridepy.commands.proteomexchange.download_px_raw_files`.""" + from pridepy.commands import proteomexchange + return proteomexchange.download_px_raw_files(px_id_or_url, output_folder, skip_if_downloaded_already) @staticmethod def _local_path_for_url(download_url: str, output_folder: str) -> str: From 1a0a9b8d540c34e2ba6e49b21655d99d50546c26 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 18:07:35 +0100 Subject: [PATCH 18/54] refactor(commands): move download_files_by_list into commands/by_list.py Moved download_files_by_list from Files into commands/by_list.py. Files keeps a shim re-export. No behaviour change. Test suite green. --- pridepy/commands/by_list.py | 58 +++++++++++++++++++++++++++++++++++++ pridepy/files/files.py | 43 ++++----------------------- 2 files changed, 64 insertions(+), 37 deletions(-) create mode 100644 pridepy/commands/by_list.py diff --git a/pridepy/commands/by_list.py b/pridepy/commands/by_list.py new file mode 100644 index 0000000..e008d6e --- /dev/null +++ b/pridepy/commands/by_list.py @@ -0,0 +1,58 @@ +"""Download a subset of project files identified by a filename list.""" +import logging +from typing import List, Optional + + +def download_files_by_list( + accession: str, + file_names: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str = "ftp", + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + parallel_files: int = 1, +) -> None: + """Download a subset of project files identified by a filename list. + + Resolves each requested filename via the project metadata API and + delegates to the provider's ``download_files`` so the existing batch + + protocol fallback engine is reused. + + :param accession: PRIDE or MassIVE project accession (public) + :param file_names: filenames to download + :param output_folder: directory to write downloaded files into + :param skip_if_downloaded_already: skip files already present locally + :param protocol: preferred protocol; falls back across others on failure + :param aspera_maximum_bandwidth: aspera ascp bandwidth cap + :param checksum_check: download project checksums and validate + :param parallel_files: number of files to download simultaneously for globus + :raises ValueError: if ``file_names`` is empty or none match the project + """ + if not file_names: + raise ValueError("file_names must contain at least one filename") + + from pridepy.providers import registry # lazy + provider = registry.resolve(accession) + all_files = provider.list_files(accession) + + requested = set(file_names) + matched = [f for f in all_files if f.get("fileName") in requested] + missing = sorted(requested - {f.get("fileName") for f in matched}) + if missing: + logging.warning("Files not found in project %s: %s", accession, missing) + if not matched: + raise ValueError( + f"No matching files in project {accession} for: {sorted(requested)}" + ) + + provider.download_files( + accession=accession, + records=matched, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + ) diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 612152b..878b4c1 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -711,48 +711,17 @@ def download_files_by_list( checksum_check: bool = False, parallel_files: int = 1, ) -> None: - """Download a subset of project files identified by a filename list. - - Resolves each requested filename via the project metadata API and - delegates to :meth:`download_files` so the existing batch + protocol - fallback engine is reused. - - :param accession: PRIDE or MassIVE project accession (public) - :param file_names: filenames to download - :param output_folder: directory to write downloaded files into - :param skip_if_downloaded_already: skip files already present locally - :param protocol: preferred protocol; falls back across others on failure - :param aspera_maximum_bandwidth: aspera ascp bandwidth cap - :param checksum_check: download project checksums and validate - :param parallel_files: number of files to download simultaneously for globus - :raises ValueError: if ``file_names`` is empty or none match the project - """ - if not file_names: - raise ValueError("file_names must contain at least one filename") - - from pridepy.providers import registry - provider = registry.resolve(accession) - all_files = provider.list_files(accession) - - requested = set(file_names) - matched = [f for f in all_files if f.get("fileName") in requested] - missing = sorted(requested - {f.get("fileName") for f in matched}) - if missing: - logging.warning("Files not found in project %s: %s", accession, missing) - if not matched: - raise ValueError( - f"No matching files in project {accession} for: {sorted(requested)}" - ) - - provider.download_files( + """Shim — see :func:`pridepy.commands.by_list.download_files_by_list`.""" + from pridepy.commands import by_list + return by_list.download_files_by_list( accession=accession, - records=matched, + file_names=file_names, output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, - parallel_files=parallel_files, - checksum_check=checksum_check, aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, ) @staticmethod From ced7415e799ce196a5ea530815da993c5c15185f Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 18:13:49 +0100 Subject: [PATCH 19/54] refactor(commands): move download_files_by_url into commands/by_url.py Moved download_files_by_url and its 6 helpers (_extract_pride_accession, _validate_urls_checksums, _http_download_url, _ftp_download_url, _dispatch_url_scheme, _download_single_url) from Files into commands/by_url.py. Files keeps shim re-exports for each. Internal calls to patch-sensitive helpers (_http_download_url, _ftp_download_url, _dispatch_url_scheme, _download_single_url) go through Files.X (lazy import) so existing test patches like patch.object(Files, '_http_download_url') keep intercepting. files.py drops below 1000 LOC. No behaviour change. Test suite green at 68 passed, 4 skipped. --- pridepy/commands/by_url.py | 254 +++++++++++++++++++++++++++++++++++++ pridepy/files/files.py | 227 ++++----------------------------- 2 files changed, 282 insertions(+), 199 deletions(-) create mode 100644 pridepy/commands/by_url.py diff --git a/pridepy/commands/by_url.py b/pridepy/commands/by_url.py new file mode 100644 index 0000000..91d6fec --- /dev/null +++ b/pridepy/commands/by_url.py @@ -0,0 +1,254 @@ +"""Download a list of explicit URLs (ftp/http/https). + +Each URL is dispatched to the matching transport based on its scheme. +PRIDE checksum validation is supported when the accession can be +inferred from the URL path. +""" +import ftplib +import logging +import os +import re +from concurrent.futures import ThreadPoolExecutor, as_completed +from ftplib import FTP +from typing import Dict, List, Optional, Tuple +from urllib.parse import urlparse + +from tqdm import tqdm + +from pridepy.util.api_handling import Util + + +def _extract_pride_accession(url: str) -> Optional[str]: + """Extract a PRIDE accession (PXD/PRD followed by digits) from a URL path. + + PRIDE archive URLs follow the pattern + ``…/pride/data/archive/YYYY/MM//filename``. + Returns ``None`` when no accession can be identified. + """ + match = re.search(r"((?:PXD|PRD)\d{4,})", url) + return match.group(1) if match else None + + +def _validate_urls_checksums(urls: List[str], output_folder: str) -> None: + """Validate downloaded files against PRIDE checksum API. + + Accessions are inferred from URL paths via + :func:`_extract_pride_accession`. URLs that do not contain a + recognisable PRIDE accession are skipped with a warning. + + :raises RuntimeError: if one or more files fail validation + """ + from pridepy.files.files import Files + + accession_urls: Dict[str, List[str]] = {} + for url in urls: + acc = _extract_pride_accession(url) + if acc: + accession_urls.setdefault(acc, []).append(url) + else: + logging.warning( + "Cannot infer PRIDE accession from URL, skipping checksum: %s", url + ) + + validation_failures: List[str] = [] + for acc, acc_urls in accession_urls.items(): + checksum_file_path = Files.save_checksum_file(acc, output_folder) + checksum_map = Files.read_checksum_file(checksum_file_path) + logging.info( + "Loaded checksums for %d files (project %s)", + len(checksum_map), acc, + ) + for url in acc_urls: + file_name = os.path.basename(urlparse(url).path) + target = os.path.join(output_folder, file_name) + expected = checksum_map.get(file_name) + logging.info("Validating %s", file_name) + valid, reason = Files.validate_download(target, expected) + if not valid: + logging.error("Validation failed for %s: %s", file_name, reason) + validation_failures.append(f"{file_name} ({reason})") + else: + logging.info("Checksum OK: %s", file_name) + + if validation_failures: + raise RuntimeError( + f"Checksum validation failed for {len(validation_failures)} file(s): " + + ", ".join(validation_failures) + ) + + +def _http_download_url(url: str, target: str) -> None: + """Stream an http/https URL into ``target`` with a progress bar.""" + session = Util.create_session_with_retries() + with session.get(url, stream=True, timeout=60) as response: + response.raise_for_status() + total = int(response.headers.get("Content-Length", 0)) + with open(target, "wb") as out, tqdm( + total=total, + unit="B", + unit_scale=True, + desc=os.path.basename(target), + ) as pbar: + for chunk in response.iter_content(chunk_size=8192): + if chunk: + out.write(chunk) + pbar.update(len(chunk)) + + +def _ftp_download_url(parsed, target: str) -> None: + """Download a single file from an ftp:// URL with a progress bar.""" + host = parsed.hostname + if not host: + raise ValueError(f"FTP URL missing host: {parsed.geturl()}") + port = parsed.port or 21 + user = parsed.username or "anonymous" + pwd = parsed.password or "anonymous@" + remote_path = parsed.path + with FTP() as ftp: + ftp.connect(host, port, timeout=60) + ftp.login(user, pwd) + try: + total = ftp.size(remote_path) or 0 + except ftplib.error_perm: + total = 0 + with open(target, "wb") as out, tqdm( + total=total, + unit="B", + unit_scale=True, + desc=os.path.basename(target), + ) as pbar: + + def _callback(data: bytes) -> None: + out.write(data) + pbar.update(len(data)) + + ftp.retrbinary(f"RETR {remote_path}", _callback) + + +def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: int = 0) -> None: + """Route a parsed URL to its protocol-specific downloader. + + ``protocol='globus'`` swaps the http/https single-connection streamer + for :func:`pridepy.files.files.Files._parallel_download` (single-connection with progress bar). + ftp:// URLs are unaffected. + """ + from pridepy.files.files import Files + + scheme = (parsed.scheme or "").lower() + if scheme in ("http", "https"): + if protocol == "globus": + Files._parallel_download(parsed.geturl(), target, position=position) + else: + Files._http_download_url(parsed.geturl(), target) + elif scheme == "ftp": + Files._ftp_download_url(parsed, target) + else: + raise ValueError(f"Unsupported URL scheme: {scheme}") + + +def _download_single_url( + url: str, + output_folder: str, + skip_if_exists: bool = False, + protocol: str = "ftp", + position: int = 0, +) -> str: + """Download one URL, dispatched by scheme; return the local file path.""" + from pridepy.files.files import Files + + parsed = urlparse(url) + if not (parsed.scheme or "").lower(): + raise ValueError(f"URL missing scheme: {url}") + + file_name = os.path.basename(parsed.path) + if not file_name: + raise ValueError(f"Cannot derive filename from URL: {url}") + + target = os.path.join(output_folder, file_name) + if skip_if_exists and os.path.isfile(target) and os.path.getsize(target) > 0: + logging.info("Skipping %s: already downloaded", file_name) + return target + + Files._dispatch_url_scheme(parsed, target, protocol, position=position) + + ok, reason = Files.validate_download(target) + if not ok: + Files._remove_if_exists(target) + raise RuntimeError(f"Download invalid: {reason} ({target})") + return target + + +def download_files_by_url( + urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool = False, + protocol: str = "ftp", + parallel_files: int = 1, + checksum_check: bool = False, +) -> None: + """Download files from a list of raw URLs, dispatched by URL scheme. + + Supported schemes: ``http``, ``https``, ``ftp``. Each URL is downloaded + independently; per-URL errors are logged, then aggregated and re-raised + as a single :class:`RuntimeError` so callers see a complete failure + summary. + + :param urls: fully-qualified URLs (each contains its scheme) + :param output_folder: directory to write downloaded files into + :param skip_if_downloaded_already: skip URLs whose target file exists + :param protocol: ``ftp`` (default) for single-connection per URL scheme; + ``globus`` for resume-capable http/https downloads (single-connection stream) + (no effect on ftp:// URLs which always use single-connection FTP) + :param checksum_check: validate downloads against PRIDE checksum API; + accessions are inferred from URL paths (only PRIDE URLs supported) + :raises ValueError: if ``urls`` is empty + :raises RuntimeError: if one or more URLs failed + """ + if not urls: + raise ValueError("urls must contain at least one URL") + + os.makedirs(output_folder, exist_ok=True) + + parallel_files = min(parallel_files, 3, len(urls)) + failures: List[Tuple[str, str]] = [] + from pridepy.files.files import Files + + if parallel_files < 2: + for url in urls: + try: + Files._download_single_url( + url, output_folder, skip_if_downloaded_already, protocol, + ) + except Exception as exc: # pylint: disable=broad-except + logging.error("Failed to download %s: %s", url, exc) + failures.append((url, str(exc))) + else: + logging.info( + "Downloading %d URL(s) with %d parallel workers", + len(urls), parallel_files, + ) + with ThreadPoolExecutor(max_workers=parallel_files) as executor: + futures = { + executor.submit( + Files._download_single_url, + url, output_folder, skip_if_downloaded_already, protocol, + position=idx, + ): url + for idx, url in enumerate(urls) + } + for future in as_completed(futures): + url = futures[future] + try: + future.result() + except Exception as exc: # pylint: disable=broad-except + logging.error("Failed to download %s: %s", url, exc) + failures.append((url, str(exc))) + + if failures: + summary = ", ".join(f"{u} ({e})" for u, e in failures) + raise RuntimeError( + f"Failed to download {len(failures)} URL(s): {summary}" + ) + + if checksum_check: + _validate_urls_checksums(urls, output_folder) diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 878b4c1..8393519 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -1,17 +1,12 @@ #!/usr/bin/env python -import ftplib import logging import os -import re import urllib import urllib.request -from concurrent.futures import ThreadPoolExecutor, as_completed from ftplib import FTP from typing import Dict, List, Optional, Tuple -from urllib.parse import urlparse import requests -from tqdm import tqdm from pridepy.util.api_handling import Util @@ -726,14 +721,9 @@ def download_files_by_list( @staticmethod def _extract_pride_accession(url: str) -> Optional[str]: - """Extract a PRIDE accession (PXD/PRD followed by digits) from a URL path. - - PRIDE archive URLs follow the pattern - ``…/pride/data/archive/YYYY/MM//filename``. - Returns ``None`` when no accession can be identified. - """ - match = re.search(r"((?:PXD|PRD)\d{4,})", url) - return match.group(1) if match else None + """Shim — see :func:`pridepy.commands.by_url._extract_pride_accession`.""" + from pridepy.commands import by_url + return by_url._extract_pride_accession(url) @staticmethod def download_files_by_url( @@ -744,116 +734,22 @@ def download_files_by_url( parallel_files: int = 1, checksum_check: bool = False, ) -> None: - """Download files from a list of raw URLs, dispatched by URL scheme. - - Supported schemes: ``http``, ``https``, ``ftp``. Each URL is downloaded - independently; per-URL errors are logged, then aggregated and re-raised - as a single :class:`RuntimeError` so callers see a complete failure - summary. - - :param urls: fully-qualified URLs (each contains its scheme) - :param output_folder: directory to write downloaded files into - :param skip_if_downloaded_already: skip URLs whose target file exists - :param protocol: ``ftp`` (default) for single-connection per URL scheme; - ``globus`` for resume-capable http/https downloads (single-connection stream) - (no effect on ftp:// URLs which always use single-connection FTP) - :param checksum_check: validate downloads against PRIDE checksum API; - accessions are inferred from URL paths (only PRIDE URLs supported) - :raises ValueError: if ``urls`` is empty - :raises RuntimeError: if one or more URLs failed - """ - if not urls: - raise ValueError("urls must contain at least one URL") - - os.makedirs(output_folder, exist_ok=True) - - parallel_files = min(parallel_files, 3, len(urls)) - failures: List[Tuple[str, str]] = [] - if parallel_files < 2: - for url in urls: - try: - Files._download_single_url( - url, output_folder, skip_if_downloaded_already, protocol, - ) - except Exception as exc: # pylint: disable=broad-except - logging.error("Failed to download %s: %s", url, exc) - failures.append((url, str(exc))) - else: - logging.info( - "Downloading %d URL(s) with %d parallel workers", - len(urls), parallel_files, - ) - with ThreadPoolExecutor(max_workers=parallel_files) as executor: - futures = { - executor.submit( - Files._download_single_url, - url, output_folder, skip_if_downloaded_already, protocol, - position=idx, - ): url - for idx, url in enumerate(urls) - } - for future in as_completed(futures): - url = futures[future] - try: - future.result() - except Exception as exc: # pylint: disable=broad-except - logging.error("Failed to download %s: %s", url, exc) - failures.append((url, str(exc))) - - if failures: - summary = ", ".join(f"{u} ({e})" for u, e in failures) - raise RuntimeError( - f"Failed to download {len(failures)} URL(s): {summary}" - ) - - if checksum_check: - Files._validate_urls_checksums(urls, output_folder) + """Shim — see :func:`pridepy.commands.by_url.download_files_by_url`.""" + from pridepy.commands import by_url + return by_url.download_files_by_url( + urls=urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + ) @staticmethod def _validate_urls_checksums(urls: List[str], output_folder: str) -> None: - """Validate downloaded files against PRIDE checksum API. - - Accessions are inferred from URL paths via - :meth:`_extract_pride_accession`. URLs that do not contain a - recognisable PRIDE accession are skipped with a warning. - - :raises RuntimeError: if one or more files fail validation - """ - accession_urls: Dict[str, List[str]] = {} - for url in urls: - acc = Files._extract_pride_accession(url) - if acc: - accession_urls.setdefault(acc, []).append(url) - else: - logging.warning( - "Cannot infer PRIDE accession from URL, skipping checksum: %s", url - ) - - validation_failures: List[str] = [] - for acc, acc_urls in accession_urls.items(): - checksum_file_path = Files.save_checksum_file(acc, output_folder) - checksum_map = Files.read_checksum_file(checksum_file_path) - logging.info( - "Loaded checksums for %d files (project %s)", - len(checksum_map), acc, - ) - for url in acc_urls: - file_name = os.path.basename(urlparse(url).path) - target = os.path.join(output_folder, file_name) - expected = checksum_map.get(file_name) - logging.info("Validating %s", file_name) - valid, reason = Files.validate_download(target, expected) - if not valid: - logging.error("Validation failed for %s: %s", file_name, reason) - validation_failures.append(f"{file_name} ({reason})") - else: - logging.info("Checksum OK: %s", file_name) - - if validation_failures: - raise RuntimeError( - f"Checksum validation failed for {len(validation_failures)} file(s): " - + ", ".join(validation_failures) - ) + """Shim — see :func:`pridepy.commands.by_url._validate_urls_checksums`.""" + from pridepy.commands import by_url + return by_url._validate_urls_checksums(urls, output_folder) @staticmethod def _download_single_url( @@ -863,94 +759,27 @@ def _download_single_url( protocol: str = "ftp", position: int = 0, ) -> str: - """Download one URL, dispatched by scheme; return the local file path.""" - parsed = urlparse(url) - if not (parsed.scheme or "").lower(): - raise ValueError(f"URL missing scheme: {url}") - - file_name = os.path.basename(parsed.path) - if not file_name: - raise ValueError(f"Cannot derive filename from URL: {url}") - - target = os.path.join(output_folder, file_name) - if skip_if_exists and os.path.isfile(target) and os.path.getsize(target) > 0: - logging.info("Skipping %s: already downloaded", file_name) - return target - - Files._dispatch_url_scheme(parsed, target, protocol, position=position) - - ok, reason = Files.validate_download(target) - if not ok: - Files._remove_if_exists(target) - raise RuntimeError(f"Download invalid: {reason} ({target})") - return target + """Shim — see :func:`pridepy.commands.by_url._download_single_url`.""" + from pridepy.commands import by_url + return by_url._download_single_url(url, output_folder, skip_if_exists, protocol, position) @staticmethod def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: int = 0) -> None: - """Route a parsed URL to its protocol-specific downloader. - - ``protocol='globus'`` swaps the http/https single-connection streamer - for :meth:`_parallel_download` (single-connection with progress bar). - ftp:// URLs are unaffected. - """ - scheme = (parsed.scheme or "").lower() - if scheme in ("http", "https"): - if protocol == "globus": - Files._parallel_download(parsed.geturl(), target, position=position) - else: - Files._http_download_url(parsed.geturl(), target) - elif scheme == "ftp": - Files._ftp_download_url(parsed, target) - else: - raise ValueError(f"Unsupported URL scheme: {scheme}") + """Shim — see :func:`pridepy.commands.by_url._dispatch_url_scheme`.""" + from pridepy.commands import by_url + return by_url._dispatch_url_scheme(parsed, target, protocol=protocol, position=position) @staticmethod def _http_download_url(url: str, target: str) -> None: - """Stream an http/https URL into ``target`` with a progress bar.""" - session = Util.create_session_with_retries() - with session.get(url, stream=True, timeout=60) as response: - response.raise_for_status() - total = int(response.headers.get("Content-Length", 0)) - with open(target, "wb") as out, tqdm( - total=total, - unit="B", - unit_scale=True, - desc=os.path.basename(target), - ) as pbar: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - out.write(chunk) - pbar.update(len(chunk)) + """Shim — see :func:`pridepy.commands.by_url._http_download_url`.""" + from pridepy.commands import by_url + return by_url._http_download_url(url, target) @staticmethod def _ftp_download_url(parsed, target: str) -> None: - """Download a single file from an ftp:// URL with a progress bar.""" - host = parsed.hostname - if not host: - raise ValueError(f"FTP URL missing host: {parsed.geturl()}") - port = parsed.port or 21 - user = parsed.username or "anonymous" - pwd = parsed.password or "anonymous@" - remote_path = parsed.path - with FTP() as ftp: - ftp.connect(host, port, timeout=60) - ftp.login(user, pwd) - try: - total = ftp.size(remote_path) or 0 - except ftplib.error_perm: - total = 0 - with open(target, "wb") as out, tqdm( - total=total, - unit="B", - unit_scale=True, - desc=os.path.basename(target), - ) as pbar: - - def _callback(data: bytes) -> None: - out.write(data) - pbar.update(len(data)) - - ftp.retrbinary(f"RETR {remote_path}", _callback) + """Shim — see :func:`pridepy.commands.by_url._ftp_download_url`.""" + from pridepy.commands import by_url + return by_url._ftp_download_url(parsed, target) def download_all_category_files( self, From e3d694b17d925799d52bc96bc84daa3af225f831 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 18:26:56 +0100 Subject: [PATCH 20/54] refactor(providers): move ProteomeXchange from commands/ to providers/ as a class ProteomeXchange behaves more like a provider than a command: it takes a PXD/PRD accession (or a ProteomeCentral URL) and returns file records, the same shape as the four other providers. Moving it to providers/ aligns the architecture. Changes: - New: pridepy/providers/proteomexchange.py with ProteomeXchangeProvider(Provider). It implements the full Provider interface (matches, list_files, download_files) plus a convenience method download_from_accession_or_url() for the download-px-raw-files CLI command's existing behaviour (skip_if_downloaded_already defaults to True, no parallel workers). - Deleted: pridepy/commands/proteomexchange.py. Its three functions (_normalize_px_xml_url, _parse_px_xml_for_raw_file_urls, download_px_raw_files) are now static/instance methods on the provider class. - Updated: Files shims for _normalize_px_xml_url, _parse_px_xml_for_raw_file_urls, and download_px_raw_files now delegate to ProteomeXchangeProvider. - Updated: commands/__init__.py docstring notes the move. Important: ProteomeXchangeProvider is NOT auto-registered with pridepy.providers.registry. PXD/PRD accessions continue to route through PrideProvider's V3 API path by default. ProteomeXchangeProvider is the explicit gateway for the cross-repository XML view, invoked via the download-px-raw-files CLI command and Files.download_px_raw_files. Full suite green at 68 passed, 4 skipped. No behaviour change for existing PXD downloads; download-px-raw-files keeps its XML-based listing flow exactly as before. --- pridepy/commands/__init__.py | 10 +- pridepy/commands/proteomexchange.py | 94 ------------- pridepy/files/files.py | 20 +-- pridepy/providers/proteomexchange.py | 192 +++++++++++++++++++++++++++ 4 files changed, 212 insertions(+), 104 deletions(-) delete mode 100644 pridepy/commands/proteomexchange.py create mode 100644 pridepy/providers/proteomexchange.py diff --git a/pridepy/commands/__init__.py b/pridepy/commands/__init__.py index c94f89e..f1312b8 100644 --- a/pridepy/commands/__init__.py +++ b/pridepy/commands/__init__.py @@ -5,7 +5,15 @@ - ``by_url``: download a list of explicit URLs (ftp/http/https) - ``by_list``: download a subset of a project's files by filename -- ``proteomexchange``: download raw files from a ProteomeXchange XML + +ProteomeXchange used to live here too but moved to +:class:`pridepy.providers.proteomexchange.ProteomeXchangeProvider` because +it conforms to the ``Provider`` interface (takes an accession or URL and +returns file records). It is deliberately not auto-registered with the +provider registry — PXD/PRD accessions continue to route through +:class:`pridepy.providers.pride.PrideProvider`; ProteomeXchangeProvider is +the explicit gateway for the cross-repository XML view, invoked via the +``download-px-raw-files`` CLI command and ``Files.download_px_raw_files``. The ``pridepy.files.files.Files`` facade keeps shim methods that delegate here, so existing test patches on ``Files.X`` keep working. diff --git a/pridepy/commands/proteomexchange.py b/pridepy/commands/proteomexchange.py deleted file mode 100644 index d86cd24..0000000 --- a/pridepy/commands/proteomexchange.py +++ /dev/null @@ -1,94 +0,0 @@ -"""ProteomeXchange XML download command. - -Given a PXD accession or a ProteomeXchange XML URL, parse the XML for -``Associated raw file URI`` cvParams and download each one over its -native scheme (ftp:// via FTP, http(s):// via HTTPS). -""" -import logging -import os -import xml.etree.ElementTree as ET -from typing import List -from urllib.parse import urlparse - -from pridepy.util.api_handling import Util - - -def _normalize_px_xml_url(px_id_or_url: str) -> str: - """ - Build the ProteomeXchange XML endpoint from a dataset accession or a dataset web URL. - Examples accepted: - - PXD039236 - - https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236 - - https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236&anything - """ - if px_id_or_url.startswith("http://") or px_id_or_url.startswith("https://"): - parsed = urlparse(px_id_or_url) - # keep the ID param value if present; otherwise fallback to the path tail - query = parsed.query or "" - if "ID=" in query: - id_value = [q.split("=", 1)[1] for q in query.split("&") if q.startswith("ID=")] - if id_value: - return ( - f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID={id_value[0]}&outputMode=XML&test=no" - ) - # If the input URL already requests XML, just ensure flags - if parsed.path.endswith("/cgi/GetDataset"): - return ( - f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?{query}&outputMode=XML&test=no" - ) - # Assume it's a plain accession if not a URL - return ( - f"https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID={px_id_or_url}&outputMode=XML&test=no" - ) - - -def _parse_px_xml_for_raw_file_urls(px_xml_url: str) -> List[str]: - """ - Parse the PX XML and return a list of associated raw file URIs. - We extract cvParam with name "Associated raw file URI" under each DatasetFile. - """ - headers = {"Accept": "application/xml"} - response = Util.get_api_call(px_xml_url, headers) - response.raise_for_status() - root = ET.fromstring(response.content) - - urls: List[str] = [] - # The XML namespace is often absent in PX XML; access elements directly - for dataset_file in root.iter("DatasetFile"): - for cv in dataset_file.findall("cvParam"): - name = cv.attrib.get("name") - value = cv.attrib.get("value") - if name == "Associated raw file URI" and value: - urls.append(value) - return urls - - -def download_px_raw_files( - px_id_or_url: str, - output_folder: str, - skip_if_downloaded_already: bool = True, -) -> None: - """Download all raw files referenced by a ProteomeXchange dataset. - - Prefers FTP when the URL is ftp://, otherwise uses HTTP(S). Supports - resume and skip. - """ - from pridepy.files.files import Files # lazy: avoid module-load cycle - - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - px_xml_url = _normalize_px_xml_url(px_id_or_url) - logging.info(f"Fetching PX XML: {px_xml_url}") - urls = _parse_px_xml_for_raw_file_urls(px_xml_url) - if not urls: - logging.info("No Associated raw file URIs found in PX XML") - return - - ftp_urls = [u for u in urls if u.lower().startswith("ftp://")] - http_urls = [u for u in urls if u.lower().startswith(("http://", "https://"))] - - if ftp_urls: - Files.download_ftp_urls(ftp_urls, output_folder, skip_if_downloaded_already) - if http_urls: - Files.download_http_urls(http_urls, output_folder, skip_if_downloaded_already) diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 8393519..b46812e 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -844,15 +844,15 @@ def get_all_category_file_list( @staticmethod def _normalize_px_xml_url(px_id_or_url: str) -> str: - """Shim — see :func:`pridepy.commands.proteomexchange._normalize_px_xml_url`.""" - from pridepy.commands import proteomexchange - return proteomexchange._normalize_px_xml_url(px_id_or_url) + """Shim — see :meth:`pridepy.providers.proteomexchange.ProteomeXchangeProvider._normalize_px_xml_url`.""" + from pridepy.providers.proteomexchange import ProteomeXchangeProvider + return ProteomeXchangeProvider._normalize_px_xml_url(px_id_or_url) @staticmethod def _parse_px_xml_for_raw_file_urls(px_xml_url: str): - """Shim — see :func:`pridepy.commands.proteomexchange._parse_px_xml_for_raw_file_urls`.""" - from pridepy.commands import proteomexchange - return proteomexchange._parse_px_xml_for_raw_file_urls(px_xml_url) + """Shim — see :meth:`pridepy.providers.proteomexchange.ProteomeXchangeProvider._parse_px_xml_for_raw_file_urls`.""" + from pridepy.providers.proteomexchange import ProteomeXchangeProvider + return ProteomeXchangeProvider._parse_px_xml_for_raw_file_urls(px_xml_url) def download_px_raw_files( self, @@ -860,9 +860,11 @@ def download_px_raw_files( output_folder: str, skip_if_downloaded_already: bool = True, ) -> None: - """Shim — see :func:`pridepy.commands.proteomexchange.download_px_raw_files`.""" - from pridepy.commands import proteomexchange - return proteomexchange.download_px_raw_files(px_id_or_url, output_folder, skip_if_downloaded_already) + """Shim — see :meth:`pridepy.providers.proteomexchange.ProteomeXchangeProvider.download_from_accession_or_url`.""" + from pridepy.providers.proteomexchange import ProteomeXchangeProvider + return ProteomeXchangeProvider().download_from_accession_or_url( + px_id_or_url, output_folder, skip_if_downloaded_already + ) @staticmethod def _local_path_for_url(download_url: str, output_folder: str) -> str: diff --git a/pridepy/providers/proteomexchange.py b/pridepy/providers/proteomexchange.py new file mode 100644 index 0000000..cef0524 --- /dev/null +++ b/pridepy/providers/proteomexchange.py @@ -0,0 +1,192 @@ +"""ProteomeXchange provider. + +ProteomeXchange is a meta-repository: a PXD/PRD accession routes through +the cross-repository XML at ``proteomecentral.proteomexchange.org``, and +the XML's ``Associated raw file URI`` cvParams point at the actual hosting +repository (PRIDE / MassIVE / JPOST / iProX / etc.). + +Unlike the other providers in this package, ``ProteomeXchangeProvider`` is +NOT auto-registered with :mod:`pridepy.providers.registry`. PXD/PRD +accessions would otherwise be ambiguous between PRIDE's V3 API listing and +ProteomeXchange's XML listing; the registry continues to route PXD/PRD via +:class:`pridepy.providers.pride.PrideProvider`. ``ProteomeXchangeProvider`` +is the explicit gateway invoked by the ``download-px-raw-files`` CLI +command and by ``Files.download_px_raw_files`` — callers who specifically +want the cross-repository XML view. + +The class accepts either: + +- a plain accession (``PXD039236``) +- a ProteomeCentral dataset URL (``https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=...``) + +…and resolves it to the XML endpoint via :meth:`_normalize_px_xml_url`. +""" +import logging +import os +import re +import xml.etree.ElementTree as ET +from typing import ClassVar, Dict, List, Optional +from urllib.parse import urlparse + +from pridepy.providers.base import Provider +from pridepy.util.api_handling import Util + + +class ProteomeXchangeProvider(Provider): + name: ClassVar[str] = "proteomexchange" + + @staticmethod + def matches(accession: str) -> bool: + """Return True for PXD/PRD accessions or ProteomeCentral URLs. + + Not used by :mod:`pridepy.providers.registry` (this provider is + deliberately not auto-registered). Provided for parity with the + ``Provider`` interface and so direct callers can introspect whether + a given input looks like something ProteomeXchange knows how to + handle. + """ + if not accession: + return False + if accession.lower().startswith(("http://", "https://")): + return "proteomexchange" in accession.lower() or "cgi/GetDataset" in accession + return bool(re.fullmatch(r"(?:PXD|PRD)\d+", accession.upper())) + + @staticmethod + def _normalize_px_xml_url(px_id_or_url: str) -> str: + """Build the ProteomeXchange XML endpoint URL from an accession or URL. + + Examples accepted: + - ``PXD039236`` + - ``https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236`` + - ``https://proteomecentral.proteomexchange.org/cgi/GetDataset?ID=PXD039236&anything`` + """ + if px_id_or_url.startswith("http://") or px_id_or_url.startswith("https://"): + parsed = urlparse(px_id_or_url) + query = parsed.query or "" + if "ID=" in query: + id_value = [ + q.split("=", 1)[1] for q in query.split("&") if q.startswith("ID=") + ] + if id_value: + return ( + "https://proteomecentral.proteomexchange.org/cgi/GetDataset" + f"?ID={id_value[0]}&outputMode=XML&test=no" + ) + if parsed.path.endswith("/cgi/GetDataset"): + return ( + "https://proteomecentral.proteomexchange.org/cgi/GetDataset" + f"?{query}&outputMode=XML&test=no" + ) + return ( + "https://proteomecentral.proteomexchange.org/cgi/GetDataset" + f"?ID={px_id_or_url}&outputMode=XML&test=no" + ) + + @staticmethod + def _parse_px_xml_for_raw_file_urls(px_xml_url: str) -> List[str]: + """Fetch the PX XML and return every ``Associated raw file URI`` value.""" + headers = {"Accept": "application/xml"} + response = Util.get_api_call(px_xml_url, headers) + response.raise_for_status() + root = ET.fromstring(response.content) + + urls: List[str] = [] + for dataset_file in root.iter("DatasetFile"): + for cv in dataset_file.findall("cvParam"): + name = cv.attrib.get("name") + value = cv.attrib.get("value") + if name == "Associated raw file URI" and value: + urls.append(value) + return urls + + def list_files(self, accession: str) -> List[Dict]: + """Return the dataset's raw-file URIs as minimal file records. + + The PX XML doesn't expose checksums or rich category labels, so + each record carries just enough to drive the downloader. + """ + px_xml_url = self._normalize_px_xml_url(accession) + logging.info(f"Fetching PX XML: {px_xml_url}") + urls = self._parse_px_xml_for_raw_file_urls(px_xml_url) + records: List[Dict] = [] + for url in urls: + parsed = urlparse(url) + records.append( + { + "accession": accession, + "fileName": os.path.basename(parsed.path), + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": url} + ], + "source": "ProteomeXchange", + } + ) + return records + + def download_files( + self, + accession: str, + records: List[Dict], + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str, + parallel_files: int = 1, + checksum_check: bool = False, + aspera_maximum_bandwidth: str = "100M", + username: Optional[str] = None, + password: Optional[str] = None, + ) -> None: + """Partition record URLs by scheme and route to the matching transport. + + Routes ftp:// records to :meth:`Files.download_ftp_urls` and + http(s):// records to :meth:`Files.download_http_urls`, going + through the Files facade so test patches like + ``patch.object(Files, "download_ftp_urls")`` continue to intercept. + """ + from pridepy.files.files import Files # lazy: avoid module-load cycle + + if not os.path.isdir(output_folder): + os.makedirs(output_folder, exist_ok=True) + + urls = [ + record["publicFileLocations"][0]["value"] + for record in records + if record.get("publicFileLocations") + ] + ftp_urls = [u for u in urls if u.lower().startswith("ftp://")] + http_urls = [u for u in urls if u.lower().startswith(("http://", "https://"))] + + if ftp_urls: + Files.download_ftp_urls( + ftp_urls, output_folder, skip_if_downloaded_already + ) + if http_urls: + Files.download_http_urls( + http_urls, output_folder, skip_if_downloaded_already + ) + + def download_from_accession_or_url( + self, + px_id_or_url: str, + output_folder: str, + skip_if_downloaded_already: bool = True, + ) -> None: + """End-to-end: resolve XML, list files, partition by scheme, download. + + Convenience for the ``download-px-raw-files`` CLI command — combines + :meth:`list_files` and :meth:`download_files` with the original + ``download_px_raw_files`` defaults (skip-if-downloaded-already + defaults to ``True``, no parallel workers). + """ + records = self.list_files(px_id_or_url) + if not records: + logging.info("No Associated raw file URIs found in PX XML") + return + self.download_files( + accession=px_id_or_url, + records=records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol="ftp", + ) From 6bc8e5a61bca33fb79a57e8438b773c1defe9ab4 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 18:56:14 +0100 Subject: [PATCH 21/54] fix(files): restore missing imports and hoist lazy imports to module top MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related fixes: 1. CI lint failure: files.py used importlib.resources, subprocess, and time but never imported them. The Task 8 refactor removed these top-level imports along with the PRIDE-specific code that used them in PrideProvider, but two methods on Files (download_files_from_aspera and the now-dead _download_range) still referenced the undefined names. Flake8 with --select=F82 fails on 4 F821 undefined-name errors. Fix: re-add the three stdlib imports at the top of files.py. Also delete the orphaned _download_range method (no callers). 2. Hoist lazy provider/command imports out of Files method bodies. The shim pattern previously did 'from pridepy.providers import X' inside every method body — ~75 occurrences. Since providers do not import Files at module load (only inside method bodies), the reverse direction is safe to hoist: Files now imports {registry, transport, util, IproxProvider, JpostProvider, MassiveProvider, PrideProvider, ProteomeXchangeProvider, by_list, by_url} at module top, and the shims reference these names directly. Lazy imports inside provider/command method bodies that go back to Files (e.g. BaseDirectDownloadProvider.download_files doing 'from pridepy.files.files import Files') are kept lazy — they are genuinely cyclic and required for backward-compat test patching. Also: commands/by_list.py's 'from pridepy.providers import registry' hoisted to module top (no Files dependency, no cycle risk). Note: 'import requests' is kept in files.py (noqa: F401) because test suites patch 'pridepy.files.files.requests.get' directly. Tests: 68 passed, 4 skipped. flake8 --select=E9,F63,F7,F82 now clean. --- pridepy/commands/by_list.py | 3 +- pridepy/files/files.py | 203 +++++++++++------------------------- 2 files changed, 61 insertions(+), 145 deletions(-) diff --git a/pridepy/commands/by_list.py b/pridepy/commands/by_list.py index e008d6e..d2d0244 100644 --- a/pridepy/commands/by_list.py +++ b/pridepy/commands/by_list.py @@ -2,6 +2,8 @@ import logging from typing import List, Optional +from pridepy.providers import registry + def download_files_by_list( accession: str, @@ -32,7 +34,6 @@ def download_files_by_list( if not file_names: raise ValueError("file_names must contain at least one filename") - from pridepy.providers import registry # lazy provider = registry.resolve(accession) all_files = provider.list_files(accession) diff --git a/pridepy/files/files.py b/pridepy/files/files.py index b46812e..3567e78 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -1,58 +1,73 @@ #!/usr/bin/env python +import importlib.resources import logging import os +import subprocess import urllib import urllib.request from ftplib import FTP from typing import Dict, List, Optional, Tuple -import requests +import requests # noqa: F401 — kept as a patch target for tests from pridepy.util.api_handling import Util - -# Re-export from providers.util so external `from pridepy.files.files import Progress` +# Module-level imports of the modular architecture. Providers and commands +# do not import Files at module level (only lazily inside method bodies), +# so hoisting these to the top is safe and avoids cluttering every shim +# method body with a local import. +from pridepy.providers import registry, transport +from pridepy.providers import util as _provider_util +from pridepy.providers.iprox import IproxProvider +from pridepy.providers.jpost import JpostProvider +from pridepy.providers.massive import MASSIVE_CATEGORY_MAP, MassiveProvider +from pridepy.providers.pride import PrideProvider +from pridepy.providers.proteomexchange import ProteomeXchangeProvider +from pridepy.commands import by_list, by_url + +# Re-export Progress so external `from pridepy.files.files import Progress` # still works. from pridepy.providers.util import Progress # noqa: F401 class Files: """ - This class handles PRIDE API files endpoint. + This class handles PRIDE API files endpoint, and dispatches to the + per-repository provider classes in :mod:`pridepy.providers`. """ - # Re-exported from providers/pride.py — kept here for back-compat. - from pridepy.providers.pride import PrideProvider as _PrideProvider - V3_API_BASE_URL = _PrideProvider.V3_API_BASE_URL - API_BASE_URL = _PrideProvider.API_BASE_URL - API_PRIVATE_URL = _PrideProvider.API_PRIVATE_URL - PRIDE_ARCHIVE_FTP = _PrideProvider.ARCHIVE_FTP - PRIDE_ARCHIVE_FTP_URL_PREFIX = _PrideProvider.ARCHIVE_FTP_URL_PREFIX - PRIDE_ARCHIVE_HTTPS_URL_PREFIX = _PrideProvider.ARCHIVE_HTTPS_URL_PREFIX - S3_URL = _PrideProvider.S3_URL - S3_BUCKET = _PrideProvider.S3_BUCKET - PROTOCOL_ORDER = _PrideProvider.PROTOCOL_ORDER - del _PrideProvider - # Re-exported from providers/massive.py — kept here for back-compat. - from pridepy.providers.massive import ( # noqa: E402 - MASSIVE_CATEGORY_MAP as _MASSIVE_CATEGORY_MAP, - MassiveProvider as _MassiveProvider, - ) - MASSIVE_CATEGORY_MAP = _MASSIVE_CATEGORY_MAP - MASSIVE_ARCHIVE_FTP = _MassiveProvider.ARCHIVE_FTP - MASSIVE_ARCHIVE_FTP_URL_PREFIX = _MassiveProvider.ARCHIVE_FTP_URL_PREFIX - del _MASSIVE_CATEGORY_MAP, _MassiveProvider - from pridepy.providers.jpost import JpostProvider as _JpostProvider - JPOST_ARCHIVE_FTP = _JpostProvider.ARCHIVE_FTP - JPOST_ARCHIVE_FTP_URL_PREFIX = _JpostProvider.ARCHIVE_FTP_URL_PREFIX - JPOST_PROXI_BASE_URL = _JpostProvider.PROXI_BASE_URL - JPOST_PROXI_CATEGORY_MAP = _JpostProvider.PROXI_CATEGORY_MAP - del _JpostProvider - from pridepy.providers.iprox import IproxProvider as _IproxProvider - IPROX_DOWNLOAD_BASE_URL = _IproxProvider.DOWNLOAD_BASE_URL - IPROX_PX_XML_URL_TEMPLATE = _IproxProvider.PX_XML_URL_TEMPLATE - IPROX_PX_CATEGORY_MAP = _IproxProvider.PX_CATEGORY_MAP - del _IproxProvider + # PRIDE class-attribute re-exports (kept here for back-compat). + V3_API_BASE_URL = PrideProvider.V3_API_BASE_URL + API_BASE_URL = PrideProvider.API_BASE_URL + API_PRIVATE_URL = PrideProvider.API_PRIVATE_URL + PRIDE_ARCHIVE_FTP = PrideProvider.ARCHIVE_FTP + PRIDE_ARCHIVE_FTP_URL_PREFIX = PrideProvider.ARCHIVE_FTP_URL_PREFIX + PRIDE_ARCHIVE_HTTPS_URL_PREFIX = PrideProvider.ARCHIVE_HTTPS_URL_PREFIX + S3_URL = PrideProvider.S3_URL + S3_BUCKET = PrideProvider.S3_BUCKET + PROTOCOL_ORDER = PrideProvider.PROTOCOL_ORDER + + # MassIVE class-attribute re-exports. + MASSIVE_ARCHIVE_FTP = MassiveProvider.ARCHIVE_FTP + MASSIVE_ARCHIVE_FTP_URL_PREFIX = MassiveProvider.ARCHIVE_FTP_URL_PREFIX + # Note: MASSIVE_CATEGORY_MAP is the module-level constant in providers/massive.py, + # re-exported on Files as a class attribute via the module-level import above. + + # JPOST class-attribute re-exports. + JPOST_ARCHIVE_FTP = JpostProvider.ARCHIVE_FTP + JPOST_ARCHIVE_FTP_URL_PREFIX = JpostProvider.ARCHIVE_FTP_URL_PREFIX + JPOST_PROXI_BASE_URL = JpostProvider.PROXI_BASE_URL + JPOST_PROXI_CATEGORY_MAP = JpostProvider.PROXI_CATEGORY_MAP + + # iProX class-attribute re-exports. + IPROX_DOWNLOAD_BASE_URL = IproxProvider.DOWNLOAD_BASE_URL + IPROX_PX_XML_URL_TEMPLATE = IproxProvider.PX_XML_URL_TEMPLATE + IPROX_PX_CATEGORY_MAP = IproxProvider.PX_CATEGORY_MAP + + # MassIVE category map re-exported. Class attribute shadowing the module-level + # constant of the same name happens cleanly in class scope. + MASSIVE_CATEGORY_MAP = MASSIVE_CATEGORY_MAP + logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") def __init__(self): @@ -61,118 +76,97 @@ def __init__(self): @staticmethod def _find_tsv_columns(header: str) -> Optional[Tuple[int, int]]: """Shim — see :func:`pridepy.providers.util._find_tsv_columns`.""" - from pridepy.providers import util - return util._find_tsv_columns(header) + return _provider_util._find_tsv_columns(header) @staticmethod def _is_md5_checksum(value: str) -> bool: """Shim — see :func:`pridepy.providers.util._is_md5_checksum`.""" - from pridepy.providers import util - return util._is_md5_checksum(value) + return _provider_util._is_md5_checksum(value) @staticmethod def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: """Shim — see :func:`pridepy.providers.util.read_checksum_file`.""" - from pridepy.providers import util - return util.read_checksum_file(checksum_file_path) + return _provider_util.read_checksum_file(checksum_file_path) @staticmethod def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: """Shim — see :func:`pridepy.providers.util.compute_md5`.""" - from pridepy.providers import util - return util.compute_md5(file_path, chunk_size) + return _provider_util.compute_md5(file_path, chunk_size) @staticmethod def validate_download(file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, str]: """Shim — see :func:`pridepy.providers.util.validate_download`.""" - from pridepy.providers import util - return util.validate_download(file_path, expected_checksum) + return _provider_util.validate_download(file_path, expected_checksum) @staticmethod def _remove_if_exists(file_path: str) -> None: """Shim — see :func:`pridepy.providers.util._remove_if_exists`.""" - from pridepy.providers import util - return util._remove_if_exists(file_path) + return _provider_util._remove_if_exists(file_path) @staticmethod def _get_download_url(file_record: Dict, protocol: str) -> str: """Shim — see :func:`pridepy.providers.util._get_download_url`.""" - from pridepy.providers import util - return util._get_download_url(file_record, protocol) + return _provider_util._get_download_url(file_record, protocol) @staticmethod def _resolve_local_path(file_record: Dict, output_folder: str) -> str: """Shim — see :func:`pridepy.providers.util._resolve_local_path`.""" - from pridepy.providers import util - return util._resolve_local_path(file_record, output_folder) + return _provider_util._resolve_local_path(file_record, output_folder) @staticmethod def _protocol_sequence(protocol: str) -> List[str]: """Shim — see :meth:`pridepy.providers.pride.PrideProvider._protocol_sequence`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider._protocol_sequence(protocol) @staticmethod def is_massive_accession(accession: str) -> bool: """Shim — see :meth:`pridepy.providers.massive.MassiveProvider.matches`.""" - from pridepy.providers.massive import MassiveProvider return MassiveProvider.matches(accession) @staticmethod def _get_massive_public_root(accession: str) -> str: - from pridepy.providers.massive import MassiveProvider return MassiveProvider._get_public_root(accession) @staticmethod def _get_massive_public_ftp_url(accession: str, remote_path: str) -> str: - from pridepy.providers.massive import MassiveProvider return MassiveProvider._get_public_ftp_url(accession, remote_path) @staticmethod def _map_massive_collection_to_category(collection: str) -> str: - from pridepy.providers.massive import MassiveProvider return MassiveProvider._map_collection_to_category(collection) @staticmethod def _build_massive_file_record(accession: str, ftp_url: str) -> Dict: - from pridepy.providers.massive import MassiveProvider return MassiveProvider._build_file_record(accession, ftp_url) @staticmethod def is_jpost_accession(accession: str) -> bool: """Shim — see :meth:`pridepy.providers.jpost.JpostProvider.matches`.""" - from pridepy.providers.jpost import JpostProvider return JpostProvider.matches(accession) @staticmethod def _get_jpost_public_root(accession: str) -> str: - from pridepy.providers.jpost import JpostProvider return JpostProvider._get_public_root(accession) @staticmethod def _get_jpost_public_ftp_url(accession: str, remote_path: str) -> str: - from pridepy.providers.jpost import JpostProvider return JpostProvider._get_public_ftp_url(accession, remote_path) @staticmethod def _build_jpost_file_record(accession, ftp_url, category_from_proxi=None): - from pridepy.providers.jpost import JpostProvider return JpostProvider._build_file_record(accession, ftp_url, category_from_proxi) @staticmethod def _build_iprox_file_record(accession, https_url, category_from_px=None): """Shim — see :meth:`pridepy.providers.iprox.IproxProvider._build_file_record`.""" - from pridepy.providers.iprox import IproxProvider return IproxProvider._build_file_record(accession, https_url, category_from_px) @staticmethod def _get_iprox_public_root(accession: str) -> str: - from pridepy.providers.iprox import IproxProvider return IproxProvider._get_public_root(accession) @staticmethod def _get_iprox_public_ftp_url(accession: str, remote_path: str) -> str: - from pridepy.providers.iprox import IproxProvider return IproxProvider._get_public_ftp_url(accession, remote_path) @staticmethod @@ -184,7 +178,6 @@ def is_direct_download_accession(accession: str) -> bool: validation and fallback), not the direct-download partitioned-by-URL- scheme path. So we filter PRIDE out here. """ - from pridepy.providers import registry try: provider = registry.resolve(accession) except ValueError: @@ -194,13 +187,11 @@ def is_direct_download_accession(accession: str) -> bool: @staticmethod def is_iprox_accession(accession: str) -> bool: """Shim — see :meth:`pridepy.providers.iprox.IproxProvider.matches`.""" - from pridepy.providers.iprox import IproxProvider return IproxProvider.matches(accession) @staticmethod def _repo_uses_tls(accession: str) -> bool: """Shim — returns the resolved provider's use_tls flag (False if unknown).""" - from pridepy.providers import registry try: provider = registry.resolve(accession) except ValueError: @@ -210,24 +201,20 @@ def _repo_uses_tls(accession: str) -> bool: @staticmethod def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: """Shim — see :func:`pridepy.providers.transport._walk_ftp_tree`.""" - from pridepy.providers import transport return transport._walk_ftp_tree(ftp=ftp, remote_dir=remote_dir) @staticmethod def _open_ftp_connection(host: str, use_tls: bool, timeout: int = 30) -> FTP: """Shim — see :func:`pridepy.providers.transport._open_ftp_connection`.""" - from pridepy.providers import transport return transport._open_ftp_connection(host=host, use_tls=use_tls, timeout=timeout) @staticmethod def _list_ftp_repo_files(host, remote_root, error_label, use_tls=False): """Shim — see :func:`pridepy.providers.transport._list_ftp_repo_files`.""" - from pridepy.providers import transport return transport._list_ftp_repo_files(host=host, remote_root=remote_root, error_label=error_label, use_tls=use_tls) def _list_massive_public_files(self, accession: str) -> List[Dict]: """Shim — see :meth:`pridepy.providers.massive.MassiveProvider.list_files`.""" - from pridepy.providers.massive import MassiveProvider return MassiveProvider().list_files(accession) def _download_massive_file_records( @@ -243,7 +230,6 @@ def _download_massive_file_records( Download public MassIVE files via anonymous FTP (now FTPS). Backward-compat shim — dispatches via the provider registry. """ - from pridepy.providers import registry registry.resolve(accession).download_files( accession=accession, records=file_records, @@ -261,7 +247,6 @@ def _list_jpost_public_files(self, accession: str) -> List[Dict]: test patches on ``_list_jpost_public_files_via_proxi`` and ``_list_ftp_repo_files`` continue to intercept. """ - from pridepy.providers.jpost import JpostProvider normalized_accession = accession.upper() try: return self._list_jpost_public_files_via_proxi(normalized_accession) @@ -286,23 +271,18 @@ def _list_jpost_public_files(self, accession: str) -> List[Dict]: def _list_jpost_public_files_via_proxi(self, accession: str) -> List[Dict]: """Shim — see :meth:`pridepy.providers.jpost.JpostProvider._list_via_proxi`.""" - from pridepy.providers.jpost import JpostProvider return JpostProvider()._list_via_proxi(accession) def _list_iprox_public_files(self, accession: str) -> List[Dict]: """Shim — see :meth:`pridepy.providers.iprox.IproxProvider.list_files`.""" - from pridepy.providers.iprox import IproxProvider return IproxProvider().list_files(accession) - async def stream_all_files_metadata(self, output_file, accession=None): """Shim — see :meth:`pridepy.providers.pride.PrideProvider.stream_all_files_metadata`.""" - from pridepy.providers.pride import PrideProvider return await PrideProvider().stream_all_files_metadata(output_file, accession) def stream_all_files_by_project(self, accession) -> List[Dict]: """Shim — see :meth:`pridepy.providers.pride.PrideProvider.stream_all_files_by_project`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider().stream_all_files_by_project(accession) def get_all_raw_file_list(self, project_accession): @@ -310,7 +290,6 @@ def get_all_raw_file_list(self, project_accession): Returns the dataset's file records filtered to fileCategory == "RAW". """ - from pridepy.providers import registry provider = registry.resolve(project_accession) records = provider.list_files(project_accession) return [r for r in records if r["fileCategory"]["value"] == "RAW"] @@ -328,7 +307,6 @@ def download_all_raw_files( """Download all RAW files for any registered provider.""" if not os.path.isdir(output_folder): os.mkdir(output_folder) - from pridepy.providers import registry provider = registry.resolve(accession) records = self.get_all_raw_file_list(accession) provider.download_files( @@ -351,7 +329,6 @@ def download_files_from_ftp( max_download_retries=3, ): """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files_from_ftp`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider.download_files_from_ftp( file_list_json, output_folder, @@ -422,45 +399,14 @@ def download_files_from_aspera( except subprocess.CalledProcessError as e: logging.error(f"Aspera download failed for {new_file_path}: {str(e)}") - @staticmethod - def _download_range(url, file_path, start, end, pbar, max_retries=3): - """Download a byte range directly into the target file using seek.""" - for attempt in range(1, max_retries + 1): - try: - session = Util.create_session_with_retries() - headers = {"Range": f"bytes={start}-{end}"} - with session.get(url, headers=headers, stream=True, timeout=(15, 15)) as r: - r.raise_for_status() - if r.status_code != 206: - raise RuntimeError(f"Server did not honor Range request: {r.status_code}") - content_range = r.headers.get("Content-Range", "") - if not content_range.lower().startswith(f"bytes {start}-{end}/"): - raise RuntimeError(f"Unexpected Content-Range header: {content_range}") - with open(file_path, "r+b") as f: - f.seek(start) - for chunk in r.iter_content(chunk_size=8 * 1024 * 1024): - if chunk: - f.write(chunk) - pbar.update(len(chunk)) - return - except (requests.RequestException, RuntimeError, OSError) as exc: - logging.warning( - f"Range {start}-{end} attempt {attempt}/{max_retries} failed: {exc}" - ) - if attempt >= max_retries: - raise - time.sleep(2 * attempt) - @staticmethod def _parallel_download(url, file_path, position=0): """Shim — see :func:`pridepy.providers.transport._parallel_download`.""" - from pridepy.providers import transport return transport._parallel_download(url=url, file_path=file_path, position=position) @staticmethod def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_retries=6, position=0): """Shim — see :meth:`pridepy.providers.pride.PrideProvider._globus_download_one`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider._globus_download_one( file, output_folder, skip_if_downloaded_already, max_retries=max_retries, position=position, @@ -473,7 +419,6 @@ def download_files_from_globus( checksum_map: Optional[Dict[str, str]] = None, ): """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files_from_globus`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider.download_files_from_globus( file_list_json, output_folder, skip_if_downloaded_already, parallel_files=parallel_files, @@ -485,14 +430,12 @@ def download_files_from_s3( file_list_json: List[Dict], output_folder: str, skip_if_downloaded_already ): """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files_from_s3`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider.download_files_from_s3( file_list_json, output_folder, skip_if_downloaded_already, ) def get_submitted_file_path_prefix(self, accession): """Shim — see :meth:`pridepy.providers.pride.PrideProvider.get_submitted_file_path_prefix`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider().get_submitted_file_path_prefix(accession) def download_file_by_name( @@ -523,7 +466,6 @@ def download_file_by_name( if not os.path.isdir(output_folder): os.mkdir(output_folder) - from pridepy.providers import registry provider = registry.resolve(accession) ## Check type of project @@ -596,7 +538,6 @@ def get_file_from_api(self, accession, file_name) -> List[Dict]: :param file_name: file name :return: file in json format """ - from pridepy.providers import registry try: records = registry.resolve(accession).list_files(accession) return [r for r in records if r["fileName"] == file_name] @@ -605,7 +546,6 @@ def get_file_from_api(self, accession, file_name) -> List[Dict]: def download_private_file_name(self, accession, file_name, output_folder, username, password): """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_private_file_name`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider().download_private_file_name( accession, file_name, output_folder, username, password, ) @@ -613,13 +553,11 @@ def download_private_file_name(self, accession, file_name, output_folder, userna @staticmethod def get_ascp_binary(): """Shim — see :meth:`pridepy.providers.pride.PrideProvider.get_ascp_binary`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider.get_ascp_binary() @staticmethod def save_checksum_file(accession, output_folder): """Shim — see :meth:`pridepy.providers.pride.PrideProvider.save_checksum_file`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider.save_checksum_file(accession, output_folder) @staticmethod @@ -638,7 +576,6 @@ def _batch_download_by_protocol( :class:`PrideProvider` calls back through ``Files.X`` so those patches keep intercepting. """ - from pridepy.providers.pride import PrideProvider return PrideProvider._batch_download_by_protocol( file_list, output_folder, @@ -660,7 +597,6 @@ def _download_with_fallback( parallel_files: int = 1, ) -> bool: """Shim — see :meth:`pridepy.providers.pride.PrideProvider._download_with_fallback`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider._download_with_fallback( file_record, output_folder, @@ -683,7 +619,6 @@ def download_files( parallel_files: int = 1, ): """Shim — see :meth:`pridepy.providers.pride.PrideProvider._download_files_batch`.""" - from pridepy.providers.pride import PrideProvider return PrideProvider._download_files_batch( file_list_json, accession, @@ -707,7 +642,6 @@ def download_files_by_list( parallel_files: int = 1, ) -> None: """Shim — see :func:`pridepy.commands.by_list.download_files_by_list`.""" - from pridepy.commands import by_list return by_list.download_files_by_list( accession=accession, file_names=file_names, @@ -722,7 +656,6 @@ def download_files_by_list( @staticmethod def _extract_pride_accession(url: str) -> Optional[str]: """Shim — see :func:`pridepy.commands.by_url._extract_pride_accession`.""" - from pridepy.commands import by_url return by_url._extract_pride_accession(url) @staticmethod @@ -735,7 +668,6 @@ def download_files_by_url( checksum_check: bool = False, ) -> None: """Shim — see :func:`pridepy.commands.by_url.download_files_by_url`.""" - from pridepy.commands import by_url return by_url.download_files_by_url( urls=urls, output_folder=output_folder, @@ -748,7 +680,6 @@ def download_files_by_url( @staticmethod def _validate_urls_checksums(urls: List[str], output_folder: str) -> None: """Shim — see :func:`pridepy.commands.by_url._validate_urls_checksums`.""" - from pridepy.commands import by_url return by_url._validate_urls_checksums(urls, output_folder) @staticmethod @@ -760,25 +691,21 @@ def _download_single_url( position: int = 0, ) -> str: """Shim — see :func:`pridepy.commands.by_url._download_single_url`.""" - from pridepy.commands import by_url return by_url._download_single_url(url, output_folder, skip_if_exists, protocol, position) @staticmethod def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: int = 0) -> None: """Shim — see :func:`pridepy.commands.by_url._dispatch_url_scheme`.""" - from pridepy.commands import by_url return by_url._dispatch_url_scheme(parsed, target, protocol=protocol, position=position) @staticmethod def _http_download_url(url: str, target: str) -> None: """Shim — see :func:`pridepy.commands.by_url._http_download_url`.""" - from pridepy.commands import by_url return by_url._http_download_url(url, target) @staticmethod def _ftp_download_url(parsed, target: str) -> None: """Shim — see :func:`pridepy.commands.by_url._ftp_download_url`.""" - from pridepy.commands import by_url return by_url._ftp_download_url(parsed, target) def download_all_category_files( @@ -808,7 +735,6 @@ def download_all_category_files( if categories is None: categories = [category] if category else ["RAW"] records = self.get_all_category_file_list(accession, categories) - from pridepy.providers import registry provider = registry.resolve(accession) provider.download_files( accession=accession, @@ -834,7 +760,6 @@ def get_all_category_file_list( if isinstance(categories, str): categories = [categories] category_set = {c.upper() for c in categories} - from pridepy.providers import registry records = registry.resolve(accession).list_files(accession) return [r for r in records if r["fileCategory"]["value"] in category_set] @@ -845,13 +770,11 @@ def get_all_category_file_list( @staticmethod def _normalize_px_xml_url(px_id_or_url: str) -> str: """Shim — see :meth:`pridepy.providers.proteomexchange.ProteomeXchangeProvider._normalize_px_xml_url`.""" - from pridepy.providers.proteomexchange import ProteomeXchangeProvider return ProteomeXchangeProvider._normalize_px_xml_url(px_id_or_url) @staticmethod def _parse_px_xml_for_raw_file_urls(px_xml_url: str): """Shim — see :meth:`pridepy.providers.proteomexchange.ProteomeXchangeProvider._parse_px_xml_for_raw_file_urls`.""" - from pridepy.providers.proteomexchange import ProteomeXchangeProvider return ProteomeXchangeProvider._parse_px_xml_for_raw_file_urls(px_xml_url) def download_px_raw_files( @@ -861,7 +784,6 @@ def download_px_raw_files( skip_if_downloaded_already: bool = True, ) -> None: """Shim — see :meth:`pridepy.providers.proteomexchange.ProteomeXchangeProvider.download_from_accession_or_url`.""" - from pridepy.providers.proteomexchange import ProteomeXchangeProvider return ProteomeXchangeProvider().download_from_accession_or_url( px_id_or_url, output_folder, skip_if_downloaded_already ) @@ -869,7 +791,6 @@ def download_px_raw_files( @staticmethod def _local_path_for_url(download_url: str, output_folder: str) -> str: """Shim — see :func:`pridepy.providers.transport._local_path_for_url`.""" - from pridepy.providers import transport return transport._local_path_for_url(download_url=download_url, output_folder=output_folder) @staticmethod @@ -882,7 +803,6 @@ def _download_one_ftp_path( position: int = 0, ) -> None: """Shim — see :func:`pridepy.providers.transport._download_one_ftp_path`.""" - from pridepy.providers import transport return transport._download_one_ftp_path( ftp=ftp, ftp_path=ftp_path, @@ -903,7 +823,6 @@ def _download_ftp_paths_serial( max_download_retries: int, ) -> None: """Shim — see :func:`pridepy.providers.transport._download_ftp_paths_serial`.""" - from pridepy.providers import transport return transport._download_ftp_paths_serial( host=host, paths=paths, @@ -926,7 +845,6 @@ def _download_ftp_paths_parallel( parallel_files: int, ) -> None: """Shim — see :func:`pridepy.providers.transport._download_ftp_paths_parallel`.""" - from pridepy.providers import transport return transport._download_ftp_paths_parallel( host=host, paths=paths, @@ -949,7 +867,6 @@ def download_ftp_urls( parallel_files: int = 1, ) -> None: """Shim — see :func:`pridepy.providers.transport.download_ftp_urls`.""" - from pridepy.providers import transport return transport.download_ftp_urls( ftp_urls=ftp_urls, output_folder=output_folder, @@ -969,7 +886,6 @@ def _http_download_one( position: int = 0, ) -> None: """Shim — see :func:`pridepy.providers.transport._http_download_one`.""" - from pridepy.providers import transport return transport._http_download_one( url=url, output_folder=output_folder, @@ -987,7 +903,6 @@ def download_http_urls( max_retries: int = 3, ) -> None: """Shim — see :func:`pridepy.providers.transport.download_http_urls`.""" - from pridepy.providers import transport return transport.download_http_urls( http_urls=http_urls, output_folder=output_folder, From c44135588c884c1874ce03249cf46b161dbaf35a Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Wed, 27 May 2026 21:48:04 +0100 Subject: [PATCH 22/54] refactor: collapse Files shim layer; providers own their logic (0.0.18) Migrated ~17 patch.object(Files, "X") test targets to canonical locations: - PRIDE methods now patched on PrideProvider - transport/util helpers patched on transport / util modules - by_url helpers patched on commands.by_url Moved PRIDE protocol workers permanently into PrideProvider: - download_files_from_{aspera,globus,s3,ftp} - _batch_download_by_protocol, _download_with_fallback, _globus_download_one - _protocol_sequence, download_private_file_name, stream_all_files_by_project - get_submitted_file_path_prefix, save_checksum_file, get_ascp_binary, get_output_file_name Eliminated every from pridepy.files.files import Files lazy import inside providers/ and commands/. Providers call self.X / transport.X / util.X directly. Zero back-into-Files coupling. Files dropped from 912 to 382 LOC: only public CLI methods + class-attribute constant re-exports. Kept ~5 one-line @staticmethod shims for likely downstream imports (compute_md5, validate_download, read_checksum_file, download_ftp_urls, download_http_urls) plus the accession-matcher helpers (is_massive_accession etc. and is_direct_download_accession) as useful API. No behaviour change. No test assertion changes. 68 passed, 4 skipped. Flake8 (--select=E9,F63,F7,F82) clean. --- pridepy/commands/by_url.py | 36 +- pridepy/files/files.py | 764 +++--------------- pridepy/providers/base.py | 22 +- pridepy/providers/pride.py | 167 ++-- pridepy/providers/proteomexchange.py | 13 +- pridepy/providers/util.py | 13 +- pridepy/tests/test_download_by_url.py | 7 +- pridepy/tests/test_download_resilience.py | 60 +- pridepy/tests/test_ftp_download_validation.py | 8 +- pridepy/tests/test_iprox_files.py | 20 +- pridepy/tests/test_jpost_files.py | 30 +- pridepy/tests/test_massive_files.py | 29 +- pyproject.toml | 2 +- 13 files changed, 340 insertions(+), 831 deletions(-) diff --git a/pridepy/commands/by_url.py b/pridepy/commands/by_url.py index 91d6fec..375f990 100644 --- a/pridepy/commands/by_url.py +++ b/pridepy/commands/by_url.py @@ -15,6 +15,9 @@ from tqdm import tqdm +from pridepy.providers import transport +from pridepy.providers import util as _provider_util +from pridepy.providers.pride import PrideProvider from pridepy.util.api_handling import Util @@ -38,8 +41,6 @@ def _validate_urls_checksums(urls: List[str], output_folder: str) -> None: :raises RuntimeError: if one or more files fail validation """ - from pridepy.files.files import Files - accession_urls: Dict[str, List[str]] = {} for url in urls: acc = _extract_pride_accession(url) @@ -52,8 +53,8 @@ def _validate_urls_checksums(urls: List[str], output_folder: str) -> None: validation_failures: List[str] = [] for acc, acc_urls in accession_urls.items(): - checksum_file_path = Files.save_checksum_file(acc, output_folder) - checksum_map = Files.read_checksum_file(checksum_file_path) + checksum_file_path = PrideProvider.save_checksum_file(acc, output_folder) + checksum_map = _provider_util.read_checksum_file(checksum_file_path) logging.info( "Loaded checksums for %d files (project %s)", len(checksum_map), acc, @@ -63,7 +64,7 @@ def _validate_urls_checksums(urls: List[str], output_folder: str) -> None: target = os.path.join(output_folder, file_name) expected = checksum_map.get(file_name) logging.info("Validating %s", file_name) - valid, reason = Files.validate_download(target, expected) + valid, reason = _provider_util.validate_download(target, expected) if not valid: logging.error("Validation failed for %s: %s", file_name, reason) validation_failures.append(f"{file_name} ({reason})") @@ -129,19 +130,17 @@ def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: i """Route a parsed URL to its protocol-specific downloader. ``protocol='globus'`` swaps the http/https single-connection streamer - for :func:`pridepy.files.files.Files._parallel_download` (single-connection with progress bar). - ftp:// URLs are unaffected. + for :func:`pridepy.providers.transport._parallel_download` (single-connection + with progress bar). ftp:// URLs are unaffected. """ - from pridepy.files.files import Files - scheme = (parsed.scheme or "").lower() if scheme in ("http", "https"): if protocol == "globus": - Files._parallel_download(parsed.geturl(), target, position=position) + transport._parallel_download(parsed.geturl(), target, position=position) else: - Files._http_download_url(parsed.geturl(), target) + _http_download_url(parsed.geturl(), target) elif scheme == "ftp": - Files._ftp_download_url(parsed, target) + _ftp_download_url(parsed, target) else: raise ValueError(f"Unsupported URL scheme: {scheme}") @@ -154,8 +153,6 @@ def _download_single_url( position: int = 0, ) -> str: """Download one URL, dispatched by scheme; return the local file path.""" - from pridepy.files.files import Files - parsed = urlparse(url) if not (parsed.scheme or "").lower(): raise ValueError(f"URL missing scheme: {url}") @@ -169,11 +166,11 @@ def _download_single_url( logging.info("Skipping %s: already downloaded", file_name) return target - Files._dispatch_url_scheme(parsed, target, protocol, position=position) + _dispatch_url_scheme(parsed, target, protocol, position=position) - ok, reason = Files.validate_download(target) + ok, reason = _provider_util.validate_download(target) if not ok: - Files._remove_if_exists(target) + _provider_util._remove_if_exists(target) raise RuntimeError(f"Download invalid: {reason} ({target})") return target @@ -211,12 +208,11 @@ def download_files_by_url( parallel_files = min(parallel_files, 3, len(urls)) failures: List[Tuple[str, str]] = [] - from pridepy.files.files import Files if parallel_files < 2: for url in urls: try: - Files._download_single_url( + _download_single_url( url, output_folder, skip_if_downloaded_already, protocol, ) except Exception as exc: # pylint: disable=broad-except @@ -230,7 +226,7 @@ def download_files_by_url( with ThreadPoolExecutor(max_workers=parallel_files) as executor: futures = { executor.submit( - Files._download_single_url, + _download_single_url, url, output_folder, skip_if_downloaded_already, protocol, position=idx, ): url diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 3567e78..7764c81 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -1,21 +1,19 @@ #!/usr/bin/env python -import importlib.resources +"""Public Files facade — thin compatibility surface over the modular +provider architecture in :mod:`pridepy.providers`. + +The provider classes own all transport/listing logic; this module exposes +a small set of high-level operations (CLI entry points + a handful of +one-line shims for downstream Python users). +""" import logging import os -import subprocess -import urllib -import urllib.request -from ftplib import FTP from typing import Dict, List, Optional, Tuple import requests # noqa: F401 — kept as a patch target for tests from pridepy.util.api_handling import Util -# Module-level imports of the modular architecture. Providers and commands -# do not import Files at module level (only lazily inside method bodies), -# so hoisting these to the top is safe and avoids cluttering every shim -# method body with a local import. from pridepy.providers import registry, transport from pridepy.providers import util as _provider_util from pridepy.providers.iprox import IproxProvider @@ -31,10 +29,7 @@ class Files: - """ - This class handles PRIDE API files endpoint, and dispatches to the - per-repository provider classes in :mod:`pridepy.providers`. - """ + """High-level facade over the per-repository providers.""" # PRIDE class-attribute re-exports (kept here for back-compat). V3_API_BASE_URL = PrideProvider.V3_API_BASE_URL @@ -50,8 +45,6 @@ class Files: # MassIVE class-attribute re-exports. MASSIVE_ARCHIVE_FTP = MassiveProvider.ARCHIVE_FTP MASSIVE_ARCHIVE_FTP_URL_PREFIX = MassiveProvider.ARCHIVE_FTP_URL_PREFIX - # Note: MASSIVE_CATEGORY_MAP is the module-level constant in providers/massive.py, - # re-exported on Files as a class attribute via the module-level import above. # JPOST class-attribute re-exports. JPOST_ARCHIVE_FTP = JpostProvider.ARCHIVE_FTP @@ -64,8 +57,7 @@ class Files: IPROX_PX_XML_URL_TEMPLATE = IproxProvider.PX_XML_URL_TEMPLATE IPROX_PX_CATEGORY_MAP = IproxProvider.PX_CATEGORY_MAP - # MassIVE category map re-exported. Class attribute shadowing the module-level - # constant of the same name happens cleanly in class scope. + # MassIVE category map re-exported. MASSIVE_CATEGORY_MAP = MASSIVE_CATEGORY_MAP logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") @@ -73,20 +65,7 @@ class Files: def __init__(self): pass - @staticmethod - def _find_tsv_columns(header: str) -> Optional[Tuple[int, int]]: - """Shim — see :func:`pridepy.providers.util._find_tsv_columns`.""" - return _provider_util._find_tsv_columns(header) - - @staticmethod - def _is_md5_checksum(value: str) -> bool: - """Shim — see :func:`pridepy.providers.util._is_md5_checksum`.""" - return _provider_util._is_md5_checksum(value) - - @staticmethod - def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: - """Shim — see :func:`pridepy.providers.util.read_checksum_file`.""" - return _provider_util.read_checksum_file(checksum_file_path) + # Pure delegating shims kept for backward compatibility. @staticmethod def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: @@ -99,201 +78,116 @@ def validate_download(file_path: str, expected_checksum: Optional[str] = None) - return _provider_util.validate_download(file_path, expected_checksum) @staticmethod - def _remove_if_exists(file_path: str) -> None: - """Shim — see :func:`pridepy.providers.util._remove_if_exists`.""" - return _provider_util._remove_if_exists(file_path) + def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: + """Shim — see :func:`pridepy.providers.util.read_checksum_file`.""" + return _provider_util.read_checksum_file(checksum_file_path) @staticmethod - def _get_download_url(file_record: Dict, protocol: str) -> str: - """Shim — see :func:`pridepy.providers.util._get_download_url`.""" - return _provider_util._get_download_url(file_record, protocol) + def download_ftp_urls( + ftp_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + max_connection_retries: int = 3, + max_download_retries: int = 3, + use_tls: bool = False, + parallel_files: int = 1, + ) -> None: + """Shim — see :func:`pridepy.providers.transport.download_ftp_urls`.""" + return transport.download_ftp_urls( + ftp_urls=ftp_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + use_tls=use_tls, + parallel_files=parallel_files, + ) @staticmethod - def _resolve_local_path(file_record: Dict, output_folder: str) -> str: - """Shim — see :func:`pridepy.providers.util._resolve_local_path`.""" - return _provider_util._resolve_local_path(file_record, output_folder) + def download_http_urls( + http_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + parallel_files: int = 1, + max_retries: int = 3, + ) -> None: + """Shim — see :func:`pridepy.providers.transport.download_http_urls`.""" + return transport.download_http_urls( + http_urls=http_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + parallel_files=parallel_files, + max_retries=max_retries, + ) - @staticmethod - def _protocol_sequence(protocol: str) -> List[str]: - """Shim — see :meth:`pridepy.providers.pride.PrideProvider._protocol_sequence`.""" - return PrideProvider._protocol_sequence(protocol) + # Accession-matcher convenience helpers (useful public API). @staticmethod def is_massive_accession(accession: str) -> bool: - """Shim — see :meth:`pridepy.providers.massive.MassiveProvider.matches`.""" return MassiveProvider.matches(accession) - @staticmethod - def _get_massive_public_root(accession: str) -> str: - return MassiveProvider._get_public_root(accession) - - @staticmethod - def _get_massive_public_ftp_url(accession: str, remote_path: str) -> str: - return MassiveProvider._get_public_ftp_url(accession, remote_path) - - @staticmethod - def _map_massive_collection_to_category(collection: str) -> str: - return MassiveProvider._map_collection_to_category(collection) - - @staticmethod - def _build_massive_file_record(accession: str, ftp_url: str) -> Dict: - return MassiveProvider._build_file_record(accession, ftp_url) - @staticmethod def is_jpost_accession(accession: str) -> bool: - """Shim — see :meth:`pridepy.providers.jpost.JpostProvider.matches`.""" return JpostProvider.matches(accession) @staticmethod - def _get_jpost_public_root(accession: str) -> str: - return JpostProvider._get_public_root(accession) - - @staticmethod - def _get_jpost_public_ftp_url(accession: str, remote_path: str) -> str: - return JpostProvider._get_public_ftp_url(accession, remote_path) - - @staticmethod - def _build_jpost_file_record(accession, ftp_url, category_from_proxi=None): - return JpostProvider._build_file_record(accession, ftp_url, category_from_proxi) - - @staticmethod - def _build_iprox_file_record(accession, https_url, category_from_px=None): - """Shim — see :meth:`pridepy.providers.iprox.IproxProvider._build_file_record`.""" - return IproxProvider._build_file_record(accession, https_url, category_from_px) - - @staticmethod - def _get_iprox_public_root(accession: str) -> str: - return IproxProvider._get_public_root(accession) - - @staticmethod - def _get_iprox_public_ftp_url(accession: str, remote_path: str) -> str: - return IproxProvider._get_public_ftp_url(accession, remote_path) + def is_iprox_accession(accession: str) -> bool: + return IproxProvider.matches(accession) @staticmethod def is_direct_download_accession(accession: str) -> bool: - """Shim — True for MassIVE/JPOST/iProX (explicitly excludes PRIDE). - - PRIDE is also a registered provider but PRIDE downloads go through - the multi-protocol orchestrator (FTP/Aspera/S3/Globus with checksum - validation and fallback), not the direct-download partitioned-by-URL- - scheme path. So we filter PRIDE out here. - """ + """True for MassIVE / JPOST / iProX (explicitly excludes PRIDE).""" try: provider = registry.resolve(accession) except ValueError: return False return provider.name != "pride" - @staticmethod - def is_iprox_accession(accession: str) -> bool: - """Shim — see :meth:`pridepy.providers.iprox.IproxProvider.matches`.""" - return IproxProvider.matches(accession) - @staticmethod def _repo_uses_tls(accession: str) -> bool: - """Shim — returns the resolved provider's use_tls flag (False if unknown).""" + """Return the resolved provider's ``use_tls`` flag (False if unknown).""" try: provider = registry.resolve(accession) except ValueError: return False return getattr(provider, "use_tls", False) - @staticmethod - def _walk_ftp_tree(ftp: FTP, remote_dir: str) -> List[str]: - """Shim — see :func:`pridepy.providers.transport._walk_ftp_tree`.""" - return transport._walk_ftp_tree(ftp=ftp, remote_dir=remote_dir) - - @staticmethod - def _open_ftp_connection(host: str, use_tls: bool, timeout: int = 30) -> FTP: - """Shim — see :func:`pridepy.providers.transport._open_ftp_connection`.""" - return transport._open_ftp_connection(host=host, use_tls=use_tls, timeout=timeout) - - @staticmethod - def _list_ftp_repo_files(host, remote_root, error_label, use_tls=False): - """Shim — see :func:`pridepy.providers.transport._list_ftp_repo_files`.""" - return transport._list_ftp_repo_files(host=host, remote_root=remote_root, error_label=error_label, use_tls=use_tls) - - def _list_massive_public_files(self, accession: str) -> List[Dict]: - """Shim — see :meth:`pridepy.providers.massive.MassiveProvider.list_files`.""" - return MassiveProvider().list_files(accession) - - def _download_massive_file_records( - self, - accession: str, - file_records: List[Dict], - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str, - parallel_files: int = 1, - ) -> None: - """ - Download public MassIVE files via anonymous FTP (now FTPS). - Backward-compat shim — dispatches via the provider registry. - """ - registry.resolve(accession).download_files( - accession=accession, - records=file_records, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - ) - - def _list_jpost_public_files(self, accession: str) -> List[Dict]: - """ - Discover all public files for a JPOST dataset. - - Delegates to JpostProvider but routes via the shim methods so that - test patches on ``_list_jpost_public_files_via_proxi`` and - ``_list_ftp_repo_files`` continue to intercept. - """ - normalized_accession = accession.upper() - try: - return self._list_jpost_public_files_via_proxi(normalized_accession) - except Exception as proxi_error: - logging.warning( - f"JPOST PROXI listing failed for {normalized_accession} " - f"({proxi_error}); falling back to FTP tree walk." - ) - remote_root = JpostProvider._get_public_root(normalized_accession) - remote_files = self._list_ftp_repo_files( - host=JpostProvider.ARCHIVE_FTP, - remote_root=remote_root, - error_label=f"JPOST dataset {normalized_accession}", - ) - return [ - self._build_jpost_file_record( - normalized_accession, - JpostProvider._get_public_ftp_url(normalized_accession, remote_file), - ) - for remote_file in remote_files - ] - - def _list_jpost_public_files_via_proxi(self, accession: str) -> List[Dict]: - """Shim — see :meth:`pridepy.providers.jpost.JpostProvider._list_via_proxi`.""" - return JpostProvider()._list_via_proxi(accession) - - def _list_iprox_public_files(self, accession: str) -> List[Dict]: - """Shim — see :meth:`pridepy.providers.iprox.IproxProvider.list_files`.""" - return IproxProvider().list_files(accession) + # Listing / metadata. async def stream_all_files_metadata(self, output_file, accession=None): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.stream_all_files_metadata`.""" + """Shim — see :meth:`PrideProvider.stream_all_files_metadata`.""" return await PrideProvider().stream_all_files_metadata(output_file, accession) - def stream_all_files_by_project(self, accession) -> List[Dict]: - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.stream_all_files_by_project`.""" - return PrideProvider().stream_all_files_by_project(accession) - def get_all_raw_file_list(self, project_accession): - """Get raw file list for any registered provider. - - Returns the dataset's file records filtered to fileCategory == "RAW". - """ + """Get raw file list for any registered provider (records with fileCategory == "RAW").""" provider = registry.resolve(project_accession) records = provider.list_files(project_accession) return [r for r in records if r["fileCategory"]["value"] == "RAW"] + def get_all_category_file_list( + self, accession: str, categories: "str | List[str]" + ) -> List[Dict]: + """Retrieve project files belonging to the given categories.""" + if isinstance(categories, str): + categories = [categories] + category_set = {c.upper() for c in categories} + records = registry.resolve(accession).list_files(accession) + return [r for r in records if r["fileCategory"]["value"] in category_set] + + def get_submitted_file_path_prefix(self, accession): + """Shim — see :meth:`PrideProvider.get_submitted_file_path_prefix`.""" + return PrideProvider().get_submitted_file_path_prefix(accession) + + def get_file_from_api(self, accession, file_name) -> List[Dict]: + """Return records matching ``file_name`` from the provider's listing.""" + try: + records = registry.resolve(accession).list_files(accession) + return [r for r in records if r["fileName"] == file_name] + except Exception as e: + raise Exception("File not found " + str(e)) + + # Download entry points. + def download_all_raw_files( self, accession, @@ -320,124 +214,34 @@ def download_all_raw_files( aspera_maximum_bandwidth=aspera_maximum_bandwidth, ) - @staticmethod - def download_files_from_ftp( - file_list_json, - output_folder, - skip_if_downloaded_already, - max_connection_retries=3, - max_download_retries=3, - ): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files_from_ftp`.""" - return PrideProvider.download_files_from_ftp( - file_list_json, - output_folder, - skip_if_downloaded_already, - max_connection_retries=max_connection_retries, - max_download_retries=max_download_retries, - ) - - @staticmethod - def get_output_file_name(download_url, file, output_folder): - public_filepath_part = download_url.rsplit("/", 1) - accession = file.get("accession", "unknown-accession") - logging.debug(accession + " -> " + public_filepath_part[1]) - new_file_path = os.path.join(output_folder, f"{public_filepath_part[1]}") - return new_file_path - - @staticmethod - def download_files_from_aspera( - file_list_json: List[Dict], + def download_all_category_files( + self, + accession: str, output_folder: str, - skip_if_downloaded_already, - maximum_bandwidth: str = "100M", - ): - """ - Download files using aspera transfer url - :param file_list_json: file list in json format - :param output_folder: folder to download the files - :param maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - """ - ascp_path = Files.get_ascp_binary() - key_full_path = importlib.resources.files("pridepy").joinpath( - "aspera/key/asperaweb_id_dsa.openssh" - ) - key_path = os.path.abspath(key_full_path) - for file in file_list_json: - if file["publicFileLocations"][0]["name"] == "Aspera Protocol": - download_url = file["publicFileLocations"][0]["value"] - else: - download_url = file["publicFileLocations"][1]["value"] - - # Create a clean filename to save the downloaded file - logging.debug(f"Downloading via Aspera: {download_url}") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) - - if skip_if_downloaded_already == True and os.path.exists(new_file_path): - logging.info("Skipping download as file already exists") - continue - - try: - # Execute the ascp command using subprocess - subprocess.run( - [ - ascp_path, - "-QT", - "-P", - "33001", - "-l", - maximum_bandwidth, # Options for Aspera: adjust as necessary - "-i", - key_path, - download_url, - new_file_path, # Source and destination - ], - check=True, - ) - logging.info(f"Successfully downloaded {new_file_path} via Aspera") - except subprocess.CalledProcessError as e: - logging.error(f"Aspera download failed for {new_file_path}: {str(e)}") - - @staticmethod - def _parallel_download(url, file_path, position=0): - """Shim — see :func:`pridepy.providers.transport._parallel_download`.""" - return transport._parallel_download(url=url, file_path=file_path, position=position) - - @staticmethod - def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_retries=6, position=0): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider._globus_download_one`.""" - return PrideProvider._globus_download_one( - file, output_folder, skip_if_downloaded_already, - max_retries=max_retries, position=position, - ) - - @staticmethod - def download_files_from_globus( - file_list_json: List[Dict], output_folder, skip_if_downloaded_already, + skip_if_downloaded_already: bool, + protocol: str, + aspera_maximum_bandwidth: str, + checksum_check: bool, + categories: List[str] = None, + category: str = None, parallel_files: int = 1, - checksum_map: Optional[Dict[str, str]] = None, ): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files_from_globus`.""" - return PrideProvider.download_files_from_globus( - file_list_json, output_folder, skip_if_downloaded_already, + """Download all files of the given categories from a project.""" + if categories is None: + categories = [category] if category else ["RAW"] + records = self.get_all_category_file_list(accession, categories) + provider = registry.resolve(accession) + provider.download_files( + accession=accession, + records=records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, parallel_files=parallel_files, - checksum_map=checksum_map, - ) - - @staticmethod - def download_files_from_s3( - file_list_json: List[Dict], output_folder: str, skip_if_downloaded_already - ): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_files_from_s3`.""" - return PrideProvider.download_files_from_s3( - file_list_json, output_folder, skip_if_downloaded_already, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, ) - def get_submitted_file_path_prefix(self, accession): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.get_submitted_file_path_prefix`.""" - return PrideProvider().get_submitted_file_path_prefix(accession) - def download_file_by_name( self, accession, @@ -450,25 +254,17 @@ def download_file_by_name( aspera_maximum_bandwidth, checksum_check, ): - """ - Download files from url - :param accession: PRIDE accession - :param file_name: file name to download - :param output_folder: folder to download the files - :param protocol: ftp, aspera, globus - :param username: Username for private datasets - :param password: Password for private datasets - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - :param aspera_maximum_bandwidth: Aspera maximum bandwidth - :param checksum_check: Download checksum for a given project. - """ + """Download a single file by name. + PRIDE supports public / private modes via the V2 private API. Other + providers (MassIVE / JPOST / iProX) only support public downloads. + """ if not os.path.isdir(output_folder): os.mkdir(output_folder) provider = registry.resolve(accession) - ## Check type of project + # Direct-download providers always use the public path. if provider.name in ("massive", "jpost", "iprox"): logging.info( "Downloading file from public direct-download dataset {}".format(accession) @@ -487,6 +283,7 @@ def download_file_by_name( ) return + # PRIDE has a public/private split that needs status interrogation. public_project = False project_status = Util.get_api_call(self.API_BASE_URL + "/status/{}".format(accession)) @@ -501,18 +298,18 @@ def download_file_by_name( if public_project: logging.info("Downloading file from public dataset {}".format(accession)) response = self.get_file_from_api(accession, file_name) - self.download_files( - response, - accession, - output_folder, - skip_if_downloaded_already, - protocol, + PrideProvider._download_files_batch( + file_list_json=response, + accession=accession, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, ) elif not public_project and (username is not None and password is not None): logging.info("Downloading file from private dataset {}".format(accession)) - self.download_private_file_name( + PrideProvider().download_private_file_name( accession=accession, file_name=file_name, output_folder=output_folder, @@ -531,105 +328,6 @@ def download_file_by_name( ) ) - def get_file_from_api(self, accession, file_name) -> List[Dict]: - """ - Fetches file from API - :param accession: PRIDE accession - :param file_name: file name - :return: file in json format - """ - try: - records = registry.resolve(accession).list_files(accession) - return [r for r in records if r["fileName"] == file_name] - except Exception as e: - raise Exception("File not found " + str(e)) - - def download_private_file_name(self, accession, file_name, output_folder, username, password): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.download_private_file_name`.""" - return PrideProvider().download_private_file_name( - accession, file_name, output_folder, username, password, - ) - - @staticmethod - def get_ascp_binary(): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.get_ascp_binary`.""" - return PrideProvider.get_ascp_binary() - - @staticmethod - def save_checksum_file(accession, output_folder): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider.save_checksum_file`.""" - return PrideProvider.save_checksum_file(accession, output_folder) - - @staticmethod - def _batch_download_by_protocol( - file_list: List[Dict], - output_folder: str, - protocol: str, - skip_if_downloaded_already: bool, - aspera_maximum_bandwidth: str, - parallel_files: int = 1, - checksum_map: Optional[Dict[str, str]] = None, - ) -> None: - """Shim — see :meth:`pridepy.providers.pride.PrideProvider._batch_download_by_protocol`. - - Tests patch this method via ``patch.object(Files, "_batch_download_by_protocol")``; - :class:`PrideProvider` calls back through ``Files.X`` so those patches - keep intercepting. - """ - return PrideProvider._batch_download_by_protocol( - file_list, - output_folder, - protocol, - skip_if_downloaded_already, - aspera_maximum_bandwidth, - parallel_files=parallel_files, - checksum_map=checksum_map, - ) - - @staticmethod - def _download_with_fallback( - file_record: Dict, - output_folder: str, - protocol_sequence: List[str], - expected_checksum: Optional[str], - aspera_maximum_bandwidth: str, - max_protocol_retries: int = 2, - parallel_files: int = 1, - ) -> bool: - """Shim — see :meth:`pridepy.providers.pride.PrideProvider._download_with_fallback`.""" - return PrideProvider._download_with_fallback( - file_record, - output_folder, - protocol_sequence, - expected_checksum, - aspera_maximum_bandwidth, - max_protocol_retries=max_protocol_retries, - parallel_files=parallel_files, - ) - - @staticmethod - def download_files( - file_list_json: List[Dict], - accession, - output_folder: str, - skip_if_downloaded_already, - protocol: str = "ftp", - aspera_maximum_bandwidth: str = "100M", # Aspera maximum bandwidth - checksum_check=False, - parallel_files: int = 1, - ): - """Shim — see :meth:`pridepy.providers.pride.PrideProvider._download_files_batch`.""" - return PrideProvider._download_files_batch( - file_list_json, - accession, - output_folder, - skip_if_downloaded_already, - protocol=protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, - parallel_files=parallel_files, - ) - def download_files_by_list( self, accession: str, @@ -641,7 +339,7 @@ def download_files_by_list( checksum_check: bool = False, parallel_files: int = 1, ) -> None: - """Shim — see :func:`pridepy.commands.by_list.download_files_by_list`.""" + """Delegate to :func:`pridepy.commands.by_list.download_files_by_list`.""" return by_list.download_files_by_list( accession=accession, file_names=file_names, @@ -653,11 +351,6 @@ def download_files_by_list( parallel_files=parallel_files, ) - @staticmethod - def _extract_pride_accession(url: str) -> Optional[str]: - """Shim — see :func:`pridepy.commands.by_url._extract_pride_accession`.""" - return by_url._extract_pride_accession(url) - @staticmethod def download_files_by_url( urls: List[str], @@ -667,7 +360,7 @@ def download_files_by_url( parallel_files: int = 1, checksum_check: bool = False, ) -> None: - """Shim — see :func:`pridepy.commands.by_url.download_files_by_url`.""" + """Delegate to :func:`pridepy.commands.by_url.download_files_by_url`.""" return by_url.download_files_by_url( urls=urls, output_folder=output_folder, @@ -677,236 +370,13 @@ def download_files_by_url( checksum_check=checksum_check, ) - @staticmethod - def _validate_urls_checksums(urls: List[str], output_folder: str) -> None: - """Shim — see :func:`pridepy.commands.by_url._validate_urls_checksums`.""" - return by_url._validate_urls_checksums(urls, output_folder) - - @staticmethod - def _download_single_url( - url: str, - output_folder: str, - skip_if_exists: bool = False, - protocol: str = "ftp", - position: int = 0, - ) -> str: - """Shim — see :func:`pridepy.commands.by_url._download_single_url`.""" - return by_url._download_single_url(url, output_folder, skip_if_exists, protocol, position) - - @staticmethod - def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: int = 0) -> None: - """Shim — see :func:`pridepy.commands.by_url._dispatch_url_scheme`.""" - return by_url._dispatch_url_scheme(parsed, target, protocol=protocol, position=position) - - @staticmethod - def _http_download_url(url: str, target: str) -> None: - """Shim — see :func:`pridepy.commands.by_url._http_download_url`.""" - return by_url._http_download_url(url, target) - - @staticmethod - def _ftp_download_url(parsed, target: str) -> None: - """Shim — see :func:`pridepy.commands.by_url._ftp_download_url`.""" - return by_url._ftp_download_url(parsed, target) - - def download_all_category_files( - self, - accession: str, - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str, - aspera_maximum_bandwidth: str, - checksum_check: bool, - categories: List[str] = None, - category: str = None, - parallel_files: int = 1, - ): - """ - Download all files of specified categories from a PRIDE project. - - :param accession: The PRIDE project accession identifier. - :param output_folder: The directory where the files will be downloaded. - :param skip_if_downloaded_already: If True, skips downloading files that already exist. - :param protocol: The transfer protocol to use (e.g., ftp, aspera, globus, s3). - :param aspera_maximum_bandwidth: Maximum bandwidth for Aspera transfers. - :param checksum_check: If True, downloads the checksum file for the project. - :param categories: List of file categories to download. - :param category: Single file category (deprecated, use categories instead). - """ - if categories is None: - categories = [category] if category else ["RAW"] - records = self.get_all_category_file_list(accession, categories) - provider = registry.resolve(accession) - provider.download_files( - accession=accession, - records=records, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - checksum_check=checksum_check, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - ) - - def get_all_category_file_list( - self, accession: str, categories: "str | List[str]" - ) -> List[Dict]: - """ - Retrieve a list of files from a specific project that belong to given categories. - - :param accession: The PRIDE project accession identifier. - :param categories: A single category string or list of categories to filter by. - :return: A list of files matching the specified categories. - """ - if isinstance(categories, str): - categories = [categories] - category_set = {c.upper() for c in categories} - records = registry.resolve(accession).list_files(accession) - return [r for r in records if r["fileCategory"]["value"] in category_set] - - # ------------------------------- - # ProteomeXchange support - # ------------------------------- - - @staticmethod - def _normalize_px_xml_url(px_id_or_url: str) -> str: - """Shim — see :meth:`pridepy.providers.proteomexchange.ProteomeXchangeProvider._normalize_px_xml_url`.""" - return ProteomeXchangeProvider._normalize_px_xml_url(px_id_or_url) - - @staticmethod - def _parse_px_xml_for_raw_file_urls(px_xml_url: str): - """Shim — see :meth:`pridepy.providers.proteomexchange.ProteomeXchangeProvider._parse_px_xml_for_raw_file_urls`.""" - return ProteomeXchangeProvider._parse_px_xml_for_raw_file_urls(px_xml_url) - def download_px_raw_files( self, px_id_or_url: str, output_folder: str, skip_if_downloaded_already: bool = True, ) -> None: - """Shim — see :meth:`pridepy.providers.proteomexchange.ProteomeXchangeProvider.download_from_accession_or_url`.""" + """Delegate to :meth:`ProteomeXchangeProvider.download_from_accession_or_url`.""" return ProteomeXchangeProvider().download_from_accession_or_url( px_id_or_url, output_folder, skip_if_downloaded_already ) - - @staticmethod - def _local_path_for_url(download_url: str, output_folder: str) -> str: - """Shim — see :func:`pridepy.providers.transport._local_path_for_url`.""" - return transport._local_path_for_url(download_url=download_url, output_folder=output_folder) - - @staticmethod - def _download_one_ftp_path( - ftp: FTP, - ftp_path: str, - local_path: str, - skip_if_downloaded_already: bool, - max_download_retries: int, - position: int = 0, - ) -> None: - """Shim — see :func:`pridepy.providers.transport._download_one_ftp_path`.""" - return transport._download_one_ftp_path( - ftp=ftp, - ftp_path=ftp_path, - local_path=local_path, - skip_if_downloaded_already=skip_if_downloaded_already, - max_download_retries=max_download_retries, - position=position, - ) - - @staticmethod - def _download_ftp_paths_serial( - host: str, - paths: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - use_tls: bool, - max_connection_retries: int, - max_download_retries: int, - ) -> None: - """Shim — see :func:`pridepy.providers.transport._download_ftp_paths_serial`.""" - return transport._download_ftp_paths_serial( - host=host, - paths=paths, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - use_tls=use_tls, - max_connection_retries=max_connection_retries, - max_download_retries=max_download_retries, - ) - - @staticmethod - def _download_ftp_paths_parallel( - host: str, - paths: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - use_tls: bool, - max_connection_retries: int, - max_download_retries: int, - parallel_files: int, - ) -> None: - """Shim — see :func:`pridepy.providers.transport._download_ftp_paths_parallel`.""" - return transport._download_ftp_paths_parallel( - host=host, - paths=paths, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - use_tls=use_tls, - max_connection_retries=max_connection_retries, - max_download_retries=max_download_retries, - parallel_files=parallel_files, - ) - - @staticmethod - def download_ftp_urls( - ftp_urls: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - max_connection_retries: int = 3, - max_download_retries: int = 3, - use_tls: bool = False, - parallel_files: int = 1, - ) -> None: - """Shim — see :func:`pridepy.providers.transport.download_ftp_urls`.""" - return transport.download_ftp_urls( - ftp_urls=ftp_urls, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - max_connection_retries=max_connection_retries, - max_download_retries=max_download_retries, - use_tls=use_tls, - parallel_files=parallel_files, - ) - - @staticmethod - def _http_download_one( - url: str, - output_folder: str, - skip_if_downloaded_already: bool, - max_retries: int = 3, - position: int = 0, - ) -> None: - """Shim — see :func:`pridepy.providers.transport._http_download_one`.""" - return transport._http_download_one( - url=url, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - max_retries=max_retries, - position=position, - ) - - @staticmethod - def download_http_urls( - http_urls: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - parallel_files: int = 1, - max_retries: int = 3, - ) -> None: - """Shim — see :func:`pridepy.providers.transport.download_http_urls`.""" - return transport.download_http_urls( - http_urls=http_urls, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - parallel_files=parallel_files, - max_retries=max_retries, - ) diff --git a/pridepy/providers/base.py b/pridepy/providers/base.py index f9fa8bc..cbd1830 100644 --- a/pridepy/providers/base.py +++ b/pridepy/providers/base.py @@ -1,7 +1,11 @@ """Abstract base classes for pridepy providers.""" +import logging from abc import ABC, abstractmethod from typing import ClassVar, Dict, List, Optional +from pridepy.providers import transport +from pridepy.providers import util as _provider_util + class Provider(ABC): """Abstract base for every repository pridepy can list and download from.""" @@ -46,10 +50,8 @@ class BaseDirectDownloadProvider(Provider): Subclasses set the ``use_tls`` class var (True for MassIVE FTPS, False for JPOST plain FTP) and override :meth:`list_files`. The shared ``download_files`` implementation partitions record URLs by scheme: - ``ftp://`` URLs are handed to :meth:`Files.download_ftp_urls`; ``http(s)://`` - URLs go to :meth:`Files.download_http_urls`. It calls **back** into - ``Files`` so that test patches on ``Files.download_ftp_urls`` / - ``Files.download_http_urls`` continue to intercept the calls. + ``ftp://`` URLs are handed to :func:`transport.download_ftp_urls`; + ``http(s)://`` URLs go to :func:`transport.download_http_urls`. """ use_tls: ClassVar[bool] = False @@ -67,31 +69,25 @@ def download_files( username: Optional[str] = None, password: Optional[str] = None, ) -> None: - # Lazy import: providers know about Files (the facade) only via the - # public attributes that tests may patch; avoid module-load cycle. - from pridepy.files.files import Files - if protocol not in ("ftp", "https", "http"): - import logging logging.warning( "Direct downloads currently use ftp / https only. " f"Ignoring requested protocol '{protocol}' for {accession}." ) - all_urls = [Files._get_download_url(record, "ftp") for record in records] + all_urls = [_provider_util._get_download_url(record, "ftp") for record in records] ftp_urls = [u for u in all_urls if u.lower().startswith("ftp://")] http_urls = [ u for u in all_urls if u.lower().startswith(("http://", "https://")) ] if not ftp_urls and not http_urls: - import logging logging.info( f"No files matched for direct-download dataset {accession}" ) return if ftp_urls: - Files.download_ftp_urls( + transport.download_ftp_urls( ftp_urls=ftp_urls, output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, @@ -99,7 +95,7 @@ def download_files( parallel_files=parallel_files, ) if http_urls: - Files.download_http_urls( + transport.download_http_urls( http_urls=http_urls, output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, diff --git a/pridepy/providers/pride.py b/pridepy/providers/pride.py index c8c40ca..5456544 100644 --- a/pridepy/providers/pride.py +++ b/pridepy/providers/pride.py @@ -3,14 +3,14 @@ PRIDE has the richest behaviour of all providers: multi-protocol batch download with aspera/s3/ftp/globus fallback, private-dataset path with username/password auth, checksum TSV validation, and submitter-path -helpers. This module hosts all of those; the :class:`Files` facade -delegates via lightweight shim methods. - -Implementation note: PRIDE-specific helpers that the existing test suite -patches via ``patch.object(Files, "X")`` are called from inside this -provider via ``Files.X(...)`` (lazy import) — never ``self.X`` — so the -patches keep intercepting. This is a deliberate backward-compat choice -documented in the refactor plan (Task 8). +helpers. This module owns all of that logic; the :class:`Files` facade +exposes a thin public surface for downstream callers. + +Implementation note: PRIDE provider methods route through other +PrideProvider methods (``PrideProvider.X(...)``) or directly through the +shared ``transport`` / ``util`` helpers — they do NOT call back into the +``Files`` facade. Tests patch the canonical locations +(``PrideProvider.X``, ``transport.X``, ``util.X``) directly. """ import ftplib import importlib.resources @@ -35,7 +35,8 @@ from tqdm import tqdm from pridepy.authentication.authentication import Authentication -from pridepy.providers import registry +from pridepy.providers import registry, transport +from pridepy.providers import util as _provider_util from pridepy.providers.base import Provider from pridepy.providers.util import Progress from pridepy.util.api_handling import Util @@ -110,10 +111,9 @@ def get_submitted_file_path_prefix(self, accession): :param accession: PRIDE accession :return: path fragment (eg: 2018/10/PXD008644) """ - # Use Files facade so test patches on get_all_raw_file_list keep working. - from pridepy.files.files import Files - results = Files().get_all_raw_file_list(accession) - first_file = results[0]["publicFileLocations"][0]["value"] + records = self.list_files(accession) + raw_files = [r for r in records if r["fileCategory"]["value"] == "RAW"] + first_file = raw_files[0]["publicFileLocations"][0]["value"] path_fragment = re.search(r"\d{4}/\d{2}/PXD\d*", first_file).group() return path_fragment @@ -157,6 +157,15 @@ def get_ascp_binary(): else: raise OSError(f"Unsupported OS or architecture: {os_type}, {arch}") + @staticmethod + def get_output_file_name(download_url, file, output_folder): + """Build the local output path for ``download_url`` inside ``output_folder``.""" + public_filepath_part = download_url.rsplit("/", 1) + accession = file.get("accession", "unknown-accession") + logging.debug(accession + " -> " + public_filepath_part[1]) + new_file_path = os.path.join(output_folder, f"{public_filepath_part[1]}") + return new_file_path + @staticmethod def save_checksum_file(accession, output_folder): """ @@ -182,11 +191,8 @@ def save_checksum_file(accession, output_folder): @staticmethod def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_retries=6, position=0): """Download a single file via globus; used as a worker target.""" - # Use Files facade so test patches on Files helpers keep working. - from pridepy.files.files import Files - - download_url = Files._get_download_url(file, "globus") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) + download_url = _provider_util._get_download_url(file, "globus") + new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) if skip_if_downloaded_already and os.path.exists(new_file_path): logging.info(f"Skipping download as file already exists: {new_file_path}") @@ -194,7 +200,7 @@ def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_re for attempt in range(1, max_retries + 1): try: - Files._parallel_download(download_url, new_file_path, position=position) + transport._parallel_download(download_url, new_file_path, position=position) return except Exception as e: logging.warning(f"Attempt {attempt}/{max_retries} failed for {file.get('fileName', '?')}: {e}") @@ -221,8 +227,6 @@ def download_files_from_ftp( :param max_connection_retries: Number of attempts to reconnect to the FTP server if the connection is lost. :param max_download_retries: Number of attempts to retry the download of a file in case of failure. """ - from pridepy.files.files import Files - if not os.path.isdir(output_folder): os.makedirs(output_folder) @@ -249,7 +253,7 @@ def connect_ftp(): logging.debug("ftp_filepath:" + download_url) # Get output file path - new_file_path = Files.get_output_file_name( + new_file_path = PrideProvider.get_output_file_name( download_url, file, output_folder ) @@ -327,6 +331,60 @@ def callback(data): ) break + @staticmethod + def download_files_from_aspera( + file_list_json: List[Dict], + output_folder: str, + skip_if_downloaded_already, + maximum_bandwidth: str = "100M", + ): + """ + Download files using aspera transfer url + :param file_list_json: file list in json format + :param output_folder: folder to download the files + :param maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. + :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. + """ + ascp_path = PrideProvider.get_ascp_binary() + key_full_path = importlib.resources.files("pridepy").joinpath( + "aspera/key/asperaweb_id_dsa.openssh" + ) + key_path = os.path.abspath(key_full_path) + for file in file_list_json: + if file["publicFileLocations"][0]["name"] == "Aspera Protocol": + download_url = file["publicFileLocations"][0]["value"] + else: + download_url = file["publicFileLocations"][1]["value"] + + # Create a clean filename to save the downloaded file + logging.debug(f"Downloading via Aspera: {download_url}") + new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) + + if skip_if_downloaded_already and os.path.exists(new_file_path): + logging.info("Skipping download as file already exists") + continue + + try: + # Execute the ascp command using subprocess + subprocess.run( + [ + ascp_path, + "-QT", + "-P", + "33001", + "-l", + maximum_bandwidth, # Options for Aspera: adjust as necessary + "-i", + key_path, + download_url, + new_file_path, # Source and destination + ], + check=True, + ) + logging.info(f"Successfully downloaded {new_file_path} via Aspera") + except subprocess.CalledProcessError as e: + logging.error(f"Aspera download failed for {new_file_path}: {str(e)}") + @staticmethod def download_files_from_globus( file_list_json: List[Dict], output_folder, skip_if_downloaded_already, @@ -346,9 +404,6 @@ def download_files_from_globus( :param parallel_files: number of files to download simultaneously :param checksum_map: mapping of file name to expected MD5 checksum """ - # Use Files facade so test patches on Files._globus_download_one etc. keep working. - from pridepy.files.files import Files - if checksum_map is None: checksum_map = {} @@ -358,12 +413,12 @@ def download_files_from_globus( # --- Phase 0: pre-filter files that need downloading ----------------- files_to_download: List[Dict] = [] for file in file_list_json: - download_url = Files._get_download_url(file, "globus") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) + download_url = _provider_util._get_download_url(file, "globus") + new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) if skip_if_downloaded_already and os.path.exists(new_file_path): expected_cs = checksum_map.get(file.get("fileName", "")) if expected_cs: - valid, reason = Files.validate_download(new_file_path, expected_cs) + valid, reason = _provider_util.validate_download(new_file_path, expected_cs) if not valid: logging.warning(f"Corrupted file detected ({reason}), will re-download: {new_file_path}") files_to_download.append(file) @@ -386,11 +441,11 @@ def download_files_from_globus( if parallel_files < 2: for file in files_to_download: try: - Files._globus_download_one( + PrideProvider._globus_download_one( file, output_folder, False ) - new_file_path = Files.get_output_file_name( - Files._get_download_url(file, "globus"), file, output_folder + new_file_path = PrideProvider.get_output_file_name( + _provider_util._get_download_url(file, "globus"), file, output_folder ) logging.info(f"Successfully downloaded {new_file_path}") except Exception as e: @@ -400,7 +455,7 @@ def download_files_from_globus( with ThreadPoolExecutor(max_workers=parallel_files) as executor: futures = { executor.submit( - Files._globus_download_one, + PrideProvider._globus_download_one, file, output_folder, False, position=idx, ): file @@ -422,8 +477,6 @@ def download_files_from_s3( :param output_folder: folder to download the files :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. """ - from pridepy.files.files import Files - if not os.path.isdir(output_folder): os.makedirs(output_folder, exist_ok=True) @@ -453,7 +506,7 @@ def download_files_from_s3( ftp_base_url = "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/" s3_path = download_url.replace(ftp_base_url, "") - new_file_path = Files.get_output_file_name(download_url, file, output_folder) + new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) if skip_if_downloaded_already == True and os.path.exists(new_file_path): logging.info("Skipping download as file already exists") @@ -587,20 +640,17 @@ def _batch_download_by_protocol( Transfer a batch of files with one protocol, reusing a single connection where the underlying helper supports it (FTP, S3). """ - # Use Files facade so test patches on each per-protocol helper keep working. - from pridepy.files.files import Files - if not file_list: return if protocol == "ftp": - Files.download_files_from_ftp( + PrideProvider.download_files_from_ftp( file_list, output_folder, skip_if_downloaded_already=skip_if_downloaded_already, ) return if protocol == "aspera": - Files.download_files_from_aspera( + PrideProvider.download_files_from_aspera( file_list, output_folder, skip_if_downloaded_already=skip_if_downloaded_already, @@ -608,7 +658,7 @@ def _batch_download_by_protocol( ) return if protocol == "globus": - Files.download_files_from_globus( + PrideProvider.download_files_from_globus( file_list, output_folder, skip_if_downloaded_already=skip_if_downloaded_already, @@ -617,7 +667,7 @@ def _batch_download_by_protocol( ) return if protocol == "s3": - Files.download_files_from_s3( + PrideProvider.download_files_from_s3( file_list, output_folder, skip_if_downloaded_already=skip_if_downloaded_already, @@ -640,10 +690,7 @@ def _download_with_fallback( after every attempt. Intended as the per-file fallback path; batch download of the primary protocol is handled separately. """ - # Patch-sensitive: call through Files so test patches intercept. - from pridepy.files.files import Files - - local_path = Files._resolve_local_path(file_record, output_folder) + local_path = _provider_util._resolve_local_path(file_record, output_folder) for protocol in protocol_sequence: for attempt in range(1, max_protocol_retries + 1): @@ -652,8 +699,8 @@ def _download_with_fallback( f"(attempt {attempt}/{max_protocol_retries})" ) try: - Files._remove_if_exists(local_path) - Files._batch_download_by_protocol( + _provider_util._remove_if_exists(local_path) + PrideProvider._batch_download_by_protocol( [file_record], output_folder, protocol, @@ -666,7 +713,7 @@ def _download_with_fallback( f"Protocol {protocol} failed for {file_record['fileName']}: {error}" ) - valid, reason = Files.validate_download(local_path, expected_checksum) + valid, reason = _provider_util.validate_download(local_path, expected_checksum) if valid: logging.info( f"File {file_record['fileName']} downloaded successfully via {protocol}" @@ -676,7 +723,7 @@ def _download_with_fallback( logging.warning( f"Validation failed for {file_record['fileName']} via {protocol}: {reason}" ) - Files._remove_if_exists(local_path) + _provider_util._remove_if_exists(local_path) logging.warning( f"Protocol {protocol} exhausted for {file_record['fileName']}, switching protocol." @@ -730,10 +777,6 @@ def _download_files_batch( :param aspera_maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. """ - # Patch-sensitive: call _batch_download_by_protocol and - # _download_with_fallback through Files so test patches intercept. - from pridepy.files.files import Files - protocols_supported = ["ftp", "aspera", "globus", "s3"] if protocol not in protocols_supported: logging.error("Protocol should be one of ftp, aspera, globus, s3") @@ -743,14 +786,14 @@ def _download_files_batch( checksum_map: Dict[str, str] = {} if checksum_check: - checksum_file_path = Files.save_checksum_file(accession, output_folder) - checksum_map = Files.read_checksum_file(checksum_file_path) + checksum_file_path = PrideProvider.save_checksum_file(accession, output_folder) + checksum_map = _provider_util.read_checksum_file(checksum_file_path) logging.info(f"Loaded checksums for {len(checksum_map)} files") if not file_list_json: return - protocol_sequence = Files._protocol_sequence(protocol) + protocol_sequence = PrideProvider._protocol_sequence(protocol) primary_protocol = protocol_sequence[0] # Retry with the primary protocol first, then fall back to others fallback_sequence = protocol_sequence @@ -762,7 +805,7 @@ def _download_files_batch( f"Downloading {len(file_list_json)} file(s) via {primary_protocol} (batch)" ) try: - Files._batch_download_by_protocol( + PrideProvider._batch_download_by_protocol( file_list_json, output_folder, primary_protocol, @@ -782,9 +825,9 @@ def _download_files_batch( failed_files: List[str] = [] for i, file_record in enumerate(file_list_json, 1): expected_checksum = checksum_map.get(file_record["fileName"]) - local_path = Files._resolve_local_path(file_record, output_folder) + local_path = _provider_util._resolve_local_path(file_record, output_folder) logging.info("Validating [%d/%d] %s", i, len(file_list_json), file_record["fileName"]) - valid, reason = Files.validate_download(local_path, expected_checksum) + valid, reason = _provider_util.validate_download(local_path, expected_checksum) if valid: continue @@ -792,13 +835,13 @@ def _download_files_batch( f"{file_record['fileName']} invalid after {primary_protocol} ({reason})" ) if "checksum mismatch" in reason: - Files._remove_if_exists(local_path) + _provider_util._remove_if_exists(local_path) if not fallback_sequence: failed_files.append(file_record.get("fileName", "")) continue - success = Files._download_with_fallback( + success = PrideProvider._download_with_fallback( file_record=file_record, output_folder=output_folder, protocol_sequence=fallback_sequence, diff --git a/pridepy/providers/proteomexchange.py b/pridepy/providers/proteomexchange.py index cef0524..f42419b 100644 --- a/pridepy/providers/proteomexchange.py +++ b/pridepy/providers/proteomexchange.py @@ -28,6 +28,7 @@ from typing import ClassVar, Dict, List, Optional from urllib.parse import urlparse +from pridepy.providers import transport from pridepy.providers.base import Provider from pridepy.util.api_handling import Util @@ -139,13 +140,9 @@ def download_files( ) -> None: """Partition record URLs by scheme and route to the matching transport. - Routes ftp:// records to :meth:`Files.download_ftp_urls` and - http(s):// records to :meth:`Files.download_http_urls`, going - through the Files facade so test patches like - ``patch.object(Files, "download_ftp_urls")`` continue to intercept. + Routes ftp:// records to :func:`transport.download_ftp_urls` and + http(s):// records to :func:`transport.download_http_urls`. """ - from pridepy.files.files import Files # lazy: avoid module-load cycle - if not os.path.isdir(output_folder): os.makedirs(output_folder, exist_ok=True) @@ -158,11 +155,11 @@ def download_files( http_urls = [u for u in urls if u.lower().startswith(("http://", "https://"))] if ftp_urls: - Files.download_ftp_urls( + transport.download_ftp_urls( ftp_urls, output_folder, skip_if_downloaded_already ) if http_urls: - Files.download_http_urls( + transport.download_http_urls( http_urls, output_folder, skip_if_downloaded_already ) diff --git a/pridepy/providers/util.py b/pridepy/providers/util.py index 0fc5791..0a4f354 100644 --- a/pridepy/providers/util.py +++ b/pridepy/providers/util.py @@ -133,7 +133,9 @@ def _get_download_url(file_record: Dict, protocol: str) -> str: arbitrary non-Aspera location would produce a URL the caller cannot actually transfer with). """ - from pridepy.files.files import Files + # Lazy import to avoid module-load cycle with PrideProvider (which lives + # in the providers package and imports back into util via _resolve_local_path). + from pridepy.providers.pride import PrideProvider locations = file_record.get("publicFileLocations", []) if not locations: @@ -159,8 +161,8 @@ def _get_download_url(file_record: Dict, protocol: str) -> str: return ftp_url if protocol == "globus": return ftp_url.replace( - Files.PRIDE_ARCHIVE_FTP_URL_PREFIX, - Files.PRIDE_ARCHIVE_HTTPS_URL_PREFIX, + PrideProvider.ARCHIVE_FTP_URL_PREFIX, + PrideProvider.ARCHIVE_HTTPS_URL_PREFIX, 1, ) if protocol == "s3": @@ -172,12 +174,13 @@ def _resolve_local_path(file_record: Dict, output_folder: str) -> str: """ Compute the canonical local path for a file regardless of transfer protocol. """ - from pridepy.files.files import Files + # Lazy import to avoid module-load cycle with PrideProvider. + from pridepy.providers.pride import PrideProvider try: canonical_url = _get_download_url(file_record, "ftp") except ValueError: canonical_url = "" if canonical_url: - return Files.get_output_file_name(canonical_url, file_record, output_folder) + return PrideProvider.get_output_file_name(canonical_url, file_record, output_folder) return os.path.join(output_folder, file_record["fileName"]) diff --git a/pridepy/tests/test_download_by_url.py b/pridepy/tests/test_download_by_url.py index fb34491..aa195dc 100644 --- a/pridepy/tests/test_download_by_url.py +++ b/pridepy/tests/test_download_by_url.py @@ -12,6 +12,7 @@ import click import pytest +from pridepy.commands import by_url from pridepy.files.files import Files from pridepy.pridepy import _read_url_arguments @@ -38,7 +39,7 @@ def fake_http(_url, target_path): _touch_valid(target_path) with patch.object( - Files, "_http_download_url", side_effect=fake_http + by_url, "_http_download_url", side_effect=fake_http ) as mock_http: Files.download_files_by_url( urls=["https://example.org/sample.raw"], @@ -56,7 +57,7 @@ def fake_ftp(_parsed, target_path): _touch_valid(target_path) with patch.object( - Files, "_ftp_download_url", side_effect=fake_ftp + by_url, "_ftp_download_url", side_effect=fake_ftp ) as mock_ftp: Files.download_files_by_url( urls=["ftp://ftp.pride.ebi.ac.uk/path/sample.raw"], @@ -86,7 +87,7 @@ def test_skip_if_exists_short_circuits(self): with tempfile.TemporaryDirectory() as tmp_dir: target = os.path.join(tmp_dir, "existing.raw") _touch_valid(target) - with patch.object(Files, "_http_download_url") as mock_http: + with patch.object(by_url, "_http_download_url") as mock_http: Files.download_files_by_url( urls=["https://example.org/existing.raw"], output_folder=tmp_dir, diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 0f86013..29b115f 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -4,7 +4,13 @@ from unittest import TestCase from unittest.mock import Mock, patch +from pridepy.commands import by_url from pridepy.files.files import Files +from pridepy.providers import transport +from pridepy.providers import util as provider_util +from pridepy.providers.massive import MassiveProvider +from pridepy.providers.pride import PrideProvider +from pridepy.providers import registry class TestDownloadResilience(TestCase): @@ -40,7 +46,7 @@ def test_get_download_url_maps_globus_to_pride_archive_https(self): ] } - download_url = Files._get_download_url(file_record, "globus") + download_url = provider_util._get_download_url(file_record, "globus") assert download_url == "https://ftp.pride.ebi.ac.uk/path/file.raw" @@ -61,10 +67,10 @@ def test_parallel_download_streams_full_file(self): session.get.return_value = stream_response with patch( - "pridepy.files.files.Util.create_session_with_retries", + "pridepy.providers.transport.Util.create_session_with_retries", return_value=session, ): - Files._parallel_download( + transport._parallel_download( "https://example.org/file.raw", output_file, ) @@ -86,10 +92,10 @@ def test_parallel_download_falls_back_when_head_fails(self): session.get.return_value = fallback_response with patch( - "pridepy.files.files.Util.create_session_with_retries", + "pridepy.providers.transport.Util.create_session_with_retries", return_value=session, ): - Files._parallel_download( + transport._parallel_download( "https://example.org/file.raw", output_file, ) @@ -114,10 +120,10 @@ def test_parallel_download_falls_back_without_accept_ranges(self): session.get.return_value = fallback_response with patch( - "pridepy.files.files.Util.create_session_with_retries", + "pridepy.providers.transport.Util.create_session_with_retries", return_value=session, ): - Files._parallel_download( + transport._parallel_download( "https://example.org/file.raw", output_file, ) @@ -142,8 +148,8 @@ def test_validate_download_rejects_empty_and_bad_checksum(self): assert "checksum mismatch" in reason def test_protocol_sequence_prefers_requested_then_fallback(self): - assert Files._protocol_sequence("ftp") == ["ftp", "aspera", "s3", "globus"] - assert Files._protocol_sequence("aspera") == ["aspera", "s3", "ftp", "globus"] + assert PrideProvider._protocol_sequence("ftp") == ["ftp", "aspera", "s3", "globus"] + assert PrideProvider._protocol_sequence("aspera") == ["aspera", "s3", "ftp", "globus"] def test_download_with_fallback_switches_protocol_after_invalid_file(self): file_record = { @@ -168,8 +174,8 @@ def fake_batch(file_list, output_folder, protocol, skip_if_downloaded_already, with open(local_path, "wb") as handle: handle.write(b"abc") - with patch.object(Files, "_batch_download_by_protocol", side_effect=fake_batch): - success = Files._download_with_fallback( + with patch.object(PrideProvider, "_batch_download_by_protocol", side_effect=fake_batch): + success = PrideProvider._download_with_fallback( file_record=file_record, output_folder=tmp_dir, protocol_sequence=["aspera", "s3"], @@ -201,9 +207,9 @@ def fake_batch(file_list, output_folder, protocol, skip_if_downloaded_already, with open(local_path, "wb") as handle: handle.write(b"data") - with patch.object(Files, "_batch_download_by_protocol", side_effect=fake_batch) as batch_mock, \ - patch.object(Files, "_download_with_fallback") as fallback_mock: - Files.download_files( + with patch.object(PrideProvider, "_batch_download_by_protocol", side_effect=fake_batch) as batch_mock, \ + patch.object(PrideProvider, "_download_with_fallback") as fallback_mock: + PrideProvider._download_files_batch( file_list_json=[file_record], accession="PXD000000", output_folder=tmp_dir, @@ -229,8 +235,8 @@ def test_globus_parallel_workers_capped_to_file_count(self): ] with tempfile.TemporaryDirectory() as tmp_dir: - with patch.object(Files, "_globus_download_one") as mock_one: - Files.download_files_from_globus( + with patch.object(PrideProvider, "_globus_download_one") as mock_one: + PrideProvider.download_files_from_globus( file_list_json=file_records, output_folder=tmp_dir, skip_if_downloaded_already=False, @@ -243,7 +249,7 @@ def test_globus_parallel_workers_capped_to_file_count(self): def test_url_parallel_workers_capped_to_url_count(self): """download_files_by_url must cap workers to len(urls).""" with tempfile.TemporaryDirectory() as tmp_dir: - with patch.object(Files, "_download_single_url") as mock_single: + with patch.object(by_url, "_download_single_url") as mock_single: Files.download_files_by_url( urls=["https://example.org/a.raw"], output_folder=tmp_dir, @@ -258,10 +264,10 @@ def test_download_files_raises_when_any_file_fails(self): with tempfile.TemporaryDirectory() as tmp_dir: file_list = [{"fileName": "missing.raw"}] - with patch.object(Files, "_batch_download_by_protocol"), \ - patch.object(Files, "_download_with_fallback", return_value=False): + with patch.object(PrideProvider, "_batch_download_by_protocol"), \ + patch.object(PrideProvider, "_download_with_fallback", return_value=False): with self.assertRaisesRegex(RuntimeError, "missing.raw"): - Files.download_files( + PrideProvider._download_files_batch( file_list_json=file_list, accession="PXD000000", output_folder=tmp_dir, @@ -274,12 +280,10 @@ def test_facade_dispatches_pride_through_registry_to_fallback(self): Files facade -> Registry.resolve -> PrideProvider.download_files -> _batch_download_by_protocol (mocked). - Patching Files._batch_download_by_protocol proves the patch intercepts - (i.e. PrideProvider calls *back* through Files, preserving the test - contract for the multi-protocol orchestrator). + Patching PrideProvider._batch_download_by_protocol proves the patch + intercepts (i.e. PrideProvider owns the multi-protocol orchestrator + and no longer routes through Files). """ - from pridepy.providers.pride import PrideProvider - fake_records = [ { "accession": "PXD000001", @@ -293,9 +297,9 @@ def test_facade_dispatches_pride_through_registry_to_fallback(self): with tempfile.TemporaryDirectory() as tmp: with patch.object(PrideProvider, "list_files", return_value=fake_records), \ - patch.object(Files, "_batch_download_by_protocol", return_value=[]) as batch_mock, \ - patch.object(Files, "validate_download", return_value=(True, "ok")), \ - patch.object(Files, "_download_with_fallback") as fallback_mock: + patch.object(PrideProvider, "_batch_download_by_protocol", return_value=[]) as batch_mock, \ + patch.object(provider_util, "validate_download", return_value=(True, "ok")), \ + patch.object(PrideProvider, "_download_with_fallback") as fallback_mock: Files().download_all_raw_files( accession="PXD000001", output_folder=tmp, diff --git a/pridepy/tests/test_ftp_download_validation.py b/pridepy/tests/test_ftp_download_validation.py index 10bbfb5..ae80ad6 100644 --- a/pridepy/tests/test_ftp_download_validation.py +++ b/pridepy/tests/test_ftp_download_validation.py @@ -12,7 +12,7 @@ import pytest -from pridepy.files.files import Files +from pridepy.providers import transport def _make_fake_ftp(expected_size, write_bytes_per_call): @@ -43,7 +43,7 @@ def test_size_mismatch_is_retried_then_succeeds(self): local_path = os.path.join(tmp, "f.bin") ftp = _make_fake_ftp(expected_size=100, write_bytes_per_call=[50, 50]) - Files._download_one_ftp_path( + transport._download_one_ftp_path( ftp=ftp, ftp_path="/JPST000001/f.bin", local_path=local_path, @@ -64,7 +64,7 @@ def test_size_mismatch_after_retries_raises(self): ftp = _make_fake_ftp(expected_size=100, write_bytes_per_call=[10, 10, 10]) with pytest.raises(RuntimeError, match="Giving up"): - Files._download_one_ftp_path( + transport._download_one_ftp_path( ftp=ftp, ftp_path="/JPST000001/f.bin", local_path=local_path, @@ -79,7 +79,7 @@ def test_correct_size_returns_without_retry(self): local_path = os.path.join(tmp, "f.bin") ftp = _make_fake_ftp(expected_size=50, write_bytes_per_call=[50]) - Files._download_one_ftp_path( + transport._download_one_ftp_path( ftp=ftp, ftp_path="/JPST000001/f.bin", local_path=local_path, diff --git a/pridepy/tests/test_iprox_files.py b/pridepy/tests/test_iprox_files.py index dfdcd21..83c6d2d 100644 --- a/pridepy/tests/test_iprox_files.py +++ b/pridepy/tests/test_iprox_files.py @@ -14,6 +14,9 @@ from unittest.mock import MagicMock, patch from pridepy.files.files import Files +from pridepy.providers import transport +from pridepy.providers.iprox import IproxProvider +from pridepy.providers.pride import PrideProvider IPROX_XML_FIXTURE = """ @@ -61,7 +64,7 @@ def test_iprox_is_a_direct_download_accession(self): assert Files.is_direct_download_accession("IPX0017413000") def test_build_iprox_file_record_maps_px_cv_to_category(self): - record = Files._build_iprox_file_record( + record = IproxProvider._build_file_record( "IPX0017413000", "http://download.iprox.org/IPX0017413000/IPX0017413001/sample.raw", category_from_px="Associated raw file URI", @@ -74,14 +77,13 @@ def test_build_iprox_file_record_maps_px_cv_to_category(self): assert record["publicFileLocations"][0]["value"].startswith("http://") def test_list_iprox_public_files_parses_px_xml(self): - files = Files() fake_response = MagicMock() fake_response.content = IPROX_XML_FIXTURE fake_response.raise_for_status = MagicMock() with patch( - "pridepy.files.files.requests.get", return_value=fake_response + "pridepy.providers.iprox.requests.get", return_value=fake_response ) as req_mock: - records = files._list_iprox_public_files("IPX0017413000") + records = IproxProvider().list_files("IPX0017413000") # The fetch hits the deterministic PX XML URL. req_mock.assert_called_once() @@ -108,8 +110,8 @@ def test_get_all_raw_file_list_filters_iprox_records(self): fake_response.content = IPROX_XML_FIXTURE fake_response.raise_for_status = MagicMock() with patch( - "pridepy.files.files.requests.get", return_value=fake_response - ), patch.object(Files, "stream_all_files_by_project") as pride_mock: + "pridepy.providers.iprox.requests.get", return_value=fake_response + ), patch.object(PrideProvider, "stream_all_files_by_project") as pride_mock: raw_files = files.get_all_raw_file_list("IPX0017413000") pride_mock.assert_not_called() @@ -121,9 +123,9 @@ def test_download_file_by_name_routes_iprox_to_http_urls(self): fake_response.content = IPROX_XML_FIXTURE fake_response.raise_for_status = MagicMock() with tempfile.TemporaryDirectory() as tmp_dir, patch( - "pridepy.files.files.requests.get", return_value=fake_response - ), patch.object(Files, "download_http_urls") as http_mock, patch.object( - Files, "download_ftp_urls" + "pridepy.providers.iprox.requests.get", return_value=fake_response + ), patch.object(transport, "download_http_urls") as http_mock, patch.object( + transport, "download_ftp_urls" ) as ftp_mock: files.download_file_by_name( accession="IPX0017413000", diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py index 1e4c652..53e1a8e 100644 --- a/pridepy/tests/test_jpost_files.py +++ b/pridepy/tests/test_jpost_files.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock, patch from pridepy.files.files import Files +from pridepy.providers import transport from pridepy.providers.jpost import JpostProvider @@ -19,7 +20,7 @@ def test_is_direct_download_accession_includes_jpost(self): assert Files.is_direct_download_accession("JPST000001") def test_build_jpost_file_record_maps_collection_to_category(self): - record = Files._build_jpost_file_record( + record = JpostProvider._build_file_record( "JPST000001", "ftp://ftp.jpostdb.org/JPST000001/peak/sample.mzML", ) @@ -30,7 +31,7 @@ def test_build_jpost_file_record_maps_collection_to_category(self): assert record["source"] == "JPOST" def test_build_jpost_file_record_marks_raw_collection_as_raw(self): - record = Files._build_jpost_file_record( + record = JpostProvider._build_file_record( "JPST000001", "ftp://ftp.jpostdb.org/JPST000001/raw/run01.raw", ) @@ -41,11 +42,11 @@ def test_build_jpost_file_record_marks_raw_collection_as_raw(self): def test_get_all_raw_file_list_filters_jpost_records(self): files = Files() jpost_records = [ - Files._build_jpost_file_record( + JpostProvider._build_file_record( "JPST000001", "ftp://ftp.jpostdb.org/JPST000001/raw/run1.raw", ), - Files._build_jpost_file_record( + JpostProvider._build_file_record( "JPST000001", "ftp://ftp.jpostdb.org/JPST000001/result/results.tsv", ), @@ -59,7 +60,7 @@ def test_get_all_raw_file_list_filters_jpost_records(self): def test_download_file_by_name_uses_jpost_ftp_listing(self): files = Files() - file_record = Files._build_jpost_file_record( + file_record = JpostProvider._build_file_record( "JPST000001", "ftp://ftp.jpostdb.org/JPST000001/raw/folder/sample.raw", ) @@ -67,7 +68,7 @@ def test_download_file_by_name_uses_jpost_ftp_listing(self): with tempfile.TemporaryDirectory() as tmp_dir: with patch.object( JpostProvider, "list_files", return_value=[file_record] - ), patch.object(Files, "download_ftp_urls") as download_mock: + ), patch.object(transport, "download_ftp_urls") as download_mock: files.download_file_by_name( accession="JPST000001", file_name="sample.raw", @@ -89,7 +90,6 @@ def test_download_file_by_name_uses_jpost_ftp_listing(self): ) def test_proxi_listing_maps_cv_name_to_category(self): - files = Files() proxi_response = { "datasetFiles": [ { @@ -117,8 +117,8 @@ def test_proxi_listing_maps_cv_name_to_category(self): fake_response = MagicMock() fake_response.content = json.dumps(proxi_response).encode("utf-8") fake_response.raise_for_status = MagicMock() - with patch("pridepy.files.files.requests.get", return_value=fake_response) as req_mock: - records = files._list_jpost_public_files_via_proxi("JPST002311") + with patch("pridepy.providers.jpost.requests.get", return_value=fake_response) as req_mock: + records = JpostProvider()._list_via_proxi("JPST002311") req_mock.assert_called_once() call_url = req_mock.call_args[0][0] @@ -132,18 +132,14 @@ def test_proxi_listing_maps_cv_name_to_category(self): assert cats["sample01.txt"] == "OTHER" def test_proxi_falls_back_to_ftp_walk_on_error(self): - files = Files() - ftp_record = Files._build_jpost_file_record( - "JPST000001", "ftp://ftp.jpostdb.org/JPST000001/raw/x.raw" - ) with patch.object( - Files, - "_list_jpost_public_files_via_proxi", + JpostProvider, + "_list_via_proxi", side_effect=RuntimeError("proxi down"), ), patch.object( - Files, "_list_ftp_repo_files", return_value=["/JPST000001/raw/x.raw"] + transport, "_list_ftp_repo_files", return_value=["/JPST000001/raw/x.raw"] ) as ftp_mock: - result = files._list_jpost_public_files("JPST000001") + result = JpostProvider().list_files("JPST000001") ftp_mock.assert_called_once() assert len(result) == 1 diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index a4e9278..290aea3 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -3,6 +3,7 @@ from unittest.mock import patch from pridepy.files.files import Files +from pridepy.providers import transport from pridepy.providers.massive import MassiveProvider @@ -14,7 +15,7 @@ def test_is_massive_accession(self): assert not Files.is_massive_accession("MSV123") def test_build_massive_file_record_maps_collection_to_category(self): - record = Files._build_massive_file_record( + record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/ccms_peak/converted/sample.mzML", ) @@ -24,7 +25,7 @@ def test_build_massive_file_record_maps_collection_to_category(self): assert record["fileCategory"]["value"] == "PEAK" def test_build_massive_file_record_marks_raw_collection_as_raw(self): - record = Files._build_massive_file_record( + record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/run01.raw", ) @@ -33,7 +34,7 @@ def test_build_massive_file_record_marks_raw_collection_as_raw(self): assert record["fileCategory"]["value"] == "RAW" def test_build_massive_file_record_keeps_non_raw_collection_even_for_raw_like_file_names(self): - record = Files._build_massive_file_record( + record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/uploads/run01.raw", ) @@ -42,7 +43,7 @@ def test_build_massive_file_record_keeps_non_raw_collection_even_for_raw_like_fi assert record["fileCategory"]["value"] == "OTHER" def test_build_massive_file_record_marks_ab_sciex_scan_sidecar_as_raw_when_under_raw(self): - record = Files._build_massive_file_record( + record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/sample.wiff.scan", ) @@ -53,15 +54,15 @@ def test_build_massive_file_record_marks_ab_sciex_scan_sidecar_as_raw_when_under def test_get_all_raw_file_list_filters_massive_records(self): files = Files() massive_records = [ - Files._build_massive_file_record( + MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/run1.raw", ), - Files._build_massive_file_record( + MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/quant/results.tsv", ), - Files._build_massive_file_record( + MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/uploads/run2.mzML", ), @@ -75,14 +76,14 @@ def test_get_all_raw_file_list_filters_massive_records(self): def test_download_file_by_name_uses_massive_ftp_listing(self): files = Files() - file_record = Files._build_massive_file_record( + file_record = MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/folder/sample.raw", ) with tempfile.TemporaryDirectory() as tmp_dir: with patch.object(MassiveProvider, "list_files", return_value=[file_record]), patch.object( - Files, "download_ftp_urls" + transport, "download_ftp_urls" ) as download_mock: files.download_file_by_name( accession="MSV000012345", @@ -112,7 +113,7 @@ def test_repo_uses_tls_true_for_massive_false_for_jpost(self): def test_download_all_raw_files_threads_parallel_files_for_massive(self): files = Files() massive_records = [ - Files._build_massive_file_record( + MassiveProvider._build_file_record( "MSV000012345", f"ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/run{i}.raw", ) @@ -122,7 +123,7 @@ def test_download_all_raw_files_threads_parallel_files_for_massive(self): with tempfile.TemporaryDirectory() as tmp_dir: with patch.object( MassiveProvider, "list_files", return_value=massive_records - ), patch.object(Files, "download_ftp_urls") as download_mock: + ), patch.object(transport, "download_ftp_urls") as download_mock: files.download_all_raw_files( accession="MSV000012345", output_folder=tmp_dir, @@ -143,7 +144,7 @@ def test_base_direct_download_provider_partitions_urls_by_scheme(self): provider = MassiveProvider() records = [ - Files._build_massive_file_record( + MassiveProvider._build_file_record( "MSV000012345", "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a.raw", ), @@ -157,8 +158,8 @@ def test_base_direct_download_provider_partitions_urls_by_scheme(self): ], }, ] - with patch.object(Files, "download_ftp_urls") as ftp_mock, \ - patch.object(Files, "download_http_urls") as http_mock: + with patch.object(transport, "download_ftp_urls") as ftp_mock, \ + patch.object(transport, "download_http_urls") as http_mock: provider.download_files( accession="MSV000012345", records=records, diff --git a/pyproject.toml b/pyproject.toml index f5b74ee..bc347c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pridepy" -version = "0.0.17" +version = "0.0.18" description = "Python Client library for PRIDE Rest API" readme = "README.md" requires-python = ">=3.9" From 0740c4b57f00cf895a8ae2cd217e0cc4bc6ec7eb Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 06:58:29 +0100 Subject: [PATCH 23/54] refactor(download): rename providers/ to download/, fold commands/ in Pure move: pridepy/providers/* -> pridepy/download/*, plus commands/by_url.py and commands/by_list.py -> download/. All imports updated from pridepy.providers / pridepy.commands to pridepy.download. download/__init__.py docstring rewritten to describe the download subsystem. No logic change. Test suite green (68 passed, 4 skipped). --- .codacy/cli.sh | 149 ++++++++++++++++++ .codacy/codacy.yaml | 15 ++ pridepy/commands/__init__.py | 20 --- pridepy/download/__init__.py | 17 ++ pridepy/{providers => download}/base.py | 4 +- pridepy/{commands => download}/by_list.py | 2 +- pridepy/{commands => download}/by_url.py | 8 +- pridepy/{providers => download}/iprox.py | 8 +- pridepy/{providers => download}/jpost.py | 8 +- pridepy/{providers => download}/massive.py | 6 +- pridepy/{providers => download}/pride.py | 8 +- .../proteomexchange.py | 10 +- pridepy/{providers => download}/registry.py | 2 +- pridepy/{providers => download}/transport.py | 0 pridepy/{providers => download}/util.py | 4 +- pridepy/files/files.py | 34 ++-- pridepy/providers/__init__.py | 7 - pridepy/tests/test_download_by_list.py | 2 +- pridepy/tests/test_download_by_url.py | 2 +- pridepy/tests/test_download_resilience.py | 18 +-- pridepy/tests/test_ftp_download_validation.py | 2 +- pridepy/tests/test_iprox_files.py | 12 +- pridepy/tests/test_jpost_files.py | 6 +- pridepy/tests/test_massive_files.py | 6 +- 24 files changed, 252 insertions(+), 98 deletions(-) create mode 100755 .codacy/cli.sh create mode 100644 .codacy/codacy.yaml delete mode 100644 pridepy/commands/__init__.py create mode 100644 pridepy/download/__init__.py rename pridepy/{providers => download}/base.py (97%) rename pridepy/{commands => download}/by_list.py (98%) rename pridepy/{commands => download}/by_url.py (97%) rename pridepy/{providers => download}/iprox.py (96%) rename pridepy/{providers => download}/jpost.py (96%) rename pridepy/{providers => download}/massive.py (95%) rename pridepy/{providers => download}/pride.py (99%) rename pridepy/{providers => download}/proteomexchange.py (96%) rename pridepy/{providers => download}/registry.py (96%) rename pridepy/{providers => download}/transport.py (100%) rename pridepy/{providers => download}/util.py (98%) delete mode 100644 pridepy/providers/__init__.py diff --git a/.codacy/cli.sh b/.codacy/cli.sh new file mode 100755 index 0000000..7057e3b --- /dev/null +++ b/.codacy/cli.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash + + +set -e +o pipefail + +# Set up paths first +bin_name="codacy-cli-v2" + +# Determine OS-specific paths +os_name=$(uname) +arch=$(uname -m) + +case "$arch" in +"x86_64") + arch="amd64" + ;; +"x86") + arch="386" + ;; +"aarch64"|"arm64") + arch="arm64" + ;; +esac + +if [ -z "$CODACY_CLI_V2_TMP_FOLDER" ]; then + if [ "$(uname)" = "Linux" ]; then + CODACY_CLI_V2_TMP_FOLDER="$HOME/.cache/codacy/codacy-cli-v2" + elif [ "$(uname)" = "Darwin" ]; then + CODACY_CLI_V2_TMP_FOLDER="$HOME/Library/Caches/Codacy/codacy-cli-v2" + else + CODACY_CLI_V2_TMP_FOLDER=".codacy-cli-v2" + fi +fi + +version_file="$CODACY_CLI_V2_TMP_FOLDER/version.yaml" + + +get_version_from_yaml() { + if [ -f "$version_file" ]; then + local version=$(grep -o 'version: *"[^"]*"' "$version_file" | cut -d'"' -f2) + if [ -n "$version" ]; then + echo "$version" + return 0 + fi + fi + return 1 +} + +get_latest_version() { + local response + if [ -n "$GH_TOKEN" ]; then + response=$(curl -Lq --header "Authorization: Bearer $GH_TOKEN" "https://api.github.com/repos/codacy/codacy-cli-v2/releases/latest" 2>/dev/null) + else + response=$(curl -Lq "https://api.github.com/repos/codacy/codacy-cli-v2/releases/latest" 2>/dev/null) + fi + + handle_rate_limit "$response" + local version=$(echo "$response" | grep -m 1 tag_name | cut -d'"' -f4) + echo "$version" +} + +handle_rate_limit() { + local response="$1" + if echo "$response" | grep -q "API rate limit exceeded"; then + fatal "Error: GitHub API rate limit exceeded. Please try again later" + fi +} + +download_file() { + local url="$1" + + echo "Downloading from URL: ${url}" + if command -v curl > /dev/null 2>&1; then + curl -# -LS "$url" -O + elif command -v wget > /dev/null 2>&1; then + wget "$url" + else + fatal "Error: Could not find curl or wget, please install one." + fi +} + +download() { + local url="$1" + local output_folder="$2" + + ( cd "$output_folder" && download_file "$url" ) +} + +download_cli() { + # OS name lower case + suffix=$(echo "$os_name" | tr '[:upper:]' '[:lower:]') + + local bin_folder="$1" + local bin_path="$2" + local version="$3" + + if [ ! -f "$bin_path" ]; then + echo "📥 Downloading CLI version $version..." + + remote_file="codacy-cli-v2_${version}_${suffix}_${arch}.tar.gz" + url="https://github.com/codacy/codacy-cli-v2/releases/download/${version}/${remote_file}" + + download "$url" "$bin_folder" + tar xzfv "${bin_folder}/${remote_file}" -C "${bin_folder}" + fi +} + +# Warn if CODACY_CLI_V2_VERSION is set and update is requested +if [ -n "$CODACY_CLI_V2_VERSION" ] && [ "$1" = "update" ]; then + echo "⚠️ Warning: Performing update with forced version $CODACY_CLI_V2_VERSION" + echo " Unset CODACY_CLI_V2_VERSION to use the latest version" +fi + +# Ensure version.yaml exists and is up to date +if [ ! -f "$version_file" ] || [ "$1" = "update" ]; then + echo "ℹ️ Fetching latest version..." + version=$(get_latest_version) + mkdir -p "$CODACY_CLI_V2_TMP_FOLDER" + echo "version: \"$version\"" > "$version_file" +fi + +# Set the version to use +if [ -n "$CODACY_CLI_V2_VERSION" ]; then + version="$CODACY_CLI_V2_VERSION" +else + version=$(get_version_from_yaml) +fi + + +# Set up version-specific paths +bin_folder="${CODACY_CLI_V2_TMP_FOLDER}/${version}" + +mkdir -p "$bin_folder" +bin_path="$bin_folder"/"$bin_name" + +# Download the tool if not already installed +download_cli "$bin_folder" "$bin_path" "$version" +chmod +x "$bin_path" + +run_command="$bin_path" +if [ -z "$run_command" ]; then + fatal "Codacy cli v2 binary could not be found." +fi + +if [ "$#" -eq 1 ] && [ "$1" = "download" ]; then + echo "Codacy cli v2 download succeeded" +else + eval "$run_command $*" +fi \ No newline at end of file diff --git a/.codacy/codacy.yaml b/.codacy/codacy.yaml new file mode 100644 index 0000000..15365c7 --- /dev/null +++ b/.codacy/codacy.yaml @@ -0,0 +1,15 @@ +runtimes: + - dart@3.7.2 + - go@1.22.3 + - java@17.0.10 + - node@22.2.0 + - python@3.11.11 +tools: + - dartanalyzer@3.7.2 + - eslint@8.57.0 + - lizard@1.17.31 + - pmd@7.11.0 + - pylint@3.3.6 + - revive@1.7.0 + - semgrep@1.78.0 + - trivy@0.66.0 diff --git a/pridepy/commands/__init__.py b/pridepy/commands/__init__.py deleted file mode 100644 index f1312b8..0000000 --- a/pridepy/commands/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -"""Cross-cutting download commands. - -Each module under this package owns one user-facing command that doesn't -fit any single provider: - -- ``by_url``: download a list of explicit URLs (ftp/http/https) -- ``by_list``: download a subset of a project's files by filename - -ProteomeXchange used to live here too but moved to -:class:`pridepy.providers.proteomexchange.ProteomeXchangeProvider` because -it conforms to the ``Provider`` interface (takes an accession or URL and -returns file records). It is deliberately not auto-registered with the -provider registry — PXD/PRD accessions continue to route through -:class:`pridepy.providers.pride.PrideProvider`; ProteomeXchangeProvider is -the explicit gateway for the cross-repository XML view, invoked via the -``download-px-raw-files`` CLI command and ``Files.download_px_raw_files``. - -The ``pridepy.files.files.Files`` facade keeps shim methods that -delegate here, so existing test patches on ``Files.X`` keep working. -""" diff --git a/pridepy/download/__init__.py b/pridepy/download/__init__.py new file mode 100644 index 0000000..f67890d --- /dev/null +++ b/pridepy/download/__init__.py @@ -0,0 +1,17 @@ +"""The pridepy download subsystem. + +This package holds everything involved in turning an accession (or URL) into +downloaded files: + +- Repository adapters — one module per repository (``pride``, ``massive``, + ``jpost``, ``iprox``, ``proteomexchange``). Each subclasses + :class:`pridepy.download.base.Provider` and implements ``matches`` + + ``list_files``; the download workflow itself is inherited from the base. +- :mod:`registry` — maps an accession to the right adapter. +- :mod:`transport` — shared FTP/FTPS/HTTPS plumbing (resume, retry, parallel). +- :mod:`util` — checksum and record helpers. +- :mod:`by_url`, :mod:`by_list` — cross-cutting download commands that take + URLs or filename lists rather than an accession. +- :mod:`client` — the :class:`~pridepy.download.client.Client` facade the CLI + drives; dispatches to adapters via the registry. +""" diff --git a/pridepy/providers/base.py b/pridepy/download/base.py similarity index 97% rename from pridepy/providers/base.py rename to pridepy/download/base.py index cbd1830..2064004 100644 --- a/pridepy/providers/base.py +++ b/pridepy/download/base.py @@ -3,8 +3,8 @@ from abc import ABC, abstractmethod from typing import ClassVar, Dict, List, Optional -from pridepy.providers import transport -from pridepy.providers import util as _provider_util +from pridepy.download import transport +from pridepy.download import util as _provider_util class Provider(ABC): diff --git a/pridepy/commands/by_list.py b/pridepy/download/by_list.py similarity index 98% rename from pridepy/commands/by_list.py rename to pridepy/download/by_list.py index d2d0244..0d2393c 100644 --- a/pridepy/commands/by_list.py +++ b/pridepy/download/by_list.py @@ -2,7 +2,7 @@ import logging from typing import List, Optional -from pridepy.providers import registry +from pridepy.download import registry def download_files_by_list( diff --git a/pridepy/commands/by_url.py b/pridepy/download/by_url.py similarity index 97% rename from pridepy/commands/by_url.py rename to pridepy/download/by_url.py index 375f990..80da1a5 100644 --- a/pridepy/commands/by_url.py +++ b/pridepy/download/by_url.py @@ -15,9 +15,9 @@ from tqdm import tqdm -from pridepy.providers import transport -from pridepy.providers import util as _provider_util -from pridepy.providers.pride import PrideProvider +from pridepy.download import transport +from pridepy.download import util as _provider_util +from pridepy.download.pride import PrideProvider from pridepy.util.api_handling import Util @@ -130,7 +130,7 @@ def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: i """Route a parsed URL to its protocol-specific downloader. ``protocol='globus'`` swaps the http/https single-connection streamer - for :func:`pridepy.providers.transport._parallel_download` (single-connection + for :func:`pridepy.download.transport._parallel_download` (single-connection with progress bar). ftp:// URLs are unaffected. """ scheme = (parsed.scheme or "").lower() diff --git a/pridepy/providers/iprox.py b/pridepy/download/iprox.py similarity index 96% rename from pridepy/providers/iprox.py rename to pridepy/download/iprox.py index 292307c..1c667b7 100644 --- a/pridepy/providers/iprox.py +++ b/pridepy/download/iprox.py @@ -20,9 +20,9 @@ import requests -from pridepy.providers import registry -from pridepy.providers.base import BaseDirectDownloadProvider -from pridepy.providers.jpost import JpostProvider +from pridepy.download import registry +from pridepy.download.base import BaseDirectDownloadProvider +from pridepy.download.jpost import JpostProvider @registry.register @@ -68,7 +68,7 @@ def _build_file_record( ``category_from_px`` is the ``cvParam`` ``name`` from the dataset's ProteomeXchange XML (e.g. ``"Associated raw file URI"``). """ - from pridepy.providers.massive import MassiveProvider + from pridepy.download.massive import MassiveProvider parsed = urlparse(https_url) root_prefix = f"/{accession.upper()}/" relative_path = parsed.path diff --git a/pridepy/providers/jpost.py b/pridepy/download/jpost.py similarity index 96% rename from pridepy/providers/jpost.py rename to pridepy/download/jpost.py index a9ab23e..c4cd0e3 100644 --- a/pridepy/providers/jpost.py +++ b/pridepy/download/jpost.py @@ -18,8 +18,8 @@ import requests -from pridepy.providers import registry -from pridepy.providers.base import BaseDirectDownloadProvider +from pridepy.download import registry +from pridepy.download.base import BaseDirectDownloadProvider @registry.register @@ -71,7 +71,7 @@ def _build_file_record( when the category isn't known. """ # Import the MassIVE collection->category map for the fallback heuristic. - from pridepy.providers.massive import MassiveProvider + from pridepy.download.massive import MassiveProvider parsed = urlparse(ftp_url) root_prefix = f"/{accession.upper()}/" relative_path = parsed.path @@ -103,7 +103,7 @@ def list_files(self, accession: str) -> List[Dict]: f"JPOST PROXI listing failed for {normalized} " f"({proxi_error}); falling back to FTP tree walk." ) - from pridepy.providers import transport + from pridepy.download import transport remote_root = self._get_public_root(normalized) remote_files = transport._list_ftp_repo_files( host=self.ARCHIVE_FTP, diff --git a/pridepy/providers/massive.py b/pridepy/download/massive.py similarity index 95% rename from pridepy/providers/massive.py rename to pridepy/download/massive.py index cdc466b..a9db248 100644 --- a/pridepy/providers/massive.py +++ b/pridepy/download/massive.py @@ -9,8 +9,8 @@ from typing import ClassVar, Dict, List from urllib.parse import urlparse -from pridepy.providers import registry -from pridepy.providers.base import BaseDirectDownloadProvider +from pridepy.download import registry +from pridepy.download.base import BaseDirectDownloadProvider MASSIVE_CATEGORY_MAP = { @@ -79,7 +79,7 @@ def _build_file_record(cls, accession: str, ftp_url: str) -> Dict: } def list_files(self, accession: str) -> List[Dict]: - from pridepy.providers import transport + from pridepy.download import transport normalized = accession.upper() remote_root = self._get_public_root(normalized) remote_files = transport._list_ftp_repo_files( diff --git a/pridepy/providers/pride.py b/pridepy/download/pride.py similarity index 99% rename from pridepy/providers/pride.py rename to pridepy/download/pride.py index 5456544..f2b0caa 100644 --- a/pridepy/providers/pride.py +++ b/pridepy/download/pride.py @@ -35,10 +35,10 @@ from tqdm import tqdm from pridepy.authentication.authentication import Authentication -from pridepy.providers import registry, transport -from pridepy.providers import util as _provider_util -from pridepy.providers.base import Provider -from pridepy.providers.util import Progress +from pridepy.download import registry, transport +from pridepy.download import util as _provider_util +from pridepy.download.base import Provider +from pridepy.download.util import Progress from pridepy.util.api_handling import Util diff --git a/pridepy/providers/proteomexchange.py b/pridepy/download/proteomexchange.py similarity index 96% rename from pridepy/providers/proteomexchange.py rename to pridepy/download/proteomexchange.py index f42419b..7cc45e8 100644 --- a/pridepy/providers/proteomexchange.py +++ b/pridepy/download/proteomexchange.py @@ -6,10 +6,10 @@ repository (PRIDE / MassIVE / JPOST / iProX / etc.). Unlike the other providers in this package, ``ProteomeXchangeProvider`` is -NOT auto-registered with :mod:`pridepy.providers.registry`. PXD/PRD +NOT auto-registered with :mod:`pridepy.download.registry`. PXD/PRD accessions would otherwise be ambiguous between PRIDE's V3 API listing and ProteomeXchange's XML listing; the registry continues to route PXD/PRD via -:class:`pridepy.providers.pride.PrideProvider`. ``ProteomeXchangeProvider`` +:class:`pridepy.download.pride.PrideProvider`. ``ProteomeXchangeProvider`` is the explicit gateway invoked by the ``download-px-raw-files`` CLI command and by ``Files.download_px_raw_files`` — callers who specifically want the cross-repository XML view. @@ -28,8 +28,8 @@ from typing import ClassVar, Dict, List, Optional from urllib.parse import urlparse -from pridepy.providers import transport -from pridepy.providers.base import Provider +from pridepy.download import transport +from pridepy.download.base import Provider from pridepy.util.api_handling import Util @@ -40,7 +40,7 @@ class ProteomeXchangeProvider(Provider): def matches(accession: str) -> bool: """Return True for PXD/PRD accessions or ProteomeCentral URLs. - Not used by :mod:`pridepy.providers.registry` (this provider is + Not used by :mod:`pridepy.download.registry` (this provider is deliberately not auto-registered). Provided for parity with the ``Provider`` interface and so direct callers can introspect whether a given input looks like something ProteomeXchange knows how to diff --git a/pridepy/providers/registry.py b/pridepy/download/registry.py similarity index 96% rename from pridepy/providers/registry.py rename to pridepy/download/registry.py index 7d2c20d..47d2786 100644 --- a/pridepy/providers/registry.py +++ b/pridepy/download/registry.py @@ -7,7 +7,7 @@ """ from typing import List, Type -from pridepy.providers.base import Provider +from pridepy.download.base import Provider _PROVIDERS: List[Type[Provider]] = [] # populated by individual provider modules diff --git a/pridepy/providers/transport.py b/pridepy/download/transport.py similarity index 100% rename from pridepy/providers/transport.py rename to pridepy/download/transport.py diff --git a/pridepy/providers/util.py b/pridepy/download/util.py similarity index 98% rename from pridepy/providers/util.py rename to pridepy/download/util.py index 0a4f354..b75ae21 100644 --- a/pridepy/providers/util.py +++ b/pridepy/download/util.py @@ -135,7 +135,7 @@ def _get_download_url(file_record: Dict, protocol: str) -> str: """ # Lazy import to avoid module-load cycle with PrideProvider (which lives # in the providers package and imports back into util via _resolve_local_path). - from pridepy.providers.pride import PrideProvider + from pridepy.download.pride import PrideProvider locations = file_record.get("publicFileLocations", []) if not locations: @@ -175,7 +175,7 @@ def _resolve_local_path(file_record: Dict, output_folder: str) -> str: Compute the canonical local path for a file regardless of transfer protocol. """ # Lazy import to avoid module-load cycle with PrideProvider. - from pridepy.providers.pride import PrideProvider + from pridepy.download.pride import PrideProvider try: canonical_url = _get_download_url(file_record, "ftp") diff --git a/pridepy/files/files.py b/pridepy/files/files.py index 7764c81..e5c3e77 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -1,6 +1,6 @@ #!/usr/bin/env python """Public Files facade — thin compatibility surface over the modular -provider architecture in :mod:`pridepy.providers`. +provider architecture in :mod:`pridepy.download`. The provider classes own all transport/listing logic; this module exposes a small set of high-level operations (CLI entry points + a handful of @@ -14,18 +14,18 @@ from pridepy.util.api_handling import Util -from pridepy.providers import registry, transport -from pridepy.providers import util as _provider_util -from pridepy.providers.iprox import IproxProvider -from pridepy.providers.jpost import JpostProvider -from pridepy.providers.massive import MASSIVE_CATEGORY_MAP, MassiveProvider -from pridepy.providers.pride import PrideProvider -from pridepy.providers.proteomexchange import ProteomeXchangeProvider -from pridepy.commands import by_list, by_url +from pridepy.download import registry, transport +from pridepy.download import util as _provider_util +from pridepy.download.iprox import IproxProvider +from pridepy.download.jpost import JpostProvider +from pridepy.download.massive import MASSIVE_CATEGORY_MAP, MassiveProvider +from pridepy.download.pride import PrideProvider +from pridepy.download.proteomexchange import ProteomeXchangeProvider +from pridepy.download import by_list, by_url # Re-export Progress so external `from pridepy.files.files import Progress` # still works. -from pridepy.providers.util import Progress # noqa: F401 +from pridepy.download.util import Progress # noqa: F401 class Files: @@ -69,17 +69,17 @@ def __init__(self): @staticmethod def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: - """Shim — see :func:`pridepy.providers.util.compute_md5`.""" + """Shim — see :func:`pridepy.download.util.compute_md5`.""" return _provider_util.compute_md5(file_path, chunk_size) @staticmethod def validate_download(file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, str]: - """Shim — see :func:`pridepy.providers.util.validate_download`.""" + """Shim — see :func:`pridepy.download.util.validate_download`.""" return _provider_util.validate_download(file_path, expected_checksum) @staticmethod def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: - """Shim — see :func:`pridepy.providers.util.read_checksum_file`.""" + """Shim — see :func:`pridepy.download.util.read_checksum_file`.""" return _provider_util.read_checksum_file(checksum_file_path) @staticmethod @@ -92,7 +92,7 @@ def download_ftp_urls( use_tls: bool = False, parallel_files: int = 1, ) -> None: - """Shim — see :func:`pridepy.providers.transport.download_ftp_urls`.""" + """Shim — see :func:`pridepy.download.transport.download_ftp_urls`.""" return transport.download_ftp_urls( ftp_urls=ftp_urls, output_folder=output_folder, @@ -111,7 +111,7 @@ def download_http_urls( parallel_files: int = 1, max_retries: int = 3, ) -> None: - """Shim — see :func:`pridepy.providers.transport.download_http_urls`.""" + """Shim — see :func:`pridepy.download.transport.download_http_urls`.""" return transport.download_http_urls( http_urls=http_urls, output_folder=output_folder, @@ -339,7 +339,7 @@ def download_files_by_list( checksum_check: bool = False, parallel_files: int = 1, ) -> None: - """Delegate to :func:`pridepy.commands.by_list.download_files_by_list`.""" + """Delegate to :func:`pridepy.download.by_list.download_files_by_list`.""" return by_list.download_files_by_list( accession=accession, file_names=file_names, @@ -360,7 +360,7 @@ def download_files_by_url( parallel_files: int = 1, checksum_check: bool = False, ) -> None: - """Delegate to :func:`pridepy.commands.by_url.download_files_by_url`.""" + """Delegate to :func:`pridepy.download.by_url.download_files_by_url`.""" return by_url.download_files_by_url( urls=urls, output_folder=output_folder, diff --git a/pridepy/providers/__init__.py b/pridepy/providers/__init__.py deleted file mode 100644 index ee1de17..0000000 --- a/pridepy/providers/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Per-repository provider classes used by :class:`pridepy.files.files.Files`. - -Each module under this package owns the listing, transport choice, and -record-construction logic for one repository: PRIDE, MassIVE, JPOST, iProX. -The :mod:`registry` module maps an accession to the right provider; the -:mod:`transport` module hosts the shared FTP/FTPS/HTTPS download plumbing. -""" diff --git a/pridepy/tests/test_download_by_list.py b/pridepy/tests/test_download_by_list.py index 5115b4e..fd31076 100644 --- a/pridepy/tests/test_download_by_list.py +++ b/pridepy/tests/test_download_by_list.py @@ -14,7 +14,7 @@ from pridepy.files.files import Files from pridepy.pridepy import _read_filename_arguments -from pridepy.providers.pride import PrideProvider +from pridepy.download.pride import PrideProvider class TestDownloadFilesByList(TestCase): diff --git a/pridepy/tests/test_download_by_url.py b/pridepy/tests/test_download_by_url.py index aa195dc..2f133ac 100644 --- a/pridepy/tests/test_download_by_url.py +++ b/pridepy/tests/test_download_by_url.py @@ -12,7 +12,7 @@ import click import pytest -from pridepy.commands import by_url +from pridepy.download import by_url from pridepy.files.files import Files from pridepy.pridepy import _read_url_arguments diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 29b115f..0a237cc 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -4,13 +4,13 @@ from unittest import TestCase from unittest.mock import Mock, patch -from pridepy.commands import by_url +from pridepy.download import by_url from pridepy.files.files import Files -from pridepy.providers import transport -from pridepy.providers import util as provider_util -from pridepy.providers.massive import MassiveProvider -from pridepy.providers.pride import PrideProvider -from pridepy.providers import registry +from pridepy.download import transport +from pridepy.download import util as provider_util +from pridepy.download.massive import MassiveProvider +from pridepy.download.pride import PrideProvider +from pridepy.download import registry class TestDownloadResilience(TestCase): @@ -67,7 +67,7 @@ def test_parallel_download_streams_full_file(self): session.get.return_value = stream_response with patch( - "pridepy.providers.transport.Util.create_session_with_retries", + "pridepy.download.transport.Util.create_session_with_retries", return_value=session, ): transport._parallel_download( @@ -92,7 +92,7 @@ def test_parallel_download_falls_back_when_head_fails(self): session.get.return_value = fallback_response with patch( - "pridepy.providers.transport.Util.create_session_with_retries", + "pridepy.download.transport.Util.create_session_with_retries", return_value=session, ): transport._parallel_download( @@ -120,7 +120,7 @@ def test_parallel_download_falls_back_without_accept_ranges(self): session.get.return_value = fallback_response with patch( - "pridepy.providers.transport.Util.create_session_with_retries", + "pridepy.download.transport.Util.create_session_with_retries", return_value=session, ): transport._parallel_download( diff --git a/pridepy/tests/test_ftp_download_validation.py b/pridepy/tests/test_ftp_download_validation.py index ae80ad6..7390045 100644 --- a/pridepy/tests/test_ftp_download_validation.py +++ b/pridepy/tests/test_ftp_download_validation.py @@ -12,7 +12,7 @@ import pytest -from pridepy.providers import transport +from pridepy.download import transport def _make_fake_ftp(expected_size, write_bytes_per_call): diff --git a/pridepy/tests/test_iprox_files.py b/pridepy/tests/test_iprox_files.py index 83c6d2d..017e0e4 100644 --- a/pridepy/tests/test_iprox_files.py +++ b/pridepy/tests/test_iprox_files.py @@ -14,9 +14,9 @@ from unittest.mock import MagicMock, patch from pridepy.files.files import Files -from pridepy.providers import transport -from pridepy.providers.iprox import IproxProvider -from pridepy.providers.pride import PrideProvider +from pridepy.download import transport +from pridepy.download.iprox import IproxProvider +from pridepy.download.pride import PrideProvider IPROX_XML_FIXTURE = """ @@ -81,7 +81,7 @@ def test_list_iprox_public_files_parses_px_xml(self): fake_response.content = IPROX_XML_FIXTURE fake_response.raise_for_status = MagicMock() with patch( - "pridepy.providers.iprox.requests.get", return_value=fake_response + "pridepy.download.iprox.requests.get", return_value=fake_response ) as req_mock: records = IproxProvider().list_files("IPX0017413000") @@ -110,7 +110,7 @@ def test_get_all_raw_file_list_filters_iprox_records(self): fake_response.content = IPROX_XML_FIXTURE fake_response.raise_for_status = MagicMock() with patch( - "pridepy.providers.iprox.requests.get", return_value=fake_response + "pridepy.download.iprox.requests.get", return_value=fake_response ), patch.object(PrideProvider, "stream_all_files_by_project") as pride_mock: raw_files = files.get_all_raw_file_list("IPX0017413000") @@ -123,7 +123,7 @@ def test_download_file_by_name_routes_iprox_to_http_urls(self): fake_response.content = IPROX_XML_FIXTURE fake_response.raise_for_status = MagicMock() with tempfile.TemporaryDirectory() as tmp_dir, patch( - "pridepy.providers.iprox.requests.get", return_value=fake_response + "pridepy.download.iprox.requests.get", return_value=fake_response ), patch.object(transport, "download_http_urls") as http_mock, patch.object( transport, "download_ftp_urls" ) as ftp_mock: diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py index 53e1a8e..d5aa1f0 100644 --- a/pridepy/tests/test_jpost_files.py +++ b/pridepy/tests/test_jpost_files.py @@ -4,8 +4,8 @@ from unittest.mock import MagicMock, patch from pridepy.files.files import Files -from pridepy.providers import transport -from pridepy.providers.jpost import JpostProvider +from pridepy.download import transport +from pridepy.download.jpost import JpostProvider class TestJPOSTFiles(TestCase): @@ -117,7 +117,7 @@ def test_proxi_listing_maps_cv_name_to_category(self): fake_response = MagicMock() fake_response.content = json.dumps(proxi_response).encode("utf-8") fake_response.raise_for_status = MagicMock() - with patch("pridepy.providers.jpost.requests.get", return_value=fake_response) as req_mock: + with patch("pridepy.download.jpost.requests.get", return_value=fake_response) as req_mock: records = JpostProvider()._list_via_proxi("JPST002311") req_mock.assert_called_once() diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index 290aea3..d4a6926 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -3,8 +3,8 @@ from unittest.mock import patch from pridepy.files.files import Files -from pridepy.providers import transport -from pridepy.providers.massive import MassiveProvider +from pridepy.download import transport +from pridepy.download.massive import MassiveProvider class TestMassIVEFiles(TestCase): @@ -140,7 +140,7 @@ def test_download_all_raw_files_threads_parallel_files_for_massive(self): def test_base_direct_download_provider_partitions_urls_by_scheme(self): """Records mixing ftp:// and http(s):// route to the right transport.""" - from pridepy.providers.massive import MassiveProvider + from pridepy.download.massive import MassiveProvider provider = MassiveProvider() records = [ From 10ddb70f7b65b27be40e4bec6c333435a4ca0c6f Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 07:04:13 +0100 Subject: [PATCH 24/54] refactor(download): move files.py to download/client.py; Files -> Client The facade moved to pridepy/download/client.py and the class renamed Files -> Client. pridepy/files/files.py is now a back-compat shim (Files = Client, plus Progress re-export). CLI imports Client (aliased as Files internally to keep the diff minimal). Test imports keep working via the shim. No behaviour change; 68 passed, 4 skipped. --- pridepy/download/client.py | 382 ++++++++++++++++++++++++++++++++++++ pridepy/files/files.py | 386 +------------------------------------ pridepy/pridepy.py | 2 +- 3 files changed, 391 insertions(+), 379 deletions(-) create mode 100644 pridepy/download/client.py diff --git a/pridepy/download/client.py b/pridepy/download/client.py new file mode 100644 index 0000000..1e0d799 --- /dev/null +++ b/pridepy/download/client.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python +"""Public ``Client`` facade — thin surface over the modular adapter +architecture in :mod:`pridepy.download`. + +The adapter classes own all transport/listing logic; this module exposes +a small set of high-level operations (CLI entry points + a handful of +one-line shims for downstream Python users). +""" +import logging +import os +from typing import Dict, List, Optional, Tuple + +import requests # noqa: F401 — kept as a patch target for tests + +from pridepy.util.api_handling import Util + +from pridepy.download import registry, transport +from pridepy.download import util as _provider_util +from pridepy.download.iprox import IproxProvider +from pridepy.download.jpost import JpostProvider +from pridepy.download.massive import MASSIVE_CATEGORY_MAP, MassiveProvider +from pridepy.download.pride import PrideProvider +from pridepy.download.proteomexchange import ProteomeXchangeProvider +from pridepy.download import by_list, by_url + +# Re-export Progress so external `from pridepy.download.client import Progress` +# (and the legacy `from pridepy.files.files import Progress`) still works. +from pridepy.download.util import Progress # noqa: F401 + + +class Client: + """High-level facade over the per-repository adapters.""" + + # PRIDE class-attribute re-exports (kept here for back-compat). + V3_API_BASE_URL = PrideProvider.V3_API_BASE_URL + API_BASE_URL = PrideProvider.API_BASE_URL + API_PRIVATE_URL = PrideProvider.API_PRIVATE_URL + PRIDE_ARCHIVE_FTP = PrideProvider.ARCHIVE_FTP + PRIDE_ARCHIVE_FTP_URL_PREFIX = PrideProvider.ARCHIVE_FTP_URL_PREFIX + PRIDE_ARCHIVE_HTTPS_URL_PREFIX = PrideProvider.ARCHIVE_HTTPS_URL_PREFIX + S3_URL = PrideProvider.S3_URL + S3_BUCKET = PrideProvider.S3_BUCKET + PROTOCOL_ORDER = PrideProvider.PROTOCOL_ORDER + + # MassIVE class-attribute re-exports. + MASSIVE_ARCHIVE_FTP = MassiveProvider.ARCHIVE_FTP + MASSIVE_ARCHIVE_FTP_URL_PREFIX = MassiveProvider.ARCHIVE_FTP_URL_PREFIX + + # JPOST class-attribute re-exports. + JPOST_ARCHIVE_FTP = JpostProvider.ARCHIVE_FTP + JPOST_ARCHIVE_FTP_URL_PREFIX = JpostProvider.ARCHIVE_FTP_URL_PREFIX + JPOST_PROXI_BASE_URL = JpostProvider.PROXI_BASE_URL + JPOST_PROXI_CATEGORY_MAP = JpostProvider.PROXI_CATEGORY_MAP + + # iProX class-attribute re-exports. + IPROX_DOWNLOAD_BASE_URL = IproxProvider.DOWNLOAD_BASE_URL + IPROX_PX_XML_URL_TEMPLATE = IproxProvider.PX_XML_URL_TEMPLATE + IPROX_PX_CATEGORY_MAP = IproxProvider.PX_CATEGORY_MAP + + # MassIVE category map re-exported. + MASSIVE_CATEGORY_MAP = MASSIVE_CATEGORY_MAP + + logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + + def __init__(self): + pass + + # Pure delegating shims kept for backward compatibility. + + @staticmethod + def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: + """Shim — see :func:`pridepy.download.util.compute_md5`.""" + return _provider_util.compute_md5(file_path, chunk_size) + + @staticmethod + def validate_download(file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, str]: + """Shim — see :func:`pridepy.download.util.validate_download`.""" + return _provider_util.validate_download(file_path, expected_checksum) + + @staticmethod + def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: + """Shim — see :func:`pridepy.download.util.read_checksum_file`.""" + return _provider_util.read_checksum_file(checksum_file_path) + + @staticmethod + def download_ftp_urls( + ftp_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + max_connection_retries: int = 3, + max_download_retries: int = 3, + use_tls: bool = False, + parallel_files: int = 1, + ) -> None: + """Shim — see :func:`pridepy.download.transport.download_ftp_urls`.""" + return transport.download_ftp_urls( + ftp_urls=ftp_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + max_connection_retries=max_connection_retries, + max_download_retries=max_download_retries, + use_tls=use_tls, + parallel_files=parallel_files, + ) + + @staticmethod + def download_http_urls( + http_urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + parallel_files: int = 1, + max_retries: int = 3, + ) -> None: + """Shim — see :func:`pridepy.download.transport.download_http_urls`.""" + return transport.download_http_urls( + http_urls=http_urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + parallel_files=parallel_files, + max_retries=max_retries, + ) + + # Accession-matcher convenience helpers (useful public API). + + @staticmethod + def is_massive_accession(accession: str) -> bool: + return MassiveProvider.matches(accession) + + @staticmethod + def is_jpost_accession(accession: str) -> bool: + return JpostProvider.matches(accession) + + @staticmethod + def is_iprox_accession(accession: str) -> bool: + return IproxProvider.matches(accession) + + @staticmethod + def is_direct_download_accession(accession: str) -> bool: + """True for MassIVE / JPOST / iProX (explicitly excludes PRIDE).""" + try: + provider = registry.resolve(accession) + except ValueError: + return False + return provider.name != "pride" + + @staticmethod + def _repo_uses_tls(accession: str) -> bool: + """Return the resolved provider's ``use_tls`` flag (False if unknown).""" + try: + provider = registry.resolve(accession) + except ValueError: + return False + return getattr(provider, "use_tls", False) + + # Listing / metadata. + + async def stream_all_files_metadata(self, output_file, accession=None): + """Shim — see :meth:`PrideProvider.stream_all_files_metadata`.""" + return await PrideProvider().stream_all_files_metadata(output_file, accession) + + def get_all_raw_file_list(self, project_accession): + """Get raw file list for any registered provider (records with fileCategory == "RAW").""" + provider = registry.resolve(project_accession) + records = provider.list_files(project_accession) + return [r for r in records if r["fileCategory"]["value"] == "RAW"] + + def get_all_category_file_list( + self, accession: str, categories: "str | List[str]" + ) -> List[Dict]: + """Retrieve project files belonging to the given categories.""" + if isinstance(categories, str): + categories = [categories] + category_set = {c.upper() for c in categories} + records = registry.resolve(accession).list_files(accession) + return [r for r in records if r["fileCategory"]["value"] in category_set] + + def get_submitted_file_path_prefix(self, accession): + """Shim — see :meth:`PrideProvider.get_submitted_file_path_prefix`.""" + return PrideProvider().get_submitted_file_path_prefix(accession) + + def get_file_from_api(self, accession, file_name) -> List[Dict]: + """Return records matching ``file_name`` from the provider's listing.""" + try: + records = registry.resolve(accession).list_files(accession) + return [r for r in records if r["fileName"] == file_name] + except Exception as e: + raise Exception("File not found " + str(e)) + + # Download entry points. + + def download_all_raw_files( + self, + accession, + output_folder, + skip_if_downloaded_already, + protocol, + aspera_maximum_bandwidth: str, + checksum_check: bool = False, + parallel_files: int = 1, + ): + """Download all RAW files for any registered provider.""" + if not os.path.isdir(output_folder): + os.mkdir(output_folder) + provider = registry.resolve(accession) + records = self.get_all_raw_file_list(accession) + provider.download_files( + accession=accession, + records=records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + ) + + def download_all_category_files( + self, + accession: str, + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str, + aspera_maximum_bandwidth: str, + checksum_check: bool, + categories: List[str] = None, + category: str = None, + parallel_files: int = 1, + ): + """Download all files of the given categories from a project.""" + if categories is None: + categories = [category] if category else ["RAW"] + records = self.get_all_category_file_list(accession, categories) + provider = registry.resolve(accession) + provider.download_files( + accession=accession, + records=records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + ) + + def download_file_by_name( + self, + accession, + file_name, + output_folder, + skip_if_downloaded_already, + protocol, + username, + password, + aspera_maximum_bandwidth, + checksum_check, + ): + """Download a single file by name. + + PRIDE supports public / private modes via the V2 private API. Other + providers (MassIVE / JPOST / iProX) only support public downloads. + """ + if not os.path.isdir(output_folder): + os.mkdir(output_folder) + + provider = registry.resolve(accession) + + # Direct-download providers always use the public path. + if provider.name in ("massive", "jpost", "iprox"): + logging.info( + "Downloading file from public direct-download dataset {}".format(accession) + ) + response = self.get_file_from_api(accession, file_name) + if not response: + raise Exception( + "File name {} not found in dataset {}".format(file_name, accession) + ) + provider.download_files( + accession=accession, + records=response, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + ) + return + + # PRIDE has a public/private split that needs status interrogation. + public_project = False + project_status = Util.get_api_call(self.API_BASE_URL + "/status/{}".format(accession)) + + if project_status.status_code == 200: + if project_status.text == "PRIVATE": + public_project = False + elif project_status.text == "PUBLIC": + public_project = True + else: + raise Exception("Dataset {} is not present in PRIDE Archive".format(accession)) + + if public_project: + logging.info("Downloading file from public dataset {}".format(accession)) + response = self.get_file_from_api(accession, file_name) + PrideProvider._download_files_batch( + file_list_json=response, + accession=accession, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + ) + elif not public_project and (username is not None and password is not None): + logging.info("Downloading file from private dataset {}".format(accession)) + PrideProvider().download_private_file_name( + accession=accession, + file_name=file_name, + output_folder=output_folder, + username=username, + password=password, + ) + else: + logging.error( + "For a private dataset {} you must provide a username and password".format( + accession + ) + ) + raise Exception( + "For a private dataset {} you must provide a username and password".format( + accession + ) + ) + + def download_files_by_list( + self, + accession: str, + file_names: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str = "ftp", + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + parallel_files: int = 1, + ) -> None: + """Delegate to :func:`pridepy.download.by_list.download_files_by_list`.""" + return by_list.download_files_by_list( + accession=accession, + file_names=file_names, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, + ) + + @staticmethod + def download_files_by_url( + urls: List[str], + output_folder: str, + skip_if_downloaded_already: bool = False, + protocol: str = "ftp", + parallel_files: int = 1, + checksum_check: bool = False, + ) -> None: + """Delegate to :func:`pridepy.download.by_url.download_files_by_url`.""" + return by_url.download_files_by_url( + urls=urls, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + ) + + def download_px_raw_files( + self, + px_id_or_url: str, + output_folder: str, + skip_if_downloaded_already: bool = True, + ) -> None: + """Delegate to :meth:`ProteomeXchangeProvider.download_from_accession_or_url`.""" + return ProteomeXchangeProvider().download_from_accession_or_url( + px_id_or_url, output_folder, skip_if_downloaded_already + ) diff --git a/pridepy/files/files.py b/pridepy/files/files.py index e5c3e77..654b46a 100644 --- a/pridepy/files/files.py +++ b/pridepy/files/files.py @@ -1,382 +1,12 @@ -#!/usr/bin/env python -"""Public Files facade — thin compatibility surface over the modular -provider architecture in :mod:`pridepy.download`. +"""Backward-compatibility shim. -The provider classes own all transport/listing logic; this module exposes -a small set of high-level operations (CLI entry points + a handful of -one-line shims for downstream Python users). +The download facade moved to :mod:`pridepy.download.client` and the class +was renamed ``Files`` -> ``Client``. This module re-exports it under the old +name so ``from pridepy.files.files import Files`` keeps working, along with +``Progress``. """ -import logging -import os -from typing import Dict, List, Optional, Tuple +from pridepy.download.client import Client, Progress # noqa: F401 -import requests # noqa: F401 — kept as a patch target for tests +Files = Client # legacy alias -from pridepy.util.api_handling import Util - -from pridepy.download import registry, transport -from pridepy.download import util as _provider_util -from pridepy.download.iprox import IproxProvider -from pridepy.download.jpost import JpostProvider -from pridepy.download.massive import MASSIVE_CATEGORY_MAP, MassiveProvider -from pridepy.download.pride import PrideProvider -from pridepy.download.proteomexchange import ProteomeXchangeProvider -from pridepy.download import by_list, by_url - -# Re-export Progress so external `from pridepy.files.files import Progress` -# still works. -from pridepy.download.util import Progress # noqa: F401 - - -class Files: - """High-level facade over the per-repository providers.""" - - # PRIDE class-attribute re-exports (kept here for back-compat). - V3_API_BASE_URL = PrideProvider.V3_API_BASE_URL - API_BASE_URL = PrideProvider.API_BASE_URL - API_PRIVATE_URL = PrideProvider.API_PRIVATE_URL - PRIDE_ARCHIVE_FTP = PrideProvider.ARCHIVE_FTP - PRIDE_ARCHIVE_FTP_URL_PREFIX = PrideProvider.ARCHIVE_FTP_URL_PREFIX - PRIDE_ARCHIVE_HTTPS_URL_PREFIX = PrideProvider.ARCHIVE_HTTPS_URL_PREFIX - S3_URL = PrideProvider.S3_URL - S3_BUCKET = PrideProvider.S3_BUCKET - PROTOCOL_ORDER = PrideProvider.PROTOCOL_ORDER - - # MassIVE class-attribute re-exports. - MASSIVE_ARCHIVE_FTP = MassiveProvider.ARCHIVE_FTP - MASSIVE_ARCHIVE_FTP_URL_PREFIX = MassiveProvider.ARCHIVE_FTP_URL_PREFIX - - # JPOST class-attribute re-exports. - JPOST_ARCHIVE_FTP = JpostProvider.ARCHIVE_FTP - JPOST_ARCHIVE_FTP_URL_PREFIX = JpostProvider.ARCHIVE_FTP_URL_PREFIX - JPOST_PROXI_BASE_URL = JpostProvider.PROXI_BASE_URL - JPOST_PROXI_CATEGORY_MAP = JpostProvider.PROXI_CATEGORY_MAP - - # iProX class-attribute re-exports. - IPROX_DOWNLOAD_BASE_URL = IproxProvider.DOWNLOAD_BASE_URL - IPROX_PX_XML_URL_TEMPLATE = IproxProvider.PX_XML_URL_TEMPLATE - IPROX_PX_CATEGORY_MAP = IproxProvider.PX_CATEGORY_MAP - - # MassIVE category map re-exported. - MASSIVE_CATEGORY_MAP = MASSIVE_CATEGORY_MAP - - logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") - - def __init__(self): - pass - - # Pure delegating shims kept for backward compatibility. - - @staticmethod - def compute_md5(file_path: str, chunk_size: int = 4 * 1024 * 1024) -> str: - """Shim — see :func:`pridepy.download.util.compute_md5`.""" - return _provider_util.compute_md5(file_path, chunk_size) - - @staticmethod - def validate_download(file_path: str, expected_checksum: Optional[str] = None) -> Tuple[bool, str]: - """Shim — see :func:`pridepy.download.util.validate_download`.""" - return _provider_util.validate_download(file_path, expected_checksum) - - @staticmethod - def read_checksum_file(checksum_file_path: str) -> Dict[str, str]: - """Shim — see :func:`pridepy.download.util.read_checksum_file`.""" - return _provider_util.read_checksum_file(checksum_file_path) - - @staticmethod - def download_ftp_urls( - ftp_urls: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - max_connection_retries: int = 3, - max_download_retries: int = 3, - use_tls: bool = False, - parallel_files: int = 1, - ) -> None: - """Shim — see :func:`pridepy.download.transport.download_ftp_urls`.""" - return transport.download_ftp_urls( - ftp_urls=ftp_urls, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - max_connection_retries=max_connection_retries, - max_download_retries=max_download_retries, - use_tls=use_tls, - parallel_files=parallel_files, - ) - - @staticmethod - def download_http_urls( - http_urls: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - parallel_files: int = 1, - max_retries: int = 3, - ) -> None: - """Shim — see :func:`pridepy.download.transport.download_http_urls`.""" - return transport.download_http_urls( - http_urls=http_urls, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - parallel_files=parallel_files, - max_retries=max_retries, - ) - - # Accession-matcher convenience helpers (useful public API). - - @staticmethod - def is_massive_accession(accession: str) -> bool: - return MassiveProvider.matches(accession) - - @staticmethod - def is_jpost_accession(accession: str) -> bool: - return JpostProvider.matches(accession) - - @staticmethod - def is_iprox_accession(accession: str) -> bool: - return IproxProvider.matches(accession) - - @staticmethod - def is_direct_download_accession(accession: str) -> bool: - """True for MassIVE / JPOST / iProX (explicitly excludes PRIDE).""" - try: - provider = registry.resolve(accession) - except ValueError: - return False - return provider.name != "pride" - - @staticmethod - def _repo_uses_tls(accession: str) -> bool: - """Return the resolved provider's ``use_tls`` flag (False if unknown).""" - try: - provider = registry.resolve(accession) - except ValueError: - return False - return getattr(provider, "use_tls", False) - - # Listing / metadata. - - async def stream_all_files_metadata(self, output_file, accession=None): - """Shim — see :meth:`PrideProvider.stream_all_files_metadata`.""" - return await PrideProvider().stream_all_files_metadata(output_file, accession) - - def get_all_raw_file_list(self, project_accession): - """Get raw file list for any registered provider (records with fileCategory == "RAW").""" - provider = registry.resolve(project_accession) - records = provider.list_files(project_accession) - return [r for r in records if r["fileCategory"]["value"] == "RAW"] - - def get_all_category_file_list( - self, accession: str, categories: "str | List[str]" - ) -> List[Dict]: - """Retrieve project files belonging to the given categories.""" - if isinstance(categories, str): - categories = [categories] - category_set = {c.upper() for c in categories} - records = registry.resolve(accession).list_files(accession) - return [r for r in records if r["fileCategory"]["value"] in category_set] - - def get_submitted_file_path_prefix(self, accession): - """Shim — see :meth:`PrideProvider.get_submitted_file_path_prefix`.""" - return PrideProvider().get_submitted_file_path_prefix(accession) - - def get_file_from_api(self, accession, file_name) -> List[Dict]: - """Return records matching ``file_name`` from the provider's listing.""" - try: - records = registry.resolve(accession).list_files(accession) - return [r for r in records if r["fileName"] == file_name] - except Exception as e: - raise Exception("File not found " + str(e)) - - # Download entry points. - - def download_all_raw_files( - self, - accession, - output_folder, - skip_if_downloaded_already, - protocol, - aspera_maximum_bandwidth: str, - checksum_check: bool = False, - parallel_files: int = 1, - ): - """Download all RAW files for any registered provider.""" - if not os.path.isdir(output_folder): - os.mkdir(output_folder) - provider = registry.resolve(accession) - records = self.get_all_raw_file_list(accession) - provider.download_files( - accession=accession, - records=records, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - checksum_check=checksum_check, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - ) - - def download_all_category_files( - self, - accession: str, - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str, - aspera_maximum_bandwidth: str, - checksum_check: bool, - categories: List[str] = None, - category: str = None, - parallel_files: int = 1, - ): - """Download all files of the given categories from a project.""" - if categories is None: - categories = [category] if category else ["RAW"] - records = self.get_all_category_file_list(accession, categories) - provider = registry.resolve(accession) - provider.download_files( - accession=accession, - records=records, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - checksum_check=checksum_check, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - ) - - def download_file_by_name( - self, - accession, - file_name, - output_folder, - skip_if_downloaded_already, - protocol, - username, - password, - aspera_maximum_bandwidth, - checksum_check, - ): - """Download a single file by name. - - PRIDE supports public / private modes via the V2 private API. Other - providers (MassIVE / JPOST / iProX) only support public downloads. - """ - if not os.path.isdir(output_folder): - os.mkdir(output_folder) - - provider = registry.resolve(accession) - - # Direct-download providers always use the public path. - if provider.name in ("massive", "jpost", "iprox"): - logging.info( - "Downloading file from public direct-download dataset {}".format(accession) - ) - response = self.get_file_from_api(accession, file_name) - if not response: - raise Exception( - "File name {} not found in dataset {}".format(file_name, accession) - ) - provider.download_files( - accession=accession, - records=response, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - ) - return - - # PRIDE has a public/private split that needs status interrogation. - public_project = False - project_status = Util.get_api_call(self.API_BASE_URL + "/status/{}".format(accession)) - - if project_status.status_code == 200: - if project_status.text == "PRIVATE": - public_project = False - elif project_status.text == "PUBLIC": - public_project = True - else: - raise Exception("Dataset {} is not present in PRIDE Archive".format(accession)) - - if public_project: - logging.info("Downloading file from public dataset {}".format(accession)) - response = self.get_file_from_api(accession, file_name) - PrideProvider._download_files_batch( - file_list_json=response, - accession=accession, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, - ) - elif not public_project and (username is not None and password is not None): - logging.info("Downloading file from private dataset {}".format(accession)) - PrideProvider().download_private_file_name( - accession=accession, - file_name=file_name, - output_folder=output_folder, - username=username, - password=password, - ) - else: - logging.error( - "For a private dataset {} you must provide a username and password".format( - accession - ) - ) - raise Exception( - "For a private dataset {} you must provide a username and password".format( - accession - ) - ) - - def download_files_by_list( - self, - accession: str, - file_names: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str = "ftp", - aspera_maximum_bandwidth: str = "100M", - checksum_check: bool = False, - parallel_files: int = 1, - ) -> None: - """Delegate to :func:`pridepy.download.by_list.download_files_by_list`.""" - return by_list.download_files_by_list( - accession=accession, - file_names=file_names, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, - parallel_files=parallel_files, - ) - - @staticmethod - def download_files_by_url( - urls: List[str], - output_folder: str, - skip_if_downloaded_already: bool = False, - protocol: str = "ftp", - parallel_files: int = 1, - checksum_check: bool = False, - ) -> None: - """Delegate to :func:`pridepy.download.by_url.download_files_by_url`.""" - return by_url.download_files_by_url( - urls=urls, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - checksum_check=checksum_check, - ) - - def download_px_raw_files( - self, - px_id_or_url: str, - output_folder: str, - skip_if_downloaded_already: bool = True, - ) -> None: - """Delegate to :meth:`ProteomeXchangeProvider.download_from_accession_or_url`.""" - return ProteomeXchangeProvider().download_from_accession_or_url( - px_id_or_url, output_folder, skip_if_downloaded_already - ) +__all__ = ["Client", "Files", "Progress"] diff --git a/pridepy/pridepy.py b/pridepy/pridepy.py index 4929954..744ba07 100644 --- a/pridepy/pridepy.py +++ b/pridepy/pridepy.py @@ -2,7 +2,7 @@ import asyncio import logging import click -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.project.project import Project PROTOCOL_CHOICES = click.Choice(["ftp", "aspera", "globus", "s3"], case_sensitive=False) From bad78ddff069ef27d28983dc225362888482da87 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 07:11:48 +0100 Subject: [PATCH 25/54] refactor(download): Template Method workflow in Provider base Fold BaseDirectDownloadProvider into a single rich Provider(ABC) that owns the shared download workflow (get_raw_files / get_category_files / find_file / download_all_raw / download_category / download_by_name / download_by_filenames) plus a default scheme-partitioning download_files. Adapters (MassIVE / JPOST / iProX / ProteomeXchange) now subclass Provider directly and drop their redundant download_files overrides. --- pridepy/download/base.py | 183 +++++++++++++++++++++++++--- pridepy/download/iprox.py | 6 +- pridepy/download/jpost.py | 4 +- pridepy/download/massive.py | 4 +- pridepy/download/proteomexchange.py | 41 +------ 5 files changed, 172 insertions(+), 66 deletions(-) diff --git a/pridepy/download/base.py b/pridepy/download/base.py index 2064004..4cf33b5 100644 --- a/pridepy/download/base.py +++ b/pridepy/download/base.py @@ -1,16 +1,31 @@ -"""Abstract base classes for pridepy providers.""" +"""Abstract base class for pridepy providers. + +The :class:`Provider` base implements the download *workflow* via the +Template Method pattern: concrete adapters only fill in the holes +(:meth:`matches`, :meth:`list_files`) while the shared listing-filter and +download-orchestration methods live here. Adapters that need different +transport behaviour (e.g. PRIDE's multi-protocol fallback) override +:meth:`download_files`; everything else routes through the inherited +default that partitions record URLs by scheme. +""" import logging from abc import ABC, abstractmethod from typing import ClassVar, Dict, List, Optional from pridepy.download import transport -from pridepy.download import util as _provider_util +from pridepy.download import util as _util class Provider(ABC): """Abstract base for every repository pridepy can list and download from.""" name: ClassVar[str] # "pride", "massive", "jpost", "iprox" + use_tls: ClassVar[bool] = False + supports_checksum: ClassVar[bool] = False + + # ------------------------------------------------------------------ + # Abstract holes — adapters must implement these. + # ------------------------------------------------------------------ @staticmethod @abstractmethod @@ -27,55 +42,185 @@ def list_files(self, accession: str) -> List[Dict]: ``{"name": ..., "value": }``). """ - @abstractmethod - def download_files( + # ------------------------------------------------------------------ + # Hook with default — adapters may override. + # ------------------------------------------------------------------ + + def get_download_url(self, record: Dict, protocol: str = "ftp") -> str: + """Resolve the download URL for ``record`` and ``protocol``.""" + return _util._get_download_url(record, protocol) + + # ------------------------------------------------------------------ + # Shared listing filters. + # ------------------------------------------------------------------ + + def get_raw_files(self, accession: str) -> List[Dict]: + """Return records whose ``fileCategory.value`` is ``"RAW"``.""" + records = self.list_files(accession) + return [r for r in records if r["fileCategory"]["value"] == "RAW"] + + def get_category_files( + self, accession: str, categories: "str | List[str]" + ) -> List[Dict]: + """Return records belonging to the given category (or categories).""" + if isinstance(categories, str): + categories = [categories] + category_set = {c.upper() for c in categories} + records = self.list_files(accession) + return [r for r in records if r["fileCategory"]["value"] in category_set] + + def find_file(self, accession: str, file_name: str) -> List[Dict]: + """Return records whose ``fileName`` equals ``file_name``.""" + records = self.list_files(accession) + return [r for r in records if r["fileName"] == file_name] + + # ------------------------------------------------------------------ + # Shared download workflow (Template Method). + # ------------------------------------------------------------------ + + def download_all_raw( self, accession: str, - records: List[Dict], output_folder: str, skip_if_downloaded_already: bool, protocol: str, - parallel_files: int = 1, + aspera_maximum_bandwidth: str = "100M", checksum_check: bool = False, + parallel_files: int = 1, + ) -> None: + """Download all RAW files for the dataset.""" + self.download_files( + accession=accession, + records=self.get_raw_files(accession), + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + ) + + def download_category( + self, + accession: str, + output_folder: str, + categories: "str | List[str]", + skip_if_downloaded_already: bool, + protocol: str, aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + parallel_files: int = 1, + ) -> None: + """Download all files of the given categories for the dataset.""" + self.download_files( + accession=accession, + records=self.get_category_files(accession, categories), + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + ) + + def download_by_name( + self, + accession: str, + file_name: str, + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str, username: Optional[str] = None, password: Optional[str] = None, + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, ) -> None: - """Download the given records into ``output_folder``.""" + """Download a single file by name from the dataset.""" + records = self.find_file(accession, file_name) + if not records: + raise Exception( + f"File name {file_name} not found in dataset {accession}" + ) + self.download_files( + accession=accession, + records=records, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + ) + def download_by_filenames( + self, + accession: str, + file_names: List[str], + output_folder: str, + skip_if_downloaded_already: bool, + protocol: str = "ftp", + aspera_maximum_bandwidth: str = "100M", + checksum_check: bool = False, + parallel_files: int = 1, + ) -> None: + """Download a subset of project files identified by a filename list. + + :raises ValueError: if ``file_names`` is empty or none match. + """ + if not file_names: + raise ValueError("file_names must contain at least one filename") -class BaseDirectDownloadProvider(Provider): - """Shared ``download_files`` for MassIVE / JPOST / iProX. + all_files = self.list_files(accession) + requested = set(file_names) + matched = [f for f in all_files if f.get("fileName") in requested] + missing = sorted(requested - {f.get("fileName") for f in matched}) + if missing: + logging.warning("Files not found in project %s: %s", accession, missing) + if not matched: + raise ValueError( + f"No matching files in project {accession} for: {sorted(requested)}" + ) - Subclasses set the ``use_tls`` class var (True for MassIVE FTPS, False for - JPOST plain FTP) and override :meth:`list_files`. The shared - ``download_files`` implementation partitions record URLs by scheme: - ``ftp://`` URLs are handed to :func:`transport.download_ftp_urls`; - ``http(s)://`` URLs go to :func:`transport.download_http_urls`. - """ + self.download_files( + accession=accession, + records=matched, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + parallel_files=parallel_files, + checksum_check=checksum_check, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + ) - use_tls: ClassVar[bool] = False + # ------------------------------------------------------------------ + # Default transport — adapters may override (e.g. PrideProvider). + # ------------------------------------------------------------------ def download_files( self, accession: str, records: List[Dict], output_folder: str, - skip_if_downloaded_already: bool, - protocol: str, + skip_if_downloaded_already: bool = False, + protocol: str = "ftp", parallel_files: int = 1, checksum_check: bool = False, aspera_maximum_bandwidth: str = "100M", username: Optional[str] = None, password: Optional[str] = None, ) -> None: + """Partition record URLs by scheme and route to the matching transport. + + ``ftp://`` URLs are handed to :func:`transport.download_ftp_urls` + (with this provider's :attr:`use_tls`); ``http(s)://`` URLs go to + :func:`transport.download_http_urls`. + """ if protocol not in ("ftp", "https", "http"): logging.warning( "Direct downloads currently use ftp / https only. " f"Ignoring requested protocol '{protocol}' for {accession}." ) - all_urls = [_provider_util._get_download_url(record, "ftp") for record in records] + all_urls = [self.get_download_url(record) for record in records] ftp_urls = [u for u in all_urls if u.lower().startswith("ftp://")] http_urls = [ u for u in all_urls if u.lower().startswith(("http://", "https://")) diff --git a/pridepy/download/iprox.py b/pridepy/download/iprox.py index 1c667b7..abb7720 100644 --- a/pridepy/download/iprox.py +++ b/pridepy/download/iprox.py @@ -21,12 +21,12 @@ import requests from pridepy.download import registry -from pridepy.download.base import BaseDirectDownloadProvider +from pridepy.download.base import Provider from pridepy.download.jpost import JpostProvider @registry.register -class IproxProvider(BaseDirectDownloadProvider): +class IproxProvider(Provider): name: ClassVar[str] = "iprox" use_tls: ClassVar[bool] = False # download.iprox.org serves over plain HTTP @@ -86,7 +86,7 @@ def _build_file_record( "fileCategory": {"value": category}, # "FTP Protocol" is the existing label the download dispatcher uses # to locate a file URL; here it actually points at HTTPS. - # BaseDirectDownloadProvider.download_files routes by URL scheme. + # Provider.download_files routes by URL scheme. "publicFileLocations": [{"name": "FTP Protocol", "value": https_url}], "relativePath": relative_path, "collection": collection, diff --git a/pridepy/download/jpost.py b/pridepy/download/jpost.py index c4cd0e3..a2abef1 100644 --- a/pridepy/download/jpost.py +++ b/pridepy/download/jpost.py @@ -19,11 +19,11 @@ import requests from pridepy.download import registry -from pridepy.download.base import BaseDirectDownloadProvider +from pridepy.download.base import Provider @registry.register -class JpostProvider(BaseDirectDownloadProvider): +class JpostProvider(Provider): name: ClassVar[str] = "jpost" use_tls: ClassVar[bool] = False diff --git a/pridepy/download/massive.py b/pridepy/download/massive.py index a9db248..6ecf03a 100644 --- a/pridepy/download/massive.py +++ b/pridepy/download/massive.py @@ -10,7 +10,7 @@ from urllib.parse import urlparse from pridepy.download import registry -from pridepy.download.base import BaseDirectDownloadProvider +from pridepy.download.base import Provider MASSIVE_CATEGORY_MAP = { @@ -28,7 +28,7 @@ @registry.register -class MassiveProvider(BaseDirectDownloadProvider): +class MassiveProvider(Provider): name: ClassVar[str] = "massive" use_tls: ClassVar[bool] = True diff --git a/pridepy/download/proteomexchange.py b/pridepy/download/proteomexchange.py index 7cc45e8..9a1d04e 100644 --- a/pridepy/download/proteomexchange.py +++ b/pridepy/download/proteomexchange.py @@ -25,10 +25,9 @@ import os import re import xml.etree.ElementTree as ET -from typing import ClassVar, Dict, List, Optional +from typing import ClassVar, Dict, List from urllib.parse import urlparse -from pridepy.download import transport from pridepy.download.base import Provider from pridepy.util.api_handling import Util @@ -125,44 +124,6 @@ def list_files(self, accession: str) -> List[Dict]: ) return records - def download_files( - self, - accession: str, - records: List[Dict], - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str, - parallel_files: int = 1, - checksum_check: bool = False, - aspera_maximum_bandwidth: str = "100M", - username: Optional[str] = None, - password: Optional[str] = None, - ) -> None: - """Partition record URLs by scheme and route to the matching transport. - - Routes ftp:// records to :func:`transport.download_ftp_urls` and - http(s):// records to :func:`transport.download_http_urls`. - """ - if not os.path.isdir(output_folder): - os.makedirs(output_folder, exist_ok=True) - - urls = [ - record["publicFileLocations"][0]["value"] - for record in records - if record.get("publicFileLocations") - ] - ftp_urls = [u for u in urls if u.lower().startswith("ftp://")] - http_urls = [u for u in urls if u.lower().startswith(("http://", "https://"))] - - if ftp_urls: - transport.download_ftp_urls( - ftp_urls, output_folder, skip_if_downloaded_already - ) - if http_urls: - transport.download_http_urls( - http_urls, output_folder, skip_if_downloaded_already - ) - def download_from_accession_or_url( self, px_id_or_url: str, From bfe06d27553e2142ae1b94ae56eca66477e49eaf Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 07:11:58 +0100 Subject: [PATCH 26/54] refactor(download): PrideProvider owns public/private download_by_name Align PrideProvider.download_files signature with the Provider base and add a download_by_name override implementing PRIDE's public/private split (V2 private API for private datasets, inherited public path otherwise). Moves this logic out of the Client facade. --- pridepy/download/pride.py | 80 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index f2b0caa..7bb18c6 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -737,15 +737,20 @@ def download_files( accession, records: List[Dict], output_folder: str, - skip_if_downloaded_already, + skip_if_downloaded_already: bool = False, protocol: str = "ftp", - aspera_maximum_bandwidth: str = "100M", - checksum_check: bool = False, parallel_files: int = 1, + checksum_check: bool = False, + aspera_maximum_bandwidth: str = "100M", username: Optional[str] = None, password: Optional[str] = None, ): - """Implement Provider.download_files — maps to the legacy static batch downloader.""" + """Override Provider.download_files with the multi-protocol orchestrator. + + Reuses the legacy batch downloader: Phase 1 batches the requested + protocol over a single connection, Phase 2 validates every file, and + Phase 3 falls back per-file across the remaining protocols. + """ PrideProvider._download_files_batch( file_list_json=records, accession=accession, @@ -757,6 +762,73 @@ def download_files( parallel_files=parallel_files, ) + def download_by_name( + self, + accession, + file_name, + output_folder, + skip_if_downloaded_already, + protocol, + username=None, + password=None, + aspera_maximum_bandwidth="100M", + checksum_check=False, + ): + """Download a single file by name, honouring PRIDE's public/private split. + + PRIDE exposes private datasets via the V2 private API (username + + password); public datasets route through the standard listing + + multi-protocol download path inherited from :class:`Provider`. + """ + public_project = False + project_status = Util.get_api_call( + self.API_BASE_URL + "/status/{}".format(accession) + ) + + if project_status.status_code == 200: + if project_status.text == "PRIVATE": + public_project = False + elif project_status.text == "PUBLIC": + public_project = True + else: + raise Exception( + "Dataset {} is not present in PRIDE Archive".format(accession) + ) + + if public_project: + logging.info("Downloading file from public dataset {}".format(accession)) + super().download_by_name( + accession=accession, + file_name=file_name, + output_folder=output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + username=username, + password=password, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + ) + elif not public_project and (username is not None and password is not None): + logging.info("Downloading file from private dataset {}".format(accession)) + self.download_private_file_name( + accession=accession, + file_name=file_name, + output_folder=output_folder, + username=username, + password=password, + ) + else: + logging.error( + "For a private dataset {} you must provide a username and password".format( + accession + ) + ) + raise Exception( + "For a private dataset {} you must provide a username and password".format( + accession + ) + ) + @staticmethod def _download_files_batch( file_list_json: List[Dict], From 1990611e0de29bccd7825f14ccdaca9f944100fd Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 07:11:58 +0100 Subject: [PATCH 27/54] refactor(download): slim Client facade to registry dispatches Reduce the public listing/download methods to one-line registry.resolve(...).(...) dispatches now that the Provider base owns the workflow. Drops the in-facade PRIDE public/private split and by_list delegation; removes newly-unused imports. --- pridepy/download/client.py | 143 +++++++++---------------------------- 1 file changed, 35 insertions(+), 108 deletions(-) diff --git a/pridepy/download/client.py b/pridepy/download/client.py index 1e0d799..dcb93ce 100644 --- a/pridepy/download/client.py +++ b/pridepy/download/client.py @@ -7,13 +7,10 @@ one-line shims for downstream Python users). """ import logging -import os from typing import Dict, List, Optional, Tuple import requests # noqa: F401 — kept as a patch target for tests -from pridepy.util.api_handling import Util - from pridepy.download import registry, transport from pridepy.download import util as _provider_util from pridepy.download.iprox import IproxProvider @@ -21,7 +18,7 @@ from pridepy.download.massive import MASSIVE_CATEGORY_MAP, MassiveProvider from pridepy.download.pride import PrideProvider from pridepy.download.proteomexchange import ProteomeXchangeProvider -from pridepy.download import by_list, by_url +from pridepy.download import by_url # Re-export Progress so external `from pridepy.download.client import Progress` # (and the legacy `from pridepy.files.files import Progress`) still works. @@ -158,21 +155,15 @@ async def stream_all_files_metadata(self, output_file, accession=None): """Shim — see :meth:`PrideProvider.stream_all_files_metadata`.""" return await PrideProvider().stream_all_files_metadata(output_file, accession) - def get_all_raw_file_list(self, project_accession): + def get_all_raw_file_list(self, accession): """Get raw file list for any registered provider (records with fileCategory == "RAW").""" - provider = registry.resolve(project_accession) - records = provider.list_files(project_accession) - return [r for r in records if r["fileCategory"]["value"] == "RAW"] + return registry.resolve(accession).get_raw_files(accession) def get_all_category_file_list( self, accession: str, categories: "str | List[str]" ) -> List[Dict]: """Retrieve project files belonging to the given categories.""" - if isinstance(categories, str): - categories = [categories] - category_set = {c.upper() for c in categories} - records = registry.resolve(accession).list_files(accession) - return [r for r in records if r["fileCategory"]["value"] in category_set] + return registry.resolve(accession).get_category_files(accession, categories) def get_submitted_file_path_prefix(self, accession): """Shim — see :meth:`PrideProvider.get_submitted_file_path_prefix`.""" @@ -181,8 +172,7 @@ def get_submitted_file_path_prefix(self, accession): def get_file_from_api(self, accession, file_name) -> List[Dict]: """Return records matching ``file_name`` from the provider's listing.""" try: - records = registry.resolve(accession).list_files(accession) - return [r for r in records if r["fileName"] == file_name] + return registry.resolve(accession).find_file(accession, file_name) except Exception as e: raise Exception("File not found " + str(e)) @@ -199,19 +189,14 @@ def download_all_raw_files( parallel_files: int = 1, ): """Download all RAW files for any registered provider.""" - if not os.path.isdir(output_folder): - os.mkdir(output_folder) - provider = registry.resolve(accession) - records = self.get_all_raw_file_list(accession) - provider.download_files( - accession=accession, - records=records, - output_folder=output_folder, + return registry.resolve(accession).download_all_raw( + accession, + output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, - parallel_files=parallel_files, - checksum_check=checksum_check, aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, ) def download_all_category_files( @@ -229,17 +214,15 @@ def download_all_category_files( """Download all files of the given categories from a project.""" if categories is None: categories = [category] if category else ["RAW"] - records = self.get_all_category_file_list(accession, categories) - provider = registry.resolve(accession) - provider.download_files( - accession=accession, - records=records, - output_folder=output_folder, + return registry.resolve(accession).download_category( + accession, + output_folder, + categories, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, - parallel_files=parallel_files, - checksum_check=checksum_check, aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + parallel_files=parallel_files, ) def download_file_by_name( @@ -256,77 +239,21 @@ def download_file_by_name( ): """Download a single file by name. - PRIDE supports public / private modes via the V2 private API. Other - providers (MassIVE / JPOST / iProX) only support public downloads. + Dispatches to the resolved provider. PRIDE overrides this to handle + its public / private split via the V2 private API; the direct-download + providers (MassIVE / JPOST / iProX) use the inherited public path. """ - if not os.path.isdir(output_folder): - os.mkdir(output_folder) - - provider = registry.resolve(accession) - - # Direct-download providers always use the public path. - if provider.name in ("massive", "jpost", "iprox"): - logging.info( - "Downloading file from public direct-download dataset {}".format(accession) - ) - response = self.get_file_from_api(accession, file_name) - if not response: - raise Exception( - "File name {} not found in dataset {}".format(file_name, accession) - ) - provider.download_files( - accession=accession, - records=response, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - ) - return - - # PRIDE has a public/private split that needs status interrogation. - public_project = False - project_status = Util.get_api_call(self.API_BASE_URL + "/status/{}".format(accession)) - - if project_status.status_code == 200: - if project_status.text == "PRIVATE": - public_project = False - elif project_status.text == "PUBLIC": - public_project = True - else: - raise Exception("Dataset {} is not present in PRIDE Archive".format(accession)) - - if public_project: - logging.info("Downloading file from public dataset {}".format(accession)) - response = self.get_file_from_api(accession, file_name) - PrideProvider._download_files_batch( - file_list_json=response, - accession=accession, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - checksum_check=checksum_check, - ) - elif not public_project and (username is not None and password is not None): - logging.info("Downloading file from private dataset {}".format(accession)) - PrideProvider().download_private_file_name( - accession=accession, - file_name=file_name, - output_folder=output_folder, - username=username, - password=password, - ) - else: - logging.error( - "For a private dataset {} you must provide a username and password".format( - accession - ) - ) - raise Exception( - "For a private dataset {} you must provide a username and password".format( - accession - ) - ) + return registry.resolve(accession).download_by_name( + accession, + file_name, + output_folder, + skip_if_downloaded_already=skip_if_downloaded_already, + protocol=protocol, + username=username, + password=password, + aspera_maximum_bandwidth=aspera_maximum_bandwidth, + checksum_check=checksum_check, + ) def download_files_by_list( self, @@ -339,11 +266,11 @@ def download_files_by_list( checksum_check: bool = False, parallel_files: int = 1, ) -> None: - """Delegate to :func:`pridepy.download.by_list.download_files_by_list`.""" - return by_list.download_files_by_list( - accession=accession, - file_names=file_names, - output_folder=output_folder, + """Download a subset of project files identified by a filename list.""" + return registry.resolve(accession).download_by_filenames( + accession, + file_names, + output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol=protocol, aspera_maximum_bandwidth=aspera_maximum_bandwidth, From 47464088ce20ac741ae0adb568b2da3e405de381 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 07:14:21 +0100 Subject: [PATCH 28/54] chore(release): bump version to 0.0.19 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index bc347c0..5456fc2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pridepy" -version = "0.0.18" +version = "0.0.19" description = "Python Client library for PRIDE Rest API" readme = "README.md" requires-python = ">=3.9" From d7ea80801415a5d0cf3d56c855f42f8c6d118bda Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 07:42:48 +0100 Subject: [PATCH 29/54] refactor(download): move PRIDE-specific helpers into PrideProvider Tier 1 + Tier 2 of the provider-specific helper audit: by_url.py (Tier 1) -> PrideProvider: - _extract_pride_accession -> PrideProvider.extract_accession_from_url - _validate_urls_checksums -> PrideProvider.validate_urls_checksums download_files_by_url now calls PrideProvider.validate_urls_checksums when checksum_check is set. by_url keeps only generic URL-download mechanics. util.py (Tier 2) -> PrideProvider: - _get_download_url (the aspera/ftp/globus/s3 protocol resolution + PRIDE archive HTTPS rewrite for globus) -> PrideProvider._get_download_url - _resolve_local_path -> PrideProvider._resolve_local_path The base Provider.get_download_url is now generic (returns the 'FTP Protocol' location value, which is what MassIVE/JPOST/iProX need); PrideProvider.get_download_url overrides it with the multi-protocol resolution. PRIDE's static download methods call PrideProvider._get_download_url directly. util.py now holds only genuinely generic helpers (Progress, compute_md5, validate_download, read_checksum_file, _find_tsv_columns, _is_md5_checksum, _remove_if_exists). Removed the lazy 'from pridepy.download.pride import PrideProvider' imports that util needed for the moved functions. Test: the one test referencing util._get_download_url now references PrideProvider._get_download_url (PrideProvider already imported). No behaviour change. 68 passed, 4 skipped. flake8 (E9,F63,F7,F82) clean. Smoke MD5 unchanged. --- pridepy/download/base.py | 17 ++- pridepy/download/by_url.py | 62 +---------- pridepy/download/pride.py | 125 +++++++++++++++++++++- pridepy/download/util.py | 65 +---------- pridepy/tests/test_download_resilience.py | 2 +- 5 files changed, 138 insertions(+), 133 deletions(-) diff --git a/pridepy/download/base.py b/pridepy/download/base.py index 4cf33b5..16aaaea 100644 --- a/pridepy/download/base.py +++ b/pridepy/download/base.py @@ -13,7 +13,6 @@ from typing import ClassVar, Dict, List, Optional from pridepy.download import transport -from pridepy.download import util as _util class Provider(ABC): @@ -47,8 +46,20 @@ def list_files(self, accession: str) -> List[Dict]: # ------------------------------------------------------------------ def get_download_url(self, record: Dict, protocol: str = "ftp") -> str: - """Resolve the download URL for ``record`` and ``protocol``.""" - return _util._get_download_url(record, protocol) + """Resolve the download URL for ``record``. + + Default: return the ``"FTP Protocol"`` public-file-location value + (direct-download adapters store their public URL there — ftp:// for + MassIVE/JPOST, http(s):// for iProX). Adapters with richer, + protocol-aware resolution (PRIDE: aspera/globus/s3) override this. + """ + locations = record.get("publicFileLocations", []) + if not locations: + raise ValueError("No public file locations present") + for location in locations: + if location.get("name") == "FTP Protocol": + return location.get("value") + return locations[0].get("value") # ------------------------------------------------------------------ # Shared listing filters. diff --git a/pridepy/download/by_url.py b/pridepy/download/by_url.py index 80da1a5..d80cda2 100644 --- a/pridepy/download/by_url.py +++ b/pridepy/download/by_url.py @@ -7,10 +7,9 @@ import ftplib import logging import os -import re from concurrent.futures import ThreadPoolExecutor, as_completed from ftplib import FTP -from typing import Dict, List, Optional, Tuple +from typing import List, Tuple from urllib.parse import urlparse from tqdm import tqdm @@ -21,63 +20,6 @@ from pridepy.util.api_handling import Util -def _extract_pride_accession(url: str) -> Optional[str]: - """Extract a PRIDE accession (PXD/PRD followed by digits) from a URL path. - - PRIDE archive URLs follow the pattern - ``…/pride/data/archive/YYYY/MM//filename``. - Returns ``None`` when no accession can be identified. - """ - match = re.search(r"((?:PXD|PRD)\d{4,})", url) - return match.group(1) if match else None - - -def _validate_urls_checksums(urls: List[str], output_folder: str) -> None: - """Validate downloaded files against PRIDE checksum API. - - Accessions are inferred from URL paths via - :func:`_extract_pride_accession`. URLs that do not contain a - recognisable PRIDE accession are skipped with a warning. - - :raises RuntimeError: if one or more files fail validation - """ - accession_urls: Dict[str, List[str]] = {} - for url in urls: - acc = _extract_pride_accession(url) - if acc: - accession_urls.setdefault(acc, []).append(url) - else: - logging.warning( - "Cannot infer PRIDE accession from URL, skipping checksum: %s", url - ) - - validation_failures: List[str] = [] - for acc, acc_urls in accession_urls.items(): - checksum_file_path = PrideProvider.save_checksum_file(acc, output_folder) - checksum_map = _provider_util.read_checksum_file(checksum_file_path) - logging.info( - "Loaded checksums for %d files (project %s)", - len(checksum_map), acc, - ) - for url in acc_urls: - file_name = os.path.basename(urlparse(url).path) - target = os.path.join(output_folder, file_name) - expected = checksum_map.get(file_name) - logging.info("Validating %s", file_name) - valid, reason = _provider_util.validate_download(target, expected) - if not valid: - logging.error("Validation failed for %s: %s", file_name, reason) - validation_failures.append(f"{file_name} ({reason})") - else: - logging.info("Checksum OK: %s", file_name) - - if validation_failures: - raise RuntimeError( - f"Checksum validation failed for {len(validation_failures)} file(s): " - + ", ".join(validation_failures) - ) - - def _http_download_url(url: str, target: str) -> None: """Stream an http/https URL into ``target`` with a progress bar.""" session = Util.create_session_with_retries() @@ -247,4 +189,4 @@ def download_files_by_url( ) if checksum_check: - _validate_urls_checksums(urls, output_folder) + PrideProvider.validate_urls_checksums(urls, output_folder) diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index 7bb18c6..c02e7ad 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -166,6 +166,121 @@ def get_output_file_name(download_url, file, output_folder): new_file_path = os.path.join(output_folder, f"{public_filepath_part[1]}") return new_file_path + @staticmethod + def _get_download_url(file_record: Dict, protocol: str) -> str: + """Resolve the PRIDE public download URL for a file and protocol. + + Raises ValueError when the requested protocol has no suitable location. + Aspera requires a dedicated "Aspera Protocol" entry; ftp/s3/globus + derive their URL from the "FTP Protocol" entry (falling back to an + arbitrary non-Aspera location would produce a URL the caller cannot + actually transfer with). The globus URL is the FTP path rewritten to + the PRIDE archive HTTPS prefix. + """ + locations = file_record.get("publicFileLocations", []) + if not locations: + raise ValueError("No public file locations present") + + aspera_url = None + ftp_url = None + for location in locations: + name = location.get("name") + if name == "Aspera Protocol": + aspera_url = location.get("value") + elif name == "FTP Protocol": + ftp_url = location.get("value") + + if protocol == "aspera": + if not aspera_url: + raise ValueError("Aspera URL not available") + return aspera_url + + if not ftp_url: + raise ValueError("FTP URL not available") + if protocol == "ftp": + return ftp_url + if protocol == "globus": + return ftp_url.replace( + PrideProvider.ARCHIVE_FTP_URL_PREFIX, + PrideProvider.ARCHIVE_HTTPS_URL_PREFIX, + 1, + ) + if protocol == "s3": + return ftp_url + raise ValueError(f"Unsupported protocol: {protocol}") + + def get_download_url(self, record: Dict, protocol: str = "ftp") -> str: + """Override the base hook with PRIDE's multi-protocol resolution.""" + return PrideProvider._get_download_url(record, protocol) + + @staticmethod + def _resolve_local_path(file_record: Dict, output_folder: str) -> str: + """Compute the canonical local path for a file regardless of protocol.""" + try: + canonical_url = PrideProvider._get_download_url(file_record, "ftp") + except ValueError: + canonical_url = "" + if canonical_url: + return PrideProvider.get_output_file_name(canonical_url, file_record, output_folder) + return os.path.join(output_folder, file_record["fileName"]) + + @staticmethod + def extract_accession_from_url(url: str) -> Optional[str]: + """Extract a PRIDE accession (PXD/PRD followed by digits) from a URL. + + PRIDE archive URLs follow the pattern + ``…/pride/data/archive/YYYY/MM//filename``. + Returns ``None`` when no accession can be identified. + """ + match = re.search(r"((?:PXD|PRD)\d{4,})", url) + return match.group(1) if match else None + + @staticmethod + def validate_urls_checksums(urls: List[str], output_folder: str) -> None: + """Validate downloaded files against the PRIDE checksum API. + + Accessions are inferred from URL paths via + :meth:`extract_accession_from_url`. URLs that do not contain a + recognisable PRIDE accession are skipped with a warning. + + :raises RuntimeError: if one or more files fail validation + """ + accession_urls: Dict[str, List[str]] = {} + for url in urls: + acc = PrideProvider.extract_accession_from_url(url) + if acc: + accession_urls.setdefault(acc, []).append(url) + else: + logging.warning( + "Cannot infer PRIDE accession from URL, skipping checksum: %s", url + ) + + validation_failures: List[str] = [] + for acc, acc_urls in accession_urls.items(): + checksum_file_path = PrideProvider.save_checksum_file(acc, output_folder) + checksum_map = _provider_util.read_checksum_file(checksum_file_path) + logging.info( + "Loaded checksums for %d files (project %s)", + len(checksum_map), acc, + ) + for url in acc_urls: + file_name = os.path.basename(urlparse(url).path) + target = os.path.join(output_folder, file_name) + expected = checksum_map.get(file_name) + logging.info("Validating %s", file_name) + valid, reason = _provider_util.validate_download(target, expected) + if not valid: + logging.error("Validation failed for %s: %s", file_name, reason) + validation_failures.append(f"{file_name} ({reason})") + else: + logging.info("Checksum OK: %s", file_name) + + if validation_failures: + raise RuntimeError( + f"Checksum validation failed for {len(validation_failures)} file(s): " + + ", ".join(validation_failures) + ) + @staticmethod def save_checksum_file(accession, output_folder): """ @@ -191,7 +306,7 @@ def save_checksum_file(accession, output_folder): @staticmethod def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_retries=6, position=0): """Download a single file via globus; used as a worker target.""" - download_url = _provider_util._get_download_url(file, "globus") + download_url = PrideProvider._get_download_url(file, "globus") new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) if skip_if_downloaded_already and os.path.exists(new_file_path): @@ -413,7 +528,7 @@ def download_files_from_globus( # --- Phase 0: pre-filter files that need downloading ----------------- files_to_download: List[Dict] = [] for file in file_list_json: - download_url = _provider_util._get_download_url(file, "globus") + download_url = PrideProvider._get_download_url(file, "globus") new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) if skip_if_downloaded_already and os.path.exists(new_file_path): expected_cs = checksum_map.get(file.get("fileName", "")) @@ -445,7 +560,7 @@ def download_files_from_globus( file, output_folder, False ) new_file_path = PrideProvider.get_output_file_name( - _provider_util._get_download_url(file, "globus"), file, output_folder + PrideProvider._get_download_url(file, "globus"), file, output_folder ) logging.info(f"Successfully downloaded {new_file_path}") except Exception as e: @@ -690,7 +805,7 @@ def _download_with_fallback( after every attempt. Intended as the per-file fallback path; batch download of the primary protocol is handled separately. """ - local_path = _provider_util._resolve_local_path(file_record, output_folder) + local_path = PrideProvider._resolve_local_path(file_record, output_folder) for protocol in protocol_sequence: for attempt in range(1, max_protocol_retries + 1): @@ -897,7 +1012,7 @@ def _download_files_batch( failed_files: List[str] = [] for i, file_record in enumerate(file_list_json, 1): expected_checksum = checksum_map.get(file_record["fileName"]) - local_path = _provider_util._resolve_local_path(file_record, output_folder) + local_path = PrideProvider._resolve_local_path(file_record, output_folder) logging.info("Validating [%d/%d] %s", i, len(file_list_json), file_record["fileName"]) valid, reason = _provider_util.validate_download(local_path, expected_checksum) if valid: diff --git a/pridepy/download/util.py b/pridepy/download/util.py index b75ae21..3dde0fc 100644 --- a/pridepy/download/util.py +++ b/pridepy/download/util.py @@ -9,7 +9,7 @@ import hashlib import logging import os -from typing import Dict, List, Optional, Tuple +from typing import Dict, Optional, Tuple from tqdm import tqdm @@ -121,66 +121,3 @@ def _remove_if_exists(file_path: str) -> None: """ if os.path.exists(file_path): os.remove(file_path) - - -def _get_download_url(file_record: Dict, protocol: str) -> str: - """ - Resolve the public download URL for a file and protocol. - - Raises ValueError when the requested protocol has no suitable location. - Aspera requires a dedicated "Aspera Protocol" entry; ftp/s3/globus - derive their URL from the "FTP Protocol" entry (falling back to an - arbitrary non-Aspera location would produce a URL the caller cannot - actually transfer with). - """ - # Lazy import to avoid module-load cycle with PrideProvider (which lives - # in the providers package and imports back into util via _resolve_local_path). - from pridepy.download.pride import PrideProvider - - locations = file_record.get("publicFileLocations", []) - if not locations: - raise ValueError("No public file locations present") - - aspera_url = None - ftp_url = None - for location in locations: - name = location.get("name") - if name == "Aspera Protocol": - aspera_url = location.get("value") - elif name == "FTP Protocol": - ftp_url = location.get("value") - - if protocol == "aspera": - if not aspera_url: - raise ValueError("Aspera URL not available") - return aspera_url - - if not ftp_url: - raise ValueError("FTP URL not available") - if protocol == "ftp": - return ftp_url - if protocol == "globus": - return ftp_url.replace( - PrideProvider.ARCHIVE_FTP_URL_PREFIX, - PrideProvider.ARCHIVE_HTTPS_URL_PREFIX, - 1, - ) - if protocol == "s3": - return ftp_url - raise ValueError(f"Unsupported protocol: {protocol}") - - -def _resolve_local_path(file_record: Dict, output_folder: str) -> str: - """ - Compute the canonical local path for a file regardless of transfer protocol. - """ - # Lazy import to avoid module-load cycle with PrideProvider. - from pridepy.download.pride import PrideProvider - - try: - canonical_url = _get_download_url(file_record, "ftp") - except ValueError: - canonical_url = "" - if canonical_url: - return PrideProvider.get_output_file_name(canonical_url, file_record, output_folder) - return os.path.join(output_folder, file_record["fileName"]) diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 0a237cc..901ca99 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -46,7 +46,7 @@ def test_get_download_url_maps_globus_to_pride_archive_https(self): ] } - download_url = provider_util._get_download_url(file_record, "globus") + download_url = PrideProvider._get_download_url(file_record, "globus") assert download_url == "https://ftp.pride.ebi.ac.uk/path/file.raw" From 121b06b82352d7ac4736eedb46a9dd2af866f326 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 07:50:47 +0100 Subject: [PATCH 30/54] refactor(download): remove the pridepy/files/ back-compat shim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pridepy/files/files.py shim (Files = Client re-export) was load-bearing only for the test suite — the CLI already imports from pridepy.download.client. Updated the 9 test files to 'from pridepy.download.client import Client as Files' (keeps the local Files alias so test bodies are otherwise untouched) and deleted the pridepy/files/ package entirely. Breaking change for any downstream code doing 'from pridepy.files.files import Files' — use 'from pridepy.download.client import Client' instead. Pre-1.0 cleanup; the class was already renamed Files -> Client in this PR. Also updated two stale docstrings (transport.py, client.py) that referenced the old pridepy.files.files path. 68 passed, 4 skipped. flake8 (E9,F63,F7,F82) clean. Smoke MD5 unchanged. --- pridepy/download/client.py | 3 +-- pridepy/download/transport.py | 6 +++--- pridepy/files/__init__.py | 0 pridepy/files/files.py | 12 ------------ pridepy/tests/test_authentication.py | 2 +- pridepy/tests/test_download_by_list.py | 2 +- pridepy/tests/test_download_by_url.py | 2 +- pridepy/tests/test_download_resilience.py | 2 +- pridepy/tests/test_iprox_files.py | 2 +- pridepy/tests/test_jpost_files.py | 2 +- pridepy/tests/test_massive_files.py | 2 +- pridepy/tests/test_raw_files.py | 2 +- pridepy/tests/test_search.py | 2 +- 13 files changed, 13 insertions(+), 26 deletions(-) delete mode 100644 pridepy/files/__init__.py delete mode 100644 pridepy/files/files.py diff --git a/pridepy/download/client.py b/pridepy/download/client.py index dcb93ce..959dc4b 100644 --- a/pridepy/download/client.py +++ b/pridepy/download/client.py @@ -20,8 +20,7 @@ from pridepy.download.proteomexchange import ProteomeXchangeProvider from pridepy.download import by_url -# Re-export Progress so external `from pridepy.download.client import Progress` -# (and the legacy `from pridepy.files.files import Progress`) still works. +# Re-export Progress so `from pridepy.download.client import Progress` works. from pridepy.download.util import Progress # noqa: F401 diff --git a/pridepy/download/transport.py b/pridepy/download/transport.py index 6649657..5693ac6 100644 --- a/pridepy/download/transport.py +++ b/pridepy/download/transport.py @@ -1,8 +1,8 @@ """Shared FTP / FTPS / HTTPS download transport. -Stateless helpers used by the per-repository providers (and re-exported on -:class:`pridepy.files.files.Files` for backward compatibility with tests that -patch ``Files.download_ftp_urls`` etc.). +Stateless helpers used by the per-repository adapters (and re-exported on +:class:`pridepy.download.client.Client` for downstream callers that use +``Client.download_ftp_urls`` etc.). """ import ftplib import logging diff --git a/pridepy/files/__init__.py b/pridepy/files/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pridepy/files/files.py b/pridepy/files/files.py deleted file mode 100644 index 654b46a..0000000 --- a/pridepy/files/files.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Backward-compatibility shim. - -The download facade moved to :mod:`pridepy.download.client` and the class -was renamed ``Files`` -> ``Client``. This module re-exports it under the old -name so ``from pridepy.files.files import Files`` keeps working, along with -``Progress``. -""" -from pridepy.download.client import Client, Progress # noqa: F401 - -Files = Client # legacy alias - -__all__ = ["Client", "Files", "Progress"] diff --git a/pridepy/tests/test_authentication.py b/pridepy/tests/test_authentication.py index d7f5e75..ec0a4f9 100644 --- a/pridepy/tests/test_authentication.py +++ b/pridepy/tests/test_authentication.py @@ -3,7 +3,7 @@ import pytest from pridepy.authentication import authentication -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.project.project import Project diff --git a/pridepy/tests/test_download_by_list.py b/pridepy/tests/test_download_by_list.py index fd31076..649017e 100644 --- a/pridepy/tests/test_download_by_list.py +++ b/pridepy/tests/test_download_by_list.py @@ -12,7 +12,7 @@ import click import pytest -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.pridepy import _read_filename_arguments from pridepy.download.pride import PrideProvider diff --git a/pridepy/tests/test_download_by_url.py b/pridepy/tests/test_download_by_url.py index 2f133ac..b8c66da 100644 --- a/pridepy/tests/test_download_by_url.py +++ b/pridepy/tests/test_download_by_url.py @@ -13,7 +13,7 @@ import pytest from pridepy.download import by_url -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.pridepy import _read_url_arguments diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 901ca99..b83e4aa 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -5,7 +5,7 @@ from unittest.mock import Mock, patch from pridepy.download import by_url -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.download import transport from pridepy.download import util as provider_util from pridepy.download.massive import MassiveProvider diff --git a/pridepy/tests/test_iprox_files.py b/pridepy/tests/test_iprox_files.py index 017e0e4..00d4997 100644 --- a/pridepy/tests/test_iprox_files.py +++ b/pridepy/tests/test_iprox_files.py @@ -13,7 +13,7 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.download import transport from pridepy.download.iprox import IproxProvider from pridepy.download.pride import PrideProvider diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py index d5aa1f0..efb0c5c 100644 --- a/pridepy/tests/test_jpost_files.py +++ b/pridepy/tests/test_jpost_files.py @@ -3,7 +3,7 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.download import transport from pridepy.download.jpost import JpostProvider diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index d4a6926..6bec7d4 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -2,7 +2,7 @@ from unittest import TestCase from unittest.mock import patch -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.download import transport from pridepy.download.massive import MassiveProvider diff --git a/pridepy/tests/test_raw_files.py b/pridepy/tests/test_raw_files.py index 1ce2ca3..3d1380b 100644 --- a/pridepy/tests/test_raw_files.py +++ b/pridepy/tests/test_raw_files.py @@ -1,6 +1,6 @@ from unittest import TestCase -from pridepy.files.files import Files +from pridepy.download.client import Client as Files class TestRawFiles(TestCase): diff --git a/pridepy/tests/test_search.py b/pridepy/tests/test_search.py index a61e83c..f21ac87 100644 --- a/pridepy/tests/test_search.py +++ b/pridepy/tests/test_search.py @@ -1,6 +1,6 @@ from unittest import TestCase -from pridepy.files.files import Files +from pridepy.download.client import Client as Files from pridepy.project.project import Project from pridepy.util.api_handling import Util import logging From b0776b06ac7b69c00027aad8f1c14cc441af2476 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 09:12:52 +0100 Subject: [PATCH 31/54] docs+refactor(download): fix review findings and restructure README Addresses the code-review findings on PR #105 and reorganizes the README command docs into collapsible sections. - README: fix broken Python-API imports (pridepy.files.files -> download.client) - README: restructure CLI docs into "PRIDE File Downloads", "Metadata and Search", and "Download from ProteomeXchange and other repositories"; wrap command/option subsections in
for an expandable layout - remove dead by_list.py (logic lives in Provider.download_by_filenames) and drop it from the download package docstring - fix stale "Files" facade references in pride.py / proteomexchange.py / util.py docstrings (renamed to Client) - correct base.py / __init__.py adapter-contract docstrings (PrideProvider also overrides get_download_url, download_files, download_by_name) - remove unwired supports_checksum ClassVar from Provider --- README.md | 324 +++++++++++++++++++--------- pridepy/download/__init__.py | 11 +- pridepy/download/base.py | 11 +- pridepy/download/by_list.py | 59 ----- pridepy/download/pride.py | 7 +- pridepy/download/proteomexchange.py | 2 +- pridepy/download/util.py | 10 +- 7 files changed, 240 insertions(+), 184 deletions(-) delete mode 100644 pridepy/download/by_list.py diff --git a/README.md b/README.md index 27bab43..7905699 100644 --- a/README.md +++ b/README.md @@ -54,9 +54,57 @@ uv sync --extra dev uv run pridepy --help ``` -## Quick Start (New Users) +## Command Overview -### 1) Download all raw files for a project (robust mode) +```bash +pridepy --help +``` + +| Command | Purpose | +| --- | --- | +| `download-all-public-raw-files` | Download every public RAW file of a dataset | +| `download-all-public-category-files` | Download files of one or more categories (RAW, SEARCH, …) | +| `download-file-by-name` | Download a single file (public or private) | +| `download-files-by-list` | Download a named subset of files from a manifest/CSV | +| `download-files-by-url` | Download files from raw `http`/`https`/`ftp` URLs | +| `download-px-raw-files` | Download RAW files resolved from a ProteomeXchange accession | +| `list-private-files` | List files of a private project (needs credentials) | +| `stream-files-metadata` | Stream file metadata (one project or all) to JSON | +| `stream-projects-metadata` | Stream all project metadata to JSON | +| `search-projects-by-keywords-and-filters` | Search projects by keyword and filters | + +The download commands work for PRIDE accessions and, transparently, for native +MassIVE (`MSV…`), JPOST (`JPST…`), and iProX (`IPX…`) accessions — see +[Download from ProteomeXchange and other repositories](#download-from-proteomexchange-and-other-repositories). + +## PRIDE File Downloads + +PRIDE downloads start with FTP and fall back across the remaining protocols +(`ftp -> aspera -> s3 -> globus`) when a transfer fails. They support resume, +per-file retries, parallel workers, and optional checksum validation. Empty or +corrupt files are retried automatically. + +
+Common download options (shared across the download commands) + +These options are shared by `download-all-public-raw-files`, +`download-all-public-category-files`, `download-file-by-name`, and +`download-files-by-list`: + +| Option | Description | Default | +| --- | --- | --- | +| `-a, --accession` | Dataset accession (e.g. `PXD008644`) | required | +| `-o, --output-folder` | Destination directory | required | +| `-p, --protocol` | Transfer protocol: `ftp`, `aspera`, `globus`, `s3` (FTP-first with fallback) | `ftp` | +| `-w, --parallel-files` | Download 1–3 files concurrently (primarily for `globus`) | `1` | +| `--skip-if-downloaded-already` | Resume: skip files already present locally | off | +| `--checksum-check` | Download PRIDE checksums and validate each file | off | +| `--aspera-maximum-bandwidth` | Aspera cap, e.g. `50M`, `100M`, `200M` (Aspera only) | `100M` | + +
+ +
+Download all raw files (robust mode) ```bash pridepy download-all-public-raw-files \ @@ -65,12 +113,7 @@ pridepy download-all-public-raw-files \ --checksum-check ``` -What this does: -- default `ftp` starts with FTP and falls back (`ftp -> aspera -> s3 -> globus`) -- `--checksum-check` downloads project checksums and validates files -- empty/corrupt files are retried automatically - -### 2) Continue interrupted downloads safely +Continue an interrupted download safely by adding `--skip-if-downloaded-already`: ```bash pridepy download-all-public-raw-files \ @@ -80,33 +123,10 @@ pridepy download-all-public-raw-files \ --checksum-check ``` -### 3) Download a public MassIVE, JPOST, or iProX dataset directly - -```bash -# MassIVE -pridepy download-all-public-raw-files \ - -a MSV000082297 \ - -o ./downloads/MSV000082297 - -# JPOST -pridepy download-all-public-raw-files \ - -a JPST002311 \ - -o ./downloads/JPST002311 - -# iProX -pridepy download-all-public-raw-files \ - -a IPX0017413000 \ - -o ./downloads/IPX0017413000 -``` - -For these direct downloads, `pridepy` enumerates the dataset from the repository: -- **MassIVE** lists files by walking the FTPS tree at `massive-ftp.ucsd.edu` (TLS is required by the server). -- **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads them from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. -- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTPS. iProX exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTPS endpoint instead so no iProX credentials are required. +
-Raw downloads follow each repository's own collection layout, so `download-all-public-raw-files` downloads the files stored under the dataset's `raw/` collection. Direct downloads support resume (REST for FTP, byte-Range for HTTPS), per-file retries, parallel workers (`-w N` up to 3), and post-transfer size verification against the server-reported size. - -### 4) Download only selected categories +
+Download only selected categories ```bash pridepy download-all-public-category-files \ @@ -115,16 +135,13 @@ pridepy download-all-public-category-files \ -c RAW,SEARCH ``` -You can also request a specific MassIVE / JPOST / iProX collection through the same category interface: +`-c, --category` takes one or more comma-separated categories. Valid values: +`RAW`, `PEAK`, `SEARCH`, `RESULT`, `SPECTRUM_LIBRARY`, `OTHER`, `FASTA`. -```bash -pridepy download-all-public-category-files \ - -a MSV000082297 \ - -o ./downloads/MSV000082297-results \ - -c RESULT -``` +
-### 5) Download one file by name +
+Download one file by name ```bash pridepy download-file-by-name \ @@ -134,15 +151,12 @@ pridepy download-file-by-name \ --checksum-check ``` -### 6) Download raw files from ProteomeXchange +`-f, --file-name` is the file to download. -```bash -pridepy download-px-raw-files \ - -a PXD039236 \ - -o ./downloads/PXD039236 -``` +
-### 6) Download a named subset of files (manifest) +
+Download a named subset of files (manifest) ```bash pridepy download-files-by-list \ @@ -153,18 +167,15 @@ pridepy download-files-by-list \ ``` `files.txt` is one filename per line (blank lines and `#` comments are -ignored). Internally each filename is resolved against the project metadata -API and downloaded via the same batch + protocol-fallback engine as -`download-all-public-raw-files`. Use `-f a.raw,b.raw,c.raw` instead of -`-F` for a small inline list. - -Useful options: +ignored). Each filename is resolved against the project metadata and downloaded +via the same batch + protocol-fallback engine as `download-all-public-raw-files`. +Use `-f a.raw,b.raw,c.raw` instead of `-F` for a small inline list (you can +combine both). -- `-p globus` — use the globus download strategy (HTTP Range + resume) -- `-w 3` — download up to 3 files in parallel (globus only, max 3) -- `--checksum-check` — validate files against PRIDE checksums after download +
-### 7) Download files from raw URLs +
+Download files from raw URLs ```bash pridepy download-files-by-url \ @@ -173,39 +184,77 @@ pridepy download-files-by-url \ ``` `urls.txt` is one fully-qualified URL per line. Schemes `http`, `https`, and -`ftp` are dispatched to the matching downloader. Use `-u/--urls` for one or -more comma-separated URLs, e.g. `--urls https://a.com/x.raw,ftp://b.com/y.raw`. -Note: URLs containing literal commas are not supported with `--urls`; use a -manifest file (`-F`) instead. +`ftp` are dispatched to the matching downloader. Use `-u, --urls` for one or +more comma-separated URLs, e.g. `--urls https://a.com/x.raw,ftp://b.com/y.raw` +(URLs containing literal commas must use a manifest file instead). -Useful options: +Command-specific options: -- `-p globus` — use globus download strategy for http/https URLs (resume-capable) -- `-w 3` — download up to 3 files in parallel (globus only, max 3) -- `--checksum-check` — validate against PRIDE checksums (accession inferred - from PRIDE URL paths; only PRIDE archive URLs are supported) +| Option | Description | Default | +| --- | --- | --- | +| `-F, --url-list` | Manifest file, one URL per line | — | +| `-u, --urls` | Comma-separated URL(s) | — | +| `-p, --protocol` | `ftp` (per-scheme) or `globus` (resume-capable http/https) | `ftp` | +| `-w, --parallel-files` | Download 1–3 files concurrently (globus only) | `1` | +| `--checksum-check` | Validate against PRIDE checksums (accession inferred from PRIDE URL paths; only PRIDE archive URLs supported) | off | -## CLI Command Overview +
+ +
+Private (restricted) files + +List the files of a private project with your PRIDE credentials: ```bash -pridepy --help +pridepy list-private-files -a PXD022105 -u YOUR_USER -p YOUR_PASSWORD ``` -Main commands: -- `download-all-public-raw-files` -- `download-all-public-category-files` -- `download-file-by-name` -- `download-files-by-list` -- `download-files-by-url` -- `download-px-raw-files` -- `list-private-files` -- `stream-files-metadata` -- `stream-projects-metadata` -- `search-projects-by-keywords-and-filters` +Download a private file by passing `--username`/`--password` to +`download-file-by-name`: -## More CLI Examples +```bash +pridepy download-file-by-name \ + -a PXD022105 \ + -f checksum.txt \ + -o ./downloads/private \ + --username YOUR_USER \ + --password YOUR_PASSWORD +``` -### Search projects +
+ +## Metadata and Search + +
+Stream all project metadata to JSON + +```bash +pridepy stream-projects-metadata -o all_pride_projects.json +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-o, --output-file` | JSON file to write all project metadata to | required | + +
+ +
+Stream file metadata + +```bash +# All file metadata for one accession +pridepy stream-files-metadata -a PXD005011 -o PXD005011_files.json +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-o, --output-file` | JSON file to write file metadata to | required | +| `-a, --accession` | Limit to one project (omit to stream all files) | optional | + +
+ +
+Search projects by keywords and filters ```bash pridepy search-projects-by-keywords-and-filters \ @@ -216,46 +265,102 @@ pridepy search-projects-by-keywords-and-filters \ -sf submissionDate ``` -### Stream all project metadata to JSON +| Option | Description | Default | +| --- | --- | --- | +| `-k, --keyword` | Keyword searched across project fields | required | +| `-f, --filters` | `field==value` filters, comma-separated (e.g. `accession==PRD000001`) | — | +| `-ps, --page-size` | Results per page (1–1000) | `100` | +| `-p, --page` | Page number (0-based) | `0` | +| `-sd, --sort-direction` | `ASC` or `DESC` | `DESC` | +| `-sf, --sort-fields` | Sort field(s), repeatable. One of: `accession`, `submissionDate`, `diseases`, `organismsPart`, `organisms`, `instruments`, `softwares`, `avgDownloadsPerFile`, `downloadCount`, `publicationDate` | `submissionDate` | -```bash -pridepy stream-projects-metadata -o all_pride_projects.json -``` +
-### Stream all file metadata for one accession +## Download from ProteomeXchange and other repositories + +A ProteomeXchange (`PXD…` / `PRD…`) accession is a cross-repository identifier: +the dataset may be hosted at PRIDE, MassIVE, JPOST, iProX, or elsewhere. +`pridepy` lets you start from the ProteomeXchange accession, or go straight to +the hosting repository using its **native** accession. + +
+Start from a ProteomeXchange accession + +`download-px-raw-files` resolves the dataset's ProteomeXchange XML and downloads +the RAW files it references, regardless of which repository hosts them: ```bash -pridepy stream-files-metadata -a PXD005011 -o PXD005011_files.json +pridepy download-px-raw-files \ + -a PXD039236 \ + -o ./downloads/PXD039236 ``` -### Download private files +| Option | Description | Default | +| --- | --- | --- | +| `-a, --accession` | ProteomeXchange accession (e.g. `PXD039236`). `--px` is a deprecated alias | required | +| `-o, --output-folder` | Destination directory | required | +| `--skip-if-downloaded-already` | Skip files already present locally | off | + +
+ +
+Go directly to the hosting repository (native MassIVE / JPOST / iProX accessions) -List files: +Datasets that do not have a ProteomeXchange accession — or where you already +know the native accession — can be downloaded directly. The standard download +commands accept MassIVE, JPOST, and iProX accessions transparently: ```bash -pridepy list-private-files -a PXD022105 -u YOUR_USER -p YOUR_PASSWORD +# MassIVE (FTPS at massive-ftp.ucsd.edu) +pridepy download-all-public-raw-files \ + -a MSV000082297 \ + -o ./downloads/MSV000082297 + +# JPOST (PROXI listing + ftp.jpostdb.org) +pridepy download-all-public-raw-files \ + -a JPST002311 \ + -o ./downloads/JPST002311 + +# iProX (ProteomeXchange XML + anonymous HTTPS at download.iprox.org) +pridepy download-all-public-raw-files \ + -a IPX0017413000 \ + -o ./downloads/IPX0017413000 ``` -Download a private file: +How each repository is enumerated: + +- **MassIVE** walks the FTPS tree at `massive-ftp.ucsd.edu` (the server requires TLS). +- **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. +- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTPS. iProX also exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTPS endpoint so no iProX credentials are required. + +Raw downloads follow each repository's own collection layout, so +`download-all-public-raw-files` retrieves the files under the dataset's `raw/` +collection. These direct downloads support resume (REST for FTP, byte-Range for +HTTPS), per-file retries, parallel workers (`-w` up to 3), and post-transfer +size verification against the server-reported size. + +You can also request a specific collection from these repositories through the +same category interface: ```bash -pridepy download-file-by-name \ - -a PXD022105 \ - -f checksum.txt \ - -o ./downloads/private \ - --username YOUR_USER \ - --password YOUR_PASSWORD +pridepy download-all-public-category-files \ + -a MSV000082297 \ + -o ./downloads/MSV000082297-results \ + -c RESULT ``` +
+ ## Python API Examples -### Example: get raw files for a project +
+Get raw files for a project ```python -from pridepy.files.files import Files +from pridepy.download.client import Client -files = Files() -raw_files = files.get_all_raw_file_list("PXD008644") +client = Client() +raw_files = client.get_all_raw_file_list("PXD008644") print(f"RAW files: {len(raw_files)}") print(raw_files[0]["fileName"]) ``` @@ -263,15 +368,18 @@ print(raw_files[0]["fileName"]) For MassIVE / JPOST / iProX accessions, the same method returns the files found under the dataset's `raw/` collection: ```python -from pridepy.files.files import Files +from pridepy.download.client import Client -files = Files() +client = Client() for accession in ("MSV000082297", "JPST002311", "IPX0017413000"): - raw_files = files.get_all_raw_file_list(accession) + raw_files = client.get_all_raw_file_list(accession) print(f"{accession} raw files: {len(raw_files)}") ``` -### Example: search projects +
+ +
+Search projects ```python from pridepy.project.project import Project @@ -288,6 +396,8 @@ results = project.search_by_keywords_and_filters( print(f"Hits: {len(results)}") ``` +
+ ## Development and Release (uv) Run tests: diff --git a/pridepy/download/__init__.py b/pridepy/download/__init__.py index f67890d..854ca75 100644 --- a/pridepy/download/__init__.py +++ b/pridepy/download/__init__.py @@ -5,13 +5,16 @@ - Repository adapters — one module per repository (``pride``, ``massive``, ``jpost``, ``iprox``, ``proteomexchange``). Each subclasses - :class:`pridepy.download.base.Provider` and implements ``matches`` + - ``list_files``; the download workflow itself is inherited from the base. + :class:`pridepy.download.base.Provider` and implements at least ``matches`` + + ``list_files``. Direct-download adapters (MassIVE / JPOST / iProX) inherit + the whole download workflow from the base; ``PrideProvider`` additionally + overrides ``get_download_url``, ``download_files``, and ``download_by_name`` + for its multi-protocol fallback and public/private split. - :mod:`registry` — maps an accession to the right adapter. - :mod:`transport` — shared FTP/FTPS/HTTPS plumbing (resume, retry, parallel). - :mod:`util` — checksum and record helpers. -- :mod:`by_url`, :mod:`by_list` — cross-cutting download commands that take - URLs or filename lists rather than an accession. +- :mod:`by_url` — cross-cutting download command that takes raw URLs rather + than an accession. - :mod:`client` — the :class:`~pridepy.download.client.Client` facade the CLI drives; dispatches to adapters via the registry. """ diff --git a/pridepy/download/base.py b/pridepy/download/base.py index 16aaaea..374e225 100644 --- a/pridepy/download/base.py +++ b/pridepy/download/base.py @@ -1,12 +1,14 @@ """Abstract base class for pridepy providers. The :class:`Provider` base implements the download *workflow* via the -Template Method pattern: concrete adapters only fill in the holes +Template Method pattern: most adapters only fill in the holes (:meth:`matches`, :meth:`list_files`) while the shared listing-filter and download-orchestration methods live here. Adapters that need different -transport behaviour (e.g. PRIDE's multi-protocol fallback) override -:meth:`download_files`; everything else routes through the inherited -default that partitions record URLs by scheme. +behaviour override the relevant hooks — e.g. PRIDE overrides +:meth:`get_download_url`, :meth:`download_files` (multi-protocol fallback), +and :meth:`download_by_name` (public/private split). Everything not +overridden routes through the inherited default that partitions record URLs +by scheme. """ import logging from abc import ABC, abstractmethod @@ -20,7 +22,6 @@ class Provider(ABC): name: ClassVar[str] # "pride", "massive", "jpost", "iprox" use_tls: ClassVar[bool] = False - supports_checksum: ClassVar[bool] = False # ------------------------------------------------------------------ # Abstract holes — adapters must implement these. diff --git a/pridepy/download/by_list.py b/pridepy/download/by_list.py deleted file mode 100644 index 0d2393c..0000000 --- a/pridepy/download/by_list.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Download a subset of project files identified by a filename list.""" -import logging -from typing import List, Optional - -from pridepy.download import registry - - -def download_files_by_list( - accession: str, - file_names: List[str], - output_folder: str, - skip_if_downloaded_already: bool, - protocol: str = "ftp", - aspera_maximum_bandwidth: str = "100M", - checksum_check: bool = False, - parallel_files: int = 1, -) -> None: - """Download a subset of project files identified by a filename list. - - Resolves each requested filename via the project metadata API and - delegates to the provider's ``download_files`` so the existing batch + - protocol fallback engine is reused. - - :param accession: PRIDE or MassIVE project accession (public) - :param file_names: filenames to download - :param output_folder: directory to write downloaded files into - :param skip_if_downloaded_already: skip files already present locally - :param protocol: preferred protocol; falls back across others on failure - :param aspera_maximum_bandwidth: aspera ascp bandwidth cap - :param checksum_check: download project checksums and validate - :param parallel_files: number of files to download simultaneously for globus - :raises ValueError: if ``file_names`` is empty or none match the project - """ - if not file_names: - raise ValueError("file_names must contain at least one filename") - - provider = registry.resolve(accession) - all_files = provider.list_files(accession) - - requested = set(file_names) - matched = [f for f in all_files if f.get("fileName") in requested] - missing = sorted(requested - {f.get("fileName") for f in matched}) - if missing: - logging.warning("Files not found in project %s: %s", accession, missing) - if not matched: - raise ValueError( - f"No matching files in project {accession} for: {sorted(requested)}" - ) - - provider.download_files( - accession=accession, - records=matched, - output_folder=output_folder, - skip_if_downloaded_already=skip_if_downloaded_already, - protocol=protocol, - parallel_files=parallel_files, - checksum_check=checksum_check, - aspera_maximum_bandwidth=aspera_maximum_bandwidth, - ) diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index c02e7ad..c9eae9a 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -3,13 +3,14 @@ PRIDE has the richest behaviour of all providers: multi-protocol batch download with aspera/s3/ftp/globus fallback, private-dataset path with username/password auth, checksum TSV validation, and submitter-path -helpers. This module owns all of that logic; the :class:`Files` facade -exposes a thin public surface for downstream callers. +helpers. This module owns all of that logic; the +:class:`~pridepy.download.client.Client` facade exposes a thin public +surface for downstream callers. Implementation note: PRIDE provider methods route through other PrideProvider methods (``PrideProvider.X(...)``) or directly through the shared ``transport`` / ``util`` helpers — they do NOT call back into the -``Files`` facade. Tests patch the canonical locations +``Client`` facade. Tests patch the canonical locations (``PrideProvider.X``, ``transport.X``, ``util.X``) directly. """ import ftplib diff --git a/pridepy/download/proteomexchange.py b/pridepy/download/proteomexchange.py index 9a1d04e..be75268 100644 --- a/pridepy/download/proteomexchange.py +++ b/pridepy/download/proteomexchange.py @@ -11,7 +11,7 @@ ProteomeXchange's XML listing; the registry continues to route PXD/PRD via :class:`pridepy.download.pride.PrideProvider`. ``ProteomeXchangeProvider`` is the explicit gateway invoked by the ``download-px-raw-files`` CLI -command and by ``Files.download_px_raw_files`` — callers who specifically +command and by ``Client.download_px_raw_files`` — callers who specifically want the cross-repository XML view. The class accepts either: diff --git a/pridepy/download/util.py b/pridepy/download/util.py index 3dde0fc..6f2f35f 100644 --- a/pridepy/download/util.py +++ b/pridepy/download/util.py @@ -1,10 +1,10 @@ -"""Cross-cutting utilities used by providers and the Files facade. +"""Cross-cutting utilities used by providers and the Client facade. Pure functions (and one tiny Progress class) for checksums, record-shape -helpers, and download progress. Originally on ``Files`` as @staticmethods; -moved here so providers can use them without depending on Files at import -time, and Files keeps shim re-exports for backward compatibility with -existing test patches. +helpers, and download progress. Originally on the facade as @staticmethods; +moved here so providers can use them without depending on the facade at +import time, while :class:`~pridepy.download.client.Client` keeps shim +re-exports for backward compatibility with existing test patches. """ import hashlib import logging From 3188719b9d0c12beb2f8680980938429897ac027 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 09:24:40 +0100 Subject: [PATCH 32/54] docs+fix: correct README/CLI accuracy issues from re-review - README: -w/--parallel-files is not available on download-file-by-name; note this in the common-options table - README + iprox.py docstring: iProX serves over plain HTTP, not HTTPS (DOWNLOAD_BASE_URL uses http://); fix the wording - README: drop the over-broad "follow the collection layout" claim (files are written by basename) and scope the post-transfer size check to FTP - fix search default sort field: submission_date -> submissionDate so the default matches the allowed click.Choice values --- README.md | 18 +++++++++--------- pridepy/download/iprox.py | 4 ++-- pridepy/pridepy.py | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7905699..04d2465 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ You can: - download public and private PRIDE files -- download public MassIVE (`MSV...`), JPOST (`JPST...`), and iProX (`IPX...`) datasets directly. MassIVE goes through FTPS at `massive-ftp.ucsd.edu`; JPOST uses the JSON PROXI endpoint at `repository.jpostdb.org` for listings and `ftp.jpostdb.org` for transfers; iProX fetches the dataset's ProteomeXchange XML from `download.iprox.org` and downloads files over anonymous HTTPS +- download public MassIVE (`MSV...`), JPOST (`JPST...`), and iProX (`IPX...`) datasets directly. MassIVE goes through FTPS at `massive-ftp.ucsd.edu`; JPOST uses the JSON PROXI endpoint at `repository.jpostdb.org` for listings and `ftp.jpostdb.org` for transfers; iProX fetches the dataset's ProteomeXchange XML from `download.iprox.org` and downloads files over anonymous HTTP - download by category (`RAW`, `SEARCH`, `RESULT`, etc.) - stream project and file metadata - search projects by keyword and filters @@ -96,7 +96,7 @@ These options are shared by `download-all-public-raw-files`, | `-a, --accession` | Dataset accession (e.g. `PXD008644`) | required | | `-o, --output-folder` | Destination directory | required | | `-p, --protocol` | Transfer protocol: `ftp`, `aspera`, `globus`, `s3` (FTP-first with fallback) | `ftp` | -| `-w, --parallel-files` | Download 1–3 files concurrently (primarily for `globus`) | `1` | +| `-w, --parallel-files` | Download 1–3 files concurrently — primarily for `globus`; not available on `download-file-by-name` | `1` | | `--skip-if-downloaded-already` | Resume: skip files already present locally | off | | `--checksum-check` | Download PRIDE checksums and validate each file | off | | `--aspera-maximum-bandwidth` | Aspera cap, e.g. `50M`, `100M`, `200M` (Aspera only) | `100M` | @@ -321,7 +321,7 @@ pridepy download-all-public-raw-files \ -a JPST002311 \ -o ./downloads/JPST002311 -# iProX (ProteomeXchange XML + anonymous HTTPS at download.iprox.org) +# iProX (ProteomeXchange XML + anonymous HTTP at download.iprox.org) pridepy download-all-public-raw-files \ -a IPX0017413000 \ -o ./downloads/IPX0017413000 @@ -331,13 +331,13 @@ How each repository is enumerated: - **MassIVE** walks the FTPS tree at `massive-ftp.ucsd.edu` (the server requires TLS). - **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. -- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTPS. iProX also exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTPS endpoint so no iProX credentials are required. +- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTP (with `Range` support for resume). iProX also exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTP endpoint so no iProX credentials are required. -Raw downloads follow each repository's own collection layout, so -`download-all-public-raw-files` retrieves the files under the dataset's `raw/` -collection. These direct downloads support resume (REST for FTP, byte-Range for -HTTPS), per-file retries, parallel workers (`-w` up to 3), and post-transfer -size verification against the server-reported size. +`download-all-public-raw-files` retrieves the files stored under the dataset's +`raw/` collection. These direct downloads support resume (REST for FTP, +byte-Range for HTTP/HTTPS), per-file retries, and parallel workers (`-w` up to +3). FTP transfers are additionally checked against the server-reported size +after each download. You can also request a specific collection from these repositories through the same category interface: diff --git a/pridepy/download/iprox.py b/pridepy/download/iprox.py index abb7720..29fdb1f 100644 --- a/pridepy/download/iprox.py +++ b/pridepy/download/iprox.py @@ -1,14 +1,14 @@ """iProX direct-download provider. iProX publishes the ProteomeXchange XML for each dataset at a -deterministic path on its anonymous HTTPS download server:: +deterministic path on its anonymous HTTP download server:: http://download.iprox.org//PX_.xml We fetch the XML, walk every ````'s ``cvParam`` entries, and turn each ``Associated raw file URI`` (and sibling URIs for search-engine output, result files, etc.) into a pridepy file record. File downloads -themselves go through plain HTTPS on the same host, which supports +themselves go through plain HTTP on the same host, which supports ``Range`` requests for resume. """ import logging diff --git a/pridepy/pridepy.py b/pridepy/pridepy.py index 744ba07..5cc41cf 100644 --- a/pridepy/pridepy.py +++ b/pridepy/pridepy.py @@ -416,10 +416,10 @@ def stream_files_metadata(accession, output_file): "-sf", "--sort-fields", required=False, - default=["submission_date"], + default=["submissionDate"], multiple=True, help="Field(s) for sorting the results on. Default for this " - "request is submission_date. More fields can be separated by " + "request is submissionDate. More fields can be separated by " "comma and passed. Example: submissionDate,accession", type=click.Choice( "accession,submissionDate,diseases,organismsPart,organisms,instruments,softwares," From 0957ec640b7a703df2e0599ac7bdbc41388f3afc Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 09:54:21 +0100 Subject: [PATCH 33/54] fix(download): preserve dataset layout and verify HTTP transfer size Addresses the two long-standing transport data-integrity issues flagged on #98/#100 for the direct-download providers (MassIVE/JPOST/iProX). - Thread each record's relativePath from Provider.download_files through to download_ftp_urls / download_http_urls, writing files to output_folder/ instead of flattening to the URL basename. Identically-named files in different sub-collections no longer overwrite each other, and --skip-if-downloaded-already no longer skips the wrong file. Parent directories are created per file; a _safe_join guard prevents path traversal outside output_folder. - Add a post-transfer size check in _parallel_download (HTTP/globus) mirroring the FTP path: a stream shorter than Content-Length now raises so the retry loop re-downloads (Range-resuming when possible) instead of silently accepting a truncated file. - Remove the now-unused _local_path_for_url helper (superseded by _dest_path). - Tests: collision-free relative_paths threading (ftp+http), truncated-stream raise, _safe_join traversal guard; update massive/jpost exact-call asserts. --- README.md | 9 +- pridepy/download/base.py | 21 +++- pridepy/download/transport.py | 130 +++++++++++++++++----- pridepy/tests/test_download_resilience.py | 94 ++++++++++++++++ pridepy/tests/test_jpost_files.py | 1 + pridepy/tests/test_massive_files.py | 1 + 6 files changed, 222 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 04d2465..a674cbb 100644 --- a/README.md +++ b/README.md @@ -334,10 +334,11 @@ How each repository is enumerated: - **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTP (with `Range` support for resume). iProX also exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTP endpoint so no iProX credentials are required. `download-all-public-raw-files` retrieves the files stored under the dataset's -`raw/` collection. These direct downloads support resume (REST for FTP, -byte-Range for HTTP/HTTPS), per-file retries, and parallel workers (`-w` up to -3). FTP transfers are additionally checked against the server-reported size -after each download. +`raw/` collection, saving them under `output_folder` with the dataset's +sub-directory layout preserved (so identically-named files in different +collections don't overwrite each other). These direct downloads support resume +(REST for FTP, byte-Range for HTTP), per-file retries, parallel workers (`-w` +up to 3), and post-transfer size verification against the server-reported size. You can also request a specific collection from these repositories through the same category interface: diff --git a/pridepy/download/base.py b/pridepy/download/base.py index 374e225..3c159d3 100644 --- a/pridepy/download/base.py +++ b/pridepy/download/base.py @@ -232,11 +232,20 @@ def download_files( f"Ignoring requested protocol '{protocol}' for {accession}." ) - all_urls = [self.get_download_url(record) for record in records] - ftp_urls = [u for u in all_urls if u.lower().startswith("ftp://")] - http_urls = [ - u for u in all_urls if u.lower().startswith(("http://", "https://")) - ] + ftp_urls: List[str] = [] + ftp_relpaths: List[Optional[str]] = [] + http_urls: List[str] = [] + http_relpaths: List[Optional[str]] = [] + for record in records: + url = self.get_download_url(record) + relpath = record.get("relativePath") + lowered = url.lower() + if lowered.startswith("ftp://"): + ftp_urls.append(url) + ftp_relpaths.append(relpath) + elif lowered.startswith(("http://", "https://")): + http_urls.append(url) + http_relpaths.append(relpath) if not ftp_urls and not http_urls: logging.info( f"No files matched for direct-download dataset {accession}" @@ -250,6 +259,7 @@ def download_files( skip_if_downloaded_already=skip_if_downloaded_already, use_tls=self.use_tls, parallel_files=parallel_files, + relative_paths=ftp_relpaths, ) if http_urls: transport.download_http_urls( @@ -257,4 +267,5 @@ def download_files( output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, parallel_files=parallel_files, + relative_paths=http_relpaths, ) diff --git a/pridepy/download/transport.py b/pridepy/download/transport.py index 5693ac6..d982ed6 100644 --- a/pridepy/download/transport.py +++ b/pridepy/download/transport.py @@ -11,7 +11,7 @@ import time from concurrent.futures import ThreadPoolExecutor, as_completed from ftplib import FTP -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from urllib.parse import urlparse import requests @@ -20,9 +20,37 @@ from pridepy.util.api_handling import Util -def _local_path_for_url(download_url: str, output_folder: str) -> str: - filename = os.path.basename(urlparse(download_url).path) - return os.path.join(output_folder, filename) +def _safe_join(output_folder: str, relative_path: str) -> str: + """Join ``output_folder`` with a dataset-relative path. + + Preserves sub-directory structure (so identically-named files in + different collections don't collide). Guards against absolute paths or + ``..`` traversal that would escape ``output_folder`` by falling back to + the basename — provider relative paths are already dataset-relative, so + this is purely defensive. + """ + relative_path = (relative_path or "").lstrip("/") + if not relative_path: + return output_folder + local_path = os.path.normpath(os.path.join(output_folder, relative_path)) + out_abs = os.path.abspath(output_folder) + local_abs = os.path.abspath(local_path) + if local_abs != out_abs and not local_abs.startswith(out_abs + os.sep): + return os.path.join(output_folder, os.path.basename(relative_path)) + return local_path + + +def _dest_path( + output_folder: str, url_path: str, relative_path: Optional[str] +) -> str: + """Resolve the local destination for a download. + + Uses the dataset-relative path when available (preserving layout), + otherwise falls back to the URL basename. + """ + if relative_path: + return _safe_join(output_folder, relative_path) + return os.path.join(output_folder, os.path.basename(url_path)) def _open_ftp_connection(host: str, use_tls: bool, timeout: int = 30) -> FTP: @@ -210,21 +238,26 @@ def callback(data): def _download_ftp_paths_serial( host: str, - paths: List[str], - output_folder: str, + items: List[Tuple[str, str]], skip_if_downloaded_already: bool, use_tls: bool, max_connection_retries: int, max_download_retries: int, ) -> None: - """Download all paths from one host over a single (reused) connection.""" + """Download all paths from one host over a single (reused) connection. + + ``items`` is a list of ``(ftp_path, local_path)`` pairs; ``local_path`` + is the precomputed destination (already including any sub-directories). + """ connection_attempt = 0 while connection_attempt < max_connection_retries: try: ftp = _open_ftp_connection(host, use_tls=use_tls) logging.info(f"Connected to FTP host: {host} (tls={use_tls})") - for ftp_path in paths: - local_path = os.path.join(output_folder, os.path.basename(ftp_path)) + for ftp_path, local_path in items: + parent = os.path.dirname(local_path) + if parent: + os.makedirs(parent, exist_ok=True) try: _download_one_ftp_path( ftp=ftp, @@ -262,8 +295,7 @@ def _download_ftp_paths_serial( def _download_ftp_paths_parallel( host: str, - paths: List[str], - output_folder: str, + items: List[Tuple[str, str]], skip_if_downloaded_already: bool, use_tls: bool, max_connection_retries: int, @@ -273,12 +305,17 @@ def _download_ftp_paths_parallel( """ Download paths concurrently using ``parallel_files`` workers; each worker opens its own FTP connection so transfers don't serialize. + + ``items`` is a list of ``(ftp_path, local_path)`` pairs. """ - def worker(ftp_path: str, position: int) -> None: - local_path = os.path.join(output_folder, os.path.basename(ftp_path)) + def worker(item: Tuple[str, str], position: int) -> None: + ftp_path, local_path = item if skip_if_downloaded_already and os.path.exists(local_path): logging.info(f"Skipping download as file already exists: {local_path}") return + parent = os.path.dirname(local_path) + if parent: + os.makedirs(parent, exist_ok=True) connection_attempt = 0 while connection_attempt < max_connection_retries: try: @@ -312,7 +349,7 @@ def worker(ftp_path: str, position: int) -> None: with ThreadPoolExecutor(max_workers=parallel_files) as executor: futures = [ - executor.submit(worker, path, idx) for idx, path in enumerate(paths) + executor.submit(worker, item, idx) for idx, item in enumerate(items) ] for future in as_completed(futures): try: @@ -329,6 +366,7 @@ def download_ftp_urls( max_download_retries: int = 3, use_tls: bool = False, parallel_files: int = 1, + relative_paths: Optional[List[str]] = None, ) -> None: """ Download a list of FTP URLs with retries, REST-based resume, and @@ -340,22 +378,33 @@ def download_ftp_urls( connection is transparently retried over TLS. :param parallel_files: When >1, downloads run concurrently with that many worker connections per host (capped at the number of files). + :param relative_paths: Optional per-URL dataset-relative destination + paths (parallel to ``ftp_urls``). When given, files are written to + ``output_folder/`` so identically-named files in + different collections don't collide. When omitted, the URL basename + is used (legacy flat layout). """ if not os.path.isdir(output_folder): os.makedirs(output_folder, exist_ok=True) - host_to_paths: Dict[str, List[str]] = {} - for url in ftp_urls: + host_to_items: Dict[str, List[Tuple[str, str]]] = {} + for idx, url in enumerate(ftp_urls): parsed = urlparse(url) - host_to_paths.setdefault(parsed.hostname, []).append(parsed.path.lstrip("/")) + remote_path = parsed.path.lstrip("/") + relpath = ( + relative_paths[idx] + if relative_paths and idx < len(relative_paths) + else None + ) + local_path = _dest_path(output_folder, remote_path, relpath) + host_to_items.setdefault(parsed.hostname, []).append((remote_path, local_path)) - for host, paths in host_to_paths.items(): - workers = max(1, min(parallel_files, len(paths))) + for host, items in host_to_items.items(): + workers = max(1, min(parallel_files, len(items))) if workers > 1: _download_ftp_paths_parallel( host=host, - paths=paths, - output_folder=output_folder, + items=items, skip_if_downloaded_already=skip_if_downloaded_already, use_tls=use_tls, max_connection_retries=max_connection_retries, @@ -365,8 +414,7 @@ def download_ftp_urls( else: _download_ftp_paths_serial( host=host, - paths=paths, - output_folder=output_folder, + items=items, skip_if_downloaded_already=skip_if_downloaded_already, use_tls=use_tls, max_connection_retries=max_connection_retries, @@ -378,6 +426,10 @@ def _parallel_download(url, file_path, position=0): """Download a file via a single-connection HTTP stream with optional resume. If a partial file exists and the server supports Range requests, resumes from where it left off; otherwise restarts from scratch.""" + parent = os.path.dirname(file_path) + if parent: + os.makedirs(parent, exist_ok=True) + session = Util.create_session_with_retries() try: head = session.head(url, timeout=(30, 30)) @@ -413,6 +465,18 @@ def _parallel_download(url, file_path, position=0): f.write(chunk) pbar.update(len(chunk)) + # Post-transfer integrity check mirroring the FTP path: the written size + # must match the server-reported Content-Length. A server that closes the + # data channel mid-stream without raising leaves a truncated file; raising + # here lets the caller's retry loop re-download (Range-resuming when able). + if total_size: + actual_size = os.path.getsize(file_path) + if actual_size != total_size: + raise RuntimeError( + f"Incomplete download for {file_path}: got {actual_size} bytes, " + f"expected {total_size}" + ) + def _http_download_one( url: str, @@ -420,14 +484,19 @@ def _http_download_one( skip_if_downloaded_already: bool, max_retries: int = 3, position: int = 0, + relative_path: Optional[str] = None, ) -> None: """ Download a single HTTP(S) URL with HEAD-then-Range resume and retry. Used as the worker target for both the serial loop and the parallel ThreadPoolExecutor path. Reuses :meth:`_parallel_download` so the same resume / restart-on-non-206 behaviour is shared with globus downloads. + + ``relative_path`` (when given) is the dataset-relative destination, so + files keep their collection layout instead of being flattened to the + URL basename. """ - local_path = _local_path_for_url(url, output_folder) + local_path = _dest_path(output_folder, urlparse(url).path, relative_path) if skip_if_downloaded_already and os.path.exists(local_path): logging.info(f"Skipping download as file already exists: {local_path}") return @@ -453,6 +522,7 @@ def download_http_urls( skip_if_downloaded_already: bool, parallel_files: int = 1, max_retries: int = 3, + relative_paths: Optional[List[str]] = None, ) -> None: """ Download a list of HTTP(S) URLs with HEAD-then-Range resume, per-file @@ -462,6 +532,9 @@ def download_http_urls( :class:`ThreadPoolExecutor`. Each worker manages its own file (a new ``requests`` session is opened inside ``_parallel_download``) so the only shared resource is the output directory. + + :param relative_paths: Optional per-URL dataset-relative destination + paths (parallel to ``http_urls``); see :func:`download_ftp_urls`. """ if not os.path.isdir(output_folder): os.makedirs(output_folder, exist_ok=True) @@ -469,6 +542,11 @@ def download_http_urls( if not http_urls: return + def _rel(idx: int) -> Optional[str]: + if relative_paths and idx < len(relative_paths): + return relative_paths[idx] + return None + workers = max(1, min(parallel_files, len(http_urls))) if workers > 1: logging.info( @@ -483,6 +561,7 @@ def download_http_urls( skip_if_downloaded_already, max_retries, idx, + _rel(idx), ) for idx, url in enumerate(http_urls) ] @@ -492,13 +571,14 @@ def download_http_urls( except Exception as e: logging.error(f"Parallel HTTP download error: {e}") else: - for url in http_urls: + for idx, url in enumerate(http_urls): try: _http_download_one( url, output_folder, skip_if_downloaded_already, max_retries, + relative_path=_rel(idx), ) except Exception as e: logging.error(f"HTTP download failed for {url}: {e}") diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index b83e4aa..f233c40 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -131,6 +131,100 @@ def test_parallel_download_falls_back_without_accept_ranges(self): with open(output_file, "rb") as handle: assert handle.read() == b"abc" + def test_parallel_download_raises_on_truncated_stream(self): + """A stream shorter than Content-Length must raise so the caller retries.""" + with tempfile.TemporaryDirectory() as tmp_dir: + output_file = os.path.join(tmp_dir, "file.raw") + session = Mock() + head = Mock() + head.headers = {"content-length": "5", "accept-ranges": "none"} + head.raise_for_status.return_value = None + session.head.return_value = head + + stream_response = Mock() + stream_response.raise_for_status.return_value = None + stream_response.iter_content.return_value = [b"ab"] # only 2 of 5 bytes + stream_response.__enter__ = Mock(return_value=stream_response) + stream_response.__exit__ = Mock(return_value=None) + session.get.return_value = stream_response + + with patch( + "pridepy.download.transport.Util.create_session_with_retries", + return_value=session, + ): + with self.assertRaisesRegex(RuntimeError, "Incomplete download"): + transport._parallel_download( + "https://example.org/file.raw", + output_file, + ) + + def test_safe_join_preserves_subdirs_and_blocks_escape(self): + out = os.path.join("/tmp", "out") + # Nested dataset-relative path is preserved under output_folder. + assert transport._safe_join(out, "raw/sub/run.raw") == os.path.join( + out, "raw", "sub", "run.raw" + ) + # Traversal that escapes output_folder falls back to the basename. + assert transport._safe_join(out, "../../etc/passwd") == os.path.join( + out, "passwd" + ) + + def test_download_files_threads_relative_paths_avoiding_collisions(self): + """Same-basename files in different collections must not flatten/collide: + base.Provider.download_files threads each record's relativePath through + to the transport layer.""" + provider = MassiveProvider() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(transport, "download_ftp_urls") as ftp_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/out", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + ) + kwargs = ftp_mock.call_args.kwargs + assert kwargs["relative_paths"] == ["raw/a/run.raw", "raw/b/run.raw"] + + def test_download_files_threads_relative_paths_for_http(self): + """The HTTP partition also forwards relativePath to download_http_urls.""" + + class _HttpProvider(MassiveProvider): + pass + + provider = _HttpProvider() + records = [ + { + "accession": "MSV000012345", + "fileName": "run.raw", + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "http://example.org/d1/run.raw"} + ], + "relativePath": "raw/d1/run.raw", + }, + ] + with patch.object(transport, "download_http_urls") as http_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/out", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + ) + assert http_mock.call_args.kwargs["relative_paths"] == ["raw/d1/run.raw"] + def test_validate_download_rejects_empty_and_bad_checksum(self): with tempfile.TemporaryDirectory() as tmp_dir: file_path = os.path.join(tmp_dir, "test.raw") diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py index efb0c5c..4227907 100644 --- a/pridepy/tests/test_jpost_files.py +++ b/pridepy/tests/test_jpost_files.py @@ -87,6 +87,7 @@ def test_download_file_by_name_uses_jpost_ftp_listing(self): skip_if_downloaded_already=False, use_tls=False, parallel_files=1, + relative_paths=["raw/folder/sample.raw"], ) def test_proxi_listing_maps_cv_name_to_category(self): diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index 6bec7d4..619c517 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -103,6 +103,7 @@ def test_download_file_by_name_uses_massive_ftp_listing(self): skip_if_downloaded_already=False, use_tls=True, parallel_files=1, + relative_paths=["raw/folder/sample.raw"], ) def test_repo_uses_tls_true_for_massive_false_for_jpost(self): From 9b2a61f7ee92d146fdc8d945bb0bd64f69fc9cd5 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 10:13:39 +0100 Subject: [PATCH 34/54] feat(massive): HTTPS fallback when FTP/FTPS is blocked Some networks block FTP/FTPS outright. MassiveProvider.list_files now tries the FTPS tree walk first and, on failure, falls back to an all-HTTPS path: - list files from the GNPS2 dataset cache file index (datasetcache.gnps2.org, CSV stream) over HTTPS - build records whose download URL is the ProteoSAFe endpoint (massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true&file=f./) Downloads then route over HTTPS automatically (scheme-based dispatch in Provider.download_files), with relativePath preserving the dataset layout. Verified live: FTPS and HTTPS copies of the same file are byte-identical (md5 match). Adds unit tests for the URL builder, the HTTPS record, and the FTPS->HTTPS listing fallback. --- README.md | 4 +- pridepy/download/massive.py | 107 +++++++++++++++++++++++++--- pridepy/tests/test_massive_files.py | 67 +++++++++++++++++ 3 files changed, 166 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a674cbb..b284a2d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ You can: - download public and private PRIDE files -- download public MassIVE (`MSV...`), JPOST (`JPST...`), and iProX (`IPX...`) datasets directly. MassIVE goes through FTPS at `massive-ftp.ucsd.edu`; JPOST uses the JSON PROXI endpoint at `repository.jpostdb.org` for listings and `ftp.jpostdb.org` for transfers; iProX fetches the dataset's ProteomeXchange XML from `download.iprox.org` and downloads files over anonymous HTTP +- download public MassIVE (`MSV...`), JPOST (`JPST...`), and iProX (`IPX...`) datasets directly. MassIVE goes through FTPS at `massive-ftp.ucsd.edu`, with an automatic HTTPS fallback (via the GNPS2 file index and the `massive.ucsd.edu` ProteoSAFe endpoint) for networks that block FTP/FTPS; JPOST uses the JSON PROXI endpoint at `repository.jpostdb.org` for listings and `ftp.jpostdb.org` for transfers; iProX fetches the dataset's ProteomeXchange XML from `download.iprox.org` and downloads files over anonymous HTTP - download by category (`RAW`, `SEARCH`, `RESULT`, etc.) - stream project and file metadata - search projects by keyword and filters @@ -329,7 +329,7 @@ pridepy download-all-public-raw-files \ How each repository is enumerated: -- **MassIVE** walks the FTPS tree at `massive-ftp.ucsd.edu` (the server requires TLS). +- **MassIVE** walks the FTPS tree at `massive-ftp.ucsd.edu` (the server requires TLS). If FTP/FTPS is blocked by the network, `pridepy` automatically falls back to HTTPS: it lists the dataset from the GNPS2 file index (`datasetcache.gnps2.org`) and downloads each file from the ProteoSAFe endpoint at `massive.ucsd.edu` (byte-identical to the FTPS copy). - **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. - **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTP (with `Range` support for resume). iProX also exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTP endpoint so no iProX credentials are required. diff --git a/pridepy/download/massive.py b/pridepy/download/massive.py index 6ecf03a..b7b7659 100644 --- a/pridepy/download/massive.py +++ b/pridepy/download/massive.py @@ -1,13 +1,21 @@ """MassIVE direct-download provider. -Lists files by walking the FTPS tree at massive-ftp.ucsd.edu (TLS is -required by the server). Downloads files via the shared transport layer -with ``use_tls=True``. +Primary path: list files by walking the FTPS tree at massive-ftp.ucsd.edu +(TLS is required by the server) and download them over FTPS. + +HTTPS fallback: some networks block FTP/FTPS entirely. When the FTPS +listing fails, fall back to the HTTPS file index at datasetcache.gnps2.org +and download each file from the ProteoSAFe HTTPS endpoint at +massive.ucsd.edu (same bytes as FTPS, verified by checksum). The fallback +keeps everything over HTTPS so it works on FTPS-blocked networks. """ +import logging import os import re from typing import ClassVar, Dict, List -from urllib.parse import urlparse +from urllib.parse import quote, urlparse + +import requests from pridepy.download import registry from pridepy.download.base import Provider @@ -35,6 +43,15 @@ class MassiveProvider(Provider): ARCHIVE_FTP: ClassVar[str] = "massive-ftp.ucsd.edu" ARCHIVE_FTP_URL_PREFIX: ClassVar[str] = "ftp://massive-ftp.ucsd.edu/v01/" + # HTTPS fallback for FTPS-blocked networks. + HTTPS_DOWNLOAD_URL: ClassVar[str] = ( + "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile" + ) + # GNPS2 dataset cache: HTTPS file index (datasette CSV stream). + HTTPS_FILE_INDEX_URL: ClassVar[str] = ( + "https://datasetcache.gnps2.org/datasette/database/filename.csv" + ) + @staticmethod def matches(accession: str) -> bool: """Return True when ``accession`` is a MassIVE dataset accession.""" @@ -78,16 +95,86 @@ def _build_file_record(cls, accession: str, ftp_url: str) -> Dict: "source": "MassIVE", } + @classmethod + def _get_https_url(cls, accession: str, relative_path: str) -> str: + """ProteoSAFe HTTPS download URL for a dataset-relative file path. + + Mirrors the FTPS file: ``f./`` in the + ProteoSAFe ftp file-space. Verified to return byte-identical content + to the FTPS copy. + """ + file_param = f"f.{accession.upper()}/{relative_path.lstrip('/')}" + return ( + f"{cls.HTTPS_DOWNLOAD_URL}?forceDownload=true" + f"&file={quote(file_param, safe='/.')}" + ) + + @classmethod + def _build_https_file_record(cls, accession: str, relative_path: str) -> Dict: + """Build a file record whose download location is the HTTPS endpoint.""" + relative_path = relative_path.lstrip("/") + collection = relative_path.split("/", 1)[0] if relative_path else "" + return { + "accession": accession.upper(), + "fileName": os.path.basename(relative_path), + "fileCategory": {"value": cls._map_collection_to_category(collection)}, + # base.Provider.download_files routes by URL scheme; an https:// + # value here sends the file through the HTTPS transport. + "publicFileLocations": [ + {"name": "HTTPS", "value": cls._get_https_url(accession, relative_path)} + ], + "relativePath": relative_path, + "collection": collection, + "source": "MassIVE", + } + + def _list_via_https(self, accession: str) -> List[Dict]: + """List dataset files over HTTPS via the GNPS2 dataset cache. + + Used when FTPS is unavailable (blocked network). Streams the file + index as CSV and builds HTTPS-download records. + """ + import csv + normalized = accession.upper() + logging.info(f"Listing MassIVE dataset {normalized} via HTTPS file index") + response = requests.get( + self.HTTPS_FILE_INDEX_URL, + params={"dataset__exact": normalized, "_stream": "on", "_col": "filepath"}, + timeout=60, + stream=True, + ) + response.raise_for_status() + lines = (line.decode("utf-8") for line in response.iter_lines() if line) + records: List[Dict] = [] + for row in csv.DictReader(lines): + file_path = (row.get("filepath") or "").strip() + if file_path: + records.append(self._build_https_file_record(normalized, file_path)) + if not records: + raise RuntimeError( + f"No files found via HTTPS file index for MassIVE dataset {normalized}" + ) + return records + def list_files(self, accession: str) -> List[Dict]: from pridepy.download import transport normalized = accession.upper() remote_root = self._get_public_root(normalized) - remote_files = transport._list_ftp_repo_files( - host=self.ARCHIVE_FTP, - remote_root=remote_root, - error_label=f"MassIVE dataset {normalized}", - use_tls=True, - ) + try: + remote_files = transport._list_ftp_repo_files( + host=self.ARCHIVE_FTP, + remote_root=remote_root, + error_label=f"MassIVE dataset {normalized}", + use_tls=True, + ) + except Exception as ftps_error: + logging.warning( + "MassIVE FTPS listing failed for %s (%s); " + "falling back to the HTTPS file index.", + normalized, + ftps_error, + ) + return self._list_via_https(normalized) return [ self._build_file_record( normalized, diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index 619c517..5498314 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -177,3 +177,70 @@ def test_base_direct_download_provider_partitions_urls_by_scheme(self): ] http_mock.assert_called_once() assert http_mock.call_args.kwargs["http_urls"] == ["http://example.org/b.raw"] + + def test_get_https_url_builds_proteosafe_endpoint(self): + url = MassiveProvider._get_https_url( + "MSV000012345", "raw/Raw spec/C 3.raw" + ) + # Path slashes/dots preserved, spaces percent-encoded. + assert url == ( + "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?forceDownload=true" + "&file=f.MSV000012345/raw/Raw%20spec/C%203.raw" + ) + + def test_build_https_file_record_sets_relpath_category_and_https_location(self): + record = MassiveProvider._build_https_file_record( + "MSV000012345", "raw/sub/run.raw" + ) + assert record["relativePath"] == "raw/sub/run.raw" + assert record["fileName"] == "run.raw" + assert record["collection"] == "raw" + assert record["fileCategory"]["value"] == "RAW" + location = record["publicFileLocations"][0] + assert location["value"].startswith( + "https://massive.ucsd.edu/ProteoSAFe/DownloadResultFile?" + ) + assert location["value"].endswith("file=f.MSV000012345/raw/sub/run.raw") + + def test_list_files_falls_back_to_https_when_ftps_blocked(self): + """When the FTPS tree walk raises (e.g. FTPS blocked), list_files must + fall back to the HTTPS file index and emit HTTPS-download records.""" + csv_text = ( + "usi,filepath\n" + "mzspec:MSV000012345:raw/a/run.raw,raw/a/run.raw\n" + "mzspec:MSV000012345:raw/b/run.raw,raw/b/run.raw\n" + "mzspec:MSV000012345:ccms_result/x.mzid,ccms_result/x.mzid\n" + ) + + class _FakeCSVResponse: + def raise_for_status(self): + return None + + def iter_lines(self): + for line in csv_text.splitlines(): + yield line.encode("utf-8") + + with patch.object( + transport, "_list_ftp_repo_files", side_effect=RuntimeError("FTPS blocked") + ), patch( + "pridepy.download.massive.requests.get", return_value=_FakeCSVResponse() + ): + records = MassiveProvider().list_files("MSV000012345") + + assert {r["relativePath"] for r in records} == { + "raw/a/run.raw", + "raw/b/run.raw", + "ccms_result/x.mzid", + } + # Same-basename files in different collections are kept distinct. + run_records = [r for r in records if r["fileName"] == "run.raw"] + assert len(run_records) == 2 + for record in records: + assert record["publicFileLocations"][0]["value"].startswith("https://") + # Downstream RAW filtering still works on the HTTPS records. + raw_names = { + rec["fileName"] + for rec in records + if rec["fileCategory"]["value"] == "RAW" + } + assert raw_names == {"run.raw"} From 9c31f152c0348a4b5426a46104869cd12155b330 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 11:03:11 +0100 Subject: [PATCH 35/54] chore(release): set version to 0.0.16 Consolidate the modular-download work (formerly split across PRs #103/#104/ #105) into a single release off master (0.0.15 -> 0.0.16). --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5456fc2..4a95f24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pridepy" -version = "0.0.19" +version = "0.0.16" description = "Python Client library for PRIDE Rest API" readme = "README.md" requires-python = ">=3.9" From cb02d519be36665c95f30ef50933711571ad95b4 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 16:50:32 +0100 Subject: [PATCH 36/54] fix(download): propagate direct-download failures; px relativePath MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses two Codex review findings. High — direct-download commands reported success even when transfers failed: transport.download_ftp_urls / download_http_urls swallowed per-file exceptions (log + continue) and returned, so Provider.download_files could not detect failures. The FTP serial/parallel helpers now collect and return failed paths, the HTTP loop collects failed URLs, and both download_ftp_urls and download_http_urls raise RuntimeError when any file fails. This restores parity with the PRIDE path (which raises) and makes the earlier HTTP/FTP post-transfer size checks actually surface as command failures. Successful downloads return empty failure lists and do not raise (verified live for MassIVE FTPS and iProX HTTP). Medium — download-px-raw-files flattened every URI to its basename, so duplicate raw filenames in different directories overwrote each other. ProteomeXchangeProvider.list_files now derives a relativePath per file by stripping the common parent directory shared by all URIs (e.g. run1/sample.raw vs run2/sample.raw), with a basename fallback for a single file or no shared prefix. Adds tests: FTP/HTTP failure propagation, Provider.download_files propagation, and PX duplicate-basename disambiguation. --- pridepy/download/proteomexchange.py | 39 ++++++++++- pridepy/download/transport.py | 76 ++++++++++++++------ pridepy/tests/test_download_resilience.py | 85 +++++++++++++++++++++++ 3 files changed, 177 insertions(+), 23 deletions(-) diff --git a/pridepy/download/proteomexchange.py b/pridepy/download/proteomexchange.py index be75268..4c44302 100644 --- a/pridepy/download/proteomexchange.py +++ b/pridepy/download/proteomexchange.py @@ -23,6 +23,7 @@ """ import logging import os +import posixpath import re import xml.etree.ElementTree as ET from typing import ClassVar, Dict, List @@ -99,17 +100,50 @@ def _parse_px_xml_for_raw_file_urls(px_xml_url: str) -> List[str]: urls.append(value) return urls + @staticmethod + def _relative_paths_for_urls(urls: List[str]) -> List[str]: + """Compute a dataset-relative destination path for each URL. + + The PX XML's raw-file URIs point at arbitrary directories on the + hosting repository, so flattening to the URL basename would let + same-named files in different directories overwrite each other. + Strip the common parent directory shared by all URIs and keep the + remainder, so e.g. ``.../run1/x.raw`` and ``.../run2/x.raw`` become + ``run1/x.raw`` and ``run2/x.raw``. A single file (or one with no + shared prefix) falls back to its basename. + """ + paths = [urlparse(url).path for url in urls] + if not paths: + return [] + dirs = [posixpath.dirname(p) for p in paths] + try: + common = dirs[0] if len(paths) == 1 else posixpath.commonpath(dirs) + except ValueError: + common = "" + rels: List[str] = [] + for path in paths: + if common and (path == common or path.startswith(common + "/")): + rel = path[len(common):].lstrip("/") + else: + rel = posixpath.basename(path) + rels.append(rel or posixpath.basename(path)) + return rels + def list_files(self, accession: str) -> List[Dict]: """Return the dataset's raw-file URIs as minimal file records. The PX XML doesn't expose checksums or rich category labels, so - each record carries just enough to drive the downloader. + each record carries just enough to drive the downloader. A + ``relativePath`` is derived per file (see + :meth:`_relative_paths_for_urls`) so the transport layer preserves + directory structure instead of colliding on duplicate basenames. """ px_xml_url = self._normalize_px_xml_url(accession) logging.info(f"Fetching PX XML: {px_xml_url}") urls = self._parse_px_xml_for_raw_file_urls(px_xml_url) + relative_paths = self._relative_paths_for_urls(urls) records: List[Dict] = [] - for url in urls: + for url, relative_path in zip(urls, relative_paths): parsed = urlparse(url) records.append( { @@ -119,6 +153,7 @@ def list_files(self, accession: str) -> List[Dict]: "publicFileLocations": [ {"name": "FTP Protocol", "value": url} ], + "relativePath": relative_path, "source": "ProteomeXchange", } ) diff --git a/pridepy/download/transport.py b/pridepy/download/transport.py index d982ed6..f99fee0 100644 --- a/pridepy/download/transport.py +++ b/pridepy/download/transport.py @@ -243,14 +243,19 @@ def _download_ftp_paths_serial( use_tls: bool, max_connection_retries: int, max_download_retries: int, -) -> None: +) -> List[str]: """Download all paths from one host over a single (reused) connection. ``items`` is a list of ``(ftp_path, local_path)`` pairs; ``local_path`` is the precomputed destination (already including any sub-directories). + + Returns the list of ``ftp_path`` values that could not be downloaded + (connection never established, or per-file giving up) so the caller can + surface a failure instead of reporting false success. """ connection_attempt = 0 while connection_attempt < max_connection_retries: + failed: List[str] = [] try: ftp = _open_ftp_connection(host, use_tls=use_tls) logging.info(f"Connected to FTP host: {host} (tls={use_tls})") @@ -270,6 +275,7 @@ def _download_ftp_paths_serial( logging.error( f"Failed to download {ftp_path} from {host}: {e}" ) + failed.append(ftp_path) try: ftp.quit() except Exception: @@ -278,7 +284,7 @@ def _download_ftp_paths_serial( except Exception: pass logging.info(f"Disconnected from FTP host: {host}") - return + return failed except (socket.timeout, ftplib.error_temp, ftplib.error_perm, OSError) as e: connection_attempt += 1 logging.error( @@ -291,6 +297,8 @@ def _download_ftp_paths_serial( logging.error( f"Giving up after {max_connection_retries} failed connection attempts to {host}." ) + return [ftp_path for ftp_path, _ in items] + return [ftp_path for ftp_path, _ in items] def _download_ftp_paths_parallel( @@ -301,18 +309,19 @@ def _download_ftp_paths_parallel( max_connection_retries: int, max_download_retries: int, parallel_files: int, -) -> None: +) -> List[str]: """ Download paths concurrently using ``parallel_files`` workers; each worker opens its own FTP connection so transfers don't serialize. - ``items`` is a list of ``(ftp_path, local_path)`` pairs. + ``items`` is a list of ``(ftp_path, local_path)`` pairs. Returns the list + of ``ftp_path`` values that failed so the caller can surface a failure. """ - def worker(item: Tuple[str, str], position: int) -> None: + def worker(item: Tuple[str, str], position: int) -> Optional[str]: ftp_path, local_path = item if skip_if_downloaded_already and os.path.exists(local_path): logging.info(f"Skipping download as file already exists: {local_path}") - return + return None parent = os.path.dirname(local_path) if parent: os.makedirs(parent, exist_ok=True) @@ -329,7 +338,7 @@ def worker(item: Tuple[str, str], position: int) -> None: max_download_retries=max_download_retries, position=position, ) - return + return None finally: try: ftp.quit() @@ -345,17 +354,27 @@ def worker(item: Tuple[str, str], position: int) -> None: ) if connection_attempt < max_connection_retries: time.sleep(5) + except Exception as e: + logging.error(f"Failed to download {ftp_path} from {host}: {e}") + return ftp_path logging.error(f"Giving up on {ftp_path} from {host}") + return ftp_path + failed: List[str] = [] with ThreadPoolExecutor(max_workers=parallel_files) as executor: - futures = [ - executor.submit(worker, item, idx) for idx, item in enumerate(items) - ] - for future in as_completed(futures): + future_to_path = { + executor.submit(worker, item, idx): item[0] + for idx, item in enumerate(items) + } + for future in as_completed(future_to_path): try: - future.result() + result = future.result() + if result is not None: + failed.append(result) except Exception as e: logging.error(f"Parallel FTP download error: {e}") + failed.append(future_to_path[future]) + return failed def download_ftp_urls( @@ -399,10 +418,11 @@ def download_ftp_urls( local_path = _dest_path(output_folder, remote_path, relpath) host_to_items.setdefault(parsed.hostname, []).append((remote_path, local_path)) + failed: List[str] = [] for host, items in host_to_items.items(): workers = max(1, min(parallel_files, len(items))) if workers > 1: - _download_ftp_paths_parallel( + failed.extend(_download_ftp_paths_parallel( host=host, items=items, skip_if_downloaded_already=skip_if_downloaded_already, @@ -410,16 +430,21 @@ def download_ftp_urls( max_connection_retries=max_connection_retries, max_download_retries=max_download_retries, parallel_files=workers, - ) + )) else: - _download_ftp_paths_serial( + failed.extend(_download_ftp_paths_serial( host=host, items=items, skip_if_downloaded_already=skip_if_downloaded_already, use_tls=use_tls, max_connection_retries=max_connection_retries, max_download_retries=max_download_retries, - ) + )) + + if failed: + raise RuntimeError( + f"Failed to download {len(failed)} FTP file(s): {failed}" + ) def _parallel_download(url, file_path, position=0): @@ -547,13 +572,14 @@ def _rel(idx: int) -> Optional[str]: return relative_paths[idx] return None + failed: List[str] = [] workers = max(1, min(parallel_files, len(http_urls))) if workers > 1: logging.info( f"Downloading {len(http_urls)} HTTP(S) file(s) with {workers} parallel workers" ) with ThreadPoolExecutor(max_workers=workers) as executor: - futures = [ + future_to_url = { executor.submit( _http_download_one, url, @@ -562,14 +588,16 @@ def _rel(idx: int) -> Optional[str]: max_retries, idx, _rel(idx), - ) + ): url for idx, url in enumerate(http_urls) - ] - for future in as_completed(futures): + } + for future in as_completed(future_to_url): + url = future_to_url[future] try: future.result() except Exception as e: - logging.error(f"Parallel HTTP download error: {e}") + logging.error(f"HTTP download failed for {url}: {e}") + failed.append(url) else: for idx, url in enumerate(http_urls): try: @@ -582,3 +610,9 @@ def _rel(idx: int) -> Optional[str]: ) except Exception as e: logging.error(f"HTTP download failed for {url}: {e}") + failed.append(url) + + if failed: + raise RuntimeError( + f"Failed to download {len(failed)} HTTP(S) file(s): {failed}" + ) diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index f233c40..9b6f7ad 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -10,6 +10,7 @@ from pridepy.download import util as provider_util from pridepy.download.massive import MassiveProvider from pridepy.download.pride import PrideProvider +from pridepy.download.proteomexchange import ProteomeXchangeProvider from pridepy.download import registry @@ -225,6 +226,90 @@ class _HttpProvider(MassiveProvider): ) assert http_mock.call_args.kwargs["relative_paths"] == ["raw/d1/run.raw"] + def test_download_http_urls_raises_when_a_file_fails(self): + """A failed HTTP transfer must surface as an exception, not be + swallowed into a false success.""" + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + transport, "_parallel_download", side_effect=RuntimeError("boom") + ): + with self.assertRaisesRegex(RuntimeError, "Failed to download"): + transport.download_http_urls( + http_urls=["https://example.org/a.raw"], + output_folder=tmp_dir, + skip_if_downloaded_already=False, + max_retries=1, + ) + + def test_download_ftp_urls_raises_when_a_file_fails(self): + """A failed FTP transfer must surface as an exception.""" + with tempfile.TemporaryDirectory() as tmp_dir: + fake_ftp = Mock() + with patch.object( + transport, "_open_ftp_connection", return_value=fake_ftp + ), patch.object( + transport, "_download_one_ftp_path", side_effect=RuntimeError("boom") + ): + with self.assertRaisesRegex(RuntimeError, "Failed to download"): + transport.download_ftp_urls( + ftp_urls=["ftp://ftp.example.org/p/a.raw"], + output_folder=tmp_dir, + skip_if_downloaded_already=False, + ) + + def test_download_files_propagates_transport_failure(self): + """Provider.download_files must propagate a transport failure so the + direct-download path doesn't report false success (parity with PRIDE).""" + provider = MassiveProvider() + record = MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a.raw", + ) + with patch.object( + transport, "download_ftp_urls", side_effect=RuntimeError("download failed") + ): + with self.assertRaises(RuntimeError): + provider.download_files( + accession="MSV000012345", + records=[record], + output_folder="/tmp/does-not-matter", + skip_if_downloaded_already=False, + protocol="ftp", + ) + + def test_proteomexchange_relative_paths_disambiguate_duplicate_basenames(self): + """download-px-raw-files must not flatten duplicate basenames from + different directories onto the same local file.""" + urls = [ + "ftp://ftp.pride.ebi.ac.uk/pride/PXD1/run1/sample.raw", + "ftp://ftp.pride.ebi.ac.uk/pride/PXD1/run2/sample.raw", + ] + with patch.object( + ProteomeXchangeProvider, "_normalize_px_xml_url", return_value="http://x" + ), patch.object( + ProteomeXchangeProvider, + "_parse_px_xml_for_raw_file_urls", + return_value=urls, + ): + records = ProteomeXchangeProvider().list_files("PXD1") + + assert {r["relativePath"] for r in records} == { + "run1/sample.raw", + "run2/sample.raw", + } + + def test_proteomexchange_single_file_relative_path_is_basename(self): + urls = ["ftp://ftp.pride.ebi.ac.uk/pride/PXD1/run1/sample.raw"] + with patch.object( + ProteomeXchangeProvider, "_normalize_px_xml_url", return_value="http://x" + ), patch.object( + ProteomeXchangeProvider, + "_parse_px_xml_for_raw_file_urls", + return_value=urls, + ): + records = ProteomeXchangeProvider().list_files("PXD1") + assert records[0]["relativePath"] == "sample.raw" + def test_validate_download_rejects_empty_and_bad_checksum(self): with tempfile.TemporaryDirectory() as tmp_dir: file_path = os.path.join(tmp_dir, "test.raw") From 15452c80db15b169e7024a8a7c24976675146b3b Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 17:44:06 +0100 Subject: [PATCH 37/54] fix(download): by_url size verification; correct stale docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - by_url: _http_download_url / _ftp_download_url now verify the written size against Content-Length / ftp.size() and raise on truncation (validate_download only checked non-empty, so a partial file slipped through and could be wrongly skipped on the next run). _download_single_url removes a partial file when the transfer raises. Brings download-files-by-url to parity with the accession transport's post-transfer size check. - iprox: iProX serves over plain HTTP, not HTTPS — fix the _build_file_record parameter name (https_url -> file_url), the location-label comment, and the "no downloadable URIs" error message. - base: clarify get_download_url docstring re: records (MassIVE HTTPS fallback) that use a location name other than "FTP Protocol" and fall through to the first location. - pride: docstrings list s3 as a supported protocol and drop the nonexistent "Phase 3" wording (the per-file fallback is part of Phase 2). - test: by_url HTTP truncation now raises. --- pridepy/download/base.py | 8 +++++--- pridepy/download/by_url.py | 18 +++++++++++++++++- pridepy/download/iprox.py | 18 ++++++++++-------- pridepy/download/pride.py | 8 ++++---- pridepy/tests/test_download_resilience.py | 20 ++++++++++++++++++++ 5 files changed, 56 insertions(+), 16 deletions(-) diff --git a/pridepy/download/base.py b/pridepy/download/base.py index 3c159d3..8c9ae4f 100644 --- a/pridepy/download/base.py +++ b/pridepy/download/base.py @@ -50,9 +50,11 @@ def get_download_url(self, record: Dict, protocol: str = "ftp") -> str: """Resolve the download URL for ``record``. Default: return the ``"FTP Protocol"`` public-file-location value - (direct-download adapters store their public URL there — ftp:// for - MassIVE/JPOST, http(s):// for iProX). Adapters with richer, - protocol-aware resolution (PRIDE: aspera/globus/s3) override this. + (most direct-download adapters store their public URL there — ftp:// + for MassIVE/JPOST, http:// for iProX). Records that use a different + location name (e.g. MassIVE's HTTPS fallback uses ``"HTTPS"``) fall + through to the first location. Adapters with richer, protocol-aware + resolution (PRIDE: aspera/globus/s3) override this. """ locations = record.get("publicFileLocations", []) if not locations: diff --git a/pridepy/download/by_url.py b/pridepy/download/by_url.py index d80cda2..c40f237 100644 --- a/pridepy/download/by_url.py +++ b/pridepy/download/by_url.py @@ -36,6 +36,11 @@ def _http_download_url(url: str, target: str) -> None: if chunk: out.write(chunk) pbar.update(len(chunk)) + if total and os.path.getsize(target) != total: + raise RuntimeError( + f"Incomplete download for {target}: got {os.path.getsize(target)} " + f"bytes, expected {total}" + ) def _ftp_download_url(parsed, target: str) -> None: @@ -66,6 +71,11 @@ def _callback(data: bytes) -> None: pbar.update(len(data)) ftp.retrbinary(f"RETR {remote_path}", _callback) + if total and os.path.getsize(target) != total: + raise RuntimeError( + f"Incomplete download for {target}: got {os.path.getsize(target)} " + f"bytes, expected {total}" + ) def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: int = 0) -> None: @@ -108,7 +118,13 @@ def _download_single_url( logging.info("Skipping %s: already downloaded", file_name) return target - _dispatch_url_scheme(parsed, target, protocol, position=position) + try: + _dispatch_url_scheme(parsed, target, protocol, position=position) + except Exception: + # Don't leave a truncated/partial file behind — a non-empty partial + # would otherwise be wrongly skipped on the next run. + _provider_util._remove_if_exists(target) + raise ok, reason = _provider_util.validate_download(target) if not ok: diff --git a/pridepy/download/iprox.py b/pridepy/download/iprox.py index 29fdb1f..8326b30 100644 --- a/pridepy/download/iprox.py +++ b/pridepy/download/iprox.py @@ -61,15 +61,16 @@ def _get_public_ftp_url(cls, accession: str, remote_path: str) -> str: @classmethod def _build_file_record( - cls, accession: str, https_url: str, category_from_px: Optional[str] = None + cls, accession: str, file_url: str, category_from_px: Optional[str] = None ) -> Dict: """Build a pridepy file record for an iProX file. - ``category_from_px`` is the ``cvParam`` ``name`` from the dataset's - ProteomeXchange XML (e.g. ``"Associated raw file URI"``). + ``file_url`` is the file URI from the PX XML (plain ``http://`` on + download.iprox.org). ``category_from_px`` is the ``cvParam`` ``name`` + from the dataset's ProteomeXchange XML (e.g. ``"Associated raw file URI"``). """ from pridepy.download.massive import MassiveProvider - parsed = urlparse(https_url) + parsed = urlparse(file_url) root_prefix = f"/{accession.upper()}/" relative_path = parsed.path if relative_path.startswith(root_prefix): @@ -85,9 +86,9 @@ def _build_file_record( "fileName": os.path.basename(parsed.path), "fileCategory": {"value": category}, # "FTP Protocol" is the existing label the download dispatcher uses - # to locate a file URL; here it actually points at HTTPS. - # Provider.download_files routes by URL scheme. - "publicFileLocations": [{"name": "FTP Protocol", "value": https_url}], + # to locate a file URL; here it actually points at HTTP + # (download.iprox.org). Provider.download_files routes by URL scheme. + "publicFileLocations": [{"name": "FTP Protocol", "value": file_url}], "relativePath": relative_path, "collection": collection, "source": "iProX", @@ -124,6 +125,7 @@ def list_files(self, accession: str) -> List[Dict]: ) if not records: raise RuntimeError( - f"iProX PX XML for {normalized} contained no downloadable HTTPS URIs" + f"iProX PX XML for {normalized} contained no downloadable " + f"HTTP/HTTPS URIs" ) return records diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index c9eae9a..a6a493f 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -864,8 +864,8 @@ def download_files( """Override Provider.download_files with the multi-protocol orchestrator. Reuses the legacy batch downloader: Phase 1 batches the requested - protocol over a single connection, Phase 2 validates every file, and - Phase 3 falls back per-file across the remaining protocols. + protocol over a single connection; Phase 2 validates every file and, + for any that fail, falls back per-file across the remaining protocols. """ PrideProvider._download_files_batch( file_list_json=records, @@ -957,11 +957,11 @@ def _download_files_batch( parallel_files: int = 1, ): """ - Download files using either FTP or Aspera transfer protocol. + Download files using the ftp, aspera, globus, or s3 transfer protocol. :param file_list_json: File list in JSON format :param accession: Project accession :param output_folder: Folder to download the files - :param protocol: ftp, aspera, globus + :param protocol: ftp, aspera, globus, s3 :param aspera_maximum_bandwidth: parameter in Aspera sets the maximum bandwidth for the transfer. :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. """ diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 9b6f7ad..e770434 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -257,6 +257,26 @@ def test_download_ftp_urls_raises_when_a_file_fails(self): skip_if_downloaded_already=False, ) + def test_by_url_http_download_raises_on_truncated_content(self): + """by_url's HTTP downloader must reject a stream shorter than + Content-Length instead of accepting a truncated file.""" + with tempfile.TemporaryDirectory() as tmp_dir: + target = os.path.join(tmp_dir, "a.raw") + session = Mock() + response = Mock() + response.raise_for_status.return_value = None + response.headers = {"Content-Length": "5"} + response.iter_content.return_value = [b"ab"] # only 2 of 5 bytes + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + session.get.return_value = response + with patch( + "pridepy.download.by_url.Util.create_session_with_retries", + return_value=session, + ): + with self.assertRaisesRegex(RuntimeError, "Incomplete download"): + by_url._http_download_url("https://example.org/a.raw", target) + def test_download_files_propagates_transport_failure(self): """Provider.download_files must propagate a transport failure so the direct-download path doesn't report false success (parity with PRIDE).""" From c224f9a93d0af40467f5f3f47feceb23a8a40c10 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 18:09:51 +0100 Subject: [PATCH 38/54] fix(download): avoid gzip false-positive size check; px root-prefix paths Follow-up review of the failure-propagation/size-verification work. - Size verification (by_url._http_download_url and transport._parallel_download) skipped the check when the response carries Content-Encoding: requests decompresses gzip/deflate transparently, so on-disk size != Content-Length (compressed) and the check would have falsely raised "Incomplete download" and deleted an intact file. Also compute the size once for the message. - ProteomeXchange._relative_paths_for_urls: when raw URIs span different top-level directories the common prefix is "/", which the previous guard rejected and collapsed to a colliding basename. Strip any non-empty common prefix (including "/") so run1/x.raw and run2/x.raw stay distinct. - Docs: document that download_ftp_urls/download_http_urls raise on failure; README's download-files-by-url -w applies to any scheme (not globus-only); iProX accepts http(s); base warning says "ftp / http(s) only". - Tests: gzip response is not treated as truncated; PX root-prefix paths disambiguate. --- README.md | 2 +- pridepy/download/base.py | 2 +- pridepy/download/by_url.py | 29 ++++++++++----- pridepy/download/iprox.py | 7 ++-- pridepy/download/proteomexchange.py | 7 +++- pridepy/download/transport.py | 8 +++- pridepy/tests/test_download_resilience.py | 45 +++++++++++++++++++++++ 7 files changed, 83 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index b284a2d..660340e 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,7 @@ Command-specific options: | `-F, --url-list` | Manifest file, one URL per line | — | | `-u, --urls` | Comma-separated URL(s) | — | | `-p, --protocol` | `ftp` (per-scheme) or `globus` (resume-capable http/https) | `ftp` | -| `-w, --parallel-files` | Download 1–3 files concurrently (globus only) | `1` | +| `-w, --parallel-files` | Download 1–3 files concurrently (any scheme) | `1` | | `--checksum-check` | Validate against PRIDE checksums (accession inferred from PRIDE URL paths; only PRIDE archive URLs supported) | off |
diff --git a/pridepy/download/base.py b/pridepy/download/base.py index 8c9ae4f..2635102 100644 --- a/pridepy/download/base.py +++ b/pridepy/download/base.py @@ -230,7 +230,7 @@ def download_files( """ if protocol not in ("ftp", "https", "http"): logging.warning( - "Direct downloads currently use ftp / https only. " + "Direct downloads currently use ftp / http(s) only. " f"Ignoring requested protocol '{protocol}' for {accession}." ) diff --git a/pridepy/download/by_url.py b/pridepy/download/by_url.py index c40f237..0b4d03a 100644 --- a/pridepy/download/by_url.py +++ b/pridepy/download/by_url.py @@ -26,6 +26,11 @@ def _http_download_url(url: str, target: str) -> None: with session.get(url, stream=True, timeout=60) as response: response.raise_for_status() total = int(response.headers.get("Content-Length", 0)) + # When the server applied Content-Encoding (gzip/deflate), requests + # decompresses transparently, so the on-disk size is the decompressed + # size while Content-Length is the compressed size — skip the size + # check to avoid a false "incomplete" on an intact file. + content_encoding = response.headers.get("Content-Encoding") with open(target, "wb") as out, tqdm( total=total, unit="B", @@ -36,11 +41,13 @@ def _http_download_url(url: str, target: str) -> None: if chunk: out.write(chunk) pbar.update(len(chunk)) - if total and os.path.getsize(target) != total: - raise RuntimeError( - f"Incomplete download for {target}: got {os.path.getsize(target)} " - f"bytes, expected {total}" - ) + if total and not content_encoding: + actual = os.path.getsize(target) + if actual != total: + raise RuntimeError( + f"Incomplete download for {target}: got {actual} bytes, " + f"expected {total}" + ) def _ftp_download_url(parsed, target: str) -> None: @@ -71,11 +78,13 @@ def _callback(data: bytes) -> None: pbar.update(len(data)) ftp.retrbinary(f"RETR {remote_path}", _callback) - if total and os.path.getsize(target) != total: - raise RuntimeError( - f"Incomplete download for {target}: got {os.path.getsize(target)} " - f"bytes, expected {total}" - ) + if total: + actual = os.path.getsize(target) + if actual != total: + raise RuntimeError( + f"Incomplete download for {target}: got {actual} bytes, " + f"expected {total}" + ) def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: int = 0) -> None: diff --git a/pridepy/download/iprox.py b/pridepy/download/iprox.py index 8326b30..dcbbe68 100644 --- a/pridepy/download/iprox.py +++ b/pridepy/download/iprox.py @@ -65,9 +65,10 @@ def _build_file_record( ) -> Dict: """Build a pridepy file record for an iProX file. - ``file_url`` is the file URI from the PX XML (plain ``http://`` on - download.iprox.org). ``category_from_px`` is the ``cvParam`` ``name`` - from the dataset's ProteomeXchange XML (e.g. ``"Associated raw file URI"``). + ``file_url`` is the file URI from the PX XML (``http://`` on + download.iprox.org; ``https://`` is also accepted if present). + ``category_from_px`` is the ``cvParam`` ``name`` from the dataset's + ProteomeXchange XML (e.g. ``"Associated raw file URI"``). """ from pridepy.download.massive import MassiveProvider parsed = urlparse(file_url) diff --git a/pridepy/download/proteomexchange.py b/pridepy/download/proteomexchange.py index 4c44302..dfe70c8 100644 --- a/pridepy/download/proteomexchange.py +++ b/pridepy/download/proteomexchange.py @@ -122,7 +122,12 @@ def _relative_paths_for_urls(urls: List[str]) -> List[str]: common = "" rels: List[str] = [] for path in paths: - if common and (path == common or path.startswith(common + "/")): + # ``common`` is the shared ancestor of every path's directory, so + # each path starts with it; strip it to keep the disambiguating + # remainder. ``common`` can legitimately be ``"/"`` (files in + # different top-level dirs) — handle that by stripping it too, + # rather than collapsing to the (colliding) basename. + if common: rel = path[len(common):].lstrip("/") else: rel = posixpath.basename(path) diff --git a/pridepy/download/transport.py b/pridepy/download/transport.py index f99fee0..97cafc5 100644 --- a/pridepy/download/transport.py +++ b/pridepy/download/transport.py @@ -402,6 +402,7 @@ def download_ftp_urls( ``output_folder/`` so identically-named files in different collections don't collide. When omitted, the URL basename is used (legacy flat layout). + :raises RuntimeError: after attempting every file, if one or more failed. """ if not os.path.isdir(output_folder): os.makedirs(output_folder, exist_ok=True) @@ -476,8 +477,10 @@ def _parallel_download(url, file_path, position=0): logging.info(f"Resuming download from {resume_size} bytes: {file_path}") headers = {"Range": f"bytes={resume_size}-"} if resume_size > 0 else {} + content_encoding = None with session.get(url, headers=headers, stream=True, timeout=(30, 60)) as r: r.raise_for_status() + content_encoding = r.headers.get("Content-Encoding") if resume_size > 0 and r.status_code != 206: logging.warning("Server did not honor Range request (status %s), restarting download", r.status_code) resume_size = 0 @@ -494,7 +497,9 @@ def _parallel_download(url, file_path, position=0): # must match the server-reported Content-Length. A server that closes the # data channel mid-stream without raising leaves a truncated file; raising # here lets the caller's retry loop re-download (Range-resuming when able). - if total_size: + # Skipped when the server applied Content-Encoding (gzip/deflate): requests + # decompresses transparently, so on-disk size won't match Content-Length. + if total_size and not content_encoding: actual_size = os.path.getsize(file_path) if actual_size != total_size: raise RuntimeError( @@ -560,6 +565,7 @@ def download_http_urls( :param relative_paths: Optional per-URL dataset-relative destination paths (parallel to ``http_urls``); see :func:`download_ftp_urls`. + :raises RuntimeError: after attempting every URL, if one or more failed. """ if not os.path.isdir(output_folder): os.makedirs(output_folder, exist_ok=True) diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index e770434..2c7d688 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -144,6 +144,7 @@ def test_parallel_download_raises_on_truncated_stream(self): stream_response = Mock() stream_response.raise_for_status.return_value = None + stream_response.headers = {} # no Content-Encoding -> size check active stream_response.iter_content.return_value = [b"ab"] # only 2 of 5 bytes stream_response.__enter__ = Mock(return_value=stream_response) stream_response.__exit__ = Mock(return_value=None) @@ -277,6 +278,50 @@ def test_by_url_http_download_raises_on_truncated_content(self): with self.assertRaisesRegex(RuntimeError, "Incomplete download"): by_url._http_download_url("https://example.org/a.raw", target) + def test_by_url_http_download_skips_size_check_when_encoded(self): + """A gzip/deflate response is decompressed by requests, so the on-disk + size won't match Content-Length — the size check must be skipped to + avoid a false 'Incomplete download' on an intact file.""" + with tempfile.TemporaryDirectory() as tmp_dir: + target = os.path.join(tmp_dir, "a.txt") + session = Mock() + response = Mock() + response.raise_for_status.return_value = None + # Content-Length is the compressed size; decompressed payload is larger. + response.headers = {"Content-Length": "5", "Content-Encoding": "gzip"} + response.iter_content.return_value = [b"abcdefghij"] # 10 decompressed bytes + response.__enter__ = Mock(return_value=response) + response.__exit__ = Mock(return_value=None) + session.get.return_value = response + with patch( + "pridepy.download.by_url.Util.create_session_with_retries", + return_value=session, + ): + by_url._http_download_url("https://example.org/a.txt", target) + with open(target, "rb") as handle: + assert handle.read() == b"abcdefghij" + + def test_proteomexchange_relative_paths_handle_root_common_prefix(self): + """When raw URIs live in different top-level directories (common + prefix is '/'), the paths must still be disambiguated, not collapsed + to a colliding basename.""" + urls = [ + "ftp://ftp.example.org/run1/sample.raw", + "ftp://ftp.example.org/run2/sample.raw", + ] + with patch.object( + ProteomeXchangeProvider, "_normalize_px_xml_url", return_value="http://x" + ), patch.object( + ProteomeXchangeProvider, + "_parse_px_xml_for_raw_file_urls", + return_value=urls, + ): + records = ProteomeXchangeProvider().list_files("PXD1") + assert {r["relativePath"] for r in records} == { + "run1/sample.raw", + "run2/sample.raw", + } + def test_download_files_propagates_transport_failure(self): """Provider.download_files must propagate a transport failure so the direct-download path doesn't report false success (parity with PRIDE).""" From 1d5b50bfff0ab56e37b5ae806fded1bf92a314e6 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 20:19:53 +0100 Subject: [PATCH 39/54] chore(download): align by-url -w help; exercise size check in tests Review follow-ups (no functional change): - download-files-by-url --help for -w said "Primarily used by globus protocol", but by_url parallelizes for any scheme; align with the README. - the three _parallel_download success-path tests left response.headers as a bare Mock, so the new Content-Encoding guard saw a truthy value and silently skipped the size check; set headers={} so the check is actually exercised. --- pridepy/pridepy.py | 2 +- pridepy/tests/test_download_resilience.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pridepy/pridepy.py b/pridepy/pridepy.py index 5cc41cf..98acfd6 100644 --- a/pridepy/pridepy.py +++ b/pridepy/pridepy.py @@ -640,7 +640,7 @@ def download_files_by_list( "--parallel-files", default=1, type=click.IntRange(1, 3), - help="Number of files to download simultaneously (1-3). Primarily used by globus protocol. Default is 1.", + help="Number of files to download simultaneously (1-3), for any URL scheme. Default is 1.", ) def download_files_by_url( url_list_path, diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 2c7d688..b533ae9 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -62,6 +62,7 @@ def test_parallel_download_streams_full_file(self): stream_response = Mock() stream_response.raise_for_status.return_value = None + stream_response.headers = {} stream_response.iter_content.return_value = [b"abc"] stream_response.__enter__ = Mock(return_value=stream_response) stream_response.__exit__ = Mock(return_value=None) @@ -87,6 +88,7 @@ def test_parallel_download_falls_back_when_head_fails(self): fallback_response = Mock() fallback_response.raise_for_status.return_value = None + fallback_response.headers = {} fallback_response.iter_content.return_value = [b"abc"] fallback_response.__enter__ = Mock(return_value=fallback_response) fallback_response.__exit__ = Mock(return_value=None) @@ -115,6 +117,7 @@ def test_parallel_download_falls_back_without_accept_ranges(self): fallback_response = Mock() fallback_response.raise_for_status.return_value = None + fallback_response.headers = {} fallback_response.iter_content.return_value = [b"abc"] fallback_response.__enter__ = Mock(return_value=fallback_response) fallback_response.__exit__ = Mock(return_value=None) From 0595f5a248a0921a7370f553b767087081d69d5a Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 20:49:40 +0100 Subject: [PATCH 40/54] fix(ci): make listing robust to API outages; deterministic raw-file tests The dev->master build (#106) failed because test_raw_files hits the live PRIDE API and a CI read timeout made Util.read_json_stream return None, which then crashed with "TypeError: 'NoneType' object is not iterable" in base.Provider.get_category_files. - base.py: add _list_files_checked() and use it in get_raw_files, get_category_files, find_file, and download_by_filenames so an unreachable API raises a clear RuntimeError instead of an opaque TypeError. - test_raw_files.py: these are live integration tests; skip them when the PRIDE API is unreachable so a transient timeout no longer fails the build. - flake8 hygiene (informational warnings): drop unused imports (registry in test_download_resilience; keep __init__ re-export with noqa), unused `e` in api_handling, `== True` -> truthy in pride.py, and wrap two over-long docstring lines in pridepy.py. --- pridepy/__init__.py | 2 +- pridepy/download/base.py | 24 +++++++++++++++++++---- pridepy/download/pride.py | 2 +- pridepy/pridepy.py | 6 ++++-- pridepy/tests/test_download_resilience.py | 1 - pridepy/tests/test_raw_files.py | 22 +++++++++++++++++++++ pridepy/util/api_handling.py | 2 +- 7 files changed, 49 insertions(+), 10 deletions(-) diff --git a/pridepy/__init__.py b/pridepy/__init__.py index ca4e345..9fee806 100644 --- a/pridepy/__init__.py +++ b/pridepy/__init__.py @@ -1 +1 @@ -from .pridepy import main +from .pridepy import main # noqa: F401 (re-exported for the `pridepy` console script) diff --git a/pridepy/download/base.py b/pridepy/download/base.py index 2635102..27d27d6 100644 --- a/pridepy/download/base.py +++ b/pridepy/download/base.py @@ -68,9 +68,25 @@ def get_download_url(self, record: Dict, protocol: str = "ftp") -> str: # Shared listing filters. # ------------------------------------------------------------------ + def _list_files_checked(self, accession: str) -> List[Dict]: + """Call :meth:`list_files` and fail clearly if it yields no listing. + + The PRIDE API helper returns ``None`` on a network error (e.g. a read + timeout), so guard here to raise an actionable error rather than a + cryptic ``TypeError: 'NoneType' object is not iterable`` downstream. + """ + records = self.list_files(accession) + if records is None: + raise RuntimeError( + f"Could not list files for {accession}: the repository API " + f"returned no data (it may be unreachable, or the accession " + f"may be invalid)." + ) + return records + def get_raw_files(self, accession: str) -> List[Dict]: """Return records whose ``fileCategory.value`` is ``"RAW"``.""" - records = self.list_files(accession) + records = self._list_files_checked(accession) return [r for r in records if r["fileCategory"]["value"] == "RAW"] def get_category_files( @@ -80,12 +96,12 @@ def get_category_files( if isinstance(categories, str): categories = [categories] category_set = {c.upper() for c in categories} - records = self.list_files(accession) + records = self._list_files_checked(accession) return [r for r in records if r["fileCategory"]["value"] in category_set] def find_file(self, accession: str, file_name: str) -> List[Dict]: """Return records whose ``fileName`` equals ``file_name``.""" - records = self.list_files(accession) + records = self._list_files_checked(accession) return [r for r in records if r["fileName"] == file_name] # ------------------------------------------------------------------ @@ -183,7 +199,7 @@ def download_by_filenames( if not file_names: raise ValueError("file_names must contain at least one filename") - all_files = self.list_files(accession) + all_files = self._list_files_checked(accession) requested = set(file_names) matched = [f for f in all_files if f.get("fileName") in requested] missing = sorted(requested - {f.get("fileName") for f in matched}) diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index a6a493f..16237e7 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -624,7 +624,7 @@ def download_files_from_s3( s3_path = download_url.replace(ftp_base_url, "") new_file_path = PrideProvider.get_output_file_name(download_url, file, output_folder) - if skip_if_downloaded_already == True and os.path.exists(new_file_path): + if skip_if_downloaded_already and os.path.exists(new_file_path): logging.info("Skipping download as file already exists") continue diff --git a/pridepy/pridepy.py b/pridepy/pridepy.py index 98acfd6..5fee488 100644 --- a/pridepy/pridepy.py +++ b/pridepy/pridepy.py @@ -258,10 +258,12 @@ def download_file_by_name( :param protocol: Protocol to use for download: ftp, aspera, globus, s3. Default is ftp. :param file_name: fileName to be downloaded :param output_folder: output folder to download or copy files - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. Default is False. + :param skip_if_downloaded_already: Boolean value to skip the download if the + file has already been downloaded. Default is False. :param username: PRIDE login username for private files :param password: PRIDE login password for private files - :param aspera_maximum_bandwidth: Aspera maximum bandwidth (e.g 50M, 100M, 200M), depending on the user's network bandwidth, default is 100M + :param aspera_maximum_bandwidth: Aspera maximum bandwidth (e.g 50M, 100M, + 200M), depending on the user's network bandwidth, default is 100M :param checksum_check: Download checksum file for project. """ diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index b533ae9..9969107 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -11,7 +11,6 @@ from pridepy.download.massive import MassiveProvider from pridepy.download.pride import PrideProvider from pridepy.download.proteomexchange import ProteomeXchangeProvider -from pridepy.download import registry class TestDownloadResilience(TestCase): diff --git a/pridepy/tests/test_raw_files.py b/pridepy/tests/test_raw_files.py index 3d1380b..3581d48 100644 --- a/pridepy/tests/test_raw_files.py +++ b/pridepy/tests/test_raw_files.py @@ -1,8 +1,30 @@ +import unittest from unittest import TestCase +import requests + from pridepy.download.client import Client as Files +_PRIDE_API_ROOT = "https://www.ebi.ac.uk/pride/ws/archive/v3/" + + +def _pride_api_reachable() -> bool: + """Return True if the live PRIDE API answers; False on a network error. + + These are integration tests that hit the real API. Skipping (rather than + failing) when the API is unreachable keeps CI deterministic instead of + flaking on a transient read timeout. + """ + try: + requests.get(_PRIDE_API_ROOT, timeout=15) + return True + except requests.RequestException: + return False + +@unittest.skipUnless( + _pride_api_reachable(), "PRIDE API not reachable (live integration test)" +) class TestRawFiles(TestCase): """ A test class to test files related methods. diff --git a/pridepy/util/api_handling.py b/pridepy/util/api_handling.py index cdf7ffc..e81622f 100644 --- a/pridepy/util/api_handling.py +++ b/pridepy/util/api_handling.py @@ -62,7 +62,7 @@ async def stream_response_to_file( pbar.update( 1 ) # Update progress bar by 1 for each detection - except PermissionError as e: + except PermissionError: print("[ERROR] No permissions to write to:", out_file) sys.exit(1) From f087d958ba99c68c4d7bf93021610aa82ddc6691 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Thu, 28 May 2026 21:04:11 +0100 Subject: [PATCH 41/54] test: make live PRIDE-API tests skip (not fail) on API outage The import-time reachability probe wasn't enough: it hit the fast API root while the tests call slower data endpoints, so the build still flaked when a data call timed out (RuntimeError "returned no data"). - Add pridepy/tests/_live_api.tolerate_api_outage: a context manager that skipTest()s on the signals an outage produces (requests.RequestException from get_api_call, the listing guard's RuntimeError, and len(None)/iterate TypeError from helpers that return None). AssertionError is not caught, so real regressions still fail. - Wrap the live calls in test_raw_files and test_search with it (replaces the import-time probe). - pride.get_submitted_file_path_prefix now routes through _list_files_checked so an outage raises the catchable RuntimeError instead of a TypeError. Verified: with the API up all six live tests run and pass; with the API helpers forced to fail, all six skip cleanly (no failures). --- pridepy/download/pride.py | 2 +- pridepy/tests/_live_api.py | 34 +++++++++++++++ pridepy/tests/test_raw_files.py | 59 ++++++++++--------------- pridepy/tests/test_search.py | 77 ++++++++++++++++++--------------- 4 files changed, 99 insertions(+), 73 deletions(-) create mode 100644 pridepy/tests/_live_api.py diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index 16237e7..d0feb2e 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -112,7 +112,7 @@ def get_submitted_file_path_prefix(self, accession): :param accession: PRIDE accession :return: path fragment (eg: 2018/10/PXD008644) """ - records = self.list_files(accession) + records = self._list_files_checked(accession) raw_files = [r for r in records if r["fileCategory"]["value"] == "RAW"] first_file = raw_files[0]["publicFileLocations"][0]["value"] path_fragment = re.search(r"\d{4}/\d{2}/PXD\d*", first_file).group() diff --git a/pridepy/tests/_live_api.py b/pridepy/tests/_live_api.py new file mode 100644 index 0000000..e04240f --- /dev/null +++ b/pridepy/tests/_live_api.py @@ -0,0 +1,34 @@ +"""Helpers for the live PRIDE-API integration tests. + +A handful of tests hit ``www.ebi.ac.uk`` directly (no mocking) to validate +real behaviour. That endpoint is occasionally slow or unreachable from CI +runners, which used to fail the build on a transient read timeout. The +:func:`tolerate_api_outage` context manager turns an API outage into a clean +skip instead of a failure, keeping CI deterministic while still exercising the +real API whenever it is available. +""" +import unittest +from contextlib import contextmanager + +import requests + + +@contextmanager +def tolerate_api_outage(testcase: unittest.TestCase): + """Skip (don't fail) the wrapped block when the live PRIDE API is down. + + Wrap only the live API call(s) and their assertions. An API outage surfaces + as one of: + * ``requests.RequestException`` — ``Util.get_api_call`` lets connection + / read timeouts propagate; + * ``RuntimeError`` — ``Provider._list_files_checked`` raises when the API + helper returned ``None``; + * ``TypeError`` — a helper that returns ``None`` on failure is then + iterated / measured (e.g. ``len(None)``). + Genuine assertion failures raise ``AssertionError``, which is *not* caught, + so real regressions still fail the test. + """ + try: + yield + except (requests.RequestException, RuntimeError, TypeError) as exc: + testcase.skipTest(f"PRIDE API unavailable: {type(exc).__name__}: {exc}") diff --git a/pridepy/tests/test_raw_files.py b/pridepy/tests/test_raw_files.py index 3581d48..f969871 100644 --- a/pridepy/tests/test_raw_files.py +++ b/pridepy/tests/test_raw_files.py @@ -1,33 +1,16 @@ -import unittest from unittest import TestCase -import requests - from pridepy.download.client import Client as Files +from pridepy.tests._live_api import tolerate_api_outage -_PRIDE_API_ROOT = "https://www.ebi.ac.uk/pride/ws/archive/v3/" - - -def _pride_api_reachable() -> bool: - """Return True if the live PRIDE API answers; False on a network error. - - These are integration tests that hit the real API. Skipping (rather than - failing) when the API is unreachable keeps CI deterministic instead of - flaking on a transient read timeout. - """ - try: - requests.get(_PRIDE_API_ROOT, timeout=15) - return True - except requests.RequestException: - return False - -@unittest.skipUnless( - _pride_api_reachable(), "PRIDE API not reachable (live integration test)" -) class TestRawFiles(TestCase): """ A test class to test files related methods. + + These hit the live PRIDE API; each call is wrapped in + :func:`tolerate_api_outage` so a transient API outage skips rather than + fails the build. """ def test_get_all_raw_file_list(self): @@ -35,10 +18,10 @@ def test_get_all_raw_file_list(self): A test method to check if it is possible to fetch the list of raw files """ raw = Files() - - # This project has only two files - result = raw.get_all_raw_file_list("PXD008644") - assert len(result) == 2 + with tolerate_api_outage(self): + # This project has only two files + result = raw.get_all_raw_file_list("PXD008644") + assert len(result) == 2 def test_get_raw_file_path_prefix(self): """ @@ -49,16 +32,17 @@ def test_get_raw_file_path_prefix(self): I.e. ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2018/10/PXD008644/7550GI_Y.raw """ raw = Files() - assert raw.get_submitted_file_path_prefix("PXD008644") == "2018/10/PXD008644" + with tolerate_api_outage(self): + assert raw.get_submitted_file_path_prefix("PXD008644") == "2018/10/PXD008644" def test_get_all_category_file_list(self): - raw = Files() - result = raw.get_all_category_file_list("PXD008644", "RAW") - assert len(result) == 2 + with tolerate_api_outage(self): + result = raw.get_all_category_file_list("PXD008644", "RAW") + assert len(result) == 2 - result = raw.get_all_category_file_list("PXD008644", "SEARCH") - assert len(result) == 2 + result = raw.get_all_category_file_list("PXD008644", "SEARCH") + assert len(result) == 2 def test_get_all_category_file_list_multiple(self): """ @@ -66,9 +50,10 @@ def test_get_all_category_file_list_multiple(self): PXD008644 has 2 RAW + 2 SEARCH = 4 files combined. """ raw = Files() - result = raw.get_all_category_file_list("PXD008644", ["RAW", "SEARCH"]) - assert len(result) == 4 + with tolerate_api_outage(self): + result = raw.get_all_category_file_list("PXD008644", ["RAW", "SEARCH"]) + assert len(result) == 4 - # Verify both categories are present - categories = {file["fileCategory"]["value"] for file in result} - assert categories == {"RAW", "SEARCH"} + # Verify both categories are present + categories = {file["fileCategory"]["value"] for file in result} + assert categories == {"RAW", "SEARCH"} diff --git a/pridepy/tests/test_search.py b/pridepy/tests/test_search.py index f21ac87..2d1e6e3 100644 --- a/pridepy/tests/test_search.py +++ b/pridepy/tests/test_search.py @@ -2,6 +2,7 @@ from pridepy.download.client import Client as Files from pridepy.project.project import Project +from pridepy.tests._live_api import tolerate_api_outage from pridepy.util.api_handling import Util import logging @@ -9,6 +10,10 @@ class TestSearch(TestCase): """ A test class to test files related methods. + + These hit the live PRIDE API; calls are wrapped in + :func:`tolerate_api_outage` so a transient API outage skips rather than + fails the build. """ def test_search_projects(self): @@ -17,46 +22,48 @@ def test_search_projects(self): """ project = Project() - result = project.search_by_keywords_and_filters( - keyword="PXD009476", - query_filter="", - page_size=100, - page=0, - sort_direction="DESC", - sort_fields="accession", - ) - assert len(result) > 0 # Search should return at least one result - assert any( - r["accession"] == "PXD009476" for r in result - ) # Search should return the queried project - - result = project.get_projects(77, 0, "ASC", "submission_date") - assert len(result) == 77 - - result = project.get_by_accession("PXD009476") - assert result["accession"] == "PXD009476" - - assert ( - len( - project.get_files_by_accession( - "PXD009476", + with tolerate_api_outage(self): + result = project.search_by_keywords_and_filters( + keyword="PXD009476", + query_filter="", + page_size=100, + page=0, + sort_direction="DESC", + sort_fields="accession", + ) + assert len(result) > 0 # Search should return at least one result + assert any( + r["accession"] == "PXD009476" for r in result + ) # Search should return the queried project + + result = project.get_projects(77, 0, "ASC", "submission_date") + assert len(result) == 77 + + result = project.get_by_accession("PXD009476") + assert result["accession"] == "PXD009476" + + assert ( + len( + project.get_files_by_accession( + "PXD009476", + ) ) + == 100 ) - == 100 - ) def test_status_dataset(self): files = Files() accession = "PXD044389" - project_status = Util.get_api_call(files.API_BASE_URL + "/status/{}".format(accession)) - public_project = False - if project_status.status_code == 200: - if project_status.text == "PRIVATE": - public_project = True - elif project_status.text == "PUBLIC": - public_project = False - else: - raise Exception("Dataset {} is not present in PRIDE Archive".format(accession)) - logging.debug(f"Public project: {public_project}") + with tolerate_api_outage(self): + project_status = Util.get_api_call(files.API_BASE_URL + "/status/{}".format(accession)) + public_project = False + if project_status.status_code == 200: + if project_status.text == "PRIVATE": + public_project = True + elif project_status.text == "PUBLIC": + public_project = False + else: + raise Exception("Dataset {} is not present in PRIDE Archive".format(accession)) + logging.debug(f"Public project: {public_project}") From 32029937b3436b7f7ed2e38b3cb7c7d2c71a5fda Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 07:53:46 +0100 Subject: [PATCH 42/54] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .codacy/cli.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.codacy/cli.sh b/.codacy/cli.sh index 7057e3b..4dba1f3 100755 --- a/.codacy/cli.sh +++ b/.codacy/cli.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash -set -e +o pipefail +set -e -o pipefail + +fatal() { + echo "$*" >&2 + exit 1 +} # Set up paths first -bin_name="codacy-cli-v2" # Determine OS-specific paths os_name=$(uname) From dff420f6e22f07b711c0d8c22e54ec6651de9e03 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 07:54:22 +0100 Subject: [PATCH 43/54] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .codacy/cli.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.codacy/cli.sh b/.codacy/cli.sh index 4dba1f3..97efd26 100755 --- a/.codacy/cli.sh +++ b/.codacy/cli.sh @@ -149,5 +149,4 @@ fi if [ "$#" -eq 1 ] && [ "$1" = "download" ]; then echo "Codacy cli v2 download succeeded" else - eval "$run_command $*" -fi \ No newline at end of file +"$run_command" "$@" \ No newline at end of file From 9609456af85aff6a9c4588e1e49edfbc40055aa7 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 09:27:30 +0100 Subject: [PATCH 44/54] fix: address Copilot PR review (remove codacy bootstrap; iProX docstrings; Files migration note) - Remove .codacy/cli.sh (resolves missing fatal() and unsafe eval findings) - Correct iProX test docstring/comments: transport is HTTP, not HTTPS - Document Files -> Client removal as a breaking change in README --- .codacy/cli.sh | 152 ------------------------------ README.md | 7 ++ pridepy/tests/test_iprox_files.py | 10 +- 3 files changed, 12 insertions(+), 157 deletions(-) delete mode 100755 .codacy/cli.sh diff --git a/.codacy/cli.sh b/.codacy/cli.sh deleted file mode 100755 index 97efd26..0000000 --- a/.codacy/cli.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env bash - - -set -e -o pipefail - -fatal() { - echo "$*" >&2 - exit 1 -} - -# Set up paths first - -# Determine OS-specific paths -os_name=$(uname) -arch=$(uname -m) - -case "$arch" in -"x86_64") - arch="amd64" - ;; -"x86") - arch="386" - ;; -"aarch64"|"arm64") - arch="arm64" - ;; -esac - -if [ -z "$CODACY_CLI_V2_TMP_FOLDER" ]; then - if [ "$(uname)" = "Linux" ]; then - CODACY_CLI_V2_TMP_FOLDER="$HOME/.cache/codacy/codacy-cli-v2" - elif [ "$(uname)" = "Darwin" ]; then - CODACY_CLI_V2_TMP_FOLDER="$HOME/Library/Caches/Codacy/codacy-cli-v2" - else - CODACY_CLI_V2_TMP_FOLDER=".codacy-cli-v2" - fi -fi - -version_file="$CODACY_CLI_V2_TMP_FOLDER/version.yaml" - - -get_version_from_yaml() { - if [ -f "$version_file" ]; then - local version=$(grep -o 'version: *"[^"]*"' "$version_file" | cut -d'"' -f2) - if [ -n "$version" ]; then - echo "$version" - return 0 - fi - fi - return 1 -} - -get_latest_version() { - local response - if [ -n "$GH_TOKEN" ]; then - response=$(curl -Lq --header "Authorization: Bearer $GH_TOKEN" "https://api.github.com/repos/codacy/codacy-cli-v2/releases/latest" 2>/dev/null) - else - response=$(curl -Lq "https://api.github.com/repos/codacy/codacy-cli-v2/releases/latest" 2>/dev/null) - fi - - handle_rate_limit "$response" - local version=$(echo "$response" | grep -m 1 tag_name | cut -d'"' -f4) - echo "$version" -} - -handle_rate_limit() { - local response="$1" - if echo "$response" | grep -q "API rate limit exceeded"; then - fatal "Error: GitHub API rate limit exceeded. Please try again later" - fi -} - -download_file() { - local url="$1" - - echo "Downloading from URL: ${url}" - if command -v curl > /dev/null 2>&1; then - curl -# -LS "$url" -O - elif command -v wget > /dev/null 2>&1; then - wget "$url" - else - fatal "Error: Could not find curl or wget, please install one." - fi -} - -download() { - local url="$1" - local output_folder="$2" - - ( cd "$output_folder" && download_file "$url" ) -} - -download_cli() { - # OS name lower case - suffix=$(echo "$os_name" | tr '[:upper:]' '[:lower:]') - - local bin_folder="$1" - local bin_path="$2" - local version="$3" - - if [ ! -f "$bin_path" ]; then - echo "📥 Downloading CLI version $version..." - - remote_file="codacy-cli-v2_${version}_${suffix}_${arch}.tar.gz" - url="https://github.com/codacy/codacy-cli-v2/releases/download/${version}/${remote_file}" - - download "$url" "$bin_folder" - tar xzfv "${bin_folder}/${remote_file}" -C "${bin_folder}" - fi -} - -# Warn if CODACY_CLI_V2_VERSION is set and update is requested -if [ -n "$CODACY_CLI_V2_VERSION" ] && [ "$1" = "update" ]; then - echo "⚠️ Warning: Performing update with forced version $CODACY_CLI_V2_VERSION" - echo " Unset CODACY_CLI_V2_VERSION to use the latest version" -fi - -# Ensure version.yaml exists and is up to date -if [ ! -f "$version_file" ] || [ "$1" = "update" ]; then - echo "ℹ️ Fetching latest version..." - version=$(get_latest_version) - mkdir -p "$CODACY_CLI_V2_TMP_FOLDER" - echo "version: \"$version\"" > "$version_file" -fi - -# Set the version to use -if [ -n "$CODACY_CLI_V2_VERSION" ]; then - version="$CODACY_CLI_V2_VERSION" -else - version=$(get_version_from_yaml) -fi - - -# Set up version-specific paths -bin_folder="${CODACY_CLI_V2_TMP_FOLDER}/${version}" - -mkdir -p "$bin_folder" -bin_path="$bin_folder"/"$bin_name" - -# Download the tool if not already installed -download_cli "$bin_folder" "$bin_path" "$version" -chmod +x "$bin_path" - -run_command="$bin_path" -if [ -z "$run_command" ]; then - fatal "Codacy cli v2 binary could not be found." -fi - -if [ "$#" -eq 1 ] && [ "$1" = "download" ]; then - echo "Codacy cli v2 download succeeded" -else -"$run_command" "$@" \ No newline at end of file diff --git a/README.md b/README.md index 660340e..3c6368c 100644 --- a/README.md +++ b/README.md @@ -354,6 +354,13 @@ pridepy download-all-public-category-files \ ## Python API Examples +> **Breaking change (0.0.16):** the legacy `pridepy.files.files.Files` class has been +> removed. Replace `from pridepy.files.files import Files` with +> `from pridepy.download.client import Client`; `Client` exposes the same public +> methods (`get_all_raw_file_list`, `download_all_raw_files`, +> `get_submitted_file_path_prefix`, `download_file_by_name`, +> `download_all_category_files`, `download_px_raw_files`, …). +
Get raw files for a project diff --git a/pridepy/tests/test_iprox_files.py b/pridepy/tests/test_iprox_files.py index 00d4997..3f9a487 100644 --- a/pridepy/tests/test_iprox_files.py +++ b/pridepy/tests/test_iprox_files.py @@ -1,11 +1,11 @@ """iProX direct-download support. iProX publishes the ProteomeXchange XML for each dataset at a deterministic -path on its anonymous HTTPS download server:: +path on its anonymous HTTP download server:: http://download.iprox.org//PX_.xml -The referenced files are served from the same host over HTTPS with byte-range +The referenced files are served from the same host over HTTP with byte-range support, so resume and parallel downloads use the same plumbing as PRIDE HTTP(S) transfers. """ @@ -73,7 +73,7 @@ def test_build_iprox_file_record_maps_px_cv_to_category(self): assert record["fileCategory"]["value"] == "RAW" assert record["source"] == "iProX" # _download_direct_download_records dispatches by URL scheme, so the - # publicFileLocations URL must still be the HTTPS download URL. + # publicFileLocations URL must still be the HTTP download URL. assert record["publicFileLocations"][0]["value"].startswith("http://") def test_list_iprox_public_files_parses_px_xml(self): @@ -92,7 +92,7 @@ def test_list_iprox_public_files_parses_px_xml(self): "http://download.iprox.org/IPX0017413000/PX_IPX0017413000.xml" ) - # 3 valid HTTPS records; the ftp:// "Other URI" cvParam was filtered out. + # 3 valid HTTP records; the ftp:// "Other URI" cvParam was filtered out. assert len(records) == 3 cats = {r["fileName"]: r["fileCategory"]["value"] for r in records} assert cats == { @@ -139,7 +139,7 @@ def test_download_file_by_name_routes_iprox_to_http_urls(self): checksum_check=False, ) - # iProX is HTTPS, not FTP — FTP path must not be called. + # iProX is HTTP, not FTP — FTP path must not be called. ftp_mock.assert_not_called() http_mock.assert_called_once() kwargs = http_mock.call_args.kwargs From 8b2e8f2bc669422f4e4521119779bf5c1e432d80 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 10:42:05 +0100 Subject: [PATCH 45/54] fix(massive): discover dataset version root instead of assuming /v01 MassIVE spreads datasets across versioned roots (/v01../vNN), so a fixed /v01 prefix made FTPS listing fail with "550 CD issue" for datasets stored elsewhere (e.g. MSV000088302 under /v04), forcing the HTTPS fallback. - Add transport._resolve_and_walk_ftp_dataset: probe top-level roots for / over one connection, then walk the match. - Prefer versioned (vNN) roots over auxiliary x01/z01 roots, which can hold only a partial peak-only copy (MSV000088302: 234 files under v04 vs 36 peak-only under z01). - Make _get_public_ftp_url / _build_file_record version-agnostic so download URLs and relative paths preserve the discovered root. - Tests for non-v01 listing, version-root preference, and URL building. --- pridepy/download/massive.py | 46 ++++++++++----- pridepy/download/transport.py | 68 ++++++++++++++++++++++ pridepy/tests/test_massive_files.py | 90 ++++++++++++++++++++++++++++- 3 files changed, 187 insertions(+), 17 deletions(-) diff --git a/pridepy/download/massive.py b/pridepy/download/massive.py index b7b7659..56d36de 100644 --- a/pridepy/download/massive.py +++ b/pridepy/download/massive.py @@ -1,7 +1,11 @@ """MassIVE direct-download provider. Primary path: list files by walking the FTPS tree at massive-ftp.ucsd.edu -(TLS is required by the server) and download them over FTPS. +(TLS is required by the server) and download them over FTPS. MassIVE spreads +datasets across versioned root directories (``/v01`` … ``/vNN``) plus auxiliary +roots (``x01`` / ``z01``) that may hold only a partial, peak-only copy; the +version is not derivable from the accession, so the correct root is discovered +at listing time and the versioned roots are preferred over the auxiliary ones. HTTPS fallback: some networks block FTP/FTPS entirely. When the FTPS listing fails, fall back to the HTTPS file index at datasetcache.gnps2.org @@ -59,17 +63,18 @@ def matches(accession: str) -> bool: return False return bool(re.fullmatch(r"R?MSV\d{9}", accession.upper())) - @staticmethod - def _get_public_root(accession: str) -> str: - return f"/v01/{accession.upper()}" - @classmethod def _get_public_ftp_url(cls, accession: str, remote_path: str) -> str: - root_path = cls._get_public_root(accession).rstrip("/") - relative_path = remote_path - if remote_path.startswith(root_path): - relative_path = remote_path[len(root_path):].lstrip("/") - return f"{cls.ARCHIVE_FTP_URL_PREFIX}{accession.upper()}/{relative_path}" + """Build the FTPS URL for an absolute server path inside the dataset. + + ``remote_path`` is the absolute path returned by the tree walk + (e.g. ``/v04/MSV000088302/ccms_peak/run.mzML``). MassIVE distributes + datasets across versioned roots, so the version is preserved as-is + rather than assumed to be ``v01``. + """ + if not remote_path.startswith("/"): + remote_path = "/" + remote_path + return f"ftp://{cls.ARCHIVE_FTP}{remote_path}" @staticmethod def _map_collection_to_category(collection: str) -> str: @@ -79,10 +84,14 @@ def _map_collection_to_category(collection: str) -> str: def _build_file_record(cls, accession: str, ftp_url: str) -> Dict: """Build a pridepy file record from an FTP URL inside the dataset.""" parsed = urlparse(ftp_url) - root_prefix = f"/v01/{accession.upper()}/" + # The version root differs per dataset (/v01../vNN), so derive the + # dataset-relative path from the accession marker rather than a fixed + # ``/v01//`` prefix. + marker = f"/{accession.upper()}/" relative_path = parsed.path - if relative_path.startswith(root_prefix): - relative_path = relative_path[len(root_prefix):] + marker_index = relative_path.find(marker) + if marker_index != -1: + relative_path = relative_path[marker_index + len(marker):] relative_path = relative_path.lstrip("/") collection = relative_path.split("/", 1)[0] if relative_path else "" return { @@ -159,13 +168,18 @@ def _list_via_https(self, accession: str) -> List[Dict]: def list_files(self, accession: str) -> List[Dict]: from pridepy.download import transport normalized = accession.upper() - remote_root = self._get_public_root(normalized) try: - remote_files = transport._list_ftp_repo_files( + # The version root (/v01../vNN) is not derivable from the + # accession, so discover which root holds the dataset instead of + # assuming /v01. + remote_files = transport._resolve_and_walk_ftp_dataset( host=self.ARCHIVE_FTP, - remote_root=remote_root, + accession=normalized, error_label=f"MassIVE dataset {normalized}", use_tls=True, + # /vNN roots hold the complete dataset; x01/z01 hold only + # partial (peak-only) copies, so prefer the versioned roots. + prefer_prefix="v", ) except Exception as ftps_error: logging.warning( diff --git a/pridepy/download/transport.py b/pridepy/download/transport.py index 97cafc5..7ff66d2 100644 --- a/pridepy/download/transport.py +++ b/pridepy/download/transport.py @@ -159,6 +159,74 @@ def _list_ftp_repo_files( pass +def _resolve_and_walk_ftp_dataset( + host: str, + accession: str, + error_label: str, + use_tls: bool = False, + prefer_prefix: str = "", +) -> List[str]: + """ + Find which top-level directory on ``host`` holds ``accession`` and walk it. + + Some repositories (e.g. MassIVE) distribute datasets across several + versioned root directories (``/v01`` … ``/vNN``, plus auxiliary roots like + ``x01`` / ``z01`` that may hold only a partial, derived copy) and the + version is not derivable from the accession. Probe each top-level directory + for ``/`` and walk the first match, reusing a single + connection for both discovery and listing. + + ``prefer_prefix`` lets the caller try the canonical roots first: roots + whose name starts with the prefix (e.g. ``"v"`` for MassIVE versioned + storage) are probed before any others, so a complete copy is chosen over + an auxiliary partial one when a dataset exists under both. + + :raises RuntimeError: on connection failure or when the accession is not + found under any top-level directory. + """ + ftp: Optional[FTP] = None + try: + ftp = _open_ftp_connection(host, use_tls=use_tls) + logging.info(f"Connected to FTP host: {host} (tls={use_tls})") + roots: List[str] = [] + ftp.retrlines("NLST /", roots.append) + # Servers may return bare names or absolute paths; keep the leaf name. + candidates = [] + for entry in roots: + name = entry.strip().strip("/").split("/")[-1] + if name and name not in {".", ".."}: + candidates.append(name) + if prefer_prefix: + prefix = prefer_prefix.lower() + candidates.sort( + key=lambda n: (not n.lower().startswith(prefix), n) + ) + for name in candidates: + dataset_root = f"/{name}/{accession}" + try: + ftp.cwd(dataset_root) + except ftplib.error_perm: + continue + logging.info(f"Found {accession} under {dataset_root} on {host}") + return _walk_ftp_tree(ftp, dataset_root) + raise RuntimeError( + f"{accession} not found under any top-level directory on {host}" + ) + except Exception as error: + raise RuntimeError( + f"Unable to list public files for {error_label}: {error}" + ) from error + finally: + if ftp is not None: + try: + ftp.quit() + except Exception: + try: + ftp.close() + except Exception: + pass + + def _download_one_ftp_path( ftp: FTP, ftp_path: str, diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index 5498314..fcdc767 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -1,3 +1,4 @@ +import ftplib import tempfile from unittest import TestCase from unittest.mock import patch @@ -188,6 +189,91 @@ def test_get_https_url_builds_proteosafe_endpoint(self): "&file=f.MSV000012345/raw/Raw%20spec/C%203.raw" ) + def test_build_massive_file_record_handles_non_v01_version_root(self): + """Datasets live under v01..vNN; records must preserve the real root.""" + record = MassiveProvider._build_file_record( + "MSV000088302", + "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/ccms_peak/run.mzML", + ) + + assert record["relativePath"] == "ccms_peak/run.mzML" + assert record["collection"] == "ccms_peak" + assert record["fileName"] == "run.mzML" + assert record["fileCategory"]["value"] == "PEAK" + assert ( + record["publicFileLocations"][0]["value"] + == "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/ccms_peak/run.mzML" + ) + + def test_get_public_ftp_url_preserves_version_root(self): + url = MassiveProvider._get_public_ftp_url( + "MSV000088302", "/v04/MSV000088302/ccms_peak/run.mzML" + ) + assert url == "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/ccms_peak/run.mzML" + + def test_list_files_discovers_version_root_when_not_v01(self): + """list_files probes top-level roots and walks the one holding the + dataset, so a dataset under /v04 is listed with v04 download URLs.""" + def fake_nlst(command, callback): + assert command == "NLST /" + for name in ["v01", "v02", "v03", "v04", "v05"]: + callback(name) + + def fake_cwd(path): + # Only the real root accepts the CWD; others 550. + if path != "/v04/MSV000088302": + raise ftplib.error_perm("550 CD issue: file does not exist") + + with patch.object(transport, "_open_ftp_connection") as open_conn, patch.object( + transport, + "_walk_ftp_tree", + return_value=["/v04/MSV000088302/ccms_peak/run.mzML"], + ) as walk_mock: + fake_ftp = open_conn.return_value + fake_ftp.retrlines.side_effect = fake_nlst + fake_ftp.cwd.side_effect = fake_cwd + + records = MassiveProvider().list_files("MSV000088302") + + walk_mock.assert_called_once_with(fake_ftp, "/v04/MSV000088302") + assert len(records) == 1 + assert records[0]["relativePath"] == "ccms_peak/run.mzML" + assert ( + records[0]["publicFileLocations"][0]["value"] + == "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/ccms_peak/run.mzML" + ) + + def test_list_files_prefers_versioned_root_over_auxiliary_root(self): + """x01/z01 can hold a partial (peak-only) copy while the full dataset + lives under a vNN root, so the versioned root must win even when both + exist.""" + def fake_nlst(command, callback): + for name in ["x01", "z01", "v04"]: + callback(name) + + existing = {"/z01/MSV000088302", "/v04/MSV000088302"} + + def fake_cwd(path): + if path not in existing: + raise ftplib.error_perm("550 CD issue: file does not exist") + + with patch.object(transport, "_open_ftp_connection") as open_conn, patch.object( + transport, + "_walk_ftp_tree", + return_value=["/v04/MSV000088302/raw/run.raw"], + ) as walk_mock: + fake_ftp = open_conn.return_value + fake_ftp.retrlines.side_effect = fake_nlst + fake_ftp.cwd.side_effect = fake_cwd + + records = MassiveProvider().list_files("MSV000088302") + + walk_mock.assert_called_once_with(fake_ftp, "/v04/MSV000088302") + assert ( + records[0]["publicFileLocations"][0]["value"] + == "ftp://massive-ftp.ucsd.edu/v04/MSV000088302/raw/run.raw" + ) + def test_build_https_file_record_sets_relpath_category_and_https_location(self): record = MassiveProvider._build_https_file_record( "MSV000012345", "raw/sub/run.raw" @@ -221,7 +307,9 @@ def iter_lines(self): yield line.encode("utf-8") with patch.object( - transport, "_list_ftp_repo_files", side_effect=RuntimeError("FTPS blocked") + transport, + "_resolve_and_walk_ftp_dataset", + side_effect=RuntimeError("FTPS blocked"), ), patch( "pridepy.download.massive.requests.get", return_value=_FakeCSVResponse() ): From 41c183e8250e6f48a379d28513f8455b9ff381d3 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 11:01:49 +0100 Subject: [PATCH 46/54] docs: design spec for flatten-by-default downloads --- .../2026-05-29-flatten-downloads-design.md | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 docs/specs/2026-05-29-flatten-downloads-design.md diff --git a/docs/specs/2026-05-29-flatten-downloads-design.md b/docs/specs/2026-05-29-flatten-downloads-design.md new file mode 100644 index 0000000..aaddc73 --- /dev/null +++ b/docs/specs/2026-05-29-flatten-downloads-design.md @@ -0,0 +1,149 @@ +# Flatten downloads into the output folder — design + +Date: 2026-05-29 +Status: Approved (pending spec review) + +## Problem + +When downloading from the direct-download repositories (MassIVE, JPOST, iProX), +pridepy recreates the dataset's subdirectory tree under the output folder, e.g.: + +``` +./downloads/MSV000088302/raw/.../sample.raw +./downloads/MSV000088302/ccms_peak/.../sample.mzML +``` + +Many users want the files dropped directly into the folder they pass with `-o`, +without the intermediate `raw/`, `ccms_peak/`, … directories. + +The subdirectory layout was introduced deliberately (PRs #98 / #100 / #105) to +stop identically-named files in different collections from overwriting each +other. Flattening reintroduces that collision risk, so it must be paired with a +collision-safe naming scheme. + +## Current behavior + +- Direct-download providers (MassIVE/JPOST/iProX) thread each file's + `relativePath` through `base.Provider.download_files` into + `transport.download_ftp_urls` / `download_http_urls`, where + `transport._dest_path` / `_safe_join` join it under the output folder, + preserving the tree. +- PRIDE downloads are **already flat**: `PrideProvider.get_output_file_name` + uses only the URL basename, so PRIDE files land directly in the output folder + today (with no collision handling — last write wins). + +## Decision + +Make **flatten the default** behavior for all repositories, with an opt-out to +preserve the directory tree. Flattening is made collision-safe with +deterministic auto-rename. + +### Behavior + +- **Default: flatten.** Each file is written directly into `output_folder` by + its basename. +- **Opt-out: `--preserve-structure`** keeps the dataset subdirectory tree + (the current behavior). +- **Collision handling (auto-rename).** When two or more files in the same + download set collapse to the same basename, the first (by sorted source + relative path) keeps the basename; subsequent ones get a numeric suffix + inserted before the final extension: `run.raw`, `run_1.raw`, `run_2.raw`. + - The mapping is computed over the **full file list** for the download, not + from what is already on disk, so it is deterministic across runs. This + keeps `--skip-if-downloaded-already` and FTP `REST` resume stable (the same + source file always maps to the same flat name). + - Extension splitting uses `os.path.splitext` (so `a.tar.gz` → `a.tar_1.gz`). + +## Architecture + +Approach A (chosen): thread a `flatten` flag into `base.Provider.download_files` +and, when set, replace each record's `relativePath` with a flat de-duplicated +name passed through the existing `relative_paths` plumbing. Because a relative +path with no slash already lands directly in `output_folder` via +`transport._safe_join`, **no change to the transport layer is required**. + +Rejected alternatives: +- **B.** Add `flatten` to `transport.download_ftp_urls` / `download_http_urls`. + Spreads naming policy across two transport signatures and mixes "how to + transfer" with "where to name"; no upside over A. +- **C.** Download with structure, then move files to a flat layout. Wastes + work, breaks resume, fragile. + +### Components + +1. **`flatten_relative_paths(relative_paths: List[str]) -> List[str]`** — new + pure helper (in `pridepy/download/util.py`). Maps a list of dataset-relative + paths to flat, de-duplicated basenames using the deterministic auto-rename + rule above. Order of the returned list matches the order of the input + (suffix assignment is decided by sorted source path, but the output aligns + positionally with the input so callers can zip it back to records). + +2. **`base.Provider.download_files(..., flatten: bool = True)`** — when + `flatten` is True, build `relative_paths` from + `flatten_relative_paths([r.get("relativePath") or basename(url) for r in records])`; + when False, use each record's `relativePath` as today. + +3. **Flag threading.** Add `flatten: bool = True` to the relevant `Client` + facade methods and the provider `download_by_*` methods so the value flows + CLI → `Client` → provider → `download_files`: + - `Client.download_all_raw_files` + - `Client.download_all_category_files` + - `Client.download_by_filenames` (download-files-by-list) + - `Client.download_px_raw_files` + +4. **CLI.** Add a `--preserve-structure` flag (Click `is_flag`, default False → + flatten on) to the multi-file download commands: + - `download-all-public-raw-files` + - `download-all-public-category-files` + - `download-files-by-list` + - `download-px-raw-files` + + `download-file-by-name` (single file) and `download-files-by-url` (already + basename-flat, no dataset structure) do not get the flag. + +### Data flow + +``` +CLI (--preserve-structure?) + -> Client.download_*(flatten = not preserve_structure) + -> provider.download_by_*(flatten=...) + -> base.Provider.download_files(flatten=...) + -> relative_paths = flat dedup names (flatten) | original relativePath (preserve) + -> transport.download_ftp_urls / download_http_urls (unchanged) +``` + +### PRIDE note + +PRIDE already writes flat via `get_output_file_name`, so the new default does +not change PRIDE behavior. `--preserve-structure` is effectively a no-op for +PRIDE (its records carry no meaningful subtree). Full parity — including +collision auto-rename for PRIDE — arrives when issue #107 routes PRIDE FTP +through the shared `transport` layer; until then PRIDE keeps its existing +flat, last-write-wins behavior. This is called out so the partial coverage is +explicit, not silent. + +## Error handling + +- Auto-rename never raises; it always produces a usable, unique name. +- `--preserve-structure` falls back to existing behavior, including + `transport._safe_join`'s defensive guard against `..`/absolute-path escape. + +## Testing + +- `flatten_relative_paths`: + - no collisions → basenames unchanged; + - collisions → deterministic `_1`, `_2` suffixes; + - ordering determinism (same input in any disk state → same output); + - multi-dot extensions (`a.tar.gz`), no-extension files, leading-slash paths. +- `base.Provider.download_files`: + - `flatten=True` passes flat de-duplicated `relative_paths`; + - `flatten=False` passes original `relativePath` values (existing tests). +- Existing MassIVE/JPOST/iProX structure-preservation tests re-run with + `flatten=False`. +- CLI: `--preserve-structure` maps to `flatten=False`; default maps to + `flatten=True` for each affected command. + +## Out of scope + +- Refactoring PRIDE FTP onto the shared transport (tracked in issue #107). +- Any change to `download-file-by-name` or `download-files-by-url`. From aca3228f1e0d167fb7dde1f98412ba5d408d2e79 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 11:18:57 +0100 Subject: [PATCH 47/54] feat(download): flatten into output folder by default, --preserve-structure to opt out Files now download directly into the output folder by basename instead of recreating the dataset's subdirectory tree (raw/.../*.raw). Colliding basenames are de-duplicated deterministically (run.raw, run_1.raw) so skip-if-downloaded and resume stay stable across runs. - Add util.flatten_relative_paths: pure, deterministic basename dedup. - base.Provider.download_files(flatten=True): compute flat dedup destination names over the whole set; flatten=False preserves relativePath layout. - Thread flatten through Client + provider download_by_* and the PX provider. - Add --preserve-structure CLI flag to download-all-public-raw-files, download-all-public-category-files, download-files-by-list, download-px-raw-files. - PRIDE accepts flatten for parity but already writes flat (see #107). --- pridepy/download/base.py | 49 ++++++++-- pridepy/download/client.py | 9 +- pridepy/download/pride.py | 7 ++ pridepy/download/proteomexchange.py | 2 + pridepy/download/util.py | 32 ++++++- pridepy/pridepy.py | 48 +++++++++- pridepy/tests/test_cli_flatten.py | 88 +++++++++++++++++ pridepy/tests/test_download_resilience.py | 13 ++- pridepy/tests/test_flatten_paths.py | 39 ++++++++ pridepy/tests/test_jpost_files.py | 2 +- pridepy/tests/test_massive_files.py | 112 +++++++++++++++++++++- 11 files changed, 381 insertions(+), 20 deletions(-) create mode 100644 pridepy/tests/test_cli_flatten.py create mode 100644 pridepy/tests/test_flatten_paths.py diff --git a/pridepy/download/base.py b/pridepy/download/base.py index 27d27d6..22915dc 100644 --- a/pridepy/download/base.py +++ b/pridepy/download/base.py @@ -13,8 +13,10 @@ import logging from abc import ABC, abstractmethod from typing import ClassVar, Dict, List, Optional +from urllib.parse import urlparse from pridepy.download import transport +from pridepy.download.util import flatten_relative_paths class Provider(ABC): @@ -117,6 +119,7 @@ def download_all_raw( aspera_maximum_bandwidth: str = "100M", checksum_check: bool = False, parallel_files: int = 1, + flatten: bool = True, ) -> None: """Download all RAW files for the dataset.""" self.download_files( @@ -128,6 +131,7 @@ def download_all_raw( parallel_files=parallel_files, checksum_check=checksum_check, aspera_maximum_bandwidth=aspera_maximum_bandwidth, + flatten=flatten, ) def download_category( @@ -140,6 +144,7 @@ def download_category( aspera_maximum_bandwidth: str = "100M", checksum_check: bool = False, parallel_files: int = 1, + flatten: bool = True, ) -> None: """Download all files of the given categories for the dataset.""" self.download_files( @@ -151,6 +156,7 @@ def download_category( parallel_files=parallel_files, checksum_check=checksum_check, aspera_maximum_bandwidth=aspera_maximum_bandwidth, + flatten=flatten, ) def download_by_name( @@ -191,6 +197,7 @@ def download_by_filenames( aspera_maximum_bandwidth: str = "100M", checksum_check: bool = False, parallel_files: int = 1, + flatten: bool = True, ) -> None: """Download a subset of project files identified by a filename list. @@ -219,6 +226,7 @@ def download_by_filenames( parallel_files=parallel_files, checksum_check=checksum_check, aspera_maximum_bandwidth=aspera_maximum_bandwidth, + flatten=flatten, ) # ------------------------------------------------------------------ @@ -237,12 +245,19 @@ def download_files( aspera_maximum_bandwidth: str = "100M", username: Optional[str] = None, password: Optional[str] = None, + flatten: bool = True, ) -> None: """Partition record URLs by scheme and route to the matching transport. ``ftp://`` URLs are handed to :func:`transport.download_ftp_urls` (with this provider's :attr:`use_tls`); ``http(s)://`` URLs go to :func:`transport.download_http_urls`. + + When ``flatten`` is True (the default) every file is written directly + into ``output_folder`` by its basename, de-duplicating colliding + basenames across the whole set (they share one folder). When False the + dataset's subdirectory layout is preserved via each record's + ``relativePath``. """ if protocol not in ("ftp", "https", "http"): logging.warning( @@ -250,26 +265,42 @@ def download_files( f"Ignoring requested protocol '{protocol}' for {accession}." ) - ftp_urls: List[str] = [] - ftp_relpaths: List[Optional[str]] = [] - http_urls: List[str] = [] - http_relpaths: List[Optional[str]] = [] + # Collect transfer entries in one pass, keeping order stable. + entries = [] # list of (scheme, url, relpath) for record in records: url = self.get_download_url(record) relpath = record.get("relativePath") lowered = url.lower() if lowered.startswith("ftp://"): - ftp_urls.append(url) - ftp_relpaths.append(relpath) + entries.append(("ftp", url, relpath)) elif lowered.startswith(("http://", "https://")): - http_urls.append(url) - http_relpaths.append(relpath) - if not ftp_urls and not http_urls: + entries.append(("http", url, relpath)) + if not entries: logging.info( f"No files matched for direct-download dataset {accession}" ) return + if flatten: + # All files share one output folder, so dedup basenames globally; + # fall back to the URL path when a record carries no relativePath. + sources = [rel if rel else urlparse(url).path for _, url, rel in entries] + dest_paths: List[Optional[str]] = flatten_relative_paths(sources) + else: + dest_paths = [rel for _, _, rel in entries] + + ftp_urls: List[str] = [] + ftp_relpaths: List[Optional[str]] = [] + http_urls: List[str] = [] + http_relpaths: List[Optional[str]] = [] + for (scheme, url, _), dest in zip(entries, dest_paths): + if scheme == "ftp": + ftp_urls.append(url) + ftp_relpaths.append(dest) + else: + http_urls.append(url) + http_relpaths.append(dest) + if ftp_urls: transport.download_ftp_urls( ftp_urls=ftp_urls, diff --git a/pridepy/download/client.py b/pridepy/download/client.py index 959dc4b..91716ea 100644 --- a/pridepy/download/client.py +++ b/pridepy/download/client.py @@ -186,6 +186,7 @@ def download_all_raw_files( aspera_maximum_bandwidth: str, checksum_check: bool = False, parallel_files: int = 1, + flatten: bool = True, ): """Download all RAW files for any registered provider.""" return registry.resolve(accession).download_all_raw( @@ -196,6 +197,7 @@ def download_all_raw_files( aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, parallel_files=parallel_files, + flatten=flatten, ) def download_all_category_files( @@ -209,6 +211,7 @@ def download_all_category_files( categories: List[str] = None, category: str = None, parallel_files: int = 1, + flatten: bool = True, ): """Download all files of the given categories from a project.""" if categories is None: @@ -222,6 +225,7 @@ def download_all_category_files( aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, parallel_files=parallel_files, + flatten=flatten, ) def download_file_by_name( @@ -264,6 +268,7 @@ def download_files_by_list( aspera_maximum_bandwidth: str = "100M", checksum_check: bool = False, parallel_files: int = 1, + flatten: bool = True, ) -> None: """Download a subset of project files identified by a filename list.""" return registry.resolve(accession).download_by_filenames( @@ -275,6 +280,7 @@ def download_files_by_list( aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, parallel_files=parallel_files, + flatten=flatten, ) @staticmethod @@ -301,8 +307,9 @@ def download_px_raw_files( px_id_or_url: str, output_folder: str, skip_if_downloaded_already: bool = True, + flatten: bool = True, ) -> None: """Delegate to :meth:`ProteomeXchangeProvider.download_from_accession_or_url`.""" return ProteomeXchangeProvider().download_from_accession_or_url( - px_id_or_url, output_folder, skip_if_downloaded_already + px_id_or_url, output_folder, skip_if_downloaded_already, flatten=flatten ) diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index d0feb2e..c3bbdd8 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -860,12 +860,19 @@ def download_files( aspera_maximum_bandwidth: str = "100M", username: Optional[str] = None, password: Optional[str] = None, + flatten: bool = True, ): """Override Provider.download_files with the multi-protocol orchestrator. Reuses the legacy batch downloader: Phase 1 batches the requested protocol over a single connection; Phase 2 validates every file and, for any that fail, falls back per-file across the remaining protocols. + + ``flatten`` is accepted for interface parity but is currently a no-op + for PRIDE: the legacy batch path already writes every file directly + into ``output_folder`` by basename. Structure-preserving downloads for + PRIDE arrive when this path is routed through the shared transport + layer (see issue #107). """ PrideProvider._download_files_batch( file_list_json=records, diff --git a/pridepy/download/proteomexchange.py b/pridepy/download/proteomexchange.py index dfe70c8..9be09b8 100644 --- a/pridepy/download/proteomexchange.py +++ b/pridepy/download/proteomexchange.py @@ -169,6 +169,7 @@ def download_from_accession_or_url( px_id_or_url: str, output_folder: str, skip_if_downloaded_already: bool = True, + flatten: bool = True, ) -> None: """End-to-end: resolve XML, list files, partition by scheme, download. @@ -187,4 +188,5 @@ def download_from_accession_or_url( output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, protocol="ftp", + flatten=flatten, ) diff --git a/pridepy/download/util.py b/pridepy/download/util.py index 6f2f35f..11a201d 100644 --- a/pridepy/download/util.py +++ b/pridepy/download/util.py @@ -9,11 +9,41 @@ import hashlib import logging import os -from typing import Dict, Optional, Tuple +from typing import Dict, List, Optional, Tuple from tqdm import tqdm +def flatten_relative_paths(relative_paths: List[str]) -> List[str]: + """Map dataset-relative paths to flat, de-duplicated basenames. + + Used when downloading into a single output folder without recreating the + dataset's subdirectory tree. Files keep their basename; when two or more + source paths collapse to the same basename, the first one (by sorted source + path) keeps the bare name and later ones get a numeric suffix inserted + before the final extension (``run.raw`` -> ``run_1.raw``). + + Suffixing is decided by the *sorted* source paths so the mapping is + deterministic across runs (independent of disk state or input order), + keeping skip-if-downloaded and resume stable. The returned list is + positionally aligned with ``relative_paths`` so callers can zip it back to + their records. + """ + assigned: Dict[str, str] = {} + seen_basenames: Dict[str, int] = {} + # Decide names in sorted-source order so the result is deterministic. + for source in sorted(relative_paths): + basename = os.path.basename((source or "").lstrip("/")) + count = seen_basenames.get(basename, 0) + if count == 0: + assigned[source] = basename + else: + stem, ext = os.path.splitext(basename) + assigned[source] = f"{stem}_{count}{ext}" + seen_basenames[basename] = count + 1 + return [assigned[source] for source in relative_paths] + + class Progress: def __init__(self, total_size, file_name): self.pbar = tqdm( diff --git a/pridepy/pridepy.py b/pridepy/pridepy.py index 5fee488..e307a61 100644 --- a/pridepy/pridepy.py +++ b/pridepy/pridepy.py @@ -58,6 +58,13 @@ def main(): type=click.IntRange(1, 3), help="Number of files to download simultaneously (1-3). Primarily used by globus protocol. Default is 1.", ) +@click.option( + "--preserve-structure", + is_flag=True, + default=False, + help="Recreate the dataset's subdirectory layout under the output folder. " + "By default files are downloaded flat into the output folder.", +) def download_all_public_raw_files( accession, protocol, @@ -66,6 +73,7 @@ def download_all_public_raw_files( aspera_maximum_bandwidth: str = "50M", checksum_check: bool = False, parallel_files: int = 1, + preserve_structure: bool = False, ): """ Command to download all public raw files from a specified PRIDE or MassIVE dataset. @@ -95,6 +103,7 @@ def download_all_public_raw_files( aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, parallel_files=parallel_files, + flatten=not preserve_structure, ) @@ -149,6 +158,13 @@ def download_all_public_raw_files( type=click.IntRange(1, 3), help="Number of files to download simultaneously (1-3). Primarily used by globus protocol. Default is 1.", ) +@click.option( + "--preserve-structure", + is_flag=True, + default=False, + help="Recreate the dataset's subdirectory layout under the output folder. " + "By default files are downloaded flat into the output folder.", +) def download_all_public_category_files( accession: str, protocol: str, @@ -158,6 +174,7 @@ def download_all_public_category_files( checksum_check: bool = False, category: str = "RAW", parallel_files: int = 1, + preserve_structure: bool = False, ): """ Command to download all public files of a specified category from a given PRIDE or MassIVE dataset. @@ -198,6 +215,7 @@ def download_all_public_category_files( checksum_check=checksum_check, categories=categories, parallel_files=parallel_files, + flatten=not preserve_structure, ) @@ -311,11 +329,28 @@ def download_file_by_name( default=False, help="Skip the download if the file has already been downloaded.", ) -def download_px_raw_files(accession: str, output_folder: str, skip_if_downloaded_already: bool): +@click.option( + "--preserve-structure", + is_flag=True, + default=False, + help="Recreate the dataset's subdirectory layout under the output folder. " + "By default files are downloaded flat into the output folder.", +) +def download_px_raw_files( + accession: str, + output_folder: str, + skip_if_downloaded_already: bool, + preserve_structure: bool = False, +): """CLI wrapper to download raw files via ProteomeXchange XML.""" files = Files() logging.info(f"PX accession/URL: {accession}") - files.download_px_raw_files(accession, output_folder, skip_if_downloaded_already) + files.download_px_raw_files( + accession, + output_folder, + skip_if_downloaded_already, + flatten=not preserve_structure, + ) @main.command("list-private-files", help="List private files by project accession") @@ -563,6 +598,13 @@ def _read_url_arguments(url_list_path, urls_csv=None): type=click.IntRange(1, 3), help="Number of files to download simultaneously (1-3). Primarily used by globus protocol. Default is 1.", ) +@click.option( + "--preserve-structure", + is_flag=True, + default=False, + help="Recreate the dataset's subdirectory layout under the output folder. " + "By default files are downloaded flat into the output folder.", +) def download_files_by_list( accession, protocol, @@ -573,6 +615,7 @@ def download_files_by_list( aspera_maximum_bandwidth, checksum_check, parallel_files, + preserve_structure: bool = False, ): """Download a named subset of files from a PRIDE project.""" file_names = _read_filename_arguments(file_list_path, files_csv) @@ -588,6 +631,7 @@ def download_files_by_list( aspera_maximum_bandwidth=aspera_maximum_bandwidth, checksum_check=checksum_check, parallel_files=parallel_files, + flatten=not preserve_structure, ) diff --git a/pridepy/tests/test_cli_flatten.py b/pridepy/tests/test_cli_flatten.py new file mode 100644 index 0000000..b068a40 --- /dev/null +++ b/pridepy/tests/test_cli_flatten.py @@ -0,0 +1,88 @@ +"""CLI wiring for the --preserve-structure flag. + +By default the download commands flatten into the output folder (flatten=True); +--preserve-structure flips that to flatten=False. +""" +from unittest import TestCase +from unittest.mock import patch + +from click.testing import CliRunner + +from pridepy.pridepy import main + + +class TestCliPreserveStructure(TestCase): + def _invoke(self, args): + return CliRunner().invoke(main, args, catch_exceptions=False) + + def test_download_all_public_raw_files_flattens_by_default(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + ["download-all-public-raw-files", "-a", "MSV000012345", "-o", "/tmp/x"] + ) + kwargs = files_cls.return_value.download_all_raw_files.call_args.kwargs + assert kwargs["flatten"] is True + + def test_download_all_public_raw_files_preserve_structure(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-all-public-raw-files", + "-a", + "MSV000012345", + "-o", + "/tmp/x", + "--preserve-structure", + ] + ) + kwargs = files_cls.return_value.download_all_raw_files.call_args.kwargs + assert kwargs["flatten"] is False + + def test_download_all_public_category_files_preserve_structure(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-all-public-category-files", + "-a", + "MSV000012345", + "-o", + "/tmp/x", + "-c", + "RAW", + "--preserve-structure", + ] + ) + kwargs = files_cls.return_value.download_all_category_files.call_args.kwargs + assert kwargs["flatten"] is False + + def test_download_files_by_list_preserve_structure(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-files-by-list", + "-a", + "MSV000012345", + "-o", + "/tmp/x", + "-f", + "a.raw", + "--preserve-structure", + ] + ) + kwargs = files_cls.return_value.download_files_by_list.call_args.kwargs + assert kwargs["flatten"] is False + + def test_download_px_raw_files_preserve_structure(self): + with patch("pridepy.pridepy.Files") as files_cls: + self._invoke( + [ + "download-px-raw-files", + "-a", + "PXD000001", + "-o", + "/tmp/x", + "--preserve-structure", + ] + ) + kwargs = files_cls.return_value.download_px_raw_files.call_args.kwargs + assert kwargs["flatten"] is False diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 9969107..6b25a5c 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -173,10 +173,10 @@ def test_safe_join_preserves_subdirs_and_blocks_escape(self): out, "passwd" ) - def test_download_files_threads_relative_paths_avoiding_collisions(self): - """Same-basename files in different collections must not flatten/collide: - base.Provider.download_files threads each record's relativePath through - to the transport layer.""" + def test_download_files_preserves_relative_paths_when_flatten_false(self): + """With flatten=False, base.Provider.download_files threads each + record's relativePath through to the transport layer so same-basename + files in different collections keep their subdirectory layout.""" provider = MassiveProvider() records = [ MassiveProvider._build_file_record( @@ -196,12 +196,14 @@ def test_download_files_threads_relative_paths_avoiding_collisions(self): skip_if_downloaded_already=False, protocol="ftp", parallel_files=1, + flatten=False, ) kwargs = ftp_mock.call_args.kwargs assert kwargs["relative_paths"] == ["raw/a/run.raw", "raw/b/run.raw"] def test_download_files_threads_relative_paths_for_http(self): - """The HTTP partition also forwards relativePath to download_http_urls.""" + """With flatten=False, the HTTP partition also forwards relativePath to + download_http_urls.""" class _HttpProvider(MassiveProvider): pass @@ -226,6 +228,7 @@ class _HttpProvider(MassiveProvider): skip_if_downloaded_already=False, protocol="ftp", parallel_files=1, + flatten=False, ) assert http_mock.call_args.kwargs["relative_paths"] == ["raw/d1/run.raw"] diff --git a/pridepy/tests/test_flatten_paths.py b/pridepy/tests/test_flatten_paths.py new file mode 100644 index 0000000..a92ba34 --- /dev/null +++ b/pridepy/tests/test_flatten_paths.py @@ -0,0 +1,39 @@ +"""Tests for flatten_relative_paths: collapsing dataset-relative paths to flat, +de-duplicated basenames for download into a single output folder.""" +from unittest import TestCase + +from pridepy.download.util import flatten_relative_paths + + +class TestFlattenRelativePaths(TestCase): + def test_distinct_basenames_are_kept_unchanged(self): + assert flatten_relative_paths( + ["raw/a.raw", "ccms_peak/b.mzML", "search/c.mzid"] + ) == ["a.raw", "b.mzML", "c.mzid"] + + def test_colliding_basenames_get_numeric_suffixes(self): + # raw/a/run.raw and raw/b/run.raw both collapse to run.raw. + result = flatten_relative_paths(["raw/a/run.raw", "raw/b/run.raw"]) + assert result == ["run.raw", "run_1.raw"] + + def test_suffix_assignment_is_deterministic_by_sorted_source_path(self): + # First by sorted source path keeps the bare name regardless of input + # order, so re-runs (any order) map a given source to the same name. + forward = flatten_relative_paths(["raw/b/run.raw", "raw/a/run.raw"]) + # Input order preserved in output; "raw/a/run.raw" (sorts first) -> run.raw + assert forward == ["run_1.raw", "run.raw"] + + def test_output_is_positionally_aligned_with_input(self): + names = flatten_relative_paths(["x/dup.txt", "y/uniq.txt", "z/dup.txt"]) + assert names == ["dup.txt", "uniq.txt", "dup_1.txt"] + + def test_multi_dot_extension_suffix_before_last_extension(self): + result = flatten_relative_paths(["a/data.tar.gz", "b/data.tar.gz"]) + assert result == ["data.tar.gz", "data.tar_1.gz"] + + def test_files_without_extension_get_plain_suffix(self): + result = flatten_relative_paths(["a/README", "b/README"]) + assert result == ["README", "README_1"] + + def test_leading_slash_is_stripped(self): + assert flatten_relative_paths(["/raw/a.raw"]) == ["a.raw"] diff --git a/pridepy/tests/test_jpost_files.py b/pridepy/tests/test_jpost_files.py index 4227907..e9546c0 100644 --- a/pridepy/tests/test_jpost_files.py +++ b/pridepy/tests/test_jpost_files.py @@ -87,7 +87,7 @@ def test_download_file_by_name_uses_jpost_ftp_listing(self): skip_if_downloaded_already=False, use_tls=False, parallel_files=1, - relative_paths=["raw/folder/sample.raw"], + relative_paths=["sample.raw"], ) def test_proxi_listing_maps_cv_name_to_category(self): diff --git a/pridepy/tests/test_massive_files.py b/pridepy/tests/test_massive_files.py index fcdc767..5ccc3d7 100644 --- a/pridepy/tests/test_massive_files.py +++ b/pridepy/tests/test_massive_files.py @@ -104,7 +104,7 @@ def test_download_file_by_name_uses_massive_ftp_listing(self): skip_if_downloaded_already=False, use_tls=True, parallel_files=1, - relative_paths=["raw/folder/sample.raw"], + relative_paths=["sample.raw"], ) def test_repo_uses_tls_true_for_massive_false_for_jpost(self): @@ -179,6 +179,116 @@ def test_base_direct_download_provider_partitions_urls_by_scheme(self): http_mock.assert_called_once() assert http_mock.call_args.kwargs["http_urls"] == ["http://example.org/b.raw"] + def test_download_files_flattens_into_output_folder_by_default(self): + """By default, files land directly in the output folder (no tree), and + colliding basenames are de-duplicated.""" + provider = MassiveProvider() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(transport, "download_ftp_urls") as ftp_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + ) + + assert ftp_mock.call_args.kwargs["relative_paths"] == ["run.raw", "run_1.raw"] + + def test_download_files_preserves_structure_when_flatten_false(self): + provider = MassiveProvider() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(transport, "download_ftp_urls") as ftp_mock: + provider.download_files( + accession="MSV000012345", + records=records, + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + parallel_files=1, + flatten=False, + ) + + assert ftp_mock.call_args.kwargs["relative_paths"] == [ + "raw/a/run.raw", + "raw/b/run.raw", + ] + + def test_client_download_all_raw_files_flattens_by_default(self): + files = Files() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(MassiveProvider, "list_files", return_value=records), patch.object( + transport, "download_ftp_urls" + ) as ftp_mock: + files.download_all_raw_files( + accession="MSV000012345", + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + aspera_maximum_bandwidth="100M", + checksum_check=False, + parallel_files=1, + ) + assert ftp_mock.call_args.kwargs["relative_paths"] == ["run.raw", "run_1.raw"] + + def test_client_download_all_raw_files_preserve_structure(self): + files = Files() + records = [ + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a/run.raw", + ), + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b/run.raw", + ), + ] + with patch.object(MassiveProvider, "list_files", return_value=records), patch.object( + transport, "download_ftp_urls" + ) as ftp_mock: + files.download_all_raw_files( + accession="MSV000012345", + output_folder="/tmp/test", + skip_if_downloaded_already=False, + protocol="ftp", + aspera_maximum_bandwidth="100M", + checksum_check=False, + parallel_files=1, + flatten=False, + ) + assert ftp_mock.call_args.kwargs["relative_paths"] == [ + "raw/a/run.raw", + "raw/b/run.raw", + ] + def test_get_https_url_builds_proteosafe_endpoint(self): url = MassiveProvider._get_https_url( "MSV000012345", "raw/Raw spec/C 3.raw" From e58a34ce0174fdd0bcded869d5226f0204fd091c Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 11:19:10 +0100 Subject: [PATCH 48/54] docs: document GitHub-branch install and flat-by-default downloads (--preserve-structure) --- README.md | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c6368c..fc30089 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,36 @@ pip install --upgrade pridepy pridepy --help ``` -### Option 3: Install from source (development) +### Option 3: Install the latest code directly from GitHub + +To get features that have not been released to PyPI yet, install straight from a +branch. `master` holds the latest stable code; `dev` holds the newest (and +potentially unstable) development work. + +With `uv`: + +```bash +# Latest stable (master) +uv tool install "git+https://github.com/PRIDE-Archive/pridepy@master" + +# Bleeding edge (dev) +uv tool install "git+https://github.com/PRIDE-Archive/pridepy@dev" +``` + +Or with `pip`: + +```bash +# Latest stable (master) +pip install --upgrade "git+https://github.com/PRIDE-Archive/pridepy@master" + +# Bleeding edge (dev) +pip install --upgrade "git+https://github.com/PRIDE-Archive/pridepy@dev" +``` + +You can pin to any branch, tag, or commit by changing the part after `@` (e.g. +`@v0.0.16` or `@`). + +### Option 4: Install from source (development) ```bash git clone https://github.com/PRIDE-Archive/pridepy @@ -100,6 +129,12 @@ These options are shared by `download-all-public-raw-files`, | `--skip-if-downloaded-already` | Resume: skip files already present locally | off | | `--checksum-check` | Download PRIDE checksums and validate each file | off | | `--aspera-maximum-bandwidth` | Aspera cap, e.g. `50M`, `100M`, `200M` (Aspera only) | `100M` | +| `--preserve-structure` | Recreate the dataset's subdirectory layout (e.g. `raw/…/`) under the output folder instead of downloading flat | off | + +By default, files are downloaded **flat** into the output folder (no +`raw/…/` subdirectories). When two files would collapse to the same name, +later ones get a numeric suffix (`run.raw`, `run_1.raw`). Pass +`--preserve-structure` to keep the dataset's original directory layout.
From d73de3e259bde1db5491cced16ca31ce2fc6ae80 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 11:30:50 +0100 Subject: [PATCH 49/54] fix(pride): route FTP batch downloads through shared transport (closes #107) The legacy PrideProvider.download_files_from_ftp used one shared connection with no resume: a single large-file read timeout (hardcoded 30s) left the control socket broken, then every remaining file in the batch failed instantly with "cannot read from timed out object", and partial files restarted from byte 0 on retry. Route the FTP batch through transport.download_ftp_urls instead, which: - downloads each file with REST-based resume (no restart-from-zero), - reconnects per file so one timeout no longer cascades to the rest, - verifies post-transfer size and retries truncated files. Files still land flat by basename, matching _resolve_local_path used by the Phase-2 validation/fallback. Remove the legacy download_files_from_ftp and its now-unused ftplib/socket/FTP imports. --- pridepy/download/pride.py | 155 +++------------------- pridepy/tests/test_download_resilience.py | 46 +++++++ 2 files changed, 67 insertions(+), 134 deletions(-) diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index c3bbdd8..cafaa48 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -13,19 +13,16 @@ ``Client`` facade. Tests patch the canonical locations (``PrideProvider.X``, ``transport.X``, ``util.X``) directly. """ -import ftplib import importlib.resources import logging import os import platform import re -import socket import subprocess import time import urllib import urllib.request from concurrent.futures import ThreadPoolExecutor, as_completed -from ftplib import FTP from typing import ClassVar, Dict, List, Optional from urllib.parse import urlparse @@ -327,126 +324,6 @@ def _globus_download_one(file, output_folder, skip_if_downloaded_already, max_re # Per-protocol batch helpers # ------------------------------------------------------------------ - @staticmethod - def download_files_from_ftp( - file_list_json, - output_folder, - skip_if_downloaded_already, - max_connection_retries=3, - max_download_retries=3, - ): - """ - Download files using a single FTP connection with a retry mechanism and a progress bar for each file. - :param file_list_json: file list in JSON format - :param output_folder: folder to download the files - :param skip_if_downloaded_already: Boolean value to skip the download if the file has already been downloaded. - :param max_connection_retries: Number of attempts to reconnect to the FTP server if the connection is lost. - :param max_download_retries: Number of attempts to retry the download of a file in case of failure. - """ - if not os.path.isdir(output_folder): - os.makedirs(output_folder) - - def connect_ftp(): - """Helper function to establish FTP connection.""" - ftp = FTP(PrideProvider.ARCHIVE_FTP, timeout=30) - ftp.login() # Anonymous login - ftp.set_pasv(True) # Enable passive mode - logging.info(f"Connected to FTP host: {PrideProvider.ARCHIVE_FTP}") - return ftp - - connection_attempt = 0 - while connection_attempt < max_connection_retries: - try: - ftp = connect_ftp() - for file in file_list_json: - try: - # Get FTP download URL - if file["publicFileLocations"][0]["name"] == "FTP Protocol": - download_url = file["publicFileLocations"][0]["value"] - else: - download_url = file["publicFileLocations"][1]["value"] - - logging.debug("ftp_filepath:" + download_url) - - # Get output file path - new_file_path = PrideProvider.get_output_file_name( - download_url, file, output_folder - ) - - if skip_if_downloaded_already and os.path.exists(new_file_path): - logging.info("Skipping download as file already exists") - continue - - # Extract file path from the download URL - parsed_url = urlparse(download_url) - ftp_file_path = urllib.parse.unquote(parsed_url.path.lstrip("/")) - - logging.info(f"Starting FTP download: {ftp_file_path}") - - # Retry download in case of failure - download_attempt = 0 - while download_attempt < max_download_retries: - try: - # Get file size for progress tracking - total_size = ftp.size(ftp_file_path) - logging.info(f"File size: {total_size} bytes") - - # Initialize progress bar - with open(new_file_path, "wb") as f: - with tqdm( - total=total_size, - unit="B", - unit_scale=True, - desc=new_file_path, - ) as pbar: - - def callback(data): - f.write(data) - pbar.update(len(data)) - - # Retrieve the file with progress callback - ftp.retrbinary(f"RETR {ftp_file_path}", callback) - - logging.info(f"Successfully downloaded {new_file_path}") - break # Exit download retry loop if successful - except ( - socket.timeout, - ftplib.error_temp, - ftplib.error_perm, - ) as e: - download_attempt += 1 - logging.error( - f"Download failed for {new_file_path} (attempt {download_attempt}): {str(e)}" - ) - if download_attempt >= max_download_retries: - logging.error( - f"Giving up on {new_file_path} after {max_download_retries} attempts." - ) - break # Give up on this file after max retries - except (KeyError, IndexError) as e: - logging.error(f"Failed to process file due to missing data: {str(e)}") - except Exception as e: - logging.error(f"Unexpected error while processing file: {str(e)}") - ftp.quit() # Close FTP connection after all files are downloaded - logging.info(f"Disconnected from FTP host: {PrideProvider.ARCHIVE_FTP}") - break # Exit connection retry loop if everything was successful - except ( - socket.timeout, - ftplib.error_temp, - ftplib.error_perm, - socket.error, - ) as e: - connection_attempt += 1 - logging.error(f"FTP connection failed (attempt {connection_attempt}): {str(e)}") - if connection_attempt < max_connection_retries: - logging.info("Retrying connection...") - time.sleep(5) # Optional delay before retrying - else: - logging.error( - f"Giving up after {max_connection_retries} failed connection attempts." - ) - break - @staticmethod def download_files_from_aspera( file_list_json: List[Dict], @@ -759,10 +636,21 @@ def _batch_download_by_protocol( if not file_list: return if protocol == "ftp": - PrideProvider.download_files_from_ftp( - file_list, - output_folder, + # Route through the shared transport, which downloads each file on + # its own connection with REST-based resume, post-transfer size + # checks, and per-file reconnect — so one slow/timed-out large file + # no longer poisons the connection and cascade-fails the rest of the + # batch (issue #107). Files land flat by basename (PRIDE archive is + # flat within a dataset), matching ``_resolve_local_path``. + ftp_urls = [ + PrideProvider._get_download_url(record, "ftp") for record in file_list + ] + transport.download_ftp_urls( + ftp_urls=ftp_urls, + output_folder=output_folder, skip_if_downloaded_already=skip_if_downloaded_already, + use_tls=False, + parallel_files=parallel_files, ) return if protocol == "aspera": @@ -864,15 +752,14 @@ def download_files( ): """Override Provider.download_files with the multi-protocol orchestrator. - Reuses the legacy batch downloader: Phase 1 batches the requested - protocol over a single connection; Phase 2 validates every file and, - for any that fail, falls back per-file across the remaining protocols. + Phase 1 batches the requested protocol (FTP routes through the shared + transport with per-file reconnect + REST resume); Phase 2 validates + every file and falls back per-file across the remaining protocols. - ``flatten`` is accepted for interface parity but is currently a no-op - for PRIDE: the legacy batch path already writes every file directly - into ``output_folder`` by basename. Structure-preserving downloads for - PRIDE arrive when this path is routed through the shared transport - layer (see issue #107). + ``flatten`` is accepted for interface parity but is a no-op for PRIDE: + a dataset's files live flat in its archive directory (no sub-tree to + preserve), so they always land directly in ``output_folder`` by + basename, which is what ``_resolve_local_path`` expects for Phase 2. """ PrideProvider._download_files_batch( file_list_json=records, diff --git a/pridepy/tests/test_download_resilience.py b/pridepy/tests/test_download_resilience.py index 6b25a5c..532f624 100644 --- a/pridepy/tests/test_download_resilience.py +++ b/pridepy/tests/test_download_resilience.py @@ -400,6 +400,52 @@ def test_protocol_sequence_prefers_requested_then_fallback(self): assert PrideProvider._protocol_sequence("ftp") == ["ftp", "aspera", "s3", "globus"] assert PrideProvider._protocol_sequence("aspera") == ["aspera", "s3", "ftp", "globus"] + def test_pride_ftp_batch_routes_through_shared_transport(self): + """PRIDE FTP batch downloads must use transport.download_ftp_urls + (per-file reconnect + REST resume + size checks) instead of the legacy + single-connection loop that cascades on one timeout (issue #107).""" + records = [ + { + "fileName": "a.raw", + "accession": "PXD000001", + "publicFileLocations": [ + { + "name": "FTP Protocol", + "value": "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2024/05/PXD000001/a.raw", + } + ], + }, + { + "fileName": "b.raw", + "accession": "PXD000001", + "publicFileLocations": [ + { + "name": "FTP Protocol", + "value": "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2024/05/PXD000001/b.raw", + } + ], + }, + ] + with patch.object(transport, "download_ftp_urls") as ftp_mock: + PrideProvider._batch_download_by_protocol( + records, + "/tmp/out", + "ftp", + skip_if_downloaded_already=False, + aspera_maximum_bandwidth="100M", + parallel_files=2, + ) + + ftp_mock.assert_called_once() + kwargs = ftp_mock.call_args.kwargs + assert kwargs["ftp_urls"] == [ + "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2024/05/PXD000001/a.raw", + "ftp://ftp.pride.ebi.ac.uk/pride/data/archive/2024/05/PXD000001/b.raw", + ] + assert kwargs["use_tls"] is False + assert kwargs["parallel_files"] == 2 + assert kwargs["skip_if_downloaded_already"] is False + def test_download_with_fallback_switches_protocol_after_invalid_file(self): file_record = { "fileName": "sample.raw", From de05b06b930a10a4d45e427de659d32b233d45df Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 11:50:33 +0100 Subject: [PATCH 50/54] chore: remove internal design doc from repo --- .../2026-05-29-flatten-downloads-design.md | 149 ------------------ 1 file changed, 149 deletions(-) delete mode 100644 docs/specs/2026-05-29-flatten-downloads-design.md diff --git a/docs/specs/2026-05-29-flatten-downloads-design.md b/docs/specs/2026-05-29-flatten-downloads-design.md deleted file mode 100644 index aaddc73..0000000 --- a/docs/specs/2026-05-29-flatten-downloads-design.md +++ /dev/null @@ -1,149 +0,0 @@ -# Flatten downloads into the output folder — design - -Date: 2026-05-29 -Status: Approved (pending spec review) - -## Problem - -When downloading from the direct-download repositories (MassIVE, JPOST, iProX), -pridepy recreates the dataset's subdirectory tree under the output folder, e.g.: - -``` -./downloads/MSV000088302/raw/.../sample.raw -./downloads/MSV000088302/ccms_peak/.../sample.mzML -``` - -Many users want the files dropped directly into the folder they pass with `-o`, -without the intermediate `raw/`, `ccms_peak/`, … directories. - -The subdirectory layout was introduced deliberately (PRs #98 / #100 / #105) to -stop identically-named files in different collections from overwriting each -other. Flattening reintroduces that collision risk, so it must be paired with a -collision-safe naming scheme. - -## Current behavior - -- Direct-download providers (MassIVE/JPOST/iProX) thread each file's - `relativePath` through `base.Provider.download_files` into - `transport.download_ftp_urls` / `download_http_urls`, where - `transport._dest_path` / `_safe_join` join it under the output folder, - preserving the tree. -- PRIDE downloads are **already flat**: `PrideProvider.get_output_file_name` - uses only the URL basename, so PRIDE files land directly in the output folder - today (with no collision handling — last write wins). - -## Decision - -Make **flatten the default** behavior for all repositories, with an opt-out to -preserve the directory tree. Flattening is made collision-safe with -deterministic auto-rename. - -### Behavior - -- **Default: flatten.** Each file is written directly into `output_folder` by - its basename. -- **Opt-out: `--preserve-structure`** keeps the dataset subdirectory tree - (the current behavior). -- **Collision handling (auto-rename).** When two or more files in the same - download set collapse to the same basename, the first (by sorted source - relative path) keeps the basename; subsequent ones get a numeric suffix - inserted before the final extension: `run.raw`, `run_1.raw`, `run_2.raw`. - - The mapping is computed over the **full file list** for the download, not - from what is already on disk, so it is deterministic across runs. This - keeps `--skip-if-downloaded-already` and FTP `REST` resume stable (the same - source file always maps to the same flat name). - - Extension splitting uses `os.path.splitext` (so `a.tar.gz` → `a.tar_1.gz`). - -## Architecture - -Approach A (chosen): thread a `flatten` flag into `base.Provider.download_files` -and, when set, replace each record's `relativePath` with a flat de-duplicated -name passed through the existing `relative_paths` plumbing. Because a relative -path with no slash already lands directly in `output_folder` via -`transport._safe_join`, **no change to the transport layer is required**. - -Rejected alternatives: -- **B.** Add `flatten` to `transport.download_ftp_urls` / `download_http_urls`. - Spreads naming policy across two transport signatures and mixes "how to - transfer" with "where to name"; no upside over A. -- **C.** Download with structure, then move files to a flat layout. Wastes - work, breaks resume, fragile. - -### Components - -1. **`flatten_relative_paths(relative_paths: List[str]) -> List[str]`** — new - pure helper (in `pridepy/download/util.py`). Maps a list of dataset-relative - paths to flat, de-duplicated basenames using the deterministic auto-rename - rule above. Order of the returned list matches the order of the input - (suffix assignment is decided by sorted source path, but the output aligns - positionally with the input so callers can zip it back to records). - -2. **`base.Provider.download_files(..., flatten: bool = True)`** — when - `flatten` is True, build `relative_paths` from - `flatten_relative_paths([r.get("relativePath") or basename(url) for r in records])`; - when False, use each record's `relativePath` as today. - -3. **Flag threading.** Add `flatten: bool = True` to the relevant `Client` - facade methods and the provider `download_by_*` methods so the value flows - CLI → `Client` → provider → `download_files`: - - `Client.download_all_raw_files` - - `Client.download_all_category_files` - - `Client.download_by_filenames` (download-files-by-list) - - `Client.download_px_raw_files` - -4. **CLI.** Add a `--preserve-structure` flag (Click `is_flag`, default False → - flatten on) to the multi-file download commands: - - `download-all-public-raw-files` - - `download-all-public-category-files` - - `download-files-by-list` - - `download-px-raw-files` - - `download-file-by-name` (single file) and `download-files-by-url` (already - basename-flat, no dataset structure) do not get the flag. - -### Data flow - -``` -CLI (--preserve-structure?) - -> Client.download_*(flatten = not preserve_structure) - -> provider.download_by_*(flatten=...) - -> base.Provider.download_files(flatten=...) - -> relative_paths = flat dedup names (flatten) | original relativePath (preserve) - -> transport.download_ftp_urls / download_http_urls (unchanged) -``` - -### PRIDE note - -PRIDE already writes flat via `get_output_file_name`, so the new default does -not change PRIDE behavior. `--preserve-structure` is effectively a no-op for -PRIDE (its records carry no meaningful subtree). Full parity — including -collision auto-rename for PRIDE — arrives when issue #107 routes PRIDE FTP -through the shared `transport` layer; until then PRIDE keeps its existing -flat, last-write-wins behavior. This is called out so the partial coverage is -explicit, not silent. - -## Error handling - -- Auto-rename never raises; it always produces a usable, unique name. -- `--preserve-structure` falls back to existing behavior, including - `transport._safe_join`'s defensive guard against `..`/absolute-path escape. - -## Testing - -- `flatten_relative_paths`: - - no collisions → basenames unchanged; - - collisions → deterministic `_1`, `_2` suffixes; - - ordering determinism (same input in any disk state → same output); - - multi-dot extensions (`a.tar.gz`), no-extension files, leading-slash paths. -- `base.Provider.download_files`: - - `flatten=True` passes flat de-duplicated `relative_paths`; - - `flatten=False` passes original `relativePath` values (existing tests). -- Existing MassIVE/JPOST/iProX structure-preservation tests re-run with - `flatten=False`. -- CLI: `--preserve-structure` maps to `flatten=False`; default maps to - `flatten=True` for each affected command. - -## Out of scope - -- Refactoring PRIDE FTP onto the shared transport (tracked in issue #107). -- Any change to `download-file-by-name` or `download-files-by-url`. From 3a3e3e804e617a3fd7e65840220e83932b7270dc Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 12:12:16 +0100 Subject: [PATCH 51/54] docs: add docs/usage.md guide; slim README to install + overview --- README.md | 347 ++----------------------------------------------- docs/usage.md | 350 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 364 insertions(+), 333 deletions(-) create mode 100644 docs/usage.md diff --git a/README.md b/README.md index fc30089..423db45 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,12 @@ uv sync --extra dev uv run pridepy --help ``` -## Command Overview +## Usage + +See the **[usage guide](docs/usage.md)** for detailed instructions and examples: +downloading data (PRIDE, MassIVE, JPOST, iProX, ProteomeXchange), category and +manifest downloads, private files, streaming metadata, searching projects, and +the Python API. ```bash pridepy --help @@ -102,344 +107,20 @@ pridepy --help | `stream-projects-metadata` | Stream all project metadata to JSON | | `search-projects-by-keywords-and-filters` | Search projects by keyword and filters | -The download commands work for PRIDE accessions and, transparently, for native -MassIVE (`MSV…`), JPOST (`JPST…`), and iProX (`IPX…`) accessions — see -[Download from ProteomeXchange and other repositories](#download-from-proteomexchange-and-other-repositories). - -## PRIDE File Downloads - -PRIDE downloads start with FTP and fall back across the remaining protocols -(`ftp -> aspera -> s3 -> globus`) when a transfer fails. They support resume, -per-file retries, parallel workers, and optional checksum validation. Empty or -corrupt files are retried automatically. - -
-Common download options (shared across the download commands) - -These options are shared by `download-all-public-raw-files`, -`download-all-public-category-files`, `download-file-by-name`, and -`download-files-by-list`: - -| Option | Description | Default | -| --- | --- | --- | -| `-a, --accession` | Dataset accession (e.g. `PXD008644`) | required | -| `-o, --output-folder` | Destination directory | required | -| `-p, --protocol` | Transfer protocol: `ftp`, `aspera`, `globus`, `s3` (FTP-first with fallback) | `ftp` | -| `-w, --parallel-files` | Download 1–3 files concurrently — primarily for `globus`; not available on `download-file-by-name` | `1` | -| `--skip-if-downloaded-already` | Resume: skip files already present locally | off | -| `--checksum-check` | Download PRIDE checksums and validate each file | off | -| `--aspera-maximum-bandwidth` | Aspera cap, e.g. `50M`, `100M`, `200M` (Aspera only) | `100M` | -| `--preserve-structure` | Recreate the dataset's subdirectory layout (e.g. `raw/…/`) under the output folder instead of downloading flat | off | - -By default, files are downloaded **flat** into the output folder (no -`raw/…/` subdirectories). When two files would collapse to the same name, -later ones get a numeric suffix (`run.raw`, `run_1.raw`). Pass -`--preserve-structure` to keep the dataset's original directory layout. - -
- -
-Download all raw files (robust mode) - -```bash -pridepy download-all-public-raw-files \ - -a PXD008644 \ - -o ./downloads/PXD008644 \ - --checksum-check -``` - -Continue an interrupted download safely by adding `--skip-if-downloaded-already`: - -```bash -pridepy download-all-public-raw-files \ - -a PXD008644 \ - -o ./downloads/PXD008644 \ - --skip-if-downloaded-already \ - --checksum-check -``` - -
- -
-Download only selected categories - -```bash -pridepy download-all-public-category-files \ - -a PXD022105 \ - -o ./downloads/PXD022105 \ - -c RAW,SEARCH -``` - -`-c, --category` takes one or more comma-separated categories. Valid values: -`RAW`, `PEAK`, `SEARCH`, `RESULT`, `SPECTRUM_LIBRARY`, `OTHER`, `FASTA`. - -
- -
-Download one file by name - -```bash -pridepy download-file-by-name \ - -a PXD022105 \ - -f checksum.txt \ - -o ./downloads/PXD022105 \ - --checksum-check -``` - -`-f, --file-name` is the file to download. - -
- -
-Download a named subset of files (manifest) - -```bash -pridepy download-files-by-list \ - -a PXD001819 \ - -F files.txt \ - -o ./downloads/PXD001819 \ - --checksum-check -``` - -`files.txt` is one filename per line (blank lines and `#` comments are -ignored). Each filename is resolved against the project metadata and downloaded -via the same batch + protocol-fallback engine as `download-all-public-raw-files`. -Use `-f a.raw,b.raw,c.raw` instead of `-F` for a small inline list (you can -combine both). - -
- -
-Download files from raw URLs - -```bash -pridepy download-files-by-url \ - -F urls.txt \ - -o ./downloads/urls -``` - -`urls.txt` is one fully-qualified URL per line. Schemes `http`, `https`, and -`ftp` are dispatched to the matching downloader. Use `-u, --urls` for one or -more comma-separated URLs, e.g. `--urls https://a.com/x.raw,ftp://b.com/y.raw` -(URLs containing literal commas must use a manifest file instead). - -Command-specific options: - -| Option | Description | Default | -| --- | --- | --- | -| `-F, --url-list` | Manifest file, one URL per line | — | -| `-u, --urls` | Comma-separated URL(s) | — | -| `-p, --protocol` | `ftp` (per-scheme) or `globus` (resume-capable http/https) | `ftp` | -| `-w, --parallel-files` | Download 1–3 files concurrently (any scheme) | `1` | -| `--checksum-check` | Validate against PRIDE checksums (accession inferred from PRIDE URL paths; only PRIDE archive URLs supported) | off | - -
- -
-Private (restricted) files - -List the files of a private project with your PRIDE credentials: - -```bash -pridepy list-private-files -a PXD022105 -u YOUR_USER -p YOUR_PASSWORD -``` - -Download a private file by passing `--username`/`--password` to -`download-file-by-name`: - -```bash -pridepy download-file-by-name \ - -a PXD022105 \ - -f checksum.txt \ - -o ./downloads/private \ - --username YOUR_USER \ - --password YOUR_PASSWORD -``` - -
- -## Metadata and Search - -
-Stream all project metadata to JSON - -```bash -pridepy stream-projects-metadata -o all_pride_projects.json -``` - -| Option | Description | Default | -| --- | --- | --- | -| `-o, --output-file` | JSON file to write all project metadata to | required | - -
- -
-Stream file metadata - -```bash -# All file metadata for one accession -pridepy stream-files-metadata -a PXD005011 -o PXD005011_files.json -``` - -| Option | Description | Default | -| --- | --- | --- | -| `-o, --output-file` | JSON file to write file metadata to | required | -| `-a, --accession` | Limit to one project (omit to stream all files) | optional | - -
- -
-Search projects by keywords and filters - -```bash -pridepy search-projects-by-keywords-and-filters \ - -k human \ - -f projectTags==ProteomeTools,organismsPart==Pancreas \ - -sd DESC \ - -sf accession \ - -sf submissionDate -``` - -| Option | Description | Default | -| --- | --- | --- | -| `-k, --keyword` | Keyword searched across project fields | required | -| `-f, --filters` | `field==value` filters, comma-separated (e.g. `accession==PRD000001`) | — | -| `-ps, --page-size` | Results per page (1–1000) | `100` | -| `-p, --page` | Page number (0-based) | `0` | -| `-sd, --sort-direction` | `ASC` or `DESC` | `DESC` | -| `-sf, --sort-fields` | Sort field(s), repeatable. One of: `accession`, `submissionDate`, `diseases`, `organismsPart`, `organisms`, `instruments`, `softwares`, `avgDownloadsPerFile`, `downloadCount`, `publicationDate` | `submissionDate` | - -
- -## Download from ProteomeXchange and other repositories - -A ProteomeXchange (`PXD…` / `PRD…`) accession is a cross-repository identifier: -the dataset may be hosted at PRIDE, MassIVE, JPOST, iProX, or elsewhere. -`pridepy` lets you start from the ProteomeXchange accession, or go straight to -the hosting repository using its **native** accession. - -
-Start from a ProteomeXchange accession - -`download-px-raw-files` resolves the dataset's ProteomeXchange XML and downloads -the RAW files it references, regardless of which repository hosts them: - -```bash -pridepy download-px-raw-files \ - -a PXD039236 \ - -o ./downloads/PXD039236 -``` - -| Option | Description | Default | -| --- | --- | --- | -| `-a, --accession` | ProteomeXchange accession (e.g. `PXD039236`). `--px` is a deprecated alias | required | -| `-o, --output-folder` | Destination directory | required | -| `--skip-if-downloaded-already` | Skip files already present locally | off | - -
- -
-Go directly to the hosting repository (native MassIVE / JPOST / iProX accessions) - -Datasets that do not have a ProteomeXchange accession — or where you already -know the native accession — can be downloaded directly. The standard download -commands accept MassIVE, JPOST, and iProX accessions transparently: +Quick examples: ```bash -# MassIVE (FTPS at massive-ftp.ucsd.edu) -pridepy download-all-public-raw-files \ - -a MSV000082297 \ - -o ./downloads/MSV000082297 - -# JPOST (PROXI listing + ftp.jpostdb.org) -pridepy download-all-public-raw-files \ - -a JPST002311 \ - -o ./downloads/JPST002311 - -# iProX (ProteomeXchange XML + anonymous HTTP at download.iprox.org) -pridepy download-all-public-raw-files \ - -a IPX0017413000 \ - -o ./downloads/IPX0017413000 -``` - -How each repository is enumerated: - -- **MassIVE** walks the FTPS tree at `massive-ftp.ucsd.edu` (the server requires TLS). If FTP/FTPS is blocked by the network, `pridepy` automatically falls back to HTTPS: it lists the dataset from the GNPS2 file index (`datasetcache.gnps2.org`) and downloads each file from the ProteoSAFe endpoint at `massive.ucsd.edu` (byte-identical to the FTPS copy). -- **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. -- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTP (with `Range` support for resume). iProX also exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTP endpoint so no iProX credentials are required. - -`download-all-public-raw-files` retrieves the files stored under the dataset's -`raw/` collection, saving them under `output_folder` with the dataset's -sub-directory layout preserved (so identically-named files in different -collections don't overwrite each other). These direct downloads support resume -(REST for FTP, byte-Range for HTTP), per-file retries, parallel workers (`-w` -up to 3), and post-transfer size verification against the server-reported size. - -You can also request a specific collection from these repositories through the -same category interface: - -```bash -pridepy download-all-public-category-files \ - -a MSV000082297 \ - -o ./downloads/MSV000082297-results \ - -c RESULT -``` - -
- -## Python API Examples - -> **Breaking change (0.0.16):** the legacy `pridepy.files.files.Files` class has been -> removed. Replace `from pridepy.files.files import Files` with -> `from pridepy.download.client import Client`; `Client` exposes the same public -> methods (`get_all_raw_file_list`, `download_all_raw_files`, -> `get_submitted_file_path_prefix`, `download_file_by_name`, -> `download_all_category_files`, `download_px_raw_files`, …). - -
-Get raw files for a project - -```python -from pridepy.download.client import Client - -client = Client() -raw_files = client.get_all_raw_file_list("PXD008644") -print(f"RAW files: {len(raw_files)}") -print(raw_files[0]["fileName"]) -``` - -For MassIVE / JPOST / iProX accessions, the same method returns the files found under the dataset's `raw/` collection: - -```python -from pridepy.download.client import Client - -client = Client() -for accession in ("MSV000082297", "JPST002311", "IPX0017413000"): - raw_files = client.get_all_raw_file_list(accession) - print(f"{accession} raw files: {len(raw_files)}") -``` - -
- -
-Search projects +# Download all public RAW files of a dataset (any repository) +pridepy download-all-public-raw-files -a PXD008644 -o ./downloads/PXD008644 --checksum-check -```python -from pridepy.project.project import Project +# Download a ProteomeXchange dataset by its PXD accession +pridepy download-px-raw-files -a PXD039236 -o ./downloads/PXD039236 -project = Project() -results = project.search_by_keywords_and_filters( - keyword="PXD009476", - query_filter="", - page_size=25, - page=0, - sort_direction="DESC", - sort_fields="accession", -) -print(f"Hits: {len(results)}") +# Download a native MassIVE / JPOST / iProX dataset +pridepy download-all-public-raw-files -a MSV000082297 -o ./downloads/MSV000082297 ``` -
+Full option tables and more examples are in [docs/usage.md](docs/usage.md). ## Development and Release (uv) diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..005d5e0 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,350 @@ +# pridepy usage guide + +This guide covers how to download data and query metadata with `pridepy`. For +installation, see the [README](../README.md#installation). + +`pridepy` works with PRIDE accessions and, transparently, with native MassIVE +(`MSV…`), JPOST (`JPST…`), and iProX (`IPX…`) accessions. The downloader +supports `ftp`, `aspera`, `s3`, and `globus`: by default it starts with FTP, +falls back across the remaining protocols when a transfer fails, and validates +downloaded files (non-empty, and checksum validation when enabled). + +## Contents + +- [Command overview](#command-overview) +- [PRIDE file downloads](#pride-file-downloads) +- [Metadata and search](#metadata-and-search) +- [Download from ProteomeXchange and other repositories](#download-from-proteomexchange-and-other-repositories) +- [Python API examples](#python-api-examples) + +## Command overview + +```bash +pridepy --help +``` + +| Command | Purpose | +| --- | --- | +| `download-all-public-raw-files` | Download every public RAW file of a dataset | +| `download-all-public-category-files` | Download files of one or more categories (RAW, SEARCH, …) | +| `download-file-by-name` | Download a single file (public or private) | +| `download-files-by-list` | Download a named subset of files from a manifest/CSV | +| `download-files-by-url` | Download files from raw `http`/`https`/`ftp` URLs | +| `download-px-raw-files` | Download RAW files resolved from a ProteomeXchange accession | +| `list-private-files` | List files of a private project (needs credentials) | +| `stream-files-metadata` | Stream file metadata (one project or all) to JSON | +| `stream-projects-metadata` | Stream all project metadata to JSON | +| `search-projects-by-keywords-and-filters` | Search projects by keyword and filters | + +The download commands work for PRIDE accessions and, transparently, for native +MassIVE (`MSV…`), JPOST (`JPST…`), and iProX (`IPX…`) accessions — see +[Download from ProteomeXchange and other repositories](#download-from-proteomexchange-and-other-repositories). + +## PRIDE file downloads + +PRIDE downloads start with FTP and fall back across the remaining protocols +(`ftp -> aspera -> s3 -> globus`) when a transfer fails. They support resume, +per-file retries, parallel workers, and optional checksum validation. Empty or +corrupt files are retried automatically. + +### Common download options + +These options are shared by `download-all-public-raw-files`, +`download-all-public-category-files`, `download-file-by-name`, and +`download-files-by-list`: + +| Option | Description | Default | +| --- | --- | --- | +| `-a, --accession` | Dataset accession (e.g. `PXD008644`) | required | +| `-o, --output-folder` | Destination directory | required | +| `-p, --protocol` | Transfer protocol: `ftp`, `aspera`, `globus`, `s3` (FTP-first with fallback) | `ftp` | +| `-w, --parallel-files` | Download 1–3 files concurrently — primarily for `globus`; not available on `download-file-by-name` | `1` | +| `--skip-if-downloaded-already` | Resume: skip files already present locally | off | +| `--checksum-check` | Download PRIDE checksums and validate each file | off | +| `--aspera-maximum-bandwidth` | Aspera cap, e.g. `50M`, `100M`, `200M` (Aspera only) | `100M` | +| `--preserve-structure` | Recreate the dataset's subdirectory layout (e.g. `raw/…/`) under the output folder instead of downloading flat | off | + +By default, files are downloaded **flat** into the output folder (no +`raw/…/` subdirectories). When two files would collapse to the same name, +later ones get a numeric suffix (`run.raw`, `run_1.raw`). Pass +`--preserve-structure` to keep the dataset's original directory layout. + +### Download all raw files (robust mode) + +```bash +pridepy download-all-public-raw-files \ + -a PXD008644 \ + -o ./downloads/PXD008644 \ + --checksum-check +``` + +Continue an interrupted download safely by adding `--skip-if-downloaded-already`: + +```bash +pridepy download-all-public-raw-files \ + -a PXD008644 \ + -o ./downloads/PXD008644 \ + --skip-if-downloaded-already \ + --checksum-check +``` + +### Download only selected categories + +```bash +pridepy download-all-public-category-files \ + -a PXD022105 \ + -o ./downloads/PXD022105 \ + -c RAW,SEARCH +``` + +`-c, --category` takes one or more comma-separated categories. Valid values: +`RAW`, `PEAK`, `SEARCH`, `RESULT`, `SPECTRUM_LIBRARY`, `OTHER`, `FASTA`. + +### Download one file by name + +```bash +pridepy download-file-by-name \ + -a PXD022105 \ + -f checksum.txt \ + -o ./downloads/PXD022105 \ + --checksum-check +``` + +`-f, --file-name` is the file to download. + +### Download a named subset of files (manifest) + +```bash +pridepy download-files-by-list \ + -a PXD001819 \ + -F files.txt \ + -o ./downloads/PXD001819 \ + --checksum-check +``` + +`files.txt` is one filename per line (blank lines and `#` comments are +ignored). Each filename is resolved against the project metadata and downloaded +via the same batch + protocol-fallback engine as `download-all-public-raw-files`. +Use `-f a.raw,b.raw,c.raw` instead of `-F` for a small inline list (you can +combine both). + +### Download files from raw URLs + +```bash +pridepy download-files-by-url \ + -F urls.txt \ + -o ./downloads/urls +``` + +`urls.txt` is one fully-qualified URL per line. Schemes `http`, `https`, and +`ftp` are dispatched to the matching downloader. Use `-u, --urls` for one or +more comma-separated URLs, e.g. `--urls https://a.com/x.raw,ftp://b.com/y.raw` +(URLs containing literal commas must use a manifest file instead). + +Command-specific options: + +| Option | Description | Default | +| --- | --- | --- | +| `-F, --url-list` | Manifest file, one URL per line | — | +| `-u, --urls` | Comma-separated URL(s) | — | +| `-p, --protocol` | `ftp` (per-scheme) or `globus` (resume-capable http/https) | `ftp` | +| `-w, --parallel-files` | Download 1–3 files concurrently (any scheme) | `1` | +| `--checksum-check` | Validate against PRIDE checksums (accession inferred from PRIDE URL paths; only PRIDE archive URLs supported) | off | + +### Private (restricted) files + +List the files of a private project with your PRIDE credentials: + +```bash +pridepy list-private-files -a PXD022105 -u YOUR_USER -p YOUR_PASSWORD +``` + +Download a private file by passing `--username`/`--password` to +`download-file-by-name`: + +```bash +pridepy download-file-by-name \ + -a PXD022105 \ + -f checksum.txt \ + -o ./downloads/private \ + --username YOUR_USER \ + --password YOUR_PASSWORD +``` + +## Metadata and search + +### Stream all project metadata to JSON + +```bash +pridepy stream-projects-metadata -o all_pride_projects.json +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-o, --output-file` | JSON file to write all project metadata to | required | + +### Stream file metadata + +```bash +# All file metadata for one accession +pridepy stream-files-metadata -a PXD005011 -o PXD005011_files.json +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-o, --output-file` | JSON file to write file metadata to | required | +| `-a, --accession` | Limit to one project (omit to stream all files) | optional | + +### Search projects by keywords and filters + +```bash +pridepy search-projects-by-keywords-and-filters \ + -k human \ + -f projectTags==ProteomeTools,organismsPart==Pancreas \ + -sd DESC \ + -sf accession \ + -sf submissionDate +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-k, --keyword` | Keyword searched across project fields | required | +| `-f, --filters` | `field==value` filters, comma-separated (e.g. `accession==PRD000001`) | — | +| `-ps, --page-size` | Results per page (1–1000) | `100` | +| `-p, --page` | Page number (0-based) | `0` | +| `-sd, --sort-direction` | `ASC` or `DESC` | `DESC` | +| `-sf, --sort-fields` | Sort field(s), repeatable. One of: `accession`, `submissionDate`, `diseases`, `organismsPart`, `organisms`, `instruments`, `softwares`, `avgDownloadsPerFile`, `downloadCount`, `publicationDate` | `submissionDate` | + +## Download from ProteomeXchange and other repositories + +A ProteomeXchange (`PXD…` / `PRD…`) accession is a cross-repository identifier: +the dataset may be hosted at PRIDE, MassIVE, JPOST, iProX, or elsewhere. +`pridepy` lets you start from the ProteomeXchange accession, or go straight to +the hosting repository using its **native** accession. + +### Start from a ProteomeXchange accession + +`download-px-raw-files` resolves the dataset's ProteomeXchange XML and downloads +the RAW files it references, regardless of which repository hosts them: + +```bash +pridepy download-px-raw-files \ + -a PXD039236 \ + -o ./downloads/PXD039236 +``` + +| Option | Description | Default | +| --- | --- | --- | +| `-a, --accession` | ProteomeXchange accession (e.g. `PXD039236`). `--px` is a deprecated alias | required | +| `-o, --output-folder` | Destination directory | required | +| `--skip-if-downloaded-already` | Skip files already present locally | off | + +### Go directly to the hosting repository (native MassIVE / JPOST / iProX accessions) + +Datasets that do not have a ProteomeXchange accession — or where you already +know the native accession — can be downloaded directly. The standard download +commands accept MassIVE, JPOST, and iProX accessions transparently: + +```bash +# MassIVE (FTPS at massive-ftp.ucsd.edu) +pridepy download-all-public-raw-files \ + -a MSV000082297 \ + -o ./downloads/MSV000082297 + +# JPOST (PROXI listing + ftp.jpostdb.org) +pridepy download-all-public-raw-files \ + -a JPST002311 \ + -o ./downloads/JPST002311 + +# iProX (ProteomeXchange XML + anonymous HTTP at download.iprox.org) +pridepy download-all-public-raw-files \ + -a IPX0017413000 \ + -o ./downloads/IPX0017413000 +``` + +How each repository is enumerated: + +- **MassIVE** walks the FTPS tree at `massive-ftp.ucsd.edu` (the server requires TLS). MassIVE distributes datasets across versioned root directories (`/v01`–`/vNN`); `pridepy` discovers the correct root automatically. If FTP/FTPS is blocked by the network, `pridepy` falls back to HTTPS: it lists the dataset from the GNPS2 file index (`datasetcache.gnps2.org`) and downloads each file from the ProteoSAFe endpoint at `massive.ucsd.edu` (byte-identical to the FTPS copy). +- **JPOST** lists files through the JSON PROXI endpoint at `https://repository.jpostdb.org/proxi/datasets/` and downloads from `ftp.jpostdb.org` over plain FTP. The PROXI listing avoids the source-IP connection limit JPOST enforces on FTP. +- **iProX** fetches the dataset's ProteomeXchange XML from `http://download.iprox.org//PX_.xml`, then downloads each referenced file from the same host over anonymous HTTP (with `Range` support for resume). iProX also exposes Aspera (`faspe://`) with username/password for very large bulk transfers; `pridepy` uses the public HTTP endpoint so no iProX credentials are required. + +`download-all-public-raw-files` retrieves the files stored under the dataset's +`raw/` collection. These direct downloads support resume (REST for FTP, +byte-Range for HTTP), per-file retries, parallel workers (`-w` up to 3), and +post-transfer size verification against the server-reported size. By default +files are written flat into the output folder; pass `--preserve-structure` to +keep the dataset's sub-directory layout. + +You can also request a specific collection from these repositories through the +same category interface: + +```bash +pridepy download-all-public-category-files \ + -a MSV000082297 \ + -o ./downloads/MSV000082297-results \ + -c RESULT +``` + +## Python API examples + +> **Breaking change (0.0.16):** the legacy `pridepy.files.files.Files` class has been +> removed. Replace `from pridepy.files.files import Files` with +> `from pridepy.download.client import Client`; `Client` exposes the same public +> methods (`get_all_raw_file_list`, `download_all_raw_files`, +> `get_submitted_file_path_prefix`, `download_file_by_name`, +> `download_all_category_files`, `download_px_raw_files`, …). + +### Get raw files for a project + +```python +from pridepy.download.client import Client + +client = Client() +raw_files = client.get_all_raw_file_list("PXD008644") +print(f"RAW files: {len(raw_files)}") +print(raw_files[0]["fileName"]) +``` + +For MassIVE / JPOST / iProX accessions, the same method returns the files found under the dataset's `raw/` collection: + +```python +from pridepy.download.client import Client + +client = Client() +for accession in ("MSV000082297", "JPST002311", "IPX0017413000"): + raw_files = client.get_all_raw_file_list(accession) + print(f"{accession} raw files: {len(raw_files)}") +``` + +### Download all raw files for a project + +```python +from pridepy.download.client import Client + +client = Client() +client.download_all_raw_files( + accession="PXD008644", + output_folder="./downloads/PXD008644", + skip_if_downloaded_already=True, + protocol="ftp", + aspera_maximum_bandwidth="100M", + checksum_check=True, +) +``` + +### Search projects + +```python +from pridepy.project.project import Project + +project = Project() +results = project.search_by_keywords_and_filters( + keyword="PXD009476", + query_filter="", + page_size=25, + page=0, + sort_direction="DESC", + sort_fields="accession", +) +print(f"Hits: {len(results)}") +``` From 14944627f72fd74f3448f803e88cc5d13624021f Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 15:20:24 +0100 Subject: [PATCH 52/54] fix(download): address PR #106 review comments (hardening) - client.get_file_from_api: chain re-raised exception with `from e`. - base.Provider: tolerate partial/None records in get_raw_files / get_category_files / find_file via defensive .get() access. - base.Provider.download_files: forward the requested protocol to get_download_url so protocol-aware adapters receive it. - pride.get_submitted_file_path_prefix: raise a clear error when no RAW file has a public location, or the path layout doesn't match (no bare IndexError / AttributeError). - pride: add timeouts to the private-file requests.get and the checksum urllib.urlopen so a stalled server can't hang indefinitely. - transport.download_ftp_urls: reject FTP URLs with no host (fail fast). - iprox / proteomexchange: parse remote XML with defusedxml (XXE/entity hardening); add defusedxml dependency. - Tests for the new guards and protocol forwarding. --- pridepy/download/base.py | 13 ++-- pridepy/download/client.py | 2 +- pridepy/download/iprox.py | 2 +- pridepy/download/pride.py | 29 +++++++-- pridepy/download/proteomexchange.py | 2 +- pridepy/download/transport.py | 4 ++ pridepy/tests/test_review_fixes.py | 98 +++++++++++++++++++++++++++++ pyproject.toml | 1 + requirements.txt | 3 +- 9 files changed, 141 insertions(+), 13 deletions(-) create mode 100644 pridepy/tests/test_review_fixes.py diff --git a/pridepy/download/base.py b/pridepy/download/base.py index 22915dc..3382f08 100644 --- a/pridepy/download/base.py +++ b/pridepy/download/base.py @@ -86,10 +86,15 @@ def _list_files_checked(self, accession: str) -> List[Dict]: ) return records + @staticmethod + def _category_value(record: Dict) -> Optional[str]: + """Safely read ``fileCategory.value`` from a (possibly partial) record.""" + return (record.get("fileCategory") or {}).get("value") + def get_raw_files(self, accession: str) -> List[Dict]: """Return records whose ``fileCategory.value`` is ``"RAW"``.""" records = self._list_files_checked(accession) - return [r for r in records if r["fileCategory"]["value"] == "RAW"] + return [r for r in records if self._category_value(r) == "RAW"] def get_category_files( self, accession: str, categories: "str | List[str]" @@ -99,12 +104,12 @@ def get_category_files( categories = [categories] category_set = {c.upper() for c in categories} records = self._list_files_checked(accession) - return [r for r in records if r["fileCategory"]["value"] in category_set] + return [r for r in records if self._category_value(r) in category_set] def find_file(self, accession: str, file_name: str) -> List[Dict]: """Return records whose ``fileName`` equals ``file_name``.""" records = self._list_files_checked(accession) - return [r for r in records if r["fileName"] == file_name] + return [r for r in records if r.get("fileName") == file_name] # ------------------------------------------------------------------ # Shared download workflow (Template Method). @@ -268,7 +273,7 @@ def download_files( # Collect transfer entries in one pass, keeping order stable. entries = [] # list of (scheme, url, relpath) for record in records: - url = self.get_download_url(record) + url = self.get_download_url(record, protocol) relpath = record.get("relativePath") lowered = url.lower() if lowered.startswith("ftp://"): diff --git a/pridepy/download/client.py b/pridepy/download/client.py index 91716ea..979c650 100644 --- a/pridepy/download/client.py +++ b/pridepy/download/client.py @@ -173,7 +173,7 @@ def get_file_from_api(self, accession, file_name) -> List[Dict]: try: return registry.resolve(accession).find_file(accession, file_name) except Exception as e: - raise Exception("File not found " + str(e)) + raise Exception("File not found " + str(e)) from e # Download entry points. diff --git a/pridepy/download/iprox.py b/pridepy/download/iprox.py index dcbbe68..db5e7fc 100644 --- a/pridepy/download/iprox.py +++ b/pridepy/download/iprox.py @@ -14,7 +14,7 @@ import logging import os import re -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET from typing import ClassVar, Dict, List, Optional from urllib.parse import urlparse diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index cafaa48..50566c2 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -110,10 +110,25 @@ def get_submitted_file_path_prefix(self, accession): :return: path fragment (eg: 2018/10/PXD008644) """ records = self._list_files_checked(accession) - raw_files = [r for r in records if r["fileCategory"]["value"] == "RAW"] + raw_files = [ + r + for r in records + if (r.get("fileCategory") or {}).get("value") == "RAW" + and r.get("publicFileLocations") + ] + if not raw_files: + raise ValueError( + f"Cannot determine submitted path prefix for {accession}: " + f"no RAW file with a public location was found." + ) first_file = raw_files[0]["publicFileLocations"][0]["value"] - path_fragment = re.search(r"\d{4}/\d{2}/PXD\d*", first_file).group() - return path_fragment + match = re.search(r"\d{4}/\d{2}/PXD\d*", first_file) + if match is None: + raise ValueError( + f"Cannot determine submitted path prefix for {accession}: " + f"unexpected file path layout ({first_file!r})." + ) + return match.group() # ------------------------------------------------------------------ # Static utilities @@ -289,7 +304,7 @@ def save_checksum_file(accession, output_folder): headers = {"accept": "text/plain"} request = urllib.request.Request(url, headers=headers, method="GET") logging.info(f"Fetching checksum file from {url}") - with urllib.request.urlopen(request) as response: + with urllib.request.urlopen(request, timeout=60) as response: data = response.read().decode("utf-8") # Save the data to a .tsv file output_path = os.path.join(output_folder, f"{accession}-checksum.tsv") @@ -554,7 +569,11 @@ def download_private_file_name(self, accession, file_name, output_folder, userna logging.info("Valid token after login: {}".format(validate_token)) url = self.API_PRIVATE_URL + "/projects/{}/files?search={}".format(accession, file_name) - content = requests.get(url, headers={"Authorization": "Bearer {}".format(auth_token)}) + content = requests.get( + url, + headers={"Authorization": "Bearer {}".format(auth_token)}, + timeout=(10, 60), + ) if content.ok and content.status_code == 200: json_file = content.json() if ( diff --git a/pridepy/download/proteomexchange.py b/pridepy/download/proteomexchange.py index 9be09b8..32ea72e 100644 --- a/pridepy/download/proteomexchange.py +++ b/pridepy/download/proteomexchange.py @@ -25,7 +25,7 @@ import os import posixpath import re -import xml.etree.ElementTree as ET +import defusedxml.ElementTree as ET from typing import ClassVar, Dict, List from urllib.parse import urlparse diff --git a/pridepy/download/transport.py b/pridepy/download/transport.py index 7ff66d2..122e7de 100644 --- a/pridepy/download/transport.py +++ b/pridepy/download/transport.py @@ -484,6 +484,10 @@ def download_ftp_urls( if relative_paths and idx < len(relative_paths) else None ) + if not parsed.hostname: + raise ValueError( + f"Cannot download FTP URL with no host: {url!r}" + ) local_path = _dest_path(output_folder, remote_path, relpath) host_to_items.setdefault(parsed.hostname, []).append((remote_path, local_path)) diff --git a/pridepy/tests/test_review_fixes.py b/pridepy/tests/test_review_fixes.py new file mode 100644 index 0000000..daeca9a --- /dev/null +++ b/pridepy/tests/test_review_fixes.py @@ -0,0 +1,98 @@ +"""Tests for hardening fixes from PR #106 code review. + +Covers: exception chaining, defensive guards for empty/partial listings, +FTP host validation, and protocol forwarding in the shared download path. +""" +import tempfile +from unittest import TestCase +from unittest.mock import patch + +import pytest + +from pridepy.download import registry, transport +from pridepy.download.client import Client +from pridepy.download.massive import MassiveProvider +from pridepy.download.pride import PrideProvider + + +class TestReviewFixes(TestCase): + def test_get_file_from_api_chains_original_exception(self): + with patch.object(registry, "resolve", side_effect=KeyError("boom")): + with pytest.raises(Exception) as exc_info: + Client().get_file_from_api("PXD000001", "x.raw") + # The original cause must be preserved for debugging. + assert isinstance(exc_info.value.__cause__, KeyError) + + def test_get_submitted_prefix_raises_clear_error_when_no_raw_files(self): + provider = PrideProvider() + records = [ + { + "fileName": "results.tsv", + "fileCategory": {"value": "SEARCH"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "ftp://h/2018/10/PXD1/results.tsv"} + ], + } + ] + with patch.object(provider, "_list_files_checked", return_value=records): + with pytest.raises(ValueError): # not a bare IndexError + provider.get_submitted_file_path_prefix("PXD1") + + def test_get_submitted_prefix_raises_clear_error_when_path_has_no_prefix(self): + provider = PrideProvider() + records = [ + { + "fileName": "a.raw", + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + {"name": "FTP Protocol", "value": "ftp://host/no-date-here/a.raw"} + ], + } + ] + with patch.object(provider, "_list_files_checked", return_value=records): + with pytest.raises(ValueError): # not a bare AttributeError on None.group() + provider.get_submitted_file_path_prefix("PXD1") + + def test_download_ftp_urls_rejects_url_without_host(self): + with tempfile.TemporaryDirectory() as tmp_dir: + with pytest.raises(ValueError, match="host"): + transport.download_ftp_urls( + ftp_urls=["ftp:///pride/data/x.raw"], # no hostname + output_folder=tmp_dir, + skip_if_downloaded_already=False, + ) + + def test_get_raw_files_tolerates_records_missing_category(self): + records = [ + {"fileName": "a.raw"}, # no fileCategory at all + MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/b.raw", + ), + ] + with patch.object(MassiveProvider, "list_files", return_value=records): + result = MassiveProvider().get_raw_files("MSV000012345") + assert {r["fileName"] for r in result} == {"b.raw"} + + def test_download_files_forwards_protocol_to_get_download_url(self): + seen = [] + + class _CapturingProvider(MassiveProvider): + def get_download_url(self, record, protocol="ftp"): + seen.append(protocol) + return record["publicFileLocations"][0]["value"] + + record = MassiveProvider._build_file_record( + "MSV000012345", + "ftp://massive-ftp.ucsd.edu/v01/MSV000012345/raw/a.raw", + ) + with patch.object(transport, "download_ftp_urls"): + _CapturingProvider().download_files( + accession="MSV000012345", + records=[record], + output_folder="/tmp/x", + skip_if_downloaded_already=False, + protocol="aspera", + parallel_files=1, + ) + assert seen == ["aspera"] diff --git a/pyproject.toml b/pyproject.toml index 4a95f24..b94d076 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "boto3>=1.34.61", "botocore>=1.34.74", "httpx>=0.27.0", + "defusedxml>=0.7.1", ] [project.optional-dependencies] diff --git a/requirements.txt b/requirements.txt index 3bae141..3a2e4c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,5 @@ boto3 botocore tqdm urllib3 -httpx \ No newline at end of file +httpx +defusedxml \ No newline at end of file From 858a118e4d18b54545bb9729aef66af2322ec776 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 15:30:27 +0100 Subject: [PATCH 53/54] fix(pride): propagate batch failures and accept PRD accessions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - download_files_from_aspera/globus/s3 now collect per-file failures and raise RuntimeError after attempting every file, instead of logging and returning success — so a fully failed batch no longer exits 0 (the multi-protocol orchestrator still catches this and falls back per file). - Use file.get("fileName") in the S3 helper to avoid KeyError on partial records. - get_submitted_file_path_prefix now matches PRD as well as PXD accessions. - Tests for batch-failure propagation and PRD support. --- pridepy/download/pride.py | 23 +++++++++- pridepy/tests/test_review_fixes.py | 70 +++++++++++++++++++++++++++++- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/pridepy/download/pride.py b/pridepy/download/pride.py index 50566c2..8ab017f 100644 --- a/pridepy/download/pride.py +++ b/pridepy/download/pride.py @@ -122,7 +122,7 @@ def get_submitted_file_path_prefix(self, accession): f"no RAW file with a public location was found." ) first_file = raw_files[0]["publicFileLocations"][0]["value"] - match = re.search(r"\d{4}/\d{2}/PXD\d*", first_file) + match = re.search(r"\d{4}/\d{2}/(?:PXD|PRD)\d*", first_file) if match is None: raise ValueError( f"Cannot determine submitted path prefix for {accession}: " @@ -358,6 +358,7 @@ def download_files_from_aspera( "aspera/key/asperaweb_id_dsa.openssh" ) key_path = os.path.abspath(key_full_path) + failed: List[str] = [] for file in file_list_json: if file["publicFileLocations"][0]["name"] == "Aspera Protocol": download_url = file["publicFileLocations"][0]["value"] @@ -392,6 +393,11 @@ def download_files_from_aspera( logging.info(f"Successfully downloaded {new_file_path} via Aspera") except subprocess.CalledProcessError as e: logging.error(f"Aspera download failed for {new_file_path}: {str(e)}") + failed.append(file.get("fileName", new_file_path)) + if failed: + raise RuntimeError( + f"Aspera download failed for {len(failed)} file(s): {failed}" + ) @staticmethod def download_files_from_globus( @@ -446,6 +452,7 @@ def download_files_from_globus( # --- Phase 1: download (skip check already done, pass False) --------- parallel_files = min(parallel_files, 3, len(files_to_download)) + failed: List[str] = [] if parallel_files < 2: for file in files_to_download: try: @@ -458,6 +465,7 @@ def download_files_from_globus( logging.info(f"Successfully downloaded {new_file_path}") except Exception as e: logging.error(f"Download from Globus failed: {str(e)}") + failed.append(file.get("fileName", "")) else: logging.info(f"Downloading {len(files_to_download)} file(s) with {parallel_files} parallel workers") with ThreadPoolExecutor(max_workers=parallel_files) as executor: @@ -474,6 +482,11 @@ def download_files_from_globus( future.result() except Exception as e: logging.error(f"Download from Globus failed: {str(e)}") + failed.append(futures[future].get("fileName", "")) + if failed: + raise RuntimeError( + f"Globus download failed for {len(failed)} file(s): {failed}" + ) @staticmethod def download_files_from_s3( @@ -503,6 +516,7 @@ def download_files_from_s3( ) bucket = s3_resource.Bucket(PrideProvider.S3_BUCKET) + failed: List[str] = [] for file in file_list_json: try: # Determine S3 or FTP path @@ -548,7 +562,12 @@ def download_files_from_s3( else: raise except Exception as e: - logging.error(f"Failed to download {file['fileName']}: {e}") + logging.error(f"Failed to download {file.get('fileName')}: {e}") + failed.append(file.get("fileName", "")) + if failed: + raise RuntimeError( + f"S3 download failed for {len(failed)} file(s): {failed}" + ) # ------------------------------------------------------------------ # Private dataset download diff --git a/pridepy/tests/test_review_fixes.py b/pridepy/tests/test_review_fixes.py index daeca9a..242941c 100644 --- a/pridepy/tests/test_review_fixes.py +++ b/pridepy/tests/test_review_fixes.py @@ -3,9 +3,10 @@ Covers: exception chaining, defensive guards for empty/partial listings, FTP host validation, and protocol forwarding in the shared download path. """ +import subprocess import tempfile from unittest import TestCase -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -15,6 +16,20 @@ from pridepy.download.pride import PrideProvider +def _pride_record(file_name="a.raw", accession="PXD000001", date="2018/10"): + return { + "fileName": file_name, + "accession": accession, + "fileCategory": {"value": "RAW"}, + "publicFileLocations": [ + { + "name": "FTP Protocol", + "value": f"ftp://ftp.pride.ebi.ac.uk/pride/data/archive/{date}/{accession}/{file_name}", + } + ], + } + + class TestReviewFixes(TestCase): def test_get_file_from_api_chains_original_exception(self): with patch.object(registry, "resolve", side_effect=KeyError("boom")): @@ -74,6 +89,59 @@ def test_get_raw_files_tolerates_records_missing_category(self): result = MassiveProvider().get_raw_files("MSV000012345") assert {r["fileName"] for r in result} == {"b.raw"} + def test_get_submitted_prefix_supports_prd_accessions(self): + provider = PrideProvider() + records = [_pride_record("a.raw", accession="PRD000123", date="2012/03")] + with patch.object(provider, "_list_files_checked", return_value=records): + assert provider.get_submitted_file_path_prefix("PRD000123") == "2012/03/PRD000123" + + def test_aspera_batch_raises_when_a_file_fails(self): + records = [ + { + "fileName": "a.raw", + "accession": "PXD000001", + "publicFileLocations": [ + {"name": "Aspera Protocol", "value": "faspe://h/a.raw"} + ], + } + ] + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object(PrideProvider, "get_ascp_binary", return_value="/bin/false"), patch( + "pridepy.download.pride.subprocess.run", + side_effect=subprocess.CalledProcessError(1, "ascp"), + ): + with pytest.raises(RuntimeError, match="Aspera"): + PrideProvider.download_files_from_aspera( + records, tmp_dir, skip_if_downloaded_already=False + ) + + def test_globus_batch_raises_when_a_file_fails(self): + records = [_pride_record("a.raw")] + with tempfile.TemporaryDirectory() as tmp_dir: + with patch.object( + PrideProvider, "_globus_download_one", side_effect=RuntimeError("boom") + ): + with pytest.raises(RuntimeError, match="Globus"): + PrideProvider.download_files_from_globus( + records, tmp_dir, skip_if_downloaded_already=False + ) + + def test_s3_batch_raises_when_a_file_fails(self): + records = [_pride_record("a.raw")] + mock_obj = Mock() + mock_obj.content_length = 10 + mock_bucket = Mock() + mock_bucket.Object.return_value = mock_obj + mock_bucket.download_file.side_effect = Exception("boom") + mock_resource = Mock() + mock_resource.Bucket.return_value = mock_bucket + with tempfile.TemporaryDirectory() as tmp_dir: + with patch("pridepy.download.pride.boto3.resource", return_value=mock_resource): + with pytest.raises(RuntimeError, match="S3"): + PrideProvider.download_files_from_s3( + records, tmp_dir, skip_if_downloaded_already=False + ) + def test_download_files_forwards_protocol_to_get_download_url(self): seen = [] From 2ebc99c5e5cc5b39ebd14ca124d957865d076073 Mon Sep 17 00:00:00 2001 From: Yasset Perez-Riverol Date: Fri, 29 May 2026 15:41:18 +0100 Subject: [PATCH 54/54] build: pin httpx>=0.27.0 in requirements.txt (GHSA-h8pj-cxx2-jfg2) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3a2e4c0..613ad5b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,5 @@ boto3 botocore tqdm urllib3 -httpx +httpx>=0.27.0 defusedxml \ No newline at end of file