diff --git a/README.md b/README.md index f6e7933..ec77e62 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,8 @@ pridepy download-px-raw-files -a PXD039236 -o ./downloads/PXD039236 # Download a native MassIVE / JPOST / iProX dataset pridepy download-all-public-raw-files -a MSV000082297 -o ./downloads/MSV000082297 -# Download PDC/CPTAC files for a study (single type, or per-row types via CSV) +# Download PDC/CPTAC files for a study (all types, or restrict with --file-type) +pridepy download-pdc-files -a PDC000109 -o ./downloads/pdc pridepy download-pdc-files -a PDC000109 --file-type psm -o ./downloads/pdc ``` diff --git a/docs/usage.md b/docs/usage.md index 005d5e0..e3885f1 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -15,6 +15,7 @@ downloaded files (non-empty, and checksum validation when enabled). - [PRIDE file downloads](#pride-file-downloads) - [Metadata and search](#metadata-and-search) - [Download from ProteomeXchange and other repositories](#download-from-proteomexchange-and-other-repositories) +- [Download CPTAC/PDC files](#download-cptacpdc-files) - [Python API examples](#python-api-examples) ## Command overview @@ -31,6 +32,7 @@ pridepy --help | `download-files-by-list` | Download a named subset of files from a manifest/CSV | | `download-files-by-url` | Download files from raw `http`/`https`/`ftp` URLs | | `download-px-raw-files` | Download RAW files resolved from a ProteomeXchange accession | +| `download-pdc-files` | Download PDC/CPTAC files via PDC GraphQL signed HTTPS URLs | | `list-private-files` | List files of a private project (needs credentials) | | `stream-files-metadata` | Stream file metadata (one project or all) to JSON | | `stream-projects-metadata` | Stream all project metadata to JSON | @@ -285,6 +287,147 @@ pridepy download-all-public-category-files \ -c RESULT ``` +## Download CPTAC/PDC files + +The [Proteomic Data Commons (PDC)](https://pdc.cancer.gov/) hosts CPTAC (Clinical +Proteomic Tumor Analysis Consortium) mass-spectrometry datasets. `pridepy` can +enumerate and download PDC files via the PDC GraphQL API, which issues short-lived +signed HTTPS URLs for each file. Files are organised by **PDC study ID** +(e.g. `PDC000109`). + +### Download all files for a study + +Omit `--file-type` to download every file available in the study: + +```bash +pridepy download-pdc-files \ + -a PDC000109 \ + -o ./downloads/PDC000109 +``` + +Files are placed under `//`. + +### Download a specific file type + +Pass `--file-type` to restrict the download to one category: + +```bash +# mzIdentML peptide-spectral-match files +pridepy download-pdc-files \ + -a PDC000109 \ + --file-type mzid \ + -o ./downloads/PDC000109 + +# PSM TSV files +pridepy download-pdc-files \ + -a PDC000109 \ + --file-type psm \ + -o ./downloads/PDC000109 + +# Vendor RAW files +pridepy download-pdc-files \ + -a PDC000109 \ + --file-type raw \ + -o ./downloads/PDC000109 + +# Processed mzML files +pridepy download-pdc-files \ + -a PDC000109 \ + --file-type mzml \ + -o ./downloads/PDC000109 +``` + +### Download multiple studies at once + +Pass a comma-separated list of study IDs or a CSV file: + +```bash +# Comma-separated list — downloads all files for each study +pridepy download-pdc-files \ + -a PDC000109,PDC000110,PDC000111 \ + -o ./downloads/pdc-batch + +# CSV with a pdc_id column — downloads all files for each row +# studies.csv: pdc_id +# PDC000109 +# PDC000110 +pridepy download-pdc-files \ + -a studies.csv \ + -o ./downloads/pdc-batch +``` + +When the CSV includes a `file-type` (or `filetype`) column, each row's file type +is used independently, allowing mixed-type batch downloads in a single command: + +```csv +pdc_id,file-type +PDC000109,raw +PDC000110,mzml +PDC000111,psm +``` + +```bash +pridepy download-pdc-files \ + -a studies.csv \ + -o ./downloads/pdc-batch +``` + +### Resume an interrupted download + +```bash +pridepy download-pdc-files \ + -a PDC000109 \ + -o ./downloads/PDC000109 \ + --skip-if-downloaded-already +``` + +### Validate checksums + +PDC-provided `md5sum` values are checked automatically. To disable: + +```bash +pridepy download-pdc-files \ + -a PDC000109 \ + -o ./downloads/PDC000109 \ + --no-checksum-check +``` + +### Speed up large files with parallel HTTP Range threads + +Use `-t / --threads` (1–32) to split each file into parallel byte-range requests: + +```bash +pridepy download-pdc-files \ + -a PDC000109 \ + --file-type raw \ + -o ./downloads/PDC000109 \ + --threads 8 +``` + +### Retry failed files (including 403 signed-URL refresh) + +Signed URLs expire after a short time. `--retry` re-fetches a fresh URL before +each retry attempt when a `403 Forbidden` is received: + +```bash +pridepy download-pdc-files \ + -a PDC000109 \ + -o ./downloads/PDC000109 \ + --retry +``` + +### Full option reference + +| Option | Description | Default | +| --- | --- | --- | +| `-a, --accession` | PDC study ID, comma-separated IDs, or a CSV with `pdc_id`/`pdc_study_id` and optional `file-type`/`filetype` column | required | +| `--file-type` | Restrict to one type: `mzid`, `psm`, `raw`, or `mzml`. Omit to download all file types. Overrides the CSV `file-type` column. | all types | +| `-o, --output-folder` | Destination directory; files are written as `//` | required | +| `--skip-if-downloaded-already` | Skip files that already exist locally and match PDC size/checksum | off | +| `--checksum-check / --no-checksum-check` | Validate downloads against PDC `md5sum` values | on | +| `-t, --threads` | Parallel HTTP Range threads per file (1–32) | `1` | +| `--retry` | Retry failed files; HTTP 403 retries refresh the PDC signed URL first | off | + ## Python API examples > **Breaking change (0.0.16):** the legacy `pridepy.files.files.Files` class has been diff --git a/pridepy/pdc/client.py b/pridepy/pdc/client.py index 73a7edb..3e217ed 100644 --- a/pridepy/pdc/client.py +++ b/pridepy/pdc/client.py @@ -44,7 +44,7 @@ class PDCFile: @dataclass(frozen=True) class PDCDownloadRequest: study_id: str - file_type: str + file_type: Optional[str] PDC_FILE_TYPE_FILTERS: Dict[str, PDCFileTypeFilter] = { @@ -182,19 +182,12 @@ def parse_download_requests(accession: str, file_type: Optional[str] = None) -> requests = [] for study_id, csv_file_type in rows: request_file_type = command_file_type - if request_file_type is None: - if csv_file_type is None: - raise ValueError( - "CSV rows must contain file-type/filetype when --file-type is not set: " - f"{study_id}" - ) + if request_file_type is None and csv_file_type is not None: request_file_type = normalize_file_type(csv_file_type) requests.append(PDCDownloadRequest(study_id, request_file_type)) return list(dict.fromkeys(requests)) study_ids = parse_accessions(accession) - if command_file_type is None: - raise ValueError("--file-type is required unless --accession is a CSV with file-type/filetype column") return [PDCDownloadRequest(study_id, command_file_type) for study_id in study_ids] @@ -258,12 +251,12 @@ def post_graphql(query: str, variables: Dict, session=None) -> Dict: return payload -def fetch_study_files(study_id: str, file_type: str, session=None) -> List[PDCFile]: +def fetch_study_files(study_id: str, file_type: Optional[str], session=None) -> List[PDCFile]: payload = post_graphql(FILES_PER_STUDY_QUERY, {"studyId": study_id}, session=session) raw_files = payload.get("data", {}).get("filesPerStudy", []) or [] files: List[PDCFile] = [] for entry in raw_files: - if not entry_matches_file_type(entry, file_type): + if file_type is not None and not entry_matches_file_type(entry, file_type): continue pdc_file = normalize_pdc_file(entry, study_id) if pdc_file is None: @@ -273,7 +266,7 @@ def fetch_study_files(study_id: str, file_type: str, session=None) -> List[PDCFi return files -def refresh_signed_url(study_id: str, file_name: str, file_type: str, session=None) -> Optional[str]: +def refresh_signed_url(study_id: str, file_name: str, file_type: Optional[str], session=None) -> Optional[str]: with _refresh_lock: for pdc_file in fetch_study_files(study_id, file_type, session=session): if pdc_file.file_name == file_name: diff --git a/pridepy/pdc/downloader.py b/pridepy/pdc/downloader.py index ecb021c..649d389 100644 --- a/pridepy/pdc/downloader.py +++ b/pridepy/pdc/downloader.py @@ -13,8 +13,8 @@ LOGGER = logging.getLogger(__name__) -FetchFiles = Callable[[str, str], List[PDCFile]] -RefreshUrl = Callable[[str, str, str], Optional[str]] +FetchFiles = Callable[[str, Optional[str]], List[PDCFile]] +RefreshUrl = Callable[[str, str, Optional[str]], Optional[str]] @dataclass @@ -131,7 +131,7 @@ def _download_with_retries( target: Path, checksum_check: bool, download_threads: int, - file_type: str, + file_type: Optional[str], refresh_url: RefreshUrl, refresh_on_403: bool, max_attempts: int, @@ -184,7 +184,7 @@ def _write_failed_files(output_folder: Path, failures: List[PDCDownloadFailure]) return failed_log -def _failure_from_result(pdc_file: PDCFile, result: PDCTransferResult, file_type: str) -> PDCDownloadFailure: +def _failure_from_result(pdc_file: PDCFile, result: PDCTransferResult, file_type: Optional[str]) -> PDCDownloadFailure: return PDCDownloadFailure( study_id=pdc_file.study_id, file_name=pdc_file.file_name, @@ -197,7 +197,7 @@ def _failure_from_result(pdc_file: PDCFile, result: PDCTransferResult, file_type def _fetch_study_file_list( study_id: str, - file_type: str, + file_type: Optional[str], fetch_files: FetchFiles, ) -> Tuple[List[PDCFile], Optional[PDCDownloadFailure]]: try: @@ -243,7 +243,7 @@ def _download_study_files( @dataclass(frozen=True) class PDCDownloadOptions: - file_type: str + file_type: Optional[str] skip_existing: bool checksum_check: bool download_threads: int @@ -343,12 +343,13 @@ def download_pdc_files( stats.total_files += len(study_files) if not study_files: - message = f"No PDC files matched file_type={request.file_type}" + file_type_label = request.file_type or "all" + message = f"No PDC files matched file_type={file_type_label}" LOGGER.warning("%s for %s", message, request.study_id) empty_match_failures.append( PDCDownloadFailure( request.study_id, - f"<{request.file_type}>", + f"<{file_type_label}>", message, file_type=request.file_type, ) diff --git a/pridepy/pridepy.py b/pridepy/pridepy.py index bdea8af..17723cf 100644 --- a/pridepy/pridepy.py +++ b/pridepy/pridepy.py @@ -729,7 +729,7 @@ def download_files_by_url( "--file-type", required=False, type=click.Choice(["mzid", "psm", "raw", "mzml"], case_sensitive=False), - help="PDC file type to download: mzid, psm, raw, or mzml. Overrides CSV file-type/filetype values.", + help="PDC file type to download: mzid, psm, raw, or mzml. Omit to download all file types. Overrides CSV file-type/filetype values.", ) @click.option( "-o", diff --git a/pridepy/tests/test_pdc_client.py b/pridepy/tests/test_pdc_client.py index 13fe0fe..472cc0b 100644 --- a/pridepy/tests/test_pdc_client.py +++ b/pridepy/tests/test_pdc_client.py @@ -107,17 +107,25 @@ def test_command_line_file_type_overrides_csv_file_type(self): assert requests == [PDCDownloadRequest("PDC000109", "raw")] assert "overrides CSV file-type values" in "\n".join(logs.output) - def test_file_type_is_required_without_csv_file_type(self): - with pytest.raises(ValueError, match="--file-type is required"): - parse_download_requests("PDC000109") + def test_no_file_type_downloads_all_files_for_plain_accession(self): + assert parse_download_requests("PDC000109") == [PDCDownloadRequest("PDC000109", None)] - def test_csv_file_type_is_required_when_command_line_file_type_is_missing(self): + def test_csv_without_file_type_column_downloads_all_files(self): with tempfile.TemporaryDirectory() as tmp_dir: path = os.path.join(tmp_dir, "studies.csv") with open(path, "w", encoding="utf-8") as handle: handle.write("pdc_id\nPDC000109\n") - with pytest.raises(ValueError, match="file-type/filetype"): - parse_download_requests(path) + assert parse_download_requests(path) == [PDCDownloadRequest("PDC000109", None)] + + def test_csv_mixed_file_type_and_no_file_type_rows(self): + with tempfile.TemporaryDirectory() as tmp_dir: + path = os.path.join(tmp_dir, "studies.csv") + with open(path, "w", encoding="utf-8") as handle: + handle.write("pdc_id,file-type\nPDC000109,raw\nPDC000110,\n") + assert parse_download_requests(path) == [ + PDCDownloadRequest("PDC000109", "raw"), + PDCDownloadRequest("PDC000110", None), + ] def test_csv_missing_column_raises(self): with tempfile.TemporaryDirectory() as tmp_dir: @@ -168,6 +176,18 @@ def test_fetch_study_files_skips_missing_signed_url(self): assert fetch_study_files("PDC000109", "psm", session=session) == [] + def test_fetch_study_files_none_file_type_returns_all_files(self): + entries = [ + _entry("sample.psm", file_format="tsv", data_category="Peptide Spectral Matches"), + _entry("sample.raw", file_format="vendor-specific", data_category="Raw Mass Spectra"), + ] + session = FakeSession(_payload(entries)) + + files = fetch_study_files("PDC000109", None, session=session) + + assert len(files) == 2 + assert {f.file_name for f in files} == {"sample.psm", "sample.raw"} + def test_refresh_signed_url_returns_matching_file(self): files = [ PDCFile("PDC000109", "1", "a.psm", "tsv", 3, "Peptide Spectral Matches", None, None, None, "old"), diff --git a/pridepy/tests/test_pdc_downloader.py b/pridepy/tests/test_pdc_downloader.py index 164d611..fb079e3 100644 --- a/pridepy/tests/test_pdc_downloader.py +++ b/pridepy/tests/test_pdc_downloader.py @@ -38,6 +38,15 @@ def fetch_files(study_id, file_type): return fetch_files +def _all_files_fetcher(files): + def fetch_files(study_id, file_type): + assert study_id == "PDC000109" + assert file_type is None + return files + + return fetch_files + + def _write_data(_url, target): with open(target, "wb") as handle: handle.write(b"abc") @@ -258,3 +267,24 @@ def fake_download(url, target): assert stats.downloaded == 1 assert seen_urls == ["https://example.org/old", "https://example.org/new"] refresh.assert_called_once_with("PDC000109", "sample.psm", "psm") + + def test_no_file_type_downloads_all_files(self): + psm_file = _pdc_file(name="sample.psm") + raw_file = _pdc_file(name="sample.raw") + with tempfile.TemporaryDirectory() as tmp_dir: + with patch( + "pridepy.pdc.downloader.transport._parallel_download", + side_effect=_write_data, + ): + stats = download_pdc_files( + accession="PDC000109", + file_type=None, + output_folder=tmp_dir, + checksum_check=True, + fetch_files=_all_files_fetcher([psm_file, raw_file]), + ) + + assert stats.downloaded == 2 + assert stats.total_files == 2 + assert os.path.exists(os.path.join(tmp_dir, "PDC000109", "sample.psm")) + assert os.path.exists(os.path.join(tmp_dir, "PDC000109", "sample.raw"))