Skip to content

CPTAC download implementation #110

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ypriverol merged 7 commits into from
Jun 8, 2026
Merged

CPTAC download implementation #110

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
07397fc
feat(download): add -t/--threads multi-segment download, replace -w
Shen-YuFei May 20, 2026
85799af
Merge branch 'dev' of https://github.com/Shen-YuFei/pridepy into dev
Shen-YuFei May 20, 2026
44be652
fix(massive/ftp): emit progress while walking the FTPS tree
ypriverol May 30, 2026
51340f5
Merge pull request #108 from PRIDE-Archive/feat/ftps-walk-progress
ypriverol May 30, 2026
f9cb6e6
Merge remote-tracking branch 'origin/dev' into dev
Shen-YuFei Jun 6, 2026
78877f9
feat(download): add PDC downloads and restore threads option
Shen-YuFei Jun 6, 2026
42d965b
Merge pull request #109 from Shen-YuFei/dev
ypriverol Jun 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@ pridepy --help
| `download-file-by-name` | Download a single file (public or private) |
| `download-files-by-list` | Download a named subset of files from a manifest/CSV |
| `download-files-by-url` | Download files from raw `http`/`https`/`ftp` URLs |
| `download-pdc-files` | Download PDC/CPTAC files via PDC signed HTTPS URLs |
| `download-px-raw-files` | Download RAW files resolved from a ProteomeXchange accession |
| `list-private-files` | List files of a private project (needs credentials) |
| `stream-files-metadata` | Stream file metadata (one project or all) to JSON |
Expand All @@ -118,6 +119,9 @@ pridepy download-px-raw-files -a PXD039236 -o ./downloads/PXD039236

# Download a native MassIVE / JPOST / iProX dataset
pridepy download-all-public-raw-files -a MSV000082297 -o ./downloads/MSV000082297

# Download PDC/CPTAC files for a study (single type, or per-row types via CSV)
pridepy download-pdc-files -a PDC000109 --file-type psm -o ./downloads/pdc
```

Full option tables and more examples are in [docs/usage.md](docs/usage.md).
Expand Down
8 changes: 8 additions & 0 deletions pridepy/download/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ def download_all_raw(
checksum_check: bool = False,
parallel_files: int = 1,
flatten: bool = True,
download_threads: int = 1,
) -> None:
"""Download all RAW files for the dataset."""
self.download_files(
Expand All @@ -137,6 +138,7 @@ def download_all_raw(
checksum_check=checksum_check,
aspera_maximum_bandwidth=aspera_maximum_bandwidth,
flatten=flatten,
download_threads=download_threads,
)

def download_category(
Expand All @@ -150,6 +152,7 @@ def download_category(
checksum_check: bool = False,
parallel_files: int = 1,
flatten: bool = True,
download_threads: int = 1,
) -> None:
"""Download all files of the given categories for the dataset."""
self.download_files(
Expand All @@ -162,6 +165,7 @@ def download_category(
checksum_check=checksum_check,
aspera_maximum_bandwidth=aspera_maximum_bandwidth,
flatten=flatten,
download_threads=download_threads,
)

def download_by_name(
Expand Down Expand Up @@ -203,6 +207,7 @@ def download_by_filenames(
checksum_check: bool = False,
parallel_files: int = 1,
flatten: bool = True,
download_threads: int = 1,
) -> None:
"""Download a subset of project files identified by a filename list.

Expand Down Expand Up @@ -232,6 +237,7 @@ def download_by_filenames(
checksum_check=checksum_check,
aspera_maximum_bandwidth=aspera_maximum_bandwidth,
flatten=flatten,
download_threads=download_threads,
)

# ------------------------------------------------------------------
Expand All @@ -251,6 +257,7 @@ def download_files(
username: Optional[str] = None,
password: Optional[str] = None,
flatten: bool = True,
download_threads: int = 1,
) -> None:
"""Partition record URLs by scheme and route to the matching transport.

Expand Down Expand Up @@ -322,4 +329,5 @@ def download_files(
skip_if_downloaded_already=skip_if_downloaded_already,
parallel_files=parallel_files,
relative_paths=http_relpaths,
download_threads=download_threads,
)
73 changes: 43 additions & 30 deletions pridepy/download/by_url.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,13 @@ def _callback(data: bytes) -> None:
)


def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: int = 0) -> None:
def _dispatch_url_scheme(
parsed,
target: str,
protocol: str = "ftp",
position: int = 0,
download_threads: int = 1,
) -> None:
"""Route a parsed URL to its protocol-specific downloader.

``protocol='globus'`` swaps the http/https single-connection streamer
Expand All @@ -96,7 +102,11 @@ def _dispatch_url_scheme(parsed, target: str, protocol: str = "ftp", position: i
"""
scheme = (parsed.scheme or "").lower()
if scheme in ("http", "https"):
if protocol == "globus":
if download_threads and download_threads > 1:
transport._multipart_download(
parsed.geturl(), target, threads=download_threads, position=position
)
elif protocol == "globus":
transport._parallel_download(parsed.geturl(), target, position=position)
else:
_http_download_url(parsed.geturl(), target)
Expand All @@ -112,6 +122,7 @@ def _download_single_url(
skip_if_exists: bool = False,
protocol: str = "ftp",
position: int = 0,
download_threads: int = 1,
) -> str:
"""Download one URL, dispatched by scheme; return the local file path."""
parsed = urlparse(url)
Expand All @@ -128,7 +139,13 @@ def _download_single_url(
return target

try:
_dispatch_url_scheme(parsed, target, protocol, position=position)
_dispatch_url_scheme(
parsed,
target,
protocol,
position=position,
download_threads=download_threads,
)
except Exception:
# Don't leave a truncated/partial file behind — a non-empty partial
# would otherwise be wrongly skipped on the next run.
Expand All @@ -149,6 +166,7 @@ def download_files_by_url(
protocol: str = "ftp",
parallel_files: int = 1,
checksum_check: bool = False,
download_threads: int = 1,
) -> None:
"""Download files from a list of raw URLs, dispatched by URL scheme.

Expand All @@ -173,39 +191,34 @@ def download_files_by_url(

os.makedirs(output_folder, exist_ok=True)

parallel_files = min(parallel_files, 3, len(urls))
workers = min(parallel_files, 3, len(urls))
failures: List[Tuple[str, str]] = []

if parallel_files < 2:
for url in urls:
if workers > 1:
logging.info(
"Downloading %d URL(s) with %d parallel workers",
len(urls), workers,
)
with ThreadPoolExecutor(max_workers=workers) as executor:
Comment on lines +194 to +202

@coderabbitai coderabbitai Bot Jun 6, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor | ⚡ Quick win

Clamp parallel_files before constructing the executor.

This path now always creates a ThreadPoolExecutor, so parallel_files=0 becomes max_workers=0 and fails immediately. download_http_urls() in pridepy/download/transport.py already uses max(1, min(...)); matching that here preserves the old serial fallback instead of turning a non-positive input into a runtime error.

Suggested fix 
-    workers = min(parallel_files, 3, len(urls))
+    workers = max(1, min(parallel_files, 3, len(urls)))
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
workers = min(parallel_files, 3, len(urls))
failures: List[Tuple[str, str]] = []
if parallel_files < 2:
for url in urls:
if workers > 1:
logging.info(
"Downloading %d URL(s) with %d parallel workers",
len(urls), workers,
)
with ThreadPoolExecutor(max_workers=workers) as executor:
workers = max(1, min(parallel_files, 3, len(urls)))
failures: List[Tuple[str, str]] = []
if workers > 1:
logging.info(
"Downloading %d URL(s) with %d parallel workers",
len(urls), workers,
)
with ThreadPoolExecutor(max_workers=workers) as executor:
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@pridepy/download/by_url.py` around lines 194 - 202, The current computation
of workers can produce 0 when parallel_files is 0, causing
ThreadPoolExecutor(max_workers=workers) to raise; change the clamp to ensure at
least one worker by computing workers = max(1, min(parallel_files, 3,
len(urls))) (matching download_http_urls()), so the serial fallback remains when
parallel_files <= 0 before creating the ThreadPoolExecutor and keeping the
existing log/conditional behavior that checks workers > 1.

futures = {
executor.submit(
_download_single_url,
url,
output_folder,
skip_if_downloaded_already,
protocol,
position=idx,
download_threads=download_threads,
): url
for idx, url in enumerate(urls)
}
for future in as_completed(futures):
url = futures[future]
try:
_download_single_url(
url, output_folder, skip_if_downloaded_already, protocol,
)
future.result()
except Exception as exc: # pylint: disable=broad-except
logging.error("Failed to download %s: %s", url, exc)
failures.append((url, str(exc)))
else:
logging.info(
"Downloading %d URL(s) with %d parallel workers",
len(urls), parallel_files,
)
with ThreadPoolExecutor(max_workers=parallel_files) as executor:
futures = {
executor.submit(
_download_single_url,
url, output_folder, skip_if_downloaded_already, protocol,
position=idx,
): url
for idx, url in enumerate(urls)
}
for future in as_completed(futures):
url = futures[future]
try:
future.result()
except Exception as exc: # pylint: disable=broad-except
logging.error("Failed to download %s: %s", url, exc)
failures.append((url, str(exc)))

if failures:
summary = ", ".join(f"{u} ({e})" for u, e in failures)
Expand Down
10 changes: 10 additions & 0 deletions pridepy/download/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ def download_http_urls(
skip_if_downloaded_already: bool,
parallel_files: int = 1,
max_retries: int = 3,
download_threads: int = 1,
) -> None:
"""Shim — see :func:`pridepy.download.transport.download_http_urls`."""
return transport.download_http_urls(
Expand All @@ -114,6 +115,7 @@ def download_http_urls(
skip_if_downloaded_already=skip_if_downloaded_already,
parallel_files=parallel_files,
max_retries=max_retries,
download_threads=download_threads,
)

# Accession-matcher convenience helpers (useful public API).
Expand Down Expand Up @@ -187,6 +189,7 @@ def download_all_raw_files(
checksum_check: bool = False,
parallel_files: int = 1,
flatten: bool = True,
download_threads: int = 1,
):
"""Download all RAW files for any registered provider."""
return registry.resolve(accession).download_all_raw(
Expand All @@ -198,6 +201,7 @@ def download_all_raw_files(
checksum_check=checksum_check,
parallel_files=parallel_files,
flatten=flatten,
download_threads=download_threads,
)

def download_all_category_files(
Expand All @@ -212,6 +216,7 @@ def download_all_category_files(
category: str = None,
parallel_files: int = 1,
flatten: bool = True,
download_threads: int = 1,
):
"""Download all files of the given categories from a project."""
if categories is None:
Expand All @@ -226,6 +231,7 @@ def download_all_category_files(
checksum_check=checksum_check,
parallel_files=parallel_files,
flatten=flatten,
download_threads=download_threads,
)

def download_file_by_name(
Expand Down Expand Up @@ -269,6 +275,7 @@ def download_files_by_list(
checksum_check: bool = False,
parallel_files: int = 1,
flatten: bool = True,
download_threads: int = 1,
) -> None:
"""Download a subset of project files identified by a filename list."""
return registry.resolve(accession).download_by_filenames(
Expand All @@ -281,6 +288,7 @@ def download_files_by_list(
checksum_check=checksum_check,
parallel_files=parallel_files,
flatten=flatten,
download_threads=download_threads,
)

@staticmethod
Expand All @@ -291,6 +299,7 @@ def download_files_by_url(
protocol: str = "ftp",
parallel_files: int = 1,
checksum_check: bool = False,
download_threads: int = 1,
) -> None:
"""Delegate to :func:`pridepy.download.by_url.download_files_by_url`."""
return by_url.download_files_by_url(
Expand All @@ -300,6 +309,7 @@ def download_files_by_url(
protocol=protocol,
parallel_files=parallel_files,
checksum_check=checksum_check,
download_threads=download_threads,
)

def download_px_raw_files(
Expand Down
Loading
Loading