Skip to content

DRS changes for _list_object() and DownloadManager.download() #304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
paulineribeyre merged 23 commits into from
Jun 10, 2026
Merged

DRS changes for _list_object() and DownloadManager.download() #304

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
ad189ee
DRS changes for _list_object() and DownloadManager.download()
krishnaa05 Jun 10, 2026
85381d4
Apply automatic documentation changes
krishnaa05 Jun 10, 2026
c206e3a
Update test_download.py
krishnaa05 Jun 10, 2026
2e04461
Update test_download.py
krishnaa05 Jun 10, 2026
88ee99c
Update drs_download.py
krishnaa05 Jun 10, 2026
4386bf8
Apply automatic documentation changes
krishnaa05 Jun 10, 2026
422de93
Update drs_download.py
krishnaa05 Jun 10, 2026
1eb1c3b
Apply automatic documentation changes
krishnaa05 Jun 10, 2026
24ac045
Update drs_download.py
krishnaa05 Jun 10, 2026
a43db9a
Update drs_download.py
krishnaa05 Jun 10, 2026
8d5ec0a
Apply automatic documentation changes
krishnaa05 Jun 10, 2026
5259625
Update drs_download.py
krishnaa05 Jun 10, 2026
9f5b5ca
Apply automatic documentation changes
krishnaa05 Jun 10, 2026
5e3fddc
Update drs_download.py
krishnaa05 Jun 10, 2026
46010cc
Update drs_download.py
krishnaa05 Jun 10, 2026
441cfc5
Apply automatic documentation changes
krishnaa05 Jun 10, 2026
e9120a4
Dummy Commit
krishnaa05 Jun 10, 2026
b757bdb
Update drs_download.py
krishnaa05 Jun 10, 2026
875fab6
Apply automatic documentation changes
krishnaa05 Jun 10, 2026
4056d83
Update drs_download.py
krishnaa05 Jun 10, 2026
9f28df4
Apply automatic documentation changes
krishnaa05 Jun 10, 2026
514c183
Update drs_download.py
krishnaa05 Jun 10, 2026
9d3e4cf
Apply automatic documentation changes
krishnaa05 Jun 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file modified docs/_build/doctrees/tools/drs_pull.doctree
View file
Open in desktop
Binary file not shown.
17 changes: 12 additions & 5 deletions docs/_build/html/_modules/gen3/tools/download/drs_download.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,7 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
<span class="n">status</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">&quot;pending&quot;</span>
<span class="n">start_time</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">end_time</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">datetime</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">status_code</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span>

<span class="k">def</span><span class="w"> </span><span class="fm">__str__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="k">return</span> <span class="p">(</span>
Expand Down Expand Up @@ -773,7 +774,7 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">

<span class="k">def</span><span class="w"> </span><span class="nf">get_download_url_using_drs</span><span class="p">(</span>
<span class="n">drs_hostname</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">object_id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">access_method</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">access_token</span><span class="p">:</span> <span class="nb">str</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">],</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
<span class="w"> </span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Returns the presigned URL for a DRS object, from a DRS hostname, via the access method</span>
<span class="sd"> Args:</span>
Expand All @@ -784,6 +785,7 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">

<span class="sd"> Returns:</span>
<span class="sd"> presigned url to object</span>
<span class="sd"> status code</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">headers</span> <span class="o">=</span> <span class="p">{</span>
<span class="s2">&quot;Content-Type&quot;</span><span class="p">:</span> <span class="s2">&quot;application/json&quot;</span><span class="p">,</span>
Expand All @@ -797,15 +799,15 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
<span class="p">)</span>
<span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
<span class="n">data</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<span class="k">return</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;url&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">)</span>

<span class="k">return</span> <span class="n">data</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="s2">&quot;url&quot;</span><span class="p">,</span> <span class="kc">None</span><span class="p">),</span> <span class="n">response</span><span class="o">.</span><span class="n">status_code</span>
<span class="k">except</span> <span class="n">requests</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">Timeout</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span><span class="sa">f</span><span class="s2">&quot;Was unable to download: </span><span class="si">{</span><span class="n">object_id</span><span class="si">}</span><span class="s2">. Timeout Error.&quot;</span><span class="p">)</span>
<span class="k">except</span> <span class="n">requests</span><span class="o">.</span><span class="n">exceptions</span><span class="o">.</span><span class="n">HTTPError</span> <span class="k">as</span> <span class="n">exc</span><span class="p">:</span>
<span class="n">logger</span><span class="o">.</span><span class="n">critical</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;HTTP Error (</span><span class="si">{</span><span class="n">exc</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span><span class="si">}</span><span class="s2">) when requesting download url from </span><span class="si">{</span><span class="n">access_method</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="p">)</span>
<span class="k">return</span> <span class="kc">None</span>
<span class="k">return</span> <span class="kc">None</span><span class="p">,</span> <span class="n">exc</span><span class="o">.</span><span class="n">response</span><span class="o">.</span><span class="n">status_code</span>
<span class="k">return</span> <span class="kc">None</span><span class="p">,</span> <span class="kc">None</span>


<span class="k">def</span><span class="w"> </span><span class="nf">get_user_auth</span><span class="p">(</span><span class="n">commons_url</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">access_token</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Optional</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]:</span>
Expand Down Expand Up @@ -1106,14 +1108,16 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
<span class="k">continue</span>
<span class="n">access_method</span> <span class="o">=</span> <span class="n">entry</span><span class="o">.</span><span class="n">access_methods</span><span class="p">[</span><span class="mi">0</span><span class="p">][</span><span class="s2">&quot;access_id&quot;</span><span class="p">]</span>

<span class="n">download_url</span> <span class="o">=</span> <span class="n">get_download_url_using_drs</span><span class="p">(</span>
<span class="n">download_url</span><span class="p">,</span> <span class="n">status_code</span> <span class="o">=</span> <span class="n">get_download_url_using_drs</span><span class="p">(</span>
<span class="n">drs_hostname</span><span class="p">,</span>
<span class="n">entry</span><span class="o">.</span><span class="n">object_id</span><span class="p">,</span>
<span class="n">access_method</span><span class="p">,</span>
<span class="n">access_token</span><span class="p">,</span>
<span class="p">)</span>

<span class="k">if</span> <span class="n">download_url</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<span class="k">if</span> <span class="n">status_code</span> <span class="o">!=</span> <span class="mi">200</span><span class="p">:</span>
<span class="n">completed</span><span class="p">[</span><span class="n">entry</span><span class="o">.</span><span class="n">object_id</span><span class="p">]</span><span class="o">.</span><span class="n">status_code</span> <span class="o">=</span> <span class="n">status_code</span>
<span class="n">completed</span><span class="p">[</span><span class="n">entry</span><span class="o">.</span><span class="n">object_id</span><span class="p">]</span><span class="o">.</span><span class="n">status</span> <span class="o">=</span> <span class="s2">&quot;error&quot;</span>
<span class="k">continue</span>

Expand Down Expand Up @@ -1351,6 +1355,9 @@ <h1>Source code for gen3.tools.download.drs_download</h1><div class="highlight">
<span class="n">hostname</span><span class="o">=</span><span class="n">hostname</span><span class="p">,</span> <span class="n">auth</span><span class="o">=</span><span class="n">auth</span><span class="p">,</span> <span class="n">download_list</span><span class="o">=</span><span class="n">object_list</span><span class="p">,</span> <span class="n">show_progress</span><span class="o">=</span><span class="kc">False</span>
<span class="p">)</span>

<span class="k">if</span> <span class="ow">not</span> <span class="n">object_list</span><span class="p">:</span>
<span class="k">return</span> <span class="kc">False</span>

<span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">object_list</span><span class="p">:</span>
<span class="nb">print</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">pprint</span><span class="p">())</span>

Expand Down
2 changes: 1 addition & 1 deletion docs/_build/html/searchindex.js
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion docs/_build/html/tools/drs_pull.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@

<dl class="py class">
<dt class="sig sig-object py" id="gen3.tools.download.drs_download.DownloadStatus">
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">gen3.tools.download.drs_download.</span></span><span class="sig-name descname"><span class="pre">DownloadStatus</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">filename</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">status</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'pending'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">start_time</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datetime</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">end_time</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datetime</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/download/drs_download.html#DownloadStatus"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.download.drs_download.DownloadStatus" title="Link to this definition">¶</a></dt>
<span class="property"><span class="k"><span class="pre">class</span></span><span class="w"> </span></span><span class="sig-prename descclassname"><span class="pre">gen3.tools.download.drs_download.</span></span><span class="sig-name descname"><span class="pre">DownloadStatus</span></span><span class="sig-paren">(</span><em class="sig-param"><span class="n"><span class="pre">filename</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">status</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">str</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">'pending'</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">start_time</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datetime</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">end_time</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">datetime</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em>, <em class="sig-param"><span class="n"><span class="pre">status_code</span></span><span class="p"><span class="pre">:</span></span><span class="w"> </span><span class="n"><span class="pre">int</span><span class="w"> </span><span class="p"><span class="pre">|</span></span><span class="w"> </span><span class="pre">None</span></span><span class="w"> </span><span class="o"><span class="pre">=</span></span><span class="w"> </span><span class="default_value"><span class="pre">None</span></span></em><span class="sig-paren">)</span><a class="reference internal" href="../_modules/gen3/tools/download/drs_download.html#DownloadStatus"><span class="viewcode-link"><span class="pre">[source]</span></span></a><a class="headerlink" href="#gen3.tools.download.drs_download.DownloadStatus" title="Link to this definition">¶</a></dt>
<dd><p>Stores the download status of objectIDs.</p>
<p>The DataManager will return a list of DownloadStatus as a result of calling the download method</p>
<p>Status is “pending” until it is downloaded or an error occurs.
Expand Down
17 changes: 12 additions & 5 deletions gen3/tools/download/drs_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,6 +279,7 @@ class DownloadStatus:
status: str = "pending"
start_time: Optional[datetime] = None
end_time: Optional[datetime] = None
status_code: Optional[int] = None

def __str__(self):
return (
Expand Down Expand Up @@ -716,7 +717,7 @@ def ensure_dirpath_exists(path: Path) -> Path:

def get_download_url_using_drs(
drs_hostname: str, object_id: str, access_method: str, access_token: str
) -> Optional[str]:
) -> Tuple[Optional[int], Optional[str]]:
"""
Returns the presigned URL for a DRS object, from a DRS hostname, via the access method
Args:
Expand All @@ -727,6 +728,7 @@ def get_download_url_using_drs(

Returns:
presigned url to object
status code
"""
headers = {
"Content-Type": "application/json",
Expand All @@ -740,15 +742,15 @@ def get_download_url_using_drs(
)
response.raise_for_status()
data = response.json()
return data.get("url", None)

return data.get("url", None), response.status_code
except requests.exceptions.Timeout:
logger.critical(f"Was unable to download: {object_id}. Timeout Error.")
except requests.exceptions.HTTPError as exc:
logger.critical(
f"HTTP Error ({exc.response.status_code}) when requesting download url from {access_method}"
)
return None
return None, exc.response.status_code
return None, None


def get_user_auth(commons_url: str, access_token: str) -> Optional[List[str]]:
Expand Down Expand Up @@ -1036,14 +1038,16 @@ def download(
continue
access_method = entry.access_methods[0]["access_id"]

download_url = get_download_url_using_drs(
download_url, status_code = get_download_url_using_drs(
drs_hostname,
entry.object_id,
access_method,
access_token,
)

if download_url is None:
if status_code != 200:
completed[entry.object_id].status_code = status_code
completed[entry.object_id].status = "error"
continue

Expand Down Expand Up @@ -1276,6 +1280,9 @@ def _list_object(hostname, auth, object_id: str) -> bool:
hostname=hostname, auth=auth, download_list=object_list, show_progress=False
)

if not object_list:
return False

for x in object_list:
print(x.pprint())

Expand Down
14 changes: 6 additions & 8 deletions tests/download_tests/test_download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -348,10 +348,9 @@ def test_download_file_from_url_failures(download_dir):
text="1234567890",
status_code=500,
)
assert (
get_download_url_using_drs("test.commons1.io", "blah", "s3", "bad token")
is None
)
assert get_download_url_using_drs(
"test.commons1.io", "blah", "s3", "bad token"
) == (None, 500)

assert (
download_file_from_url(
Expand All @@ -366,10 +365,9 @@ def test_download_file_from_url_failures(download_dir):
url=f"https://test.commons1.io/ga4gh/drs/v1/objects/blah/access/s3",
exc=requests.exceptions.Timeout,
)
assert (
get_download_url_using_drs("test.commons1.io", "blah", "s3", "bad token")
is None
)
assert get_download_url_using_drs(
"test.commons1.io", "blah", "s3", "bad token"
) == (None, None)

try:
download_file_from_url(
Expand Down
Loading