Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions doc/changes/DM-55646.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
add a list of sites exempted from using davix move and add RAL to it
121 changes: 83 additions & 38 deletions python/lsst/resources/davutils.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@
# Use the same logger than `dav.py`.
log = getLogger(f"""{__name__.replace(".davutils", ".dav")}""")

move_exempted_sites = ["webdav.echo.stfc.ac.uk"]


def normalize_path(path: str | None) -> str:
"""Normalize a path intended to be part of a URL.
Expand Down Expand Up @@ -2170,20 +2172,33 @@ def write(self, url: str, data: BinaryIO | bytes) -> int | None:
# exist before we can write to it. So create it first and then
# upload.
self.mkcol(self._parent(url))

try:
# Upload to a temporary file and rename to the final name.
temporary_url = self._make_temporary_url(url)
size = self.put(temporary_url, data=data)
self.rename(temporary_url, url, overwrite=True, create_parent=False)

# Update the file size cache with this size
self._file_size_cache.update_size(url, size)
return size
except Exception:
# Upload failed. Attempt to remove the temporary file.
self.delete(temporary_url)
raise
# upload directly if the site is in the list of sites that
# are exempted from the move operation
if any(site in url for site in move_exempted_sites):
try:
self.delete(url)
except Exception:
pass
try:
size = self.put(url, data=data)
return size
except Exception:
self.delete(url)
raise
else:
try:
# Upload to a temporary file and rename to the final name.
temporary_url = self._make_temporary_url(url)
size = self.put(temporary_url, data=data)
self.rename(temporary_url, url, overwrite=True, create_parent=False)

# Update the file size cache with this size
self._file_size_cache.update_size(url, size)
return size
except Exception:
# Upload failed. Attempt to remove the temporary file.
self.delete(temporary_url)
raise

def checksums(self, url: str) -> dict[str, str]:
"""Return the checksums of the contents of file located at `url`.
Expand Down Expand Up @@ -2936,19 +2951,34 @@ def write(self, url: str, data: BinaryIO | bytes) -> int | None:
# to RFC 4918, this is advantageous because it avoids several
# round-trips to the server for creating all the directories
# before actually uploading the data.
try:
# Upload to a temporary file and rename to the final name.
temporary_url = self._make_temporary_url(url)
size = self.put(temporary_url, data=data)
self.rename(temporary_url, url, overwrite=True, create_parent=False)

# Update the file size cache with this size
self._file_size_cache.update_size(url, size)
return size
except Exception:
# Upload failed. Attempt to remove the temporary file.
self.delete(temporary_url)
raise
# upload directly if the site is in the list of sites that
# are exempted from the move operation
if any(site in url for site in move_exempted_sites):
try:
self.delete(url)
except Exception:
pass
try:
size = self.put(url, data=data)
return size
except Exception:
self.delete(url)
raise
else:
try:
# Upload to a temporary file and rename to the final name.
temporary_url = self._make_temporary_url(url)
size = self.put(temporary_url, data=data)
self.rename(temporary_url, url, overwrite=True, create_parent=False)

# Update the file size cache with this size
self._file_size_cache.update_size(url, size)
return size
except Exception:
# Upload failed. Attempt to remove the temporary file.
self.delete(temporary_url)
raise

@override
def mkcol(self, url: str) -> None:
Expand Down Expand Up @@ -3178,19 +3208,34 @@ def write(self, url: str, data: BinaryIO | bytes) -> int | None:
# to RFC 4918, this is advantageous because it avoids several
# round-trips to the server for creating all the directories
# before actually uploading the data.
try:
# Upload to a temporary file and rename to the final name.
temporary_url = self._make_temporary_url(url)
size = self.put(temporary_url, data=data)
self.rename(temporary_url, url, overwrite=True, create_parent=False)

# Update the file size cache with this size
self._file_size_cache.update_size(url, size)
return size
except Exception:
# Upload failed. Attempt to remove the temporary file.
self.delete(temporary_url)
raise
# upload directly if the site is in the list of sites that
# are exempted from the move operation
if any(site in url for site in move_exempted_sites):
try:
self.delete(url)
except Exception:
pass
try:
size = self.put(url, data=data)
return size
except Exception:
self.delete(url)
raise
else:
try:
# Upload to a temporary file and rename to the final name.
temporary_url = self._make_temporary_url(url)
size = self.put(temporary_url, data=data)
self.rename(temporary_url, url, overwrite=True, create_parent=False)

# Update the file size cache with this size
self._file_size_cache.update_size(url, size)
return size
except Exception:
# Upload failed. Attempt to remove the temporary file.
self.delete(temporary_url)
raise

@override
def mkcol(self, url: str) -> None:
Expand Down