From 34826c9d83506651f64746953dfd4036bc5231ca Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Thu, 13 Aug 2026 19:38:05 -0700 Subject: [PATCH 1/4] Fix file implementation to allow for subclasses --- python/lsst/resources/file.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/python/lsst/resources/file.py b/python/lsst/resources/file.py index 657b7bbc..5bef1b73 100644 --- a/python/lsst/resources/file.py +++ b/python/lsst/resources/file.py @@ -222,6 +222,17 @@ def transfer_from( transfer, ) + # Short circuit if the URIs are identical. The inode comparison below + # only runs for a local source, so a non-local source that happens to + # name this same resource would otherwise be reported as a clash. + if self == src: + log.debug( + "Target and destination URIs are identical: %s, returning immediately." + " No further action required.", + self, + ) + return + # The output location should not exist unless overwrite=True. # Rather than use `exists()`, use os.stat since we might need # the full answer later. @@ -461,7 +472,13 @@ def walk( # Filter by the regex if file_filter is not None: files = [f for f in files if file_filter.search(f)] - yield type(self)(root, forceAbsolute=False, forceDirectory=True), dirs, files + # Rebuild from the parsed URI rather than from the OS path, so + # that a subclass keeps its own scheme and netloc. Constructing + # from a plain path always resolves to a file URI. + path = os2posix(root) + if self.quotePaths: + path = urllib.parse.quote(path) + yield self.replace(path=path, forceDirectory=True), dirs, files @classmethod def _fixupPathUri( From 636c5aea29456bd90aae75b7773bc8b1638a9408 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Thu, 13 Aug 2026 19:37:35 -0700 Subject: [PATCH 2/4] Add new remote-test URI scheme for local files that pretend to be remote --- python/lsst/resources/_resourcePath.py | 5 ++ python/lsst/resources/remote_test.py | 56 +++++++++++++++++++++ tests/test_remote_test.py | 67 ++++++++++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 python/lsst/resources/remote_test.py create mode 100644 tests/test_remote_test.py diff --git a/python/lsst/resources/_resourcePath.py b/python/lsst/resources/_resourcePath.py index 8156ff13..692d8142 100644 --- a/python/lsst/resources/_resourcePath.py +++ b/python/lsst/resources/_resourcePath.py @@ -417,6 +417,11 @@ def __new__( from .eups import EupsResourcePath subclass = EupsResourcePath + elif parsed.scheme == "remote-test": + # EUPS package root. + from .remote_test import RemoteTestResourcePath + + subclass = RemoteTestResourcePath else: raise NotImplementedError( f"No URI support for scheme: '{parsed.scheme}' in {parsed.geturl()}" diff --git a/python/lsst/resources/remote_test.py b/python/lsst/resources/remote_test.py new file mode 100644 index 00000000..ac50cca2 --- /dev/null +++ b/python/lsst/resources/remote_test.py @@ -0,0 +1,56 @@ +# This file is part of lsst-resources. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# Use of this source code is governed by a 3-clause BSD-style +# license that can be found in the LICENSE file. +from __future__ import annotations + +__all__ = ("RemoteTestResourcePath",) + +import contextlib +import logging +import shutil +from collections.abc import Generator + +from ._resourcePath import ResourcePath +from .file import FileResourcePath + +log = logging.getLogger(__name__) + + +class RemoteTestResourcePath(FileResourcePath): + """A local file resource path that pretends it is a remote resource.""" + + # By definition a remote file. + isLocal = False + + @contextlib.contextmanager + def _as_local( + self, multithreaded: bool = True, tmpdir: ResourcePath | None = None + ) -> Generator[ResourcePath, None, None]: + """Copy file to a new location in a temporary directory. + + Parameters + ---------- + multithreaded : `bool`, optional + Unused. + tmpdir : `ResourcePath` or `None`, optional + Explicit override of the temporary directory to use for remote + downloads. + + Returns + ------- + local_uri : `ResourcePath` + A URI to a local POSIX file corresponding to a local temporary + downloaded copy of the resource. + """ + with ( + ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=True) as tmp_uri, + ): + shutil.copy(self.ospath, tmp_uri.ospath) + yield tmp_uri diff --git a/tests/test_remote_test.py b/tests/test_remote_test.py new file mode 100644 index 00000000..1bca60f4 --- /dev/null +++ b/tests/test_remote_test.py @@ -0,0 +1,67 @@ +# This file is part of lsst-resources. +# +# Developed for the LSST Data Management System. +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# Use of this source code is governed by a 3-clause BSD-style +# license that can be found in the LICENSE file. + +import os +import unittest + +from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase +from lsst.resources.utils import makeTestTempDir, removeTestTempDir + +TESTDIR = os.path.abspath(os.path.dirname(__file__)) + + +class RemoteTestTestCase(GenericTestCase, unittest.TestCase): + """File-specific generic test cases.""" + + scheme = "remote-test" + netloc = "localhost" + + +class RemoteTestReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase): + """File tests involving reading and writing of data.""" + + scheme = "remote-test" + netloc = "localhost" + testdir = TESTDIR + # transfer_modes deliberately left at the remote default of copy/move. + # Link modes are refused for a non-local resource, which is the whole + # point of this scheme. + + @classmethod + def setUpClass(cls) -> None: + # This scheme is backed by the local file system, so the URI path has + # to name a real writable directory. The netloc is not used. + cls._tmproot = makeTestTempDir(TESTDIR) + cls.base_path = cls._tmproot + super().setUpClass() + + @classmethod + def tearDownClass(cls) -> None: + removeTestTempDir(cls._tmproot) + super().tearDownClass() + + def test_not_local(self) -> None: + """Test that the resource is not local and localizes to temp.""" + test_file = self.root_uri.join("test_file.txt") + test_file.write(b"abc") + + self.assertEqual(test_file.scheme, "remote-test") + self.assertEqual(test_file.read(), b"abc") + self.assertFalse(test_file.isLocal) + + with test_file.as_local() as loc: + # Tests that ospath does not raise and that the "local" version + # of the file is at a different location. + self.assertNotEqual(loc.ospath, test_file.ospath) + + +if __name__ == "__main__": + unittest.main() From c246e8bab7871be767cf54d900592096fe791fe4 Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Fri, 14 Aug 2026 10:42:52 -0700 Subject: [PATCH 3/4] Add a test helper function to create a remote-test URI The key problem is that naively creating remote-test URI from an arbitrary path fails to do the correct URI escapes. --- python/lsst/resources/tests.py | 34 ++++++++++++++++++++++++++- tests/test_remote_test.py | 42 +++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/python/lsst/resources/tests.py b/python/lsst/resources/tests.py index b6c73fac..c489f42b 100644 --- a/python/lsst/resources/tests.py +++ b/python/lsst/resources/tests.py @@ -10,7 +10,7 @@ # license that can be found in the LICENSE file. from __future__ import annotations -__all__ = ["GenericReadWriteTestCase", "GenericTestCase"] +__all__ = ["GenericReadWriteTestCase", "GenericTestCase", "make_remote_test_uri"] import datetime import logging @@ -1142,3 +1142,35 @@ def test_mexists(self) -> None: # Clean up a subset of files that are already gone, but this can # trigger a different code path. ResourcePath.mremove(expected_uris[:5], do_raise=False) + + +def make_remote_test_uri(path: str, *, forceDirectory: bool = True) -> ResourcePath: + """Return a ``remote-test`` URI corresponding to a local file system path. + + Parameters + ---------- + path : `str` + Local file system path to wrap. + forceDirectory : `bool`, optional + If `True`, the returned URI is treated as a directory. + + Returns + ------- + uri : `lsst.resources.ResourcePath` + URI using the ``remote-test`` scheme, which reports itself as not + local while being backed by ``path``. + + Notes + ----- + `~lsst.resources.ResourcePath` percent-encodes a path only when it is given + a schemeless path; a string that already includes a scheme is used + verbatim. The URI is therefore built from the encoded path of the + equivalent ``file`` URI. Interpolating ``path`` directly would leave any + space in the path unencoded, giving a malformed URI that. + + When a ``file`` URI is constructed only the ``path`` is copied to the + new test URI. Any fragments are dropped (a ``#`` in the final component of + the URI is always treated as a fragment by package convention). + """ + file_uri = ResourcePath(path, forceDirectory=forceDirectory, forceAbsolute=True) + return ResourcePath(f"remote-test://localhost{file_uri.path}", forceDirectory=forceDirectory) diff --git a/tests/test_remote_test.py b/tests/test_remote_test.py index 1bca60f4..581d9c0e 100644 --- a/tests/test_remote_test.py +++ b/tests/test_remote_test.py @@ -12,7 +12,8 @@ import os import unittest -from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase +from lsst.resources import ResourcePath +from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase, make_remote_test_uri from lsst.resources.utils import makeTestTempDir, removeTestTempDir TESTDIR = os.path.abspath(os.path.dirname(__file__)) @@ -63,5 +64,44 @@ def test_not_local(self) -> None: self.assertNotEqual(loc.ospath, test_file.ospath) +class RemoteTestUriTestCase(unittest.TestCase): + """Tests for the remote-test URI helper.""" + + def setUp(self): + self.root = makeTestTempDir(TESTDIR) + + def tearDown(self): + removeTestTempDir(self.root) + + def testScheme(self): + uri = make_remote_test_uri(self.root) + self.assertEqual(uri.scheme, "remote-test") + self.assertFalse(uri.isLocal) + self.assertEqual(uri.ospath.rstrip("/"), self.root) + + def testSpecialCharacters(self): + awkward = os.path.join(self.root, "a dir with spaces") + os.makedirs(awkward) + uri = make_remote_test_uri(awkward) + + # Interpolating the path into a string that already has a scheme + # leaves the spaces unencoded, so the helper has to encode them. + naive = ResourcePath(f"remote-test://localhost{awkward}/", forceDirectory=True) + self.assertIn(" ", naive.geturl()) + self.assertNotIn(" ", uri.geturl()) + + self.assertEqual(uri.ospath.rstrip("/"), awkward) + + child = uri.join("file.json", forceDirectory=False) + child.write(b"{}") + self.assertEqual(child.read(), b"{}") + + def testFile(self): + uri = make_remote_test_uri(os.path.join(self.root, "a file.json"), forceDirectory=False) + self.assertFalse(uri.isdir()) + self.assertIn("a%20file.json", uri.geturl()) + self.assertTrue(uri.ospath.endswith("a file.json")) + + if __name__ == "__main__": unittest.main() From 0100b0dcc453ad4fedfad1fa8bc66ff728367d8b Mon Sep 17 00:00:00 2001 From: Tim Jenness Date: Fri, 14 Aug 2026 11:34:05 -0700 Subject: [PATCH 4/4] Add news fragment --- doc/changes/DM-55824.feature.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 doc/changes/DM-55824.feature.rst diff --git a/doc/changes/DM-55824.feature.rst b/doc/changes/DM-55824.feature.rst new file mode 100644 index 00000000..be8be337 --- /dev/null +++ b/doc/changes/DM-55824.feature.rst @@ -0,0 +1,2 @@ +Added a new ``remote-test`` URI schema that can be used in test code to to allow you to run tests where a local file URI is pretending to be a remote URI. +This comes with a helper ``lsst.resources.tests.make_remote_test_uri`` to easily construct these URIs from a local file path.