DM-55824: Add remote-test URI scheme

doc/changes/DM-55824.feature.rst

@@ -0,0 +1,2 @@
+Added a new ``remote-test`` URI schema that can be used in test code to to allow you to run tests where a local file URI is pretending to be a remote URI.
+This comes with a helper ``lsst.resources.tests.make_remote_test_uri`` to easily construct these URIs from a local file path.

python/lsst/resources/_resourcePath.py

@@ -417,6 +417,11 @@ def __new__(
                 from .eups import EupsResourcePath
 
                 subclass = EupsResourcePath
+            elif parsed.scheme == "remote-test":
+                # EUPS package root.
+                from .remote_test import RemoteTestResourcePath
+
+                subclass = RemoteTestResourcePath
             else:
                 raise NotImplementedError(
                     f"No URI support for scheme: '{parsed.scheme}' in {parsed.geturl()}"

python/lsst/resources/file.py

@@ -222,6 +222,17 @@ def transfer_from(
                 transfer,
             )
 
+        # Short circuit if the URIs are identical. The inode comparison below
+        # only runs for a local source, so a non-local source that happens to
+        # name this same resource would otherwise be reported as a clash.
+        if self == src:
+            log.debug(
+                "Target and destination URIs are identical: %s, returning immediately."
+                " No further action required.",
+                self,
+            )
+            return
+
         # The output location should not exist unless overwrite=True.
         # Rather than use `exists()`, use os.stat since we might need
         # the full answer later.
@@ -461,7 +472,13 @@ def walk(
             # Filter by the regex
             if file_filter is not None:
                 files = [f for f in files if file_filter.search(f)]
-            yield type(self)(root, forceAbsolute=False, forceDirectory=True), dirs, files
+            # Rebuild from the parsed URI rather than from the OS path, so
+            # that a subclass keeps its own scheme and netloc. Constructing
+            # from a plain path always resolves to a file URI.
+            path = os2posix(root)
+            if self.quotePaths:
+                path = urllib.parse.quote(path)
+            yield self.replace(path=path, forceDirectory=True), dirs, files
 
     @classmethod
     def _fixupPathUri(

python/lsst/resources/remote_test.py

@@ -0,0 +1,56 @@
+# This file is part of lsst-resources.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# Use of this source code is governed by a 3-clause BSD-style
+# license that can be found in the LICENSE file.
+from __future__ import annotations
+
+__all__ = ("RemoteTestResourcePath",)
+
+import contextlib
+import logging
+import shutil
+from collections.abc import Generator
+
+from ._resourcePath import ResourcePath
+from .file import FileResourcePath
+
+log = logging.getLogger(__name__)
+
+
+class RemoteTestResourcePath(FileResourcePath):
+    """A local file resource path that pretends it is a remote resource."""
+
+    # By definition a remote file.
+    isLocal = False
+
+    @contextlib.contextmanager
+    def _as_local(
+        self, multithreaded: bool = True, tmpdir: ResourcePath | None = None
+    ) -> Generator[ResourcePath, None, None]:
+        """Copy file to a new location in a temporary directory.
+
+        Parameters
+        ----------
+        multithreaded : `bool`, optional
+            Unused.
+        tmpdir : `ResourcePath` or `None`, optional
+            Explicit override of the temporary directory to use for remote
+            downloads.
+
+        Returns
+        -------
+        local_uri : `ResourcePath`
+            A URI to a local POSIX file corresponding to a local temporary
+            downloaded copy of the resource.
+        """
+        with (
+            ResourcePath.temporary_uri(prefix=tmpdir, suffix=self.getExtension(), delete=True) as tmp_uri,
+        ):
+            shutil.copy(self.ospath, tmp_uri.ospath)
+            yield tmp_uri

python/lsst/resources/tests.py

@@ -10,7 +10,7 @@
 # license that can be found in the LICENSE file.
 from __future__ import annotations
 
-__all__ = ["GenericReadWriteTestCase", "GenericTestCase"]
+__all__ = ["GenericReadWriteTestCase", "GenericTestCase", "make_remote_test_uri"]
 
 import datetime
 import logging
@@ -1142,3 +1142,35 @@ def test_mexists(self) -> None:
         # Clean up a subset of files that are already gone, but this can
         # trigger a different code path.
         ResourcePath.mremove(expected_uris[:5], do_raise=False)
+
+
+def make_remote_test_uri(path: str, *, forceDirectory: bool = True) -> ResourcePath:
+    """Return a ``remote-test`` URI corresponding to a local file system path.
+
+    Parameters
+    ----------
+    path : `str`
+        Local file system path to wrap.
+    forceDirectory : `bool`, optional
+        If `True`, the returned URI is treated as a directory.
+
+    Returns
+    -------
+    uri : `lsst.resources.ResourcePath`
+        URI using the ``remote-test`` scheme, which reports itself as not
+        local while being backed by ``path``.
+
+    Notes
+    -----
+    `~lsst.resources.ResourcePath` percent-encodes a path only when it is given
+    a schemeless path; a string that already includes a scheme is used
+    verbatim. The URI is therefore built from the encoded path of the
+    equivalent ``file`` URI. Interpolating ``path`` directly would leave any
+    space in the path unencoded, giving a malformed URI that.
+
+    When a ``file`` URI is constructed only the ``path`` is copied to the
+    new test URI. Any fragments are dropped (a ``#`` in the final component of
+    the  URI is always treated as a fragment by package convention).
+    """
+    file_uri = ResourcePath(path, forceDirectory=forceDirectory, forceAbsolute=True)
+    return ResourcePath(f"remote-test://localhost{file_uri.path}", forceDirectory=forceDirectory)

tests/test_remote_test.py

@@ -0,0 +1,107 @@
+# This file is part of lsst-resources.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (https://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# Use of this source code is governed by a 3-clause BSD-style
+# license that can be found in the LICENSE file.
+
+import os
+import unittest
+
+from lsst.resources import ResourcePath
+from lsst.resources.tests import GenericReadWriteTestCase, GenericTestCase, make_remote_test_uri
+from lsst.resources.utils import makeTestTempDir, removeTestTempDir
+
+TESTDIR = os.path.abspath(os.path.dirname(__file__))
+
+
+class RemoteTestTestCase(GenericTestCase, unittest.TestCase):
+    """File-specific generic test cases."""
+
+    scheme = "remote-test"
+    netloc = "localhost"
+
+
+class RemoteTestReadWriteTestCase(GenericReadWriteTestCase, unittest.TestCase):
+    """File tests involving reading and writing of data."""
+
+    scheme = "remote-test"
+    netloc = "localhost"
+    testdir = TESTDIR
+    # transfer_modes deliberately left at the remote default of copy/move.
+    # Link modes are refused for a non-local resource, which is the whole
+    # point of this scheme.
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        # This scheme is backed by the local file system, so the URI path has
+        # to name a real writable directory. The netloc is not used.
+        cls._tmproot = makeTestTempDir(TESTDIR)
+        cls.base_path = cls._tmproot
+        super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        removeTestTempDir(cls._tmproot)
+        super().tearDownClass()
+
+    def test_not_local(self) -> None:
+        """Test that the resource is not local and localizes to temp."""
+        test_file = self.root_uri.join("test_file.txt")
+        test_file.write(b"abc")
+
+        self.assertEqual(test_file.scheme, "remote-test")
+        self.assertEqual(test_file.read(), b"abc")
+        self.assertFalse(test_file.isLocal)
+
+        with test_file.as_local() as loc:
+            # Tests that ospath does not raise and that the "local" version
+            # of the file is at a different location.
+            self.assertNotEqual(loc.ospath, test_file.ospath)
+
+
+class RemoteTestUriTestCase(unittest.TestCase):
+    """Tests for the remote-test URI helper."""
+
+    def setUp(self):
+        self.root = makeTestTempDir(TESTDIR)
+
+    def tearDown(self):
+        removeTestTempDir(self.root)
+
+    def testScheme(self):
+        uri = make_remote_test_uri(self.root)
+        self.assertEqual(uri.scheme, "remote-test")
+        self.assertFalse(uri.isLocal)
+        self.assertEqual(uri.ospath.rstrip("/"), self.root)
+
+    def testSpecialCharacters(self):
+        awkward = os.path.join(self.root, "a dir with spaces")
+        os.makedirs(awkward)
+        uri = make_remote_test_uri(awkward)
+
+        # Interpolating the path into a string that already has a scheme
+        # leaves the spaces unencoded, so the helper has to encode them.
+        naive = ResourcePath(f"remote-test://localhost{awkward}/", forceDirectory=True)
+        self.assertIn(" ", naive.geturl())
+        self.assertNotIn(" ", uri.geturl())
+
+        self.assertEqual(uri.ospath.rstrip("/"), awkward)
+
+        child = uri.join("file.json", forceDirectory=False)
+        child.write(b"{}")
+        self.assertEqual(child.read(), b"{}")
+
+    def testFile(self):
+        uri = make_remote_test_uri(os.path.join(self.root, "a file.json"), forceDirectory=False)
+        self.assertFalse(uri.isdir())
+        self.assertIn("a%20file.json", uri.geturl())
+        self.assertTrue(uri.ospath.endswith("a file.json"))
+
+
+if __name__ == "__main__":
+    unittest.main()