From f48b4e6f3b287ff41b2a83c5eb170e45ab5d1737 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Wed, 22 Apr 2026 23:00:47 +0100 Subject: [PATCH 1/3] Support meta_only query param on /list-remote-directories to skip directory scan --- pydatalab/src/pydatalab/routes/v0_1/remotes.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pydatalab/src/pydatalab/routes/v0_1/remotes.py b/pydatalab/src/pydatalab/routes/v0_1/remotes.py index be50502f9..2f4662092 100644 --- a/pydatalab/src/pydatalab/routes/v0_1/remotes.py +++ b/pydatalab/src/pydatalab/routes/v0_1/remotes.py @@ -70,17 +70,24 @@ def list_remote_directories(): 400, ) + meta_only = request.args.get("meta_only") in ("1", "true") + + response: dict[str, Any] = {} + response["meta"] = {} + response["meta"]["remotes"] = [json.loads(d.json()) for d in CONFIG.REMOTE_FILESYSTEMS] + + if meta_only: + return jsonify(response), 200 + all_directory_structures = get_directory_structures( CONFIG.REMOTE_FILESYSTEMS, invalidate_cache=invalidate_cache ) - response = {} - response["meta"] = {} - response["meta"]["remotes"] = [json.loads(d.json()) for d in CONFIG.REMOTE_FILESYSTEMS] if all_directory_structures: oldest_update = min(d["last_updated"] for d in all_directory_structures) response["meta"]["oldest_cache_update"] = oldest_update.isoformat() response["data"] = all_directory_structures + return jsonify(response), 200 From 75395ea54ad54abecb40ee1cb93214566699039b Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Wed, 22 Apr 2026 23:00:55 +0100 Subject: [PATCH 2/3] Split remote FS fetches per-remote in store and fetch helpers --- webapp/src/server_fetch_utils.js | 41 ++++++++++++++++++++++++++------ webapp/src/store/index.js | 36 +++++++++++++++++++++------- 2 files changed, 61 insertions(+), 16 deletions(-) diff --git a/webapp/src/server_fetch_utils.js b/webapp/src/server_fetch_utils.js index 3e245e2e2..4caca881c 100644 --- a/webapp/src/server_fetch_utils.js +++ b/webapp/src/server_fetch_utils.js @@ -1083,26 +1083,53 @@ export function deleteFileFromSample(item_id, file_id) { }); } -export async function fetchRemoteTree(invalidate_cache) { +export async function fetchRemotesList() { + if (!(await waitForUserAuth())) return; + + var url = new URL(`${API_URL}/list-remote-directories?meta_only=1`); + store.commit("setRemoteFilesystemsIsLoading", true); + return fetch_get(url) + .then(function (response_json) { + var remotes = (response_json && response_json.meta && response_json.meta.remotes) || []; + store.commit("setRemoteFilesystems", remotes); + store.commit("setRemoteFilesystemsIsLoading", false); + return response_json; + }) + .catch((error) => { + DialogService.error({ + title: "Remote filesystems retrieval failed", + message: `Error retrieving remote filesystems list from API: ${error}`, + }); + store.commit("setRemoteFilesystemsIsLoading", false); + throw error; + }); +} + +export async function fetchRemoteTree(remoteName, invalidate_cache) { if (!(await waitForUserAuth())) return; var invalidate_cache_param = invalidate_cache ? "1" : "0"; var url = new URL( - `${API_URL}/list-remote-directories?invalidate_cache=${invalidate_cache_param}`, + `${API_URL}/remotes/${encodeURIComponent(remoteName)}?invalidate_cache=${invalidate_cache_param}`, ); - store.commit("setRemoteDirectoryTreeIsLoading", true); + store.commit("setRemoteDirectoryLoading", { name: remoteName, isLoading: true }); return fetch_get(url) .then(function (response_json) { - store.commit("setRemoteDirectoryTree", response_json); - store.commit("setRemoteDirectoryTreeIsLoading", false); + var data = response_json && response_json.data; + var lastUpdated = data && data.last_updated ? data.last_updated : null; + store.commit("setRemoteDirectoryData", { + name: remoteName, + data: data, + lastUpdated: lastUpdated, + }); return response_json; }) .catch((error) => { + store.commit("setRemoteDirectoryError", { name: remoteName, error: String(error) }); DialogService.error({ title: "Remote directory retrieval failed", - message: `Error retrieving remote directory tree from API: ${error}`, + message: `Error retrieving remote directory ${remoteName}: ${error}`, }); - store.commit("setRemoteDirectoryTreeIsLoading", false); throw error; }); } diff --git a/webapp/src/store/index.js b/webapp/src/store/index.js index 210e78b46..6d798df3c 100644 --- a/webapp/src/store/index.js +++ b/webapp/src/store/index.js @@ -23,11 +23,11 @@ export default createStore({ saved_status_collections: {}, updating: {}, updatingDelayed: {}, - remoteDirectoryTree: {}, - remoteDirectoryTreeSecondsSinceLastUpdate: null, + remoteFilesystems: [], + remoteDirectories: {}, itemGraphData: null, itemGraphIsLoading: false, - remoteDirectoryTreeIsLoading: false, + remoteFilesystemsIsLoading: false, fileSelectModalIsOpen: false, currentUserDisplayName: null, currentUserID: null, @@ -218,11 +218,29 @@ export default createStore({ files.splice(index, 1); } }, - setRemoteDirectoryTree(state, remoteDirectoryTree) { - state.remoteDirectoryTree = remoteDirectoryTree.data; + setRemoteFilesystems(state, remotes) { + state.remoteFilesystems = remotes; }, - setRemoteDirectoryTreeSecondsSinceLastUpdate(state, secondsSinceLastUpdate) { - state.remoteDirectoryTreeSecondsSinceLastUpdate = secondsSinceLastUpdate; + setRemoteDirectoryLoading(state, { name, isLoading }) { + const existing = state.remoteDirectories[name] || {}; + state.remoteDirectories = { + ...state.remoteDirectories, + [name]: { ...existing, isLoading }, + }; + }, + setRemoteDirectoryData(state, { name, data, lastUpdated }) { + const existing = state.remoteDirectories[name] || {}; + state.remoteDirectories = { + ...state.remoteDirectories, + [name]: { ...existing, data, lastUpdated, error: null, isLoading: false }, + }; + }, + setRemoteDirectoryError(state, { name, error }) { + const existing = state.remoteDirectories[name] || {}; + state.remoteDirectories = { + ...state.remoteDirectories, + [name]: { ...existing, error, isLoading: false }, + }; }, addABlock(state, { item_id, new_block_obj, new_block_insert_index }) { // payload: item_id, new_block_obj, new_display_order @@ -347,8 +365,8 @@ export default createStore({ setFileSelectModalOpenStatus(state, isOpen) { state.fileSelectModalIsOpen = isOpen; }, - setRemoteDirectoryTreeIsLoading(state, isLoading) { - state.remoteDirectoryTreeIsLoading = isLoading; + setRemoteFilesystemsIsLoading(state, isLoading) { + state.remoteFilesystemsIsLoading = isLoading; }, setItemGraph(state, payload) { state.itemGraphData = payload; From 96750d70d0259ee2efb5735c3717da94b7bcbfb9 Mon Sep 17 00:00:00 2001 From: Matthew Evans Date: Wed, 22 Apr 2026 23:01:06 +0100 Subject: [PATCH 3/3] Load remote FS tree lazily on modal open and on per-remote expansion --- webapp/src/components/FileSelectModal.vue | 50 ++++---------------- webapp/src/components/SelectableFileTree.vue | 35 +++++++++++++- webapp/src/components/TreeMenu.vue | 42 +++++++++++++--- 3 files changed, 80 insertions(+), 47 deletions(-) diff --git a/webapp/src/components/FileSelectModal.vue b/webapp/src/components/FileSelectModal.vue index 515e18d66..25c5d33da 100644 --- a/webapp/src/components/FileSelectModal.vue +++ b/webapp/src/components/FileSelectModal.vue @@ -1,17 +1,7 @@