Enhance remote directory handling and optimise fetching

pydatalab/src/pydatalab/routes/v0_1/remotes.py

@@ -70,17 +70,24 @@ def list_remote_directories():
             400,
         )
 
+    meta_only = request.args.get("meta_only") in ("1", "true")
+
+    response: dict[str, Any] = {}
+    response["meta"] = {}
+    response["meta"]["remotes"] = [json.loads(d.json()) for d in CONFIG.REMOTE_FILESYSTEMS]
+
+    if meta_only:
+        return jsonify(response), 200
+
     all_directory_structures = get_directory_structures(
         CONFIG.REMOTE_FILESYSTEMS, invalidate_cache=invalidate_cache
     )
 
-    response = {}
-    response["meta"] = {}
-    response["meta"]["remotes"] = [json.loads(d.json()) for d in CONFIG.REMOTE_FILESYSTEMS]
     if all_directory_structures:
         oldest_update = min(d["last_updated"] for d in all_directory_structures)
         response["meta"]["oldest_cache_update"] = oldest_update.isoformat()
         response["data"] = all_directory_structures
+
     return jsonify(response), 200
 
 

webapp/src/components/FileSelectModal.vue

@@ -1,17 +1,7 @@
 <template>
   <div class="modal-enclosure">
     <Modal v-model="fileSelectModalIsOpen">
-      <template #header>
-        Select files to add
-        <button
-          :disabled="isLoadingRemoteTree"
-          class="ml-4 btn btn-small btn-default"
-          @click="fetchRemoteTree"
-        >
-          <font-awesome-icon v-show="isLoadingRemoteTree" :icon="['fa', 'sync']" class="fa-spin" />
-          Update tree
-        </button>
-      </template>
+      <template #header> Select files to add </template>
 
       <template #body>
         <SelectableFileTree
@@ -47,7 +37,7 @@
 import Modal from "@/components/Modal";
 import SelectableFileTree from "@/components/SelectableFileTree";
 
-import { fetchRemoteTree, addRemoteFileToSample } from "@/server_fetch_utils";
+import { fetchRemotesList, addRemoteFileToSample } from "@/server_fetch_utils";
 
 export default {
   components: {
@@ -64,12 +54,10 @@ export default {
     return {
       selectedRemoteFiles: [],
       isLoadingRemoteFiles: false,
+      hasLoadedRemotesList: false,
     };
   },
   computed: {
-    isLoadingRemoteTree() {
-      return this.$store.state.remoteDirectoryTreeIsLoading;
-    },
     // Change the text on the load files button based for singular/plural file uploads
     loadFilesButtonValue() {
       const len = this.selectedRemoteFiles.length;
@@ -91,33 +79,15 @@ export default {
       },
     },
   },
-  mounted() {
-    this.loadCachedTree();
-  },
-  methods: {
-    fetchRemoteTree: fetchRemoteTree, // imported directly from server_fetch_utils. Note: automatically sets and usets the remote tree loading status.
-    async loadCachedTree() {
-      var response_json = await fetchRemoteTree(false);
-      if (!response_json) {
-        return;
-      }
-      var oldest_cache_update = null;
-      var seconds_since_oldest_update = null;
-      if ("meta" in response_json) {
-        if ("oldest_cache_update" in response_json.meta) {
-          oldest_cache_update = new Date(response_json.meta.oldest_cache_update + "Z");
-          seconds_since_oldest_update = (new Date() - oldest_cache_update) / 1000;
-          console.log(
-            `loadCachedTree received, oldest dir update was ${seconds_since_oldest_update} s ago`,
-          );
-        }
-      }
-
-      if (seconds_since_oldest_update == null || seconds_since_oldest_update > 3600) {
-        console.log("cache is probably more than 1 hr out of date. Fetching new sample tree");
-        this.fetchRemoteTree(true);
+  watch: {
+    fileSelectModalIsOpen(isOpen) {
+      if (isOpen && !this.hasLoadedRemotesList) {
+        this.hasLoadedRemotesList = true;
+        fetchRemotesList();
       }
     },
+  },
+  methods: {
     async loadSelectedRemoteFiles() {
       this.isLoadingRemoteFiles = true;
       var promises = [];

webapp/src/components/SelectableFileTree.vue

@@ -13,6 +13,8 @@
           :search-term="searchTerm"
           @set-selected-entry="setSelectedEntry($event, toplevel.name)"
           @append-to-selected-entries="appendToSelectedEntries($event, toplevel.name)"
+          @toplevel-expand="handleToplevelExpand(toplevel.name)"
+          @toplevel-refresh="handleToplevelRefresh(toplevel.name)"
         />
       </div>
 
@@ -55,6 +57,7 @@
 
 <script>
 import TreeMenu from "@/components/TreeMenu.vue";
+import { fetchRemoteTree } from "@/server_fetch_utils";
 
 export default {
   components: {
@@ -75,7 +78,26 @@ export default {
   },
   computed: {
     remoteTree() {
-      return this.$store.state.remoteDirectoryTree;
+      const remotes = this.$store.state.remoteFilesystems || [];
+      const dirs = this.$store.state.remoteDirectories || {};
+      return remotes.map((remote) => {
+        const state = dirs[remote.name];
+        if (state && state.data) {
+          return {
+            ...state.data,
+            isLoading: !!state.isLoading,
+            error: state.error || null,
+          };
+        }
+        return {
+          name: remote.name,
+          type: "toplevel",
+          contents: [],
+          isLoading: !!(state && state.isLoading),
+          error: (state && state.error) || null,
+          isUnloaded: !(state && state.data),
+        };
+      });
     },
   },
   watch: {
@@ -117,6 +139,17 @@ export default {
         this.selectedEntries.push(entry);
       } else this.selectedEntries.splice(index, 1);
     },
+    handleToplevelExpand(remoteName) {
+      const state = this.$store.state.remoteDirectories[remoteName];
+      if (!state || (!state.data && !state.isLoading)) {
+        fetchRemoteTree(remoteName, false);
+      }
+    },
+    handleToplevelRefresh(remoteName) {
+      const state = this.$store.state.remoteDirectories[remoteName];
+      if (state && state.isLoading) return;
+      fetchRemoteTree(remoteName, true);
+    },
     unselectEntry(event, entry) {
       console.log("calling unselectEntry with");
       console.log(event);

webapp/src/components/TreeMenu.vue

@@ -1,10 +1,33 @@
 <template>
   <div class="tree-menu">
-    <div v-if="entry.type == 'toplevel'" class="my-2" @click="toggleChildren">
-      <span :class="{ expanded: showChildren }" class="directory-arrow">&#9656;</span>
-      <font-awesome-icon :icon="['fas', 'hdd']" class="toplevel-icon" />
-      <span class="toplevel-name" :class="{ 'search-match': searchMatched }">
-        {{ entry.name }}
+    <div v-if="entry.type == 'toplevel'" class="my-2 toplevel-row">
+      <span class="toplevel-click" @click="toggleToplevel">
+        <span :class="{ expanded: showChildren }" class="directory-arrow">&#9656;</span>
+        <font-awesome-icon :icon="['fas', 'hdd']" class="toplevel-icon" />
+        <span class="toplevel-name" :class="{ 'search-match': searchMatched }">
+          {{ entry.name }}
+        </span>
+      </span>
+      <font-awesome-icon
+        v-if="entry.isLoading"
+        :icon="['fa', 'sync']"
+        class="fa-spin ml-2 toplevel-status"
+      />
+      <button
+        v-else-if="!entry.isUnloaded"
+        type="button"
+        class="btn btn-link btn-sm toplevel-refresh"
+        title="Refresh this remote"
+        @click.stop="$emit('toplevelRefresh')"
+      >
+        <font-awesome-icon :icon="['fa', 'sync']" />
+      </button>
+      <span v-if="entry.error" class="ml-2 text-danger toplevel-error">
+        <font-awesome-icon :icon="['fas', 'exclamation-circle']" />
+        {{ entry.error }}
+      </span>
+      <span v-else-if="entry.isUnloaded && !entry.isLoading" class="ml-2 text-muted toplevel-hint">
+        (click to load)
       </span>
     </div>
     <div v-else-if="entry.type == 'directory'" @click="toggleChildren">
@@ -92,7 +115,7 @@ export default {
       required: true,
     },
   },
-  emits: ["setSelectedEntry", "appendToSelectedEntries"],
+  emits: ["setSelectedEntry", "appendToSelectedEntries", "toplevelExpand", "toplevelRefresh"],
   data() {
     return {
       showChildren: false,
@@ -117,6 +140,13 @@ export default {
     toggleChildren() {
       this.showChildren = !this.showChildren;
     },
+    toggleToplevel() {
+      const willExpand = !this.showChildren;
+      this.showChildren = willExpand;
+      if (willExpand) {
+        this.$emit("toplevelExpand");
+      }
+    },
     selectFile() {
       console.log("emmiting setSelectedEntry up the tree");
       this.$emit("setSelectedEntry", this.entry);

webapp/src/server_fetch_utils.js

@@ -1083,26 +1083,53 @@ export function deleteFileFromSample(item_id, file_id) {
     });
 }
 
-export async function fetchRemoteTree(invalidate_cache) {
+export async function fetchRemotesList() {
+  if (!(await waitForUserAuth())) return;
+
+  var url = new URL(`${API_URL}/list-remote-directories?meta_only=1`);
+  store.commit("setRemoteFilesystemsIsLoading", true);
+  return fetch_get(url)
+    .then(function (response_json) {
+      var remotes = (response_json && response_json.meta && response_json.meta.remotes) || [];
+      store.commit("setRemoteFilesystems", remotes);
+      store.commit("setRemoteFilesystemsIsLoading", false);
+      return response_json;
+    })
+    .catch((error) => {
+      DialogService.error({
+        title: "Remote filesystems retrieval failed",
+        message: `Error retrieving remote filesystems list from API: ${error}`,
+      });
+      store.commit("setRemoteFilesystemsIsLoading", false);
+      throw error;
+    });
+}
+
+export async function fetchRemoteTree(remoteName, invalidate_cache) {
   if (!(await waitForUserAuth())) return;
 
   var invalidate_cache_param = invalidate_cache ? "1" : "0";
   var url = new URL(
-    `${API_URL}/list-remote-directories?invalidate_cache=${invalidate_cache_param}`,
+    `${API_URL}/remotes/${encodeURIComponent(remoteName)}?invalidate_cache=${invalidate_cache_param}`,
   );
-  store.commit("setRemoteDirectoryTreeIsLoading", true);
+  store.commit("setRemoteDirectoryLoading", { name: remoteName, isLoading: true });
   return fetch_get(url)
     .then(function (response_json) {
-      store.commit("setRemoteDirectoryTree", response_json);
-      store.commit("setRemoteDirectoryTreeIsLoading", false);
+      var data = response_json && response_json.data;
+      var lastUpdated = data && data.last_updated ? data.last_updated : null;
+      store.commit("setRemoteDirectoryData", {
+        name: remoteName,
+        data: data,
+        lastUpdated: lastUpdated,
+      });
       return response_json;
     })
     .catch((error) => {
+      store.commit("setRemoteDirectoryError", { name: remoteName, error: String(error) });
       DialogService.error({
         title: "Remote directory retrieval failed",
-        message: `Error retrieving remote directory tree from API: ${error}`,
+        message: `Error retrieving remote directory ${remoteName}: ${error}`,
       });
-      store.commit("setRemoteDirectoryTreeIsLoading", false);
       throw error;
     });
 }

webapp/src/store/index.js

@@ -23,11 +23,11 @@ export default createStore({
     saved_status_collections: {},
     updating: {},
     updatingDelayed: {},
-    remoteDirectoryTree: {},
-    remoteDirectoryTreeSecondsSinceLastUpdate: null,
+    remoteFilesystems: [],
+    remoteDirectories: {},
     itemGraphData: null,
     itemGraphIsLoading: false,
-    remoteDirectoryTreeIsLoading: false,
+    remoteFilesystemsIsLoading: false,
     fileSelectModalIsOpen: false,
     currentUserDisplayName: null,
     currentUserID: null,
@@ -218,11 +218,29 @@ export default createStore({
         files.splice(index, 1);
       }
     },
-    setRemoteDirectoryTree(state, remoteDirectoryTree) {
-      state.remoteDirectoryTree = remoteDirectoryTree.data;
+    setRemoteFilesystems(state, remotes) {
+      state.remoteFilesystems = remotes;
     },
-    setRemoteDirectoryTreeSecondsSinceLastUpdate(state, secondsSinceLastUpdate) {
-      state.remoteDirectoryTreeSecondsSinceLastUpdate = secondsSinceLastUpdate;
+    setRemoteDirectoryLoading(state, { name, isLoading }) {
+      const existing = state.remoteDirectories[name] || {};
+      state.remoteDirectories = {
+        ...state.remoteDirectories,
+        [name]: { ...existing, isLoading },
+      };
+    },
+    setRemoteDirectoryData(state, { name, data, lastUpdated }) {
+      const existing = state.remoteDirectories[name] || {};
+      state.remoteDirectories = {
+        ...state.remoteDirectories,
+        [name]: { ...existing, data, lastUpdated, error: null, isLoading: false },
+      };
+    },
+    setRemoteDirectoryError(state, { name, error }) {
+      const existing = state.remoteDirectories[name] || {};
+      state.remoteDirectories = {
+        ...state.remoteDirectories,
+        [name]: { ...existing, error, isLoading: false },
+      };
     },
     addABlock(state, { item_id, new_block_obj, new_block_insert_index }) {
       // payload: item_id, new_block_obj, new_display_order
@@ -347,8 +365,8 @@ export default createStore({
     setFileSelectModalOpenStatus(state, isOpen) {
       state.fileSelectModalIsOpen = isOpen;
     },
-    setRemoteDirectoryTreeIsLoading(state, isLoading) {
-      state.remoteDirectoryTreeIsLoading = isLoading;
+    setRemoteFilesystemsIsLoading(state, isLoading) {
+      state.remoteFilesystemsIsLoading = isLoading;
     },
     setItemGraph(state, payload) {
       state.itemGraphData = payload;