Refactor virtual drive sync engine

.eslintrc.js

@@ -13,7 +13,7 @@ module.exports = {
     },
   ],
   rules: {
-    'no-await-in-loop': 'warn',
+    'no-await-in-loop': 'off',
     '@typescript-eslint/no-use-before-define': ['warn', { functions: false, classes: true, variables: true }],
     'array-callback-return': 'warn',
     'max-len': [

src/apps/drive/dependency-injection/offline-drive/registerTemporalFilesServices.ts

@@ -9,12 +9,15 @@ import { TemporalFileByFolderFinder } from '../../../../context/storage/Temporal
 import { TemporalFileByPathFinder } from '../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder';
 import { TemporalFilePathsByFolderFinder } from '../../../../context/storage/TemporalFiles/application/find/TemporalFilePathsByFolderFinder';
 import { TemporalFileTruncater } from '../../../../context/storage/TemporalFiles/application/truncate/TemporalFileTruncater';
+import { createTemporalFileUploadQueueService } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/create-temporal-file-upload-queue-service';
+import { TemporalFileUploadQueue } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types';
 import { TemporalFileUploader } from '../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploader';
 import { TemporalFileWriter } from '../../../../context/storage/TemporalFiles/application/write/TemporalFileWriter';
 import { TemporalFileRepository } from '../../../../context/storage/TemporalFiles/domain/TemporalFileRepository';
 import { TemporalFileUploaderFactory } from '../../../../context/storage/TemporalFiles/domain/upload/TemporalFileUploaderFactory';
 import { NodeTemporalFileRepository } from '../../../../context/storage/TemporalFiles/infrastructure/NodeTemporalFileRepository';
 import { EnvironmentTemporalFileUploaderFactory } from '../../../../context/storage/TemporalFiles/infrastructure/upload/EnvironmentTemporalFileUploaderFactory';
+import { FirstsFileSearcher } from '../../../../context/virtual-drive/files/application/search/FirstsFileSearcher';
 import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider';
 import { PATHS } from '../../../../core/electron/paths';
 
@@ -52,6 +55,17 @@ export async function registerTemporalFilesServices(builder: ContainerBuilder) {
   builder.registerAndUse(TemporalFileTruncater);
   builder.registerAndUse(TemporalFileByteByByteComparator);
   builder.registerAndUse(TemporalFileByFolderFinder);
+  builder
+    .register(TemporalFileUploadQueue)
+    .useFactory((container) =>
+      createTemporalFileUploadQueueService({
+        repository: container.get(TemporalFileRepository),
+        uploader: container.get(TemporalFileUploader),
+        deleter: container.get(TemporalFileDeleter),
+        fileSearcher: container.get(FirstsFileSearcher),
+      }),
+    )
+    .asSingleton();
 
   // Event handlers
   builder.registerAndUse(DeleteTemporalFileOnFileCreated).addTag('event-handler');

src/apps/drive/dependency-injection/virtual-drive/registerFilesServices.ts

@@ -25,7 +25,7 @@ import { Environment } from '@internxt/inxt-js';
 export async function registerFilesServices(builder: ContainerBuilder): Promise<void> {
   // Infra
 
-  builder.register(FileRepository).use(InMemoryFileRepository).asSingleton().private();
+  builder.register(FileRepository).use(InMemoryFileRepository).asSingleton();
 
   const user = DependencyInjectionUserProvider.get();
 

src/apps/drive/dependency-injection/virtual-drive/registerFolderServices.ts

@@ -27,7 +27,7 @@ export async function registerFolderServices(builder: ContainerBuilder): Promise
   builder.register(SyncFolderMessenger).use(MainProcessSyncFolderMessenger);
   // TODO: can be private?
 
-  builder.register(FolderRepository).use(InMemoryFolderRepository).asSingleton().private();
+  builder.register(FolderRepository).use(InMemoryFolderRepository).asSingleton();
 
   builder.register(RemoteFileSystem).use(HttpRemoteFileSystem).private();
 

src/apps/drive/dependency-injection/virtual-drive/registerVirtualDriveSharedServices.ts

@@ -1,11 +1,35 @@
 import { ContainerBuilder } from 'diod';
 import { AbsolutePathToRelativeConverter } from '../../../../context/virtual-drive/shared/application/AbsolutePathToRelativeConverter';
 import { RelativePathToAbsoluteConverter } from '../../../../context/virtual-drive/shared/application/RelativePathToAbsoluteConverter';
+import { FileRepository } from '../../../../context/virtual-drive/files/domain/FileRepository';
+import { FolderRepository } from '../../../../context/virtual-drive/folders/domain/FolderRepository';
 import { PATHS } from '../../../../core/electron/paths';
+import {
+  createDirectoryStateSqliteRepository,
+  DirectoryStateSqliteRepository,
+} from '../../../../backend/features/virtual-drive/services/lazy/DirectoryStateSqliteRepository';
+import {
+  createLazyVirtualDriveHydratorService,
+  LazyVirtualDriveHydrator,
+} from '../../../../backend/features/virtual-drive/services/lazy/virtual-drive-hydrator/create-lazy-virtual-drive-hydrator-service';
 
 export async function registerVirtualDriveSharedServices(builder: ContainerBuilder): Promise<void> {
   const downloaded = PATHS.DOWNLOADED;
 
   builder.register(RelativePathToAbsoluteConverter).useFactory(() => new RelativePathToAbsoluteConverter(downloaded));
   builder.register(AbsolutePathToRelativeConverter).useFactory(() => new AbsolutePathToRelativeConverter(downloaded));
+  builder
+    .register(DirectoryStateSqliteRepository)
+    .useFactory(() => createDirectoryStateSqliteRepository())
+    .asSingleton();
+  builder
+    .register(LazyVirtualDriveHydrator)
+    .useFactory((container) =>
+      createLazyVirtualDriveHydratorService({
+        folderRepository: container.get(FolderRepository),
+        fileRepository: container.get(FileRepository),
+        directoryStateRepository: container.get(DirectoryStateSqliteRepository),
+      }),
+    )
+    .asSingleton();
 }

src/apps/main/database/data-source.test.ts

@@ -1,15 +1,17 @@
 import { call, calls, partialSpyOn } from 'tests/vitest/utils.helper';
-import { AppDataSource, resetAppDataSourceOnLogout } from './data-source';
+import { AppDataSource, initializeVirtualDriveSqlite, resetAppDataSourceOnLogout } from './data-source';
 
 describe('data-source', () => {
   const dropDatabaseMock = partialSpyOn(AppDataSource, 'dropDatabase');
   const destroyMock = partialSpyOn(AppDataSource, 'destroy');
+  const queryMock = partialSpyOn(AppDataSource, 'query');
   const isInitializedMock = vi.spyOn(AppDataSource, 'isInitialized', 'get');
 
   beforeEach(() => {
     isInitializedMock.mockReturnValue(true);
     dropDatabaseMock.mockResolvedValue(undefined);
     destroyMock.mockResolvedValue(undefined);
+    queryMock.mockResolvedValue(undefined);
   });
 
   it('should drop the database before destroying the connection on logout', async () => {
@@ -36,4 +38,26 @@ describe('data-source', () => {
 
     call(destroyMock).toStrictEqual([]);
   });
+
+  it('should run the sqlite bootstrap statements when initialized', async () => {
+    await initializeVirtualDriveSqlite();
+
+    expect(queryMock).toHaveBeenCalledTimes(12);
+  });
+
+  it('should skip the sqlite bootstrap when the datasource is not initialized', async () => {
+    isInitializedMock.mockReturnValue(false);
+
+    await initializeVirtualDriveSqlite();
+
+    calls(queryMock).toHaveLength(0);
+  });
+
+  it('should continue bootstrapping even if one statement fails', async () => {
+    queryMock.mockRejectedValueOnce(new Error('boom')).mockResolvedValue(undefined);
+
+    await initializeVirtualDriveSqlite();
+
+    expect(queryMock).toHaveBeenCalledTimes(12);
+  });
 });

src/apps/main/database/data-source.ts

@@ -6,6 +6,28 @@ import { TypeOrmStorageFile } from '../../../context/storage/StorageFiles/infras
 import { logger } from '@internxt/drive-desktop-core/build/backend';
 import { PATHS } from '../../../core/electron/paths';
 
+const SQLITE_BOOTSTRAP_STATEMENTS = [
+  'PRAGMA journal_mode=WAL;',
+  'PRAGMA synchronous=NORMAL;',
+  'PRAGMA temp_store=MEMORY;',
+  'PRAGMA foreign_keys=ON;',
+  'PRAGMA busy_timeout=5000;',
+  'PRAGMA wal_autocheckpoint=1000;',
+  'PRAGMA mmap_size=268435456;',
+  `CREATE TABLE IF NOT EXISTS drive_directory_state (
+      folder_id INTEGER NOT NULL,
+      status_scope TEXT NOT NULL,
+      children_loaded_at TEXT NULL,
+      children_last_error_at TEXT NULL,
+      fetch_state TEXT NOT NULL DEFAULT 'idle',
+      PRIMARY KEY (folder_id, status_scope)
+    );`,
+  'CREATE INDEX IF NOT EXISTS idx_drive_file_folder_status_updated ON drive_file(folderId, status, updatedAt);',
+  'CREATE INDEX IF NOT EXISTS idx_drive_file_status_updated ON drive_file(status, updatedAt);',
+  'CREATE INDEX IF NOT EXISTS idx_drive_folder_parent_status_updated ON drive_folder(parentId, status, updatedAt);',
+  'CREATE INDEX IF NOT EXISTS idx_drive_folder_status_updated ON drive_folder(status, updatedAt);',
+];
+
 export const AppDataSource = new DataSource({
   type: 'better-sqlite3',
   database: PATHS.DATABASE,
@@ -16,6 +38,20 @@ export const AppDataSource = new DataSource({
 
 logger.debug({ msg: `Using database file at ${PATHS.DATABASE}` });
 
+export async function initializeVirtualDriveSqlite() {
+  if (!AppDataSource.isInitialized) {
+    return;
+  }
+
+  for (const statement of SQLITE_BOOTSTRAP_STATEMENTS) {
+    try {
+      await AppDataSource.query(statement);
+    } catch (error) {
+      logger.error({ msg: 'Error running SQLite bootstrap statement', statement, error });
+    }
+  }
+}
+
 export async function resetAppDataSourceOnLogout() {
   if (!AppDataSource.isInitialized) {
     return;

src/apps/main/remote-sync/RemoteSyncManager.test.ts

@@ -158,6 +158,7 @@ describe('RemoteSyncManager', () => {
       await sut.startRemoteSync();
 
       expect(mockedGet).toBeCalledTimes(2);
+      expect(mockedGet.mock.calls[0]?.[1]).toMatchObject({ query: { status: 'EXISTS', limit: 2 } });
       expect(sut.getSyncStatus()).toBe('SYNCED');
     });
     it('Should sync all the folders', async () => {
@@ -207,6 +208,7 @@ describe('RemoteSyncManager', () => {
       await sut.startRemoteSync();
 
       expect(mockedGet).toBeCalledTimes(3);
+      expect(mockedGet.mock.calls[0]?.[1]).toMatchObject({ query: { status: 'EXISTS', limit: 2 } });
       expect(sut.getSyncStatus()).toBe('SYNCED');
     });
 
@@ -252,7 +254,7 @@ describe('RemoteSyncManager', () => {
 
       expect(mockedGet).toBeCalledTimes(6);
       expect(sut.getSyncStatus()).toBe('SYNC_FAILED');
-    });
+    }, 15000);
 
     it('Should fail the sync if some files or folders cannot be retrieved', async () => {
       mockedGet.mockResolvedValueOnce({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') });
@@ -261,7 +263,7 @@ describe('RemoteSyncManager', () => {
 
       expect(mockedGet).toBeCalledTimes(6);
       expect(sut.getSyncStatus()).toBe('SYNC_FAILED');
-    });
+    }, 15000);
 
     it('should handle the error while syncing files by calling the error handler properly', async () => {
       const sut = new RemoteSyncManager(

src/apps/main/remote-sync/RemoteSyncManager.ts

@@ -93,20 +93,19 @@ export class RemoteSyncManager {
     logger.debug({ tag: 'SYNC-ENGINE', msg: 'Starting' });
     this.changeStatus('SYNCING');
     try {
-      await Promise.all([
-        this.config.syncFiles
-          ? this.syncRemoteFiles({
-              retry: 1,
-              maxRetries: 3,
-            })
-          : Promise.resolve(),
-        this.config.syncFolders
-          ? this.syncRemoteFolders({
-              retry: 1,
-              maxRetries: 3,
-            })
-          : Promise.resolve(),
-      ]);
+      if (this.config.syncFolders) {
+        await this.syncRemoteFolders({
+          retry: 1,
+          maxRetries: 3,
+        });
+      }
+
+      if (this.config.syncFiles) {
+        await this.syncRemoteFiles({
+          retry: 1,
+          maxRetries: 3,
+        });
+      }
     } catch (error) {
       this.changeStatus('SYNC_FAILED');
       logger.error({
@@ -333,7 +332,7 @@ export class RemoteSyncManager {
     const { data, error } = await fetchFiles({
       limit: this.config.fetchFilesLimitPerRequest,
       offset: 0,
-      status: 'ALL',
+      status: 'EXISTS',
       updatedAt: updatedAtCheckpoint?.toISOString(),
     });
 
@@ -357,9 +356,9 @@ export class RemoteSyncManager {
     result: RemoteSyncedFolder[];
   }> {
     const { data, error } = await fetchFolders({
-      limit: this.config.fetchFilesLimitPerRequest,
+      limit: this.config.fetchFoldersLimitPerRequest,
       offset: 0,
-      status: 'ALL',
+      status: 'EXISTS',
       updatedAt: updatedAtCheckpoint?.toISOString(),
     });
 

src/apps/main/remote-sync/service.ts

@@ -19,8 +19,8 @@ export const remoteSyncManager = new RemoteSyncManager(
     folders: driveFoldersCollection,
   },
   {
-    fetchFilesLimitPerRequest: 1000,
-    fetchFoldersLimitPerRequest: 1000,
+    fetchFilesLimitPerRequest: 250,
+    fetchFoldersLimitPerRequest: 500,
     syncFiles: true,
     syncFolders: true,
   },

src/backend/features/user/file-size-limit/add-max-file-size-rejection.test.ts

@@ -1,11 +1,10 @@
+import { addMaxFileSizeRejection, clearMaxFileSizeRejectionModal } from './add-max-file-size-rejection';
+import * as modalModule from './show-max-file-size-rejection-modal';
 import {
-  addMaxFileSizeRejection,
-  clearMaxFileSizeRejectionModal,
   clearUploadSizeLimitBlockedPath,
   isUploadSizeLimitBlockedPath,
   markUploadSizeLimitBlockedPath,
-} from './add-max-file-size-rejection';
-import * as modalModule from './show-max-file-size-rejection-modal';
+} from './upload-size-limit-blocked-paths';
 
 describe('addMaxFileSizeRejection', () => {
   const showModalMock = vi.spyOn(modalModule, 'showMaxFileSizeRejectionModal');

src/backend/features/user/file-size-limit/add-max-file-size-rejection.ts

@@ -1,8 +1,7 @@
 import { MODAL_DEBOUNCE_MS } from './constants';
 import { showMaxFileSizeRejectionModal } from './show-max-file-size-rejection-modal';
 import { type UploadFileSizeValidation } from './validate-upload-file-size';
-
-const uploadSizeLimitBlockedPaths = new Set<string>();
+import { markUploadSizeLimitBlockedPath } from './upload-size-limit-blocked-paths';
 
 type MaxFileSizeRejectionModalState = {
   rejectedFilesCount: number;
@@ -23,18 +22,6 @@ type MaxFileSizeRejectionModalDraft = Omit<MaxFileSizeRejectionModalState, 'time
 
 let state: MaxFileSizeRejectionModalState | undefined;
 
-export function markUploadSizeLimitBlockedPath(path: string): void {
-  uploadSizeLimitBlockedPaths.add(path);
-}
-
-export function isUploadSizeLimitBlockedPath(path: string): boolean {
-  return uploadSizeLimitBlockedPaths.has(path);
-}
-
-export function clearUploadSizeLimitBlockedPath(path: string): void {
-  uploadSizeLimitBlockedPaths.delete(path);
-}
-
 export function addMaxFileSizeRejection(rejection: MaxFileSizeRejection): void {
   if (rejection.blockUploadPath ?? true) {
     markUploadSizeLimitBlockedPath(rejection.path);

src/backend/features/user/file-size-limit/index.ts

@@ -1,10 +1,9 @@
+export { addMaxFileSizeRejection, clearMaxFileSizeRejectionModal } from './add-max-file-size-rejection';
 export {
-  addMaxFileSizeRejection,
-  clearMaxFileSizeRejectionModal,
   clearUploadSizeLimitBlockedPath,
   isUploadSizeLimitBlockedPath,
   markUploadSizeLimitBlockedPath,
-} from './add-max-file-size-rejection';
+} from './upload-size-limit-blocked-paths';
 export { ABSOLUTE_UPLOAD_FILE_SIZE_LIMIT } from './constants';
 export { calculateProjectedWriteSize } from './calculate-projected-write-size';
 export { preserveRejectedFileSizeTooBig } from './rejected-file-size-too-big/preserve-rejected-file-size-too-big';

src/backend/features/user/file-size-limit/upload-size-limit-blocked-paths.ts

@@ -0,0 +1,13 @@
+const uploadSizeLimitBlockedPaths = new Set<string>();
+
+export function markUploadSizeLimitBlockedPath(path: string): void {
+  uploadSizeLimitBlockedPaths.add(path);
+}
+
+export function isUploadSizeLimitBlockedPath(path: string): boolean {
+  return uploadSizeLimitBlockedPaths.has(path);
+}
+
+export function clearUploadSizeLimitBlockedPath(path: string): void {
+  uploadSizeLimitBlockedPaths.delete(path);
+}

src/backend/features/virtual-drive/controllers/operations/release.controller.test.ts

@@ -6,6 +6,9 @@ import * as releaseServiceModule from '../../services/operations/release.service
 import { partialSpyOn } from '../../../../../../tests/vitest/utils.helper';
 import { FuseError } from '../../../../../apps/drive/fuse/callbacks/FuseErrors';
 import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes';
+import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder';
+import { TemporalFileDeleter } from '../../../../../context/storage/TemporalFiles/application/deletion/TemporalFileDeleter';
+import { TemporalFileUploadQueue } from '../../../../../context/storage/TemporalFiles/application/upload/TemporalFileUploadQueue/types';
 
 vi.mock(import('@internxt/drive-desktop-core/build/backend'));
 
@@ -19,6 +22,11 @@ describe('releaseController', () => {
     req = mockDeep<Request>();
     res = mockDeep<Response>();
     container = mockDeep<Container>();
+    container.get.calledWith(TemporalFileByPathFinder).mockReturnValue({ run: vi.fn() } as TemporalFileByPathFinder);
+    container.get.calledWith(TemporalFileDeleter).mockReturnValue({ run: vi.fn() } as TemporalFileDeleter);
+    container.get.calledWith(TemporalFileUploadQueue).mockReturnValue({
+      enqueue: vi.fn(),
+    } as unknown as { enqueue: (props: unknown) => Promise<void> });
   });
 
   it('should return errno 0 when release succeeds', async () => {