diff --git a/.vscodeignore b/.vscodeignore index 84bed7d3..18e7cf0d 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -4,6 +4,7 @@ src/** patches/** docs/** .gitignore +AGENTS.md .yarnrc **/tsconfig.json **/.eslintrc.json diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000..f21a557d --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,25 @@ +# Overleaf Workshop Local Fork + +## Local Replica Sync + +- Runtime local changes are event-driven through VS Code `FileSystemWatcher`; do not add periodic directory or hash polling. +- `.overleaf/sync-state.json` stores the remote history version and SHA-256 content baseline used only for startup/reconnect reconciliation. +- `.overleaf/settings.json` is the authoritative local-replica association. It must contain enough project URI and SCM settings metadata to rebuild the transient per-login project/SCM state after authentication expires. +- Batch startup state changes into one write and skip writes when serialized state is unchanged; do not rewrite the state file once per synchronized path. The file is a disposable cache, so write it in place to avoid delete/create watcher events. +- History API failures such as HTTP 429 must not be interpreted as a missing version. Leave the checkpoint unchanged and retry on a later reconnect. +- Reuse recent history updates for both version discovery and changed-path collection. Serialize unavoidable history requests and honor `Retry-After` instead of issuing immediate parallel probes. +- Route every HTTP request from every project through one process-wide queue. A 429 response pauses the entire queue for `Retry-After`; local watcher bursts are debounced before upload. +- A failed path must make incremental startup sync fail explicitly; never log completion or advance `remoteVersion` after partial failure. +- Report final user-actionable failures through a deduplicated VS Code notification with access to the Output log; individual retries remain log-only. +- Connection and SCM creation logs must include project ID, connection scheme, retry attempt, local base URI, and the original structured error. Never swallow `joinProject` or trigger-initialization errors behind a generic reconnecting message. +- Disposing a cached VFS is terminal: disconnect handlers must not schedule reconnects after disposal. Successful `Open Project Locally` creation must leave its provider-owned VFS alive. +- A background project used by `Open Project Locally` must not register workspace-global commands, views, status items, or compile actions. Those features belong only to the project identified by the active workspace; deterministic feature-registration failures must not enter the connection retry loop. +- Ignore every path containing a dot-prefixed component, including `.output`, before any stat/read/write work. +- Ignore symbolic links and paths below symbolic-link directories in both directions. Never upload them, overwrite/delete them during a pull, or include them in sync state. +- Never choose a winner or synthesize a merge when both sides changed. Pause that path, preserve both sides, notify the user, and require manual resolution followed by a window reload. Do not create conflict-copy files. +- Full remote-to-local sync is automatic only for an empty replica or when all local hashes still match the checkpoint. Missing/invalid history plus uncheckpointed local changes pauses all replica watchers. + +## Verification + +- Run `npm run compile`, `npm run lint`, and `git diff --check` after synchronization changes. +- Package local builds with `npx @vscode/vsce package --out overleaf-workshop-local-0.15.10.vsix` and install with `code --install-extension --force`. diff --git a/docs/anatomy.md b/docs/anatomy.md index 35a0dda4..25ac85b7 100644 --- a/docs/anatomy.md +++ b/docs/anatomy.md @@ -283,10 +283,11 @@ It also proxies commands for `class HistoryDataProvider` via `(get) triggers`. #### `src/scm/localReplicaSCM.ts` The exported `LocalReplicaSCMProvider` implements the `BaseSCM` interface and supports the ["Open Project Locally"](wiki.md#open-project-locally) feature. -Since the local filesystem keeps no version information, at the start of project open, it always apply `this.overwrite` to overwrite local changes with overleaf server version. -Therefore, if there is a network disturbance, your local changes or remote changes (by other collaborators) will be lost depending on `syncFromVFS` is called via `this.vfsWatcher` firstly, or `syncToVFS` is called via `this.localWatcher` firstly. +The provider persists the last synchronized Overleaf project version and SHA-256 hashes in `.overleaf/sync-state.json`. On project open or reconnect, it reuses recent project-history updates to determine both the current version and changed paths. A separate file-tree diff request is only made when those updates do not cover the stored checkpoint. All HTTP requests share one process-wide queue, and rate-limit responses pause that queue according to `Retry-After`. A full remote-to-local synchronization is allowed only when the local replica is empty or still matches its checkpoint. If history is unavailable while local files changed, or if the same path changed on both sides, synchronization pauses without modifying either side. -A smarter solution is proposed in `(private async) overwrite`, but is not applied in the `this.writeFile`. +The local workspace association is stored in `.overleaf/settings.json`, independently of authentication credentials. When a local replica is reopened after login expiry, the extension rebuilds its transient project/SCM entry from this file and reconnects after the user logs in again; the user does not need to repeat `Open Project Locally`. + +If the same text file changed locally and remotely, the existing `diff-match-patch` merge strategy is applied using the previous remote version as the base. If a reliable base cannot be retrieved, the remote version remains authoritative, matching the previous initialization behavior. #### `src/scm/localGitBridgeSCM.ts` > Not completed now. Target to provide local git bridge via [isomorphic-git](https://github.com/isomorphic-git/isomorphic-git) without local file system or git client binary needed. diff --git a/docs/wiki.md b/docs/wiki.md index 1e3df7d3..8399a805 100644 --- a/docs/wiki.md +++ b/docs/wiki.md @@ -323,6 +323,10 @@ In the Local Replica configuration, you can choose to enable/disable the Local R ![screenshot-config-local-replica](assets/screenshot-config-local-replica.png) +Changes made to files in the local replica by other applications are also detected and uploaded automatically, including while the VS Code window is in the background. Synchronization is driven by VS Code file-system events and does not require an editor save or window focus. Files matching the Local Replica ignore patterns are excluded. Synchronization diagnostics are available in the `Overleaf Workshop` channel in the Output view. + +After the first synchronization, `.overleaf/sync-state.json` records the last Overleaf project version and SHA-256 hashes of synchronized files. Subsequent project opens and reconnects reuse the latest history update response for both the current version and changed paths when possible, so unchanged file contents are not downloaded and an additional file-tree diff request is avoided. History requests are serialized and rate-limit responses honor `Retry-After`. State changes produced by startup reconciliation are batched into one write, unchanged state is not rewritten, and the disposable cache is written in place so it does not repeatedly appear as delete/create events. If synchronization ultimately fails, a deduplicated VS Code error notification links to the `Overleaf Workshop` output log. Local symbolic links and paths below symbolic-link directories are excluded from upload, download, deletion, and sync state. + The project-related metadata for local replica are located in `.overleaf/settings.json` in the following format: ```json { diff --git a/package.nls.json b/package.nls.json index 2838547c..0b0e76ea 100644 --- a/package.nls.json +++ b/package.nls.json @@ -69,4 +69,4 @@ "views.explorer.overleaf-workshop.chatWebview.contextualTitle": "Overleaf Chat", "customEditors.overleaf-workshop.pdfViewer.displayName": "Overleaf Workshop PDF Viewer" -} \ No newline at end of file +} diff --git a/src/api/base.ts b/src/api/base.ts index 0cc4230d..df65bf4e 100644 --- a/src/api/base.ts +++ b/src/api/base.ts @@ -1,9 +1,54 @@ /* eslint-disable @typescript-eslint/naming-convention */ -import * as stream from 'stream'; -import * as FormData from 'form-data'; +import { Blob } from 'buffer'; import { v4 as uuidv4 } from 'uuid'; -import { fetch } from 'undici'; +import { fetch, FormData } from 'undici'; import { FileEntity, FileType, FolderEntity, OutputFileEntity } from '../core/remoteFileSystemProvider'; +import { log } from '../utils/outputChannel'; + +// Overleaf rate limits are applied to the authenticated user, not to a single +// BaseAPI instance. Keep every HTTP request in one process-wide queue so +// several projects cannot collectively trigger a burst of 429 responses. +let globalRequestChain: Promise = Promise.resolve(); +let globalNextRequestAt = 0; +let globalRateLimitedUntil = 0; +const GLOBAL_MIN_REQUEST_INTERVAL_MS = 300; + +function retryAfterHeaderMs(response: any): number { + const retryAfter = response.headers?.get?.('retry-after'); + if (retryAfter!==undefined && retryAfter!==null) { + const seconds = Number(retryAfter); + if (Number.isFinite(seconds)) { + return Math.max(1000, seconds*1000); + } + const retryAt = Date.parse(retryAfter); + if (Number.isFinite(retryAt)) { return Math.max(1000, retryAt-Date.now()); } + } + return 5000; +} + +async function queuedFetch(url: string, init: any): Promise { + const previous = globalRequestChain; + let release!: () => void; + globalRequestChain = new Promise(resolve => { release = resolve; }); + await previous; + try { + const delayMs = Math.max( + 0, + globalNextRequestAt-Date.now(), + globalRateLimitedUntil-Date.now(), + ); + if (delayMs>0) { await new Promise(resolve => setTimeout(resolve, delayMs)); } + globalNextRequestAt = Date.now() + GLOBAL_MIN_REQUEST_INTERVAL_MS; + const response = await fetch(url, init); + if (response.status===429) { + globalRateLimitedUntil = Math.max(globalRateLimitedUntil, Date.now() + retryAfterHeaderMs(response)); + log(`Global HTTP request queue entered rate-limit cooldown until ${new Date(globalRateLimitedUntil).toISOString()}.`); + } + return response; + } finally { + release(); + } +} /** Extract set-cookie headers from an undici/Response object. */ function getSetCookie(res: any): string[] { @@ -11,7 +56,9 @@ function getSetCookie(res: any): string[] { return res.headers.getSetCookie(); } const raw = res.headers?.raw?.()?.['set-cookie']; - if (raw) return raw; + if (raw) { + return raw; + } return []; } @@ -177,6 +224,7 @@ export interface ProjectSettingsSchema { export interface ResponseSchema { type: 'success' | 'error'; + statusCode?: number; raw?: ArrayBuffer; message?: string; userInfo?: {userId:string, userEmail:string}; @@ -202,13 +250,51 @@ export interface ResponseSchema { export class BaseAPI { private url: string; private identity?: Identity; + private historyRequestChain: Promise = Promise.resolve(); + private lastHistoryRequestAt = 0; + + private isHistoryRoute(route: string): boolean { + return route.includes('/updates?') || route.includes('/filetree/diff?') || route.includes('/diff?'); + } + + private async waitForHistoryRequest() { + const previous = this.historyRequestChain; + let release!: () => void; + this.historyRequestChain = new Promise(resolve => { release = resolve; }); + await previous; + try { + const minimumIntervalMs = 3000; + const delayMs = Math.max(0, minimumIntervalMs - (Date.now() - this.lastHistoryRequestAt)); + if (delayMs>0) { + await new Promise(resolve => setTimeout(resolve, delayMs)); + } + this.lastHistoryRequestAt = Date.now(); + } finally { + release(); + } + } + + private retryAfterMs(response: any, attempt: number): number { + const retryAfter = response.headers?.get?.('retry-after'); + if (retryAfter!==undefined && retryAfter!==null) { + const seconds = Number(retryAfter); + if (Number.isFinite(seconds)) { + return Math.max(1000, seconds*1000); + } + const retryAt = Date.parse(retryAfter); + if (Number.isFinite(retryAt)) { + return Math.max(1000, retryAt-Date.now()); + } + } + return 5000 * Math.pow(2, attempt); + } constructor(url:string) { this.url = url; } private async getCsrfToken(): Promise { - const res = await fetch(this.url+'login', { + const res = await queuedFetch(this.url+'login', { method: 'GET', redirect: 'manual', }); const body = await res.text(); @@ -223,7 +309,7 @@ export class BaseAPI { } private async getUserId(cookies:string) { - const res = await fetch(this.url+'project', { + const res = await queuedFetch(this.url+'project', { method: 'GET', redirect:'manual', headers: { 'Connection': 'keep-alive', @@ -267,7 +353,7 @@ export class BaseAPI { async passportLogin(email:string, password:string): Promise { const identity = await this.getCsrfToken(); - const res = await fetch(this.url+'login', { + const res = await queuedFetch(this.url+'login', { method: 'POST', redirect: 'manual', headers: { 'Accept': '*/*', @@ -329,7 +415,7 @@ export class BaseAPI { } async updateCookies(identity: Identity) { - const res = await fetch(this.url + 'socket.io/socket.io.js', { + const res = await queuedFetch(this.url + 'socket.io/socket.io.js', { method: 'GET', redirect: 'manual', headers: { @@ -383,10 +469,13 @@ export class BaseAPI { for (let attempt = 0; attempt <= MAX_HTTP_RETRIES; attempt++) { try { + if (this.isHistoryRoute(route)) { + await this.waitForHistoryRequest(); + } let res = undefined; switch(type) { case 'GET': - res = await fetch(this.url+route, { + res = await queuedFetch(this.url+route, { method: 'GET', redirect: 'manual', headers: { 'Connection': 'keep-alive', @@ -401,7 +490,7 @@ export class BaseAPI { _csrf: this.identity!.csrfToken, ...body }); - res = await fetch(this.url+route, { + res = await queuedFetch(this.url+route, { method: 'POST', redirect: 'manual', headers: { 'Connection': 'keep-alive', @@ -415,7 +504,7 @@ export class BaseAPI { case 'PUT': break; case 'DELETE': - res = await fetch(this.url+route, { + res = await queuedFetch(this.url+route, { method: 'DELETE', redirect: 'manual', headers: { 'Connection': 'keep-alive', @@ -436,8 +525,8 @@ export class BaseAPI { } as ResponseSchema; } else if (res && this.isTransientError(res.status) && attempt < MAX_HTTP_RETRIES) { // Transient error: retry with backoff - const delayMs = Math.min(1000 * Math.pow(2, attempt), 4000); - console.log(`HTTP ${res.status} on ${route}, retrying in ${delayMs}ms (attempt ${attempt + 1}/${MAX_HTTP_RETRIES})`); + const delayMs = res.status===429 ? this.retryAfterMs(res, attempt) : Math.min(1000 * Math.pow(2, attempt), 4000); + log(`HTTP ${res.status} on ${route}, retrying in ${delayMs}ms (attempt ${attempt + 1}/${MAX_HTTP_RETRIES})`); lastError = {statusCode: res.status, message: await res.text().catch(() => '')}; await new Promise(r => setTimeout(r, delayMs)); continue; @@ -447,6 +536,7 @@ export class BaseAPI { try { errorBody = await resOrFallback.text(); } catch { errorBody = ''; } return { type: 'error', + statusCode: typeof resOrFallback.status==='number' ? resOrFallback.status : undefined, message: `${resOrFallback.status}: ${errorBody}` }; } @@ -454,12 +544,13 @@ export class BaseAPI { const errMsg = err?.message || String(err); if (this.isTransientError(undefined, errMsg) && attempt < MAX_HTTP_RETRIES) { const delayMs = Math.min(1000 * Math.pow(2, attempt), 4000); - console.log(`HTTP fetch error on ${route}: ${errMsg}, retrying in ${delayMs}ms (attempt ${attempt + 1}/${MAX_HTTP_RETRIES})`); + log(`HTTP fetch error on ${route}: ${errMsg}, retrying in ${delayMs}ms (attempt ${attempt + 1}/${MAX_HTTP_RETRIES})`); await new Promise(r => setTimeout(r, delayMs)); continue; } return { type: 'error', + statusCode: undefined, message: errMsg }; } @@ -468,6 +559,7 @@ export class BaseAPI { // All retries exhausted return { type: 'error', + statusCode: lastError.statusCode, message: lastError.message || `Request failed after ${MAX_HTTP_RETRIES + 1} attempts` }; } @@ -477,7 +569,7 @@ export class BaseAPI { let content: Buffer[] = []; while(true) { - const res = await fetch(this.url+route, { + const res = await queuedFetch(this.url+route, { method: 'GET', redirect: 'manual', headers: { 'Connection': 'keep-alive', @@ -598,13 +690,14 @@ export class BaseAPI { } async uploadFile(identity:Identity, projectId:string, parentFolderId:string, filename:string, fileContent:Uint8Array) { - const fileStream = stream.Readable.from(fileContent); const formData = new FormData(); const mimeType = require('mime-types').lookup(filename); formData.append('targetFolderId', parentFolderId); formData.append('name', filename); formData.append('type', mimeType? mimeType : 'text/plain'); - formData.append('qqfile', fileStream, {filename}); + formData.append('qqfile', new Blob([Buffer.from(fileContent)], { + type: mimeType ? mimeType : 'application/octet-stream', + }), filename); this.setIdentity(identity); return this.request('POST', `project/${projectId}/upload?folder_id=${parentFolderId}`, formData, (res) => { @@ -616,9 +709,10 @@ export class BaseAPI { async uploadProject(identity:Identity, filename:string, fileContent:Uint8Array) { const uuid = uuidv4(); - const fileStream = stream.Readable.from(fileContent); const formData = new FormData(); - formData.append('qqfile', fileStream, {filename}); + formData.append('qqfile', new Blob([Buffer.from(fileContent)], { + type: 'application/zip', + }), filename); this.setIdentity(identity); return this.request('POST', `project/new/upload?_csrf=${identity.csrfToken}&qquuid=${uuid}&qqfilename=${filename}&qqtotalfilesize=${fileContent.length}`, formData, (res) => { @@ -786,7 +880,7 @@ export class BaseAPI { } let content: Buffer[] = []; while (true) { - const res = await fetch(absoluteUrl, { + const res = await queuedFetch(absoluteUrl, { method: 'GET', redirect: 'manual', headers }); diff --git a/src/api/socketio.ts b/src/api/socketio.ts index 2cfd9e6c..00859bcc 100644 --- a/src/api/socketio.ts +++ b/src/api/socketio.ts @@ -2,6 +2,7 @@ import { Identity, BaseAPI, ProjectMessageResponseSchema } from './base'; import { FileEntity, DocumentEntity, FileRefEntity, FileType, FolderEntity, ProjectEntity } from '../core/remoteFileSystemProvider'; import { EventBus } from '../utils/eventBus'; +import { log, notifyError } from '../utils/outputChannel'; import { SocketIOAlt } from './socketioAlt'; function decodePackedUtf8(text: string): string { @@ -75,7 +76,7 @@ export interface EventsHandler { type ConnectionScheme = 'Alt' | 'v1' | 'v2'; export class SocketIOAPI { - private scheme: ConnectionScheme = 'v1'; + private scheme: ConnectionScheme; private record?: Promise; private _handlers: Array = []; /** Track EventBus listeners for cleanup to prevent MaxListenersExceededWarning */ @@ -91,6 +92,11 @@ export class SocketIOAPI { private readonly identity:Identity, private readonly projectId:string) { + try { + this.scheme = new URL(url).hostname==='www.overleaf.com' ? 'v2' : 'v1'; + } catch { + this.scheme = 'v1'; + } this.init(); } @@ -165,6 +171,10 @@ export class SocketIOAPI { return this._socketInitScheme !== this.scheme || !this.socket; } + get connectionScheme(): ConnectionScheme { + return this.scheme; + } + /** Clean up any accumulated EventBus listeners */ private _cleanupEventBusListeners() { for (const cleanup of this._eventBusCleanups) { @@ -175,19 +185,31 @@ export class SocketIOAPI { private initInternalHandlers() { this.socket.on('connect', () => { - console.log('SocketIOAPI: connected'); + log('SocketIOAPI: connected', {scheme: this.scheme, projectId: this.projectId}); + }); + this.socket.on('connect_failed', (connectionError:any) => { + log('SocketIOAPI: connect_failed', {scheme: this.scheme, projectId: this.projectId, error: connectionError}); + }); + this.socket.on('disconnect', (reason:any) => { + log('SocketIOAPI: disconnect event', {scheme: this.scheme, projectId: this.projectId, reason}); + }); + this.socket.on('reconnecting', (delay:any, attempt:any) => { + log('SocketIOAPI: reconnecting', {scheme: this.scheme, projectId: this.projectId, delay, attempt}); + }); + this.socket.on('reconnect', (transport:any, attempts:any) => { + log('SocketIOAPI: reconnected', {scheme: this.scheme, projectId: this.projectId, transport, attempts}); }); - this.socket.on('connect_failed', () => { - console.log('SocketIOAPI: connect_failed'); + this.socket.on('reconnect_failed', (error:any) => { + log('SocketIOAPI: reconnect_failed', {scheme: this.scheme, projectId: this.projectId, error}); }); this.socket.on('forceDisconnect', (message:string, delay=10) => { - console.log('SocketIOAPI: forceDisconnect', message); + log('SocketIOAPI: forceDisconnect', {message, delay, projectId: this.projectId}); }); this.socket.on('connectionRejected', (err:any) => { - console.log('SocketIOAPI: connectionRejected.', err?.message || err); + log('SocketIOAPI: connectionRejected', {scheme: this.scheme, projectId: this.projectId, error: err}); // If v2 also gets rejected, fall back to v1 rather than staying stuck if (this.scheme === 'v2') { - console.log('SocketIOAPI: v2 rejected, falling back to v1'); + log('SocketIOAPI: v2 rejected, falling back to v1'); this.scheme = 'v1'; } // Disable auto-reconnect on this socket: the server explicitly rejected @@ -199,17 +221,21 @@ export class SocketIOAPI { }); this.socket.on('error', (err:any) => { // Log error instead of throwing to avoid crashing the extension - console.error('SocketIOAPI: socket error', err?.message || err); + const message = err?.message || String(err); + notifyError(`Overleaf connection error: ${message}`, undefined, 'socketio-error'); }); if (this.scheme==='v2') { - this.record = new Promise(resolve => { + this.record = new Promise((resolve, reject) => { this.socket.on('joinProjectResponse', (res:any) => { const publicId = res.publicId as string; const project = res.project as ProjectEntity; EventBus.fire('socketioConnectedEvent', {publicId}); resolve(project); }); + this.socket.on('connectionRejected', (err:any) => { + reject(err?.message || err); + }); }); } } @@ -408,10 +434,19 @@ export class SocketIOAPI { * @returns {Promise} */ async applyOtUpdate(docId:string, update:UpdateSchema) { - return this.emit('applyOtUpdate', docId, update) - .then(() => { - return; - }); + try { + await this.emit('applyOtUpdate', docId, update); + } catch (error) { + let detail: string; + if (error instanceof Error) { + detail = error.message; + } else if (typeof error==='string') { + detail = error; + } else { + try { detail = JSON.stringify(error); } catch { detail = String(error); } + } + throw new Error(`Overleaf document update rejected: ${detail}`); + } } /** diff --git a/src/api/socketioAlt.ts b/src/api/socketioAlt.ts index b87fc7d4..38d890a4 100644 --- a/src/api/socketioAlt.ts +++ b/src/api/socketioAlt.ts @@ -1,5 +1,6 @@ /* eslint-disable @typescript-eslint/naming-convention */ import * as vscode from 'vscode'; +import { error as logError, notifyError } from '../utils/outputChannel'; import * as DiffMatchPatch from 'diff-match-patch'; import { EventEmitter } from 'events'; import { BaseAPI, Identity, ProjectMessageResponseSchema, ProjectSettingsSchema } from './base'; @@ -57,14 +58,21 @@ class SyncTimer { private _interval: number, private readonly _callback: () => Promise, ) { - this._callback().then(() => this.trigger()); + this.runCallback(); } - private trigger() { - this.timer = setTimeout(async () => { + private async runCallback() { + try { await this._callback(); + } catch (error) { + notifyError('Overleaf background refresh failed. See the Overleaf Workshop output log.', error, 'alternative-refresh-failed'); + } finally { this.trigger(); - }, this._interval); + } + } + + private trigger() { + this.timer = setTimeout(() => this.runCallback(), this._interval); } set interval(value: number) { @@ -146,11 +154,13 @@ export class SocketIOAlt { private async refreshVFS() { const vfs = await this.vfs; const latestVersion = await vfs.getCurrentVersion(); + if (latestVersion===undefined) { return; } if (this.vfsLocalVersion===undefined) { this.vfsLocalVersion = latestVersion; } if (latestVersion===this.vfsLocalVersion) { return; } const activeUsers = []; - const diffs = (await vfs.getFileTreeDiff(this.vfsLocalVersion, latestVersion))?.diff; + const cachedDiff = vfs.getRecentFileTreeDiff(this.vfsLocalVersion, latestVersion); + const diffs = (cachedDiff ?? await vfs.getFileTreeDiff(this.vfsLocalVersion, latestVersion))?.diff; for (const diff of diffs || []) { if (diff.operation===undefined) { continue; } @@ -161,7 +171,7 @@ export class SocketIOAlt { const {fileType, fileEntity, fileId} = await vfs._resolveUri(vfsUri); [entityId, entity] = [fileId, fileEntity]; } catch (error) { - console.error(error); + logError(error); } // handle vfs update @@ -339,7 +349,10 @@ export class SocketIOAlt { } } // update local version - this.vfsLocalVersion = await vfs.getCurrentVersion(); + const latestVersion = await vfs.getCurrentVersion(); + if (latestVersion!==undefined) { + this.vfsLocalVersion = latestVersion; + } return Promise.resolve(); }); } diff --git a/src/compile/compileManager.ts b/src/compile/compileManager.ts index afe2d04e..9086f57a 100644 --- a/src/compile/compileManager.ts +++ b/src/compile/compileManager.ts @@ -5,6 +5,7 @@ import { PdfDocument } from '../core/pdfViewEditorProvider'; import { LatexParser, ErrorSchema } from './compileLogParser'; import { EventBus } from '../utils/eventBus'; import { LocalReplicaSCMProvider } from '../scm/localReplicaSCM'; +import { error as logError, warn } from '../utils/outputChannel'; // map string level to severity const severityMap: Record = { @@ -297,7 +298,7 @@ export class CompileManager { const lineIndex = targetLine - 1; if (lineIndex < 0 || lineIndex >= editor.document.lineCount) { - console.warn(`${ELEGANT_NAME}: Invalid line number ${targetLine} for revealing in editor. Document has ${editor.document.lineCount} lines.`); + warn(`${ELEGANT_NAME}: Invalid line number ${targetLine} for revealing in editor. Document has ${editor.document.lineCount} lines.`); // Optionally, just focus the editor if the line is invalid vscode.window.showTextDocument(editor.document, { viewColumn: editor.viewColumn, preserveFocus: false }); return; @@ -352,13 +353,13 @@ export class CompileManager { } }, (error) => { - console.error(`${ELEGANT_NAME}: Failed to open document ${fileUri.fsPath} for syncPdf:`, error); + logError(`${ELEGANT_NAME}: Failed to open document ${fileUri.fsPath} for syncPdf:`, error); } ); } }) .catch(error => { - console.error(`${ELEGANT_NAME}: Error in syncPdf promise chain:`, error); + logError(`${ELEGANT_NAME}: Error in syncPdf promise chain:`, error); }); } } diff --git a/src/core/projectManagerProvider.ts b/src/core/projectManagerProvider.ts index 5f8688a3..96665af9 100644 --- a/src/core/projectManagerProvider.ts +++ b/src/core/projectManagerProvider.ts @@ -4,6 +4,7 @@ import { ProjectTagsResponseSchema } from '../api/base'; import { GlobalStateManager } from '../utils/globalStateManager'; import { VirtualFileSystem, parseUri } from './remoteFileSystemProvider'; import { LocalReplicaSCMProvider } from '../scm/localReplicaSCM'; +import { log, notifyError } from '../utils/outputChannel'; class DataItem extends vscode.TreeItem { constructor( @@ -226,6 +227,7 @@ export class ProjectManagerProvider implements vscode.TreeDataProvider .then(success => { if (success) { this.refresh(); + vscode.commands.executeCommand(`${ROOT_NAME}.localReplica.activate`, true); } else { vscode.window.showErrorMessage( vscode.l10n.t('Login failed.') ); } @@ -244,6 +246,7 @@ export class ProjectManagerProvider implements vscode.TreeDataProvider .then(success => { if (success) { this.refresh(); + vscode.commands.executeCommand(`${ROOT_NAME}.localReplica.activate`, true); } else { vscode.window.showErrorMessage( vscode.l10n.t('Login failed.') ); } @@ -566,12 +569,84 @@ export class ProjectManagerProvider implements vscode.TreeDataProvider }); } + private async promptForLocalReplicaPath(projectName: string): Promise { + const inputBox = LocalReplicaSCMProvider.baseUriInputBox; + inputBox.ignoreFocusOut = true; + inputBox.title = vscode.l10n.t('Create Source Control: {scm}', {scm:LocalReplicaSCMProvider.label}); + inputBox.buttons = [{iconPath: new vscode.ThemeIcon('check')}]; + + const selectedPath = await new Promise(resolve => { + let settled = false; + const finish = (value?: string) => { + if (settled) { return; } + settled = true; + resolve(value); + inputBox.hide(); + }; + inputBox.onDidTriggerButton(() => finish(inputBox.value)); + inputBox.onDidAccept(() => { + if (inputBox.activeItems.length===0) { + finish(inputBox.value); + } + }); + inputBox.onDidHide(() => finish()); + inputBox.show(); + }); + inputBox.dispose(); + if (!selectedPath) { return undefined; } + + log('ProjectManager: validating local replica path', {projectName, path:selectedPath}); + return LocalReplicaSCMProvider.validateBaseUri(selectedPath, projectName); + } + + private async createLocalReplica(vfs: VirtualFileSystem): Promise { + const baseUri = await this.promptForLocalReplicaPath(vfs.projectName); + if (baseUri===undefined) { return false; } + + const scm = new LocalReplicaSCMProvider(vfs, baseUri); + vfs.setProjectSCMPersist(scm.scmKey, { + enabled: true, + label: LocalReplicaSCMProvider.label, + baseUri: scm.baseUri.path, + settings: {} as JSON, + }); + + let triggers: vscode.Disposable[] = []; + try { + triggers = await scm.triggers; + log('ProjectManager: local replica created', { + projectId: vfs.projectId, + baseUri: baseUri.toString(), + triggerCount: triggers.length, + }); + vscode.window.showInformationMessage(vscode.l10n.t('"{scm}" created: {uri}.', { + scm: LocalReplicaSCMProvider.label, + uri: decodeURI(baseUri.toString()), + })); + return true; + } catch (error) { + vfs.setProjectSCMPersist(scm.scmKey, undefined); + throw error; + } finally { + triggers.forEach(trigger => trigger.dispose()); + } + } + async openProjectLocalReplica(project: ProjectItem) { - // should close other open vfs firstly + log('ProjectManager: Open Project Locally started', {projectId: project.pid, projectName: project.label, uri: project.uri}); + let openInNewWindow = false; + // A remote Overleaf folder cannot share the same workspace with a local + // replica. Offer a new window instead of silently returning before the + // local-replica creation prompt is reached. const vfsFolder = vscode.workspace.workspaceFolders?.find(folder => folder.uri.scheme===ROOT_NAME); if (vfsFolder) { - vscode.window.showWarningMessage( vscode.l10n.t('Please close the open remote overleaf folder firstly.') ); - return; + const answer = await vscode.window.showWarningMessage( + vscode.l10n.t('An Overleaf project is already open in this window.'), + vscode.l10n.t('Open Local Replica in New Window'), + vscode.l10n.t('Cancel') + ); + if (answer!==vscode.l10n.t('Open Local Replica in New Window')) { return; } + openInNewWindow = true; } const uri = vscode.Uri.parse(project.uri); @@ -579,25 +654,53 @@ export class ProjectManagerProvider implements vscode.TreeDataProvider // fetch existing local replica scm let scmPersists = GlobalStateManager.getServerProjectSCMPersists(this.context, serverName, projectId); let replicas = Object.values(scmPersists).filter(scmPersist => scmPersist.label===LocalReplicaSCMProvider.label); + const replicaUri = (scmPersist: typeof replicas[number]) => { + const parsed = vscode.Uri.parse(scmPersist.baseUri); + return parsed.scheme==='' ? vscode.Uri.file(scmPersist.baseUri) : parsed; + }; + const usableReplicaRecords = async (candidates: typeof replicas) => { + const usable = [] as typeof replicas; + for (const replica of candidates) { + try { + const stat = await vscode.workspace.fs.stat(replicaUri(replica)); + if (stat.type===vscode.FileType.Directory) { usable.push(replica); } + } catch { + // Keep the persisted entry untouched; the user may recreate it. + } + } + return usable; + }; + // A persisted SCM entry can outlive its directory. Treat that as no + // usable replica so the normal create-local-folder prompt is shown. + replicas = await usableReplicaRecords(replicas); // if not exist, create new one if (replicas.length===0) { const vfs = (await (await vscode.commands.executeCommand('remoteFileSystem.prefetch', uri))) as VirtualFileSystem; - await vfs.init(); - const answer = await vscode.window.showWarningMessage( vscode.l10n.t('No local replica found, create one for project "{label}" ?', {label:project.label}), "Yes", "No"); - if (answer === "Yes") { - await (await vscode.commands.executeCommand(`${ROOT_NAME}.projectSCM.newSCM`, LocalReplicaSCMProvider)); - // fetch local replica scm again - scmPersists = GlobalStateManager.getServerProjectSCMPersists(this.context, serverName, projectId); - replicas = Object.values(scmPersists).filter(scmPersist => scmPersist.label===LocalReplicaSCMProvider.label); - } else { - vfs.dispose(); - return; + try { + await vfs.init({activateWorkspaceFeatures:false}); + log('ProjectManager: remote project initialized for local replica creation', {projectId, projectName: project.label}); + const answer = await vscode.window.showWarningMessage( vscode.l10n.t('No local replica found, create one for project "{label}" ?', {label:project.label}), "Yes", "No"); + if (answer === "Yes") { + const created = await this.createLocalReplica(vfs); + if (!created) { return; } + // fetch local replica scm again + scmPersists = GlobalStateManager.getServerProjectSCMPersists(this.context, serverName, projectId); + replicas = Object.values(scmPersists).filter(scmPersist => scmPersist.label===LocalReplicaSCMProvider.label); + replicas = await usableReplicaRecords(replicas); + if (replicas.length===0) { + notifyError('Local replica creation did not produce a usable SCM record. See the Overleaf Workshop output for details.', undefined, 'local-replica-create-empty'); + return; + } + } else { + return; + } + } finally { + await vscode.commands.executeCommand('remoteFileSystem.reset', uri); } - vfs.dispose(); } // open local replica - const replicasPath = replicas.map(scmPersist => vscode.Uri.parse(scmPersist.baseUri).fsPath); + const replicasPath = replicas.map(replicaUri).map(uri => uri.fsPath); if (replicasPath.length===0) { return; } const quickPickItems = replicasPath.map(path => { let label = path; @@ -623,7 +726,7 @@ export class ProjectManagerProvider implements vscode.TreeDataProvider const scmKey = Object.keys(scmPersists).find(key => vscode.Uri.parse(scmPersists[key].baseUri).fsPath===item.label)!; GlobalStateManager.updateServerProjectSCMPersist(this.context, serverName, projectId, scmKey); // remove entry from quick pick - quickPick.items = quickPick.items.filter(item => item.label!==item.label); + quickPick.items = quickPick.items.filter(quickPickItem => quickPickItem.label!==item.label); } }); } @@ -640,7 +743,7 @@ export class ProjectManagerProvider implements vscode.TreeDataProvider .then(path => { const uri = vscode.Uri.file(path as string); // always open in current window - vscode.commands.executeCommand('vscode.openFolder', uri, false); + vscode.commands.executeCommand('vscode.openFolder', uri, openInNewWindow); vscode.commands.executeCommand('workbench.view.explorer'); }); } @@ -714,7 +817,9 @@ export class ProjectManagerProvider implements vscode.TreeDataProvider this.openProjectInNewWindow(item); }), vscode.commands.registerCommand(`${ROOT_NAME}.projectManager.openProjectLocalReplica`, (item) => { - this.openProjectLocalReplica(item); + return this.openProjectLocalReplica(item).catch(error => { + notifyError('Open Project Locally failed. See the Overleaf Workshop output for details.', error, 'open-project-locally-failed'); + }); }), ]; } diff --git a/src/core/remoteFileSystemProvider.ts b/src/core/remoteFileSystemProvider.ts index 0746f732..fae96df6 100644 --- a/src/core/remoteFileSystemProvider.ts +++ b/src/core/remoteFileSystemProvider.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/naming-convention */ import * as vscode from 'vscode'; import * as DiffMatchPatch from 'diff-match-patch'; -import { BaseAPI, MemberEntity, ProjectSettingsSchema } from '../api/base'; +import { BaseAPI, MemberEntity, ProjectFileTreeDiffResponseSchema, ProjectSettingsSchema, ProjectUpdateResponseSchema } from '../api/base'; import { SocketIOAPI, UpdateSchema } from '../api/socketio'; import { OUTPUT_FOLDER_NAME, ROOT_NAME } from '../consts'; import { GlobalStateManager } from '../utils/globalStateManager'; @@ -9,6 +9,7 @@ import { ClientManager } from '../collaboration/clientManager'; import { EventBus } from '../utils/eventBus'; import { SCMCollectionProvider } from '../scm/scmCollectionProvider'; import { ExtendedBaseAPI, ProjectLinkedFileProvider, UrlLinkedFileProvider } from '../api/extendedBase'; +import { error, log, notifyError, warn } from '../utils/outputChannel'; const __OUTPUTS_ID = `${ROOT_NAME}-outputs`; @@ -109,6 +110,7 @@ export function parseUri(uri: vscode.Uri) { export class VirtualFileSystem extends vscode.Disposable { private root?: ProjectEntity; private currentVersion?: number; + private recentUpdates?: ProjectUpdateResponseSchema; private context: vscode.ExtensionContext; private api: BaseAPI; private socket: SocketIOAPI; @@ -120,6 +122,9 @@ export class VirtualFileSystem extends vscode.Disposable { private retryTimer?: NodeJS.Timeout; /** Whether a "Reconnecting..." notification is currently shown */ private reconnectingNotification: boolean = false; + private disposed: boolean = false; + private workspaceFeaturesRequested: boolean = false; + private workspaceFeaturesSuppressed: boolean = false; /** Timestamp of last disconnect for debounce */ private lastDisconnectTime: number = 0; /** Whether event handlers have been registered on the current socket */ @@ -140,6 +145,7 @@ export class VirtualFileSystem extends vscode.Disposable { constructor(context: vscode.ExtensionContext, uri: vscode.Uri, notify: (events:vscode.FileChangeEvent[])=>void) { // define the dispose behavior super(() => { + this.disposed = true; // dispose all triggers of clientManager this.clientManagerItem?.triggers.forEach((trigger) => trigger.dispose()); this.clientManagerItem = undefined; @@ -147,7 +153,11 @@ export class VirtualFileSystem extends vscode.Disposable { this.scmCollectionItem?.triggers.forEach((trigger) => trigger.dispose()); this.scmCollectionItem = undefined; // disconnect socketio - // this.socket.disconnect(); + if (this.retryTimer!==undefined) { + clearTimeout(this.retryTimer); + this.retryTimer = undefined; + } + this.socket?.disconnect(); }); const {userId,projectId,serverName,projectName} = parseUri(uri); @@ -172,9 +182,24 @@ export class VirtualFileSystem extends vscode.Disposable { return this.userId; } - async init() : Promise { + async init(options: {activateWorkspaceFeatures?: boolean} = {}) : Promise { + if (options.activateWorkspaceFeatures===false) { + this.workspaceFeaturesSuppressed = true; + this.workspaceFeaturesRequested = false; + } else if (options.activateWorkspaceFeatures===true) { + this.workspaceFeaturesSuppressed = false; + this.workspaceFeaturesRequested = true; + } else if (!this.workspaceFeaturesSuppressed) { + this.workspaceFeaturesRequested = true; + } + if (this.disposed) { + throw new Error('VirtualFileSystem has been disposed.'); + } if (this.root) { - return Promise.resolve(this.root); + if (this.workspaceFeaturesRequested) { + await this.activateWorkspaceFeatures(); + } + return this.root; } if (!this.initializing) { @@ -183,7 +208,59 @@ export class VirtualFileSystem extends vscode.Disposable { return this.initializing; } + private async belongsToActiveWorkspace(): Promise { + const workspaceFolders = vscode.workspace.workspaceFolders; + if (workspaceFolders===undefined || workspaceFolders.length===0) { return true; } + if (workspaceFolders.length!==1) { return false; } + + const workspaceUri = workspaceFolders[0].uri; + if (workspaceUri.scheme===ROOT_NAME) { + return workspaceUri.authority===this.origin.authority && workspaceUri.query===this.origin.query; + } + if (workspaceUri.scheme!=='file') { return false; } + + try { + const settingUri = vscode.Uri.joinPath(workspaceUri, '.overleaf/settings.json'); + const content = await vscode.workspace.fs.readFile(settingUri); + const setting = JSON.parse(new TextDecoder().decode(content)); + if (typeof setting.uri!=='string') { return false; } + const configuredUri = vscode.Uri.parse(setting.uri); + const configured = parseUri(configuredUri); + return configured.serverName===this.serverName && configured.projectId===this.projectId; + } catch { + return false; + } + } + + private async activateWorkspaceFeatures(): Promise { + if (!await this.belongsToActiveWorkspace()) { + log('VirtualFileSystem: skipped workspace feature registration for background project', { + serverName: this.serverName, + projectId: this.projectId, + }); + return; + } + + if (this.clientManagerItem===undefined) { + const clientManager = new ClientManager(this, this.context, this.publicId||'', this.socket); + this.clientManagerItem = { + manager: clientManager, + triggers: clientManager.triggers, + }; + } + if (this.scmCollectionItem===undefined) { + const scmCollection = new SCMCollectionProvider(this, this.context); + this.scmCollectionItem = { + collection: scmCollection, + triggers: scmCollection.triggers, + }; + } + } + private get initializingPromise(): Promise { + if (this.disposed) { + return Promise.reject(new Error('VirtualFileSystem has been disposed.')); + } const MAX_RETRIES = 5; const BASE_DELAY_MS = 1000; // 1 second base delay @@ -242,6 +319,14 @@ export class VirtualFileSystem extends vscode.Disposable { await new Promise(resolve => setTimeout(resolve, delayMs)); } + log('VirtualFileSystem: initializing project connection', { + serverName: this.serverName, + projectId: this.projectId, + attempt: this.retryConnection + 1, + maxAttempts: MAX_RETRIES, + scheme: this.socket.connectionScheme, + }); + // Only recreate the socket when the connection scheme has changed // (e.g., v1→v2 after connectionRejected). For transient disconnects, // socket.io's built-in auto-reconnect handles re-establishing the TCP @@ -258,46 +343,52 @@ export class VirtualFileSystem extends vscode.Disposable { } this.root = undefined; - return this.socket.joinProject(this.projectId).then(async (project) => { - // Reset retry counter on success - this.retryConnection = 0; - this.reconnectingNotification = false; - // fetch project settings + const attempt = this.retryConnection + 1; + let project: ProjectEntity; + try { + project = await this.socket.joinProject(this.projectId); + log('VirtualFileSystem: joinProject succeeded', { + serverName: this.serverName, + projectId: this.projectId, + scheme: this.socket.connectionScheme, + }); const identity = await GlobalStateManager.authenticate(this.context, this.serverName); project.settings = (await this.api.getProjectSettings(identity, this.projectId)).settings!; - this.root = project; - const activeCondition = (vscode.workspace.workspaceFolders===undefined) || (vscode.workspace.workspaceFolders?.[0].uri.scheme!==ROOT_NAME) || (vscode.workspace.workspaceFolders?.[0].uri===this.origin); - // Register: [collaboration] ClientManager on Statusbar - if (activeCondition) { - if (this.clientManagerItem?.triggers) { - this.clientManagerItem.triggers.forEach((trigger) => trigger.dispose()); - delete this.clientManagerItem; - } - const clientManager = new ClientManager(this, this.context, this.publicId||'', this.socket); - this.clientManagerItem = { - manager: clientManager, - triggers: clientManager.triggers, - }; - } - // Register: [scm] SCMCollectionProvider in explorer - if (activeCondition) { - if (this.scmCollectionItem?.triggers) { - this.scmCollectionItem.triggers.forEach((trigger) => trigger.dispose()); - delete this.scmCollectionItem; - } - const scmCollection = new SCMCollectionProvider(this, this.context); - this.scmCollectionItem = { - collection: scmCollection, - triggers: scmCollection.triggers, - }; - } - // trigger the first compile - vscode.commands.executeCommand(`${ROOT_NAME}.compileManager.compile`); - return project; - }).catch((err) => { + } catch (err) { + error('VirtualFileSystem: project initialization failed', { + serverName: this.serverName, + projectId: this.projectId, + attempt, + scheme: this.socket.connectionScheme, + error: err, + }); + if (this.disposed) { throw err; } this.retryConnection += 1; return this.initializingPromise; - }); + } + + this.root = project; + if (this.workspaceFeaturesRequested) { + try { + await this.activateWorkspaceFeatures(); + } catch (err) { + error('VirtualFileSystem: workspace feature initialization failed', { + serverName: this.serverName, + projectId: this.projectId, + scheme: this.socket.connectionScheme, + error: err, + }); + this.initializing = undefined; + throw err; + } + } + + this.retryConnection = 0; + this.reconnectingNotification = false; + if (this.clientManagerItem!==undefined || this.scmCollectionItem!==undefined) { + vscode.commands.executeCommand(`${ROOT_NAME}.compileManager.compile`); + } + return project; }; return attemptReconnect(); @@ -438,13 +529,14 @@ export class VirtualFileSystem extends vscode.Disposable { private remoteWatch(): void { this.socket.updateEventHandlers({ - onDisconnected: () => { + onDisconnected: (reason?: any) => { + if (this.disposed) { return; } if (this.root===undefined) { return; } // bypass the first initialization - console.log("Disconnected"); + log('VirtualFileSystem: disconnected', {serverName: this.serverName, projectId: this.projectId, reason}); // Debounce: ignore rapid disconnect/reconnect cycles (within 2 seconds) const now = Date.now(); if (now - this.lastDisconnectTime < 2000) { - console.log("Disconnected: debounced (too soon since last disconnect)"); + log("Disconnected: debounced (too soon since last disconnect)"); return; } this.lastDisconnectTime = now; @@ -536,15 +628,20 @@ export class VirtualFileSystem extends vscode.Disposable { // if doc dirty, local cache should diverge from remote cache if (_doc && !_doc.isDirty) {doc.localCache = content;} doc.remoteCache = content; - this.isDirty = true; - this.notify([ - {type: vscode.FileChangeType.Changed, uri: this.pathToUri(res.path)} - ]); + } else { + // The document has not been opened yet. Invalidate its + // lazy cache so the file watcher can fetch it on demand. + doc.remoteCache = undefined; + doc.localCache = undefined; } } else { doc.remoteCache = undefined; doc.localCache = undefined; } + this.isDirty = true; + this.notify([ + {type: vscode.FileChangeType.Changed, uri: this.pathToUri(res.path)} + ]); }, onSpellCheckLanguageUpdated: (language:string) => { if (this.root) { @@ -660,13 +757,16 @@ export class VirtualFileSystem extends vscode.Disposable { throw vscode.FileSystemError.FileExists(uri); } - let res = undefined; + let res: FileEntity | undefined; + let failureMessage: string | undefined; const identity = await GlobalStateManager.authenticate(this.context, this.serverName); if (content.length===0) { const _res = await this.api.addDoc(identity, this.projectId, parentFolder._id, fileName); if (_res.type==='success') { res = _res.entity; + } else { + failureMessage = _res.message; } } else { const parentFolderId = parentFolder._id; @@ -674,9 +774,7 @@ export class VirtualFileSystem extends vscode.Disposable { if (_res.type==='success' && _res.entity!==undefined) { res = _res.entity; } else { - if (_res.message!==undefined) { - vscode.window.showErrorMessage(_res.message); - } + failureMessage = _res.message; } } if (res && res._type) { @@ -684,7 +782,11 @@ export class VirtualFileSystem extends vscode.Disposable { this.notify([ {type: vscode.FileChangeType.Created, uri: uri}, ]); + return; } + throw vscode.FileSystemError.Unavailable( + failureMessage || vscode.l10n.t('Failed to create {fileName}', {fileName}) + ); } async refreshLinkedFile(uri: vscode.Uri) { @@ -836,14 +938,25 @@ export class VirtualFileSystem extends vscode.Disposable { if (fileType && fileType==='doc' && fileEntity) { const doc = fileEntity as DocumentEntity; const _content = new TextDecoder().decode(content); + if (doc.version===undefined || doc.localCache===undefined || doc.remoteCache===undefined) { + await this.openFile(uri); + } if (doc.version===undefined || doc.localCache===undefined || doc.remoteCache===undefined) { return; } + // `content` is the caller's complete desired file. Applying a + // local-to-remote patch to it reverses the direction and can send + // stale remote text back to Overleaf. Treat a divergent remote + // cache as a conflict instead of guessing how to merge it. + if (doc.localCache!==doc.remoteCache) { + if (_content===doc.remoteCache) { + doc.localCache = _content; + return; + } + throw vscode.FileSystemError.Unavailable('The remote document changed while the local document was being updated.'); + } const dmp = new DiffMatchPatch(); - const patches = dmp.patch_make(doc.localCache, doc.remoteCache); - - const mergeResArray = dmp.patch_apply(patches, _content); - const mergeRes = mergeResArray[0] as string; + const mergeRes = _content; const update = { doc: doc._id, lastV: doc.lastVersion, @@ -878,7 +991,12 @@ export class VirtualFileSystem extends vscode.Disposable { .filter(x => x) as any; })(), }; - this.isDirty = (update.op && update.op.length) ? true : false; + if (!update.op || update.op.length===0) { + doc.localCache = mergeRes; + doc.remoteCache = mergeRes; + return; + } + this.isDirty = true; await this.socket.applyOtUpdate(doc._id, update); doc.localCache = mergeRes; doc.remoteCache = mergeRes; @@ -989,7 +1107,7 @@ export class VirtualFileSystem extends vscode.Disposable { if (rootEntry?.path) { rootResourcePath = rootEntry.path.replace(/^\//, ''); } else { - console.warn(`Unable to resolve root document id '${resolvedRootDocId}' to a path; compiling without explicit rootResourcePath.`); + warn(`Unable to resolve root document id '${resolvedRootDocId}' to a path; compiling without explicit rootResourcePath.`); } } const res = await this.api.compile(identity, this.projectId, rootResourcePath, draft, stopOnFirstError); @@ -1002,7 +1120,7 @@ export class VirtualFileSystem extends vscode.Disposable { return true; } else { if (res.message!==undefined) { - console.error('Compile failure.', res.message); + error('Compile failure.', res.message); } return false; } @@ -1210,58 +1328,79 @@ export class VirtualFileSystem extends vscode.Disposable { const res = await this.api.proxyToHistoryApiAndGetFileDiff(identity, this.projectId, pathname, from, to); if (res.type==='success') { return res.diff; - } else { + } else if (res.statusCode===404) { return undefined; + } else { + const message = `Failed to fetch file history: ${res.message || 'unknown error'}`; + notifyError('Overleaf history request failed. See the Overleaf Workshop output log.', message, `file-history:${res.statusCode || 'unknown'}`); + throw new Error(message); } } async getFileTreeDiff(from:number, to:number) { + if (from>=to) { + return {diff: []}; + } const identity = await GlobalStateManager.authenticate(this.context, this.serverName); const res = await this.api.proxyToHistoryApiAndGetFileTreeDiff(identity, this.projectId, from, to); if (res.type==='success') { return res.treeDiff; - } else { + } else if (res.statusCode===404) { return undefined; + } else { + const message = `Failed to fetch file tree history: ${res.message || 'unknown error'}`; + notifyError('Overleaf history request failed. See the Overleaf Workshop output log.', message, `file-tree-history:${res.statusCode || 'unknown'}`); + throw new Error(message); } } async getCurrentVersion() { - const base = this.currentVersion ?? 0; - let lb = base; - let rb = base+2**4; - // firstly try: a) no update `+1`, b) one update `+2` - const res = await this.getFileTreeDiff(base+1, base+1); - if (res===undefined) { - this.currentVersion = base; - return base; + const identity = await GlobalStateManager.authenticate(this.context, this.serverName); + const res = await this.api.proxyToHistoryApiAndGetUpdates(identity, this.projectId); + if (res.type!=='success' || res.updates===undefined) { + notifyError('Overleaf could not determine the current project version. Startup sync was paused.', res.message, 'current-version-unavailable'); + return undefined; } - const res2 = await this.getFileTreeDiff(base+2, base+2); - if (res2===undefined) { - this.currentVersion = base+1; - return this.currentVersion; + + this.recentUpdates = res.updates; + const latestVersion = res.updates.updates.at(0)?.toV ?? 0; + if (!Number.isInteger(latestVersion) || latestVersion<0) { + notifyError('Overleaf returned an invalid project version. Startup sync was paused.', undefined, 'current-version-invalid'); + return undefined; } - // locate the actual upper bound - do { - const res = await this.getFileTreeDiff(rb, rb); - if (res!==undefined) { - rb = lb + (rb-lb)*2; - } else { - break; + this.currentVersion = latestVersion; + return this.currentVersion; + } + + getRecentFileTreeDiff(from: number, to: number): ProjectFileTreeDiffResponseSchema | undefined { + if (from>=to) { return {diff: []}; } + const updates = this.recentUpdates?.updates + ?.filter(update => update.toV>from && update.fromV<=to) + .sort((a, b) => a.fromV-b.fromV); + if (updates===undefined || updates.length===0) { return undefined; } + + let coveredUntil = from; + for (const update of updates) { + if (update.fromV>coveredUntil) { return undefined; } + coveredUntil = Math.max(coveredUntil, update.toV); + } + if (coveredUntil(); + for (const update of updates) { + for (const pathname of update.pathnames || []) { + operations.set(pathname, 'edited'); } - } while (true); - // binary search the current version - while (lb ({pathname, operation}))}; } async createLabel(comment: string, version: number) { @@ -1337,6 +1476,15 @@ export class RemoteFileSystemProvider implements vscode.FileSystemProvider { return this.getVFS(uri).then((vfs) => {return vfs;}); } + reset(uri: vscode.Uri) { + const key = uri.query; + const vfs = this.vfss[key]; + if (vfs!==undefined) { + vfs.dispose(); + delete this.vfss[key]; + } + } + notify(events :vscode.FileChangeEvent[]) { this._emitter.fire(events); } @@ -1395,6 +1543,9 @@ export class RemoteFileSystemProvider implements vscode.FileSystemProvider { vscode.commands.registerCommand('remoteFileSystem.prefetch', (uri: vscode.Uri) => { return this.prefetch(uri); }), + vscode.commands.registerCommand('remoteFileSystem.reset', (uri: vscode.Uri) => { + this.reset(uri); + }), ]; } } diff --git a/src/extension.ts b/src/extension.ts index a17b99f2..b9e90842 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -1,14 +1,22 @@ import * as vscode from 'vscode'; import { ROOT_NAME, ELEGANT_NAME } from './consts'; -import { RemoteFileSystemProvider, VirtualFileSystem } from './core/remoteFileSystemProvider'; +import { parseUri, RemoteFileSystemProvider, VirtualFileSystem } from './core/remoteFileSystemProvider'; import { ProjectManagerProvider } from './core/projectManagerProvider'; import { PdfViewEditorProvider } from './core/pdfViewEditorProvider'; import { CompileManager } from './compile/compileManager'; import { LangIntellisenseProvider } from './intellisense'; import { LocalReplicaSCMProvider } from './scm/localReplicaSCM'; +import { GlobalStateManager } from './utils/globalStateManager'; +import { initOutputChannel, log, notifyError } from './utils/outputChannel'; + +let localReplicaActivation: Promise | undefined; export function activate(context: vscode.ExtensionContext) { + // Keep extension diagnostics in a selectable channel in the Output view. + initOutputChannel(context); + log('Overleaf Workshop local sync build 2026-08-17.8 activated.'); + // Register: [core] RemoteFileSystemProvider const remoteFileSystemProvider = new RemoteFileSystemProvider(context); context.subscriptions.push( ...remoteFileSystemProvider.triggers ); @@ -29,26 +37,61 @@ export function activate(context: vscode.ExtensionContext) { const langIntellisenseProvider = new LangIntellisenseProvider(context, remoteFileSystemProvider); context.subscriptions.push( ...langIntellisenseProvider.triggers ); - // activate vfs for local replica - LocalReplicaSCMProvider.readSettings() - .then(async setting => { - if (setting?.uri) { + const activateLocalReplica = (forceReset=false): Promise => { + if (localReplicaActivation!==undefined) { return localReplicaActivation; } + localReplicaActivation = (async () => { + const setting = await LocalReplicaSCMProvider.readSettings(); + if (!setting?.uri) { return; } const uri = vscode.Uri.parse(setting.uri); - if (uri.scheme===ROOT_NAME) { - // activate vfs - const vfs = (await (await vscode.commands.executeCommand('remoteFileSystem.prefetch', uri))) as VirtualFileSystem; - await vfs.init(); - vscode.commands.executeCommand('setContext', `${ROOT_NAME}.activate`, true); - // activate compile & preview - if (setting?.enableCompileNPreview) { - vscode.commands.executeCommand('setContext', `${ROOT_NAME}.activateCompile`, true); - } + if (uri.scheme!==ROOT_NAME) { return; } + const workspaceRoot = vscode.workspace.workspaceFolders?.[0]?.uri; + if (workspaceRoot===undefined || workspaceRoot.scheme!=='file') { return; } + + const {serverName, projectId, projectName, userId} = parseUri(uri); + const existing = GlobalStateManager.getServerProjectSCMPersists(context, serverName, projectId); + const existingPersist = Object.values(existing).find(persist => { + const baseUri = vscode.Uri.parse(persist.baseUri); + return (baseUri.scheme==='' ? vscode.Uri.file(persist.baseUri) : baseUri).toString()===workspaceRoot.toString(); + }); + const restored = await GlobalStateManager.restoreLocalReplicaSCM( + context, + serverName, + projectId, + projectName, + userId, + workspaceRoot.toString(), + { + enabled: true, + label: LocalReplicaSCMProvider.label, + baseUri: workspaceRoot.toString(), + settings: setting.localReplica?.settings ?? existingPersist?.settings ?? {} as JSON, + }, + ); + if (!restored) { + throw new Error(`Not logged in to ${serverName}`); } - } - }); + if (forceReset) { + await vscode.commands.executeCommand('remoteFileSystem.reset', uri); + } + const vfs = (await vscode.commands.executeCommand('remoteFileSystem.prefetch', uri)) as VirtualFileSystem; + await vfs.init(); + await vscode.commands.executeCommand('setContext', `${ROOT_NAME}.activate`, true); + await vscode.commands.executeCommand('setContext', `${ROOT_NAME}.activateCompile`, Boolean(setting.enableCompileNPreview)); + })() + .catch(error => { + notifyError('The local Overleaf project could not reconnect. Please verify that you are logged in, then retry.', error, 'local-replica-reconnect'); + }) + .finally(() => { localReplicaActivation = undefined; }); + return localReplicaActivation; + }; + + context.subscriptions.push(vscode.commands.registerCommand(`${ROOT_NAME}.localReplica.activate`, (forceReset?: boolean) => { + return activateLocalReplica(forceReset===true); + })); + activateLocalReplica(); } export function deactivate() { vscode.commands.executeCommand('setContext', `${ROOT_NAME}.activate`, false); vscode.commands.executeCommand('setContext', `${ROOT_NAME}.activateCompile`, false); -} \ No newline at end of file +} diff --git a/src/intellisense/texDocumentParseUtility.ts b/src/intellisense/texDocumentParseUtility.ts index 30ced12e..2bfb135a 100644 --- a/src/intellisense/texDocumentParseUtility.ts +++ b/src/intellisense/texDocumentParseUtility.ts @@ -1,5 +1,6 @@ import type * as Ast from '@unified-latex/unified-latex-types'; import * as unifiedLaTeXParse from '@unified-latex/unified-latex-util-parse'; +import { error } from '../utils/outputChannel'; // eslint-disable-next-line @typescript-eslint/naming-convention export enum TeXElementType { Environment, Command, Section, SectionAst, SubFile, BibItem, BibField, BibFile}; @@ -317,7 +318,7 @@ export async function genTexElements(documentText: string): Promise = Promise.resolve(); + private syncStateBatchDepth = 0; + private lastPersistedSyncState?: string; + private ignoredLocalSymbolicLinks = new Set(); + private conflictPaths = new Set(); + private syncReady = false; + private livePushRetryTimers = new Map(); + private livePushRetryAttempts = new Map(); + private localEventTimers = new Map(); + private syncQueue: Promise = Promise.resolve(); private ignorePatterns: string[] = [ '**/.*', '**/.*/**', @@ -120,6 +146,7 @@ export class LocalReplicaSCMProvider extends BaseSCM { public static async pathToUri(path: string): Promise { const workspaceRoot = vscode.workspace.workspaceFolders?.[0].uri; if (workspaceRoot===undefined || workspaceRoot?.scheme!=='file') { return undefined; } + if (path.replace(/\\/g, '/').split('/').some(part => part.startsWith('.'))) { return undefined; } const settingUri = vscode.Uri.joinPath(workspaceRoot, '.overleaf/settings.json'); try { @@ -137,7 +164,9 @@ export class LocalReplicaSCMProvider extends BaseSCM { const settingUri = vscode.Uri.joinPath(workspaceRoot, '.overleaf/settings.json'); try { await vscode.workspace.fs.stat(settingUri); - return uri.path.slice(workspaceRoot.path.length); + const path = uri.path.slice(workspaceRoot.path.length); + if (path.replace(/\\/g, '/').split('/').some(part => part.startsWith('.'))) { return undefined; } + return path; } catch (error) { return undefined; } @@ -158,15 +187,219 @@ export class LocalReplicaSCMProvider extends BaseSCM { } private matchIgnorePatterns(path: string): boolean { + const normalizedPath = this.normalizeRelPath(path); + if (normalizedPath.split('/').some(part => part.startsWith('.'))) { + return true; + } const ignorePatterns = this.getSetting(IGNORE_SETTING_KEY) || this.ignorePatterns; for (const pattern of ignorePatterns) { - if (minimatch(path, pattern, {dot:true})) { + if (minimatch(normalizedPath, pattern, {dot:true})) { return true; } } return false; } + private normalizeRelPath(path: string): string { + const normalized = path.replace(/\\/g, '/'); + return normalized.startsWith('/') ? normalized : `/${normalized}`; + } + + private async statOrUndefined(uri: vscode.Uri): Promise { + try { + return await vscode.workspace.fs.stat(uri); + } catch { + return undefined; + } + } + + private isConflictPath(relPath: string): boolean { + const normalizedPath = this.normalizeRelPath(relPath); + return [...this.conflictPaths].some(path => + normalizedPath===path || normalizedPath.startsWith(`${path}/`) || path.startsWith(`${normalizedPath}/`) + ); + } + + private setContainsOverlappingPath(paths: Set, relPath: string): boolean { + const normalizedPath = this.normalizeRelPath(relPath); + return [...paths].some(path => + normalizedPath===path || normalizedPath.startsWith(`${path}/`) || path.startsWith(`${normalizedPath}/`) + ); + } + + private markConflict(relPath: string, reason: string) { + const normalizedPath = this.normalizeRelPath(relPath); + this.conflictPaths.add(normalizedPath); + notifyError( + `Overleaf sync paused for "${normalizedPath}" because both local and remote changed. Resolve it manually, then reload the window.`, + reason, + `local-replica-conflict:${normalizedPath}` + ); + } + + private async hasUncheckpointedLocalChange(relPath: string, type: 'update'|'delete'): Promise { + const normalizedPath = this.normalizeRelPath(relPath); + const localUri = vscode.Uri.joinPath(this.baseUri, normalizedPath); + const localStat = await this.statOrUndefined(localUri); + if (localStat===undefined) { return false; } + if (type==='update' && localStat.type===vscode.FileType.Directory) { return true; } + if (localStat.type!==vscode.FileType.File) { return true; } + const checkpointHash = this.syncState?.files[normalizedPath] ?? + (this.baseCache[normalizedPath]===undefined ? undefined : sha256(this.baseCache[normalizedPath])); + if (checkpointHash===undefined) { return true; } + return sha256(await vscode.workspace.fs.readFile(localUri))!==checkpointHash; + } + + private async findLocalSymbolicLink(uri: vscode.Uri): Promise { + const basePath = this.baseUri.path.replace(/\/$/, ''); + if (uri.scheme!==this.baseUri.scheme || uri.authority!==this.baseUri.authority || + (uri.path!==basePath && !uri.path.startsWith(`${basePath}/`))) { + return undefined; + } + + let currentUri = this.baseUri; + let currentPath = ''; + const parts = uri.path.slice(basePath.length).split('/').filter(Boolean); + for (const part of parts) { + currentUri = vscode.Uri.joinPath(currentUri, part); + currentPath = `${currentPath}/${part}`; + const stat = await this.statOrUndefined(currentUri); + if (stat!==undefined && (stat.type & vscode.FileType.SymbolicLink)!==0) { + return currentPath; + } + } + return undefined; + } + + private async ignoreLocalSymbolicLink(uri: vscode.Uri, deleted: boolean = false): Promise { + const relPath = this.normalizeRelPath(uri.path.slice(this.baseUri.path.replace(/\/$/, '').length)); + const symbolicLink = await this.findLocalSymbolicLink(uri); + if (symbolicLink!==undefined) { + if (!this.ignoredLocalSymbolicLinks.has(symbolicLink)) { + warn(`Ignoring local symbolic link "${symbolicLink}"; it will not be synchronized with Overleaf.`); + } + this.ignoredLocalSymbolicLinks.add(symbolicLink); + return true; + } + + if (deleted) { + const rememberedLink = [...this.ignoredLocalSymbolicLinks] + .find(path => relPath===path || relPath.startsWith(`${path}/`)); + if (rememberedLink!==undefined) { + if (relPath===rememberedLink) { + this.ignoredLocalSymbolicLinks.delete(rememberedLink); + } + return true; + } + } else { + // A regular file may intentionally replace an earlier link. + this.ignoredLocalSymbolicLinks.delete(relPath); + } + return false; + } + + private async loadSyncState(): Promise { + const stateUri = vscode.Uri.joinPath(this.baseUri, SYNC_STATE_PATH); + try { + const content = await vscode.workspace.fs.readFile(stateUri); + const state = JSON.parse(new TextDecoder().decode(content)) as LocalReplicaSyncState; + if (state.schemaVersion!==SYNC_STATE_SCHEMA_VERSION || + state.projectUri!==this.vfs.origin.toString() || + !Number.isInteger(state.remoteVersion) || + typeof state.files!=='object' || state.files===null) { + return undefined; + } + this.lastPersistedSyncState = JSON.stringify(state, null, 2); + return state; + } catch { + return undefined; + } + } + + private persistSyncState(): Promise { + if (this.syncState===undefined) { return Promise.resolve(); } + const stateUri = vscode.Uri.joinPath(this.baseUri, SYNC_STATE_PATH); + const serializedState = JSON.stringify(this.syncState, null, 2); + const content = new TextEncoder().encode(serializedState); + this.syncStateWritePromise = this.syncStateWritePromise + .catch(() => {}) + .then(async () => { + if (serializedState===this.lastPersistedSyncState) { return; } + // This is a disposable cache, not user content. Writing in place + // avoids delete/create events from the temporary-file rename. + await vscode.workspace.fs.writeFile(stateUri, content); + this.lastPersistedSyncState = serializedState; + }); + return this.syncStateWritePromise; + } + + private scheduleSyncStateWrite() { + if (this.syncStateBatchDepth>0) { + return; + } + if (this.syncStateWriteTimer!==undefined) { + clearTimeout(this.syncStateWriteTimer); + } + this.syncStateWriteTimer = setTimeout(() => { + this.syncStateWriteTimer = undefined; + this.persistSyncState().catch(logError); + }, 250); + } + + private updateSyncStateFile(relPath: string, content?: Uint8Array) { + if (this.syncState===undefined) { return; } + const normalizedPath = this.normalizeRelPath(relPath); + if (content===undefined) { + for (const path of Object.keys(this.syncState.files)) { + if (path===normalizedPath || path.startsWith(`${normalizedPath}/`)) { + delete this.syncState.files[path]; + } + } + } else { + this.syncState.files[normalizedPath] = sha256(content); + } + this.scheduleSyncStateWrite(); + } + + private async scanLocalFileHashes(): Promise> { + const files = new Map(); + const queue: Array<[vscode.Uri,string]> = [[this.baseUri, '/']]; + while (queue.length!==0) { + const [directoryUri, directoryPath] = queue.shift()!; + const entries = await vscode.workspace.fs.readDirectory(directoryUri); + for (const [name, type] of entries) { + const relPath = this.normalizeRelPath(`${directoryPath}${name}`); + if (this.matchIgnorePatterns(relPath)) { continue; } + const uri = vscode.Uri.joinPath(directoryUri, name); + if ((type & vscode.FileType.SymbolicLink)!==0) { + this.ignoredLocalSymbolicLinks.add(relPath); + } else if (type===vscode.FileType.Directory) { + queue.push([uri, `${relPath}/`]); + } else if (type===vscode.FileType.File) { + files.set(relPath, sha256(await vscode.workspace.fs.readFile(uri))); + } + } + } + return files; + } + + private enqueueSync(operation: () => Promise): Promise { + const result = this.syncQueue.then(operation, operation); + this.syncQueue = result.then(() => undefined, () => undefined); + return result; + } + + private async ensureParentDirectories(baseUri: vscode.Uri, relPath: string) { + const parts = this.normalizeRelPath(relPath).split('/').filter(Boolean).slice(0, -1); + let currentUri = baseUri; + for (const part of parts) { + currentUri = vscode.Uri.joinPath(currentUri, part); + if (await this.statOrUndefined(currentUri)===undefined) { + await vscode.workspace.fs.createDirectory(currentUri); + } + } + } + private setBypassCache(relPath: string, content?: Uint8Array, action?: 'push'|'pull') { const date = Date.now(); const hash = hashCode(content); @@ -194,6 +427,17 @@ export class LocalReplicaSCMProvider extends BaseSCM { // console.log(action, relPath, `[${cache[0].hash}, ${cache[1].hash}]`, thisHash); if (action==='push' && cache[0].hash===thisHash) { return false; } if (action==='pull' && cache[1].hash===thisHash) { return false; } + // A remote update is mirrored to the local replica and reported by + // the local watcher as well. Do not upload that mirror operation. + if (action==='push' && cache[1].hash===thisHash) { + this.setBypassCache(relPath, content, action); + return false; + } + // Likewise, ignore the VFS watcher event caused by our own upload. + if (action==='pull' && cache[0].hash===thisHash) { + this.setBypassCache(relPath, content, action); + return false; + } if (cache[0].hash!==cache[1].hash) { if (action==='push' && now-cache[0].date<500 || action==='pull' && now-cache[1].date<500) { this.setBypassCache(relPath, content, action); @@ -207,14 +451,14 @@ export class LocalReplicaSCMProvider extends BaseSCM { return true; } - private async overwrite(root: string='/'): Promise { + private async overwrite(remoteVersion: number, localHashes: Map, root: string='/'): Promise { return await vscode.window.withProgress({ location: vscode.ProgressLocation.Notification, title: vscode.l10n.t('Sync Files'), cancellable: true, }, async (progress, token) => { // breadth-first search for the files - const files: [string,string][] = []; + const files: string[] = []; const queue: string[] = [root]; while (queue.length!==0) { const nextRoot = queue.shift(); @@ -230,48 +474,328 @@ export class LocalReplicaSCMProvider extends BaseSCM { if (type === vscode.FileType.Directory) { queue.push(relPath+'/'); } else { - files.push([name, relPath]); + files.push(this.normalizeRelPath(relPath)); } } } - // sync the files + const stateFiles: {[path:string]: string} = {}; const total = files.length; for (let i=0; i, + remoteDiff?: ProjectFileTreeDiffResponseSchema, + ): Promise { + this.syncState = state; + const localChangedPaths = new Set(); + const knownPaths = new Set([...Object.keys(state.files), ...localHashes.keys()]); + for (const path of knownPaths) { + if (state.files[path]!==localHashes.get(path)) { + localChangedPaths.add(path); + } + } + + const remoteChangedPaths = new Set(); + for (const change of remoteDiff?.diff || []) { + if (change.operation===undefined) { continue; } + const oldPath = this.normalizeRelPath(change.pathname); + if (!this.matchIgnorePatterns(oldPath)) { + remoteChangedPaths.add(oldPath); + } + if (change.operation==='renamed' && change.newPathname!==undefined) { + const newPath = this.normalizeRelPath(change.newPathname); + if (!this.matchIgnorePatterns(newPath)) { + remoteChangedPaths.add(newPath); + } + } + } + + const changedPaths = [...new Set([...localChangedPaths, ...remoteChangedPaths])] + .sort((a, b) => a.split('/').length-b.split('/').length || a.localeCompare(b)); + if (changedPaths.length===0) { + state.remoteVersion = currentRemoteVersion; + await this.persistSyncState(); + log(`Local replica is current: version ${currentRemoteVersion}, no file content transferred.`); + return true; + } + + const failedPaths: string[] = []; + this.syncStateBatchDepth += 1; + try { + await vscode.window.withProgress({ + location: vscode.ProgressLocation.Notification, + title: vscode.l10n.t('Sync Files'), + cancellable: true, + }, async (progress, token) => { + for (const relPath of changedPaths) { + if (token.isCancellationRequested) { + failedPaths.push(...changedPaths.slice(changedPaths.indexOf(relPath))); + break; + } + progress.report({increment: 100/changedPaths.length, message: relPath}); + try { + if (this.isConflictPath(relPath)) { continue; } + const localChanged = this.setContainsOverlappingPath(localChangedPaths, relPath); + const remoteChanged = this.setContainsOverlappingPath(remoteChangedPaths, relPath); + if (localChanged && remoteChanged) { + await this.syncConcurrentPath(relPath); + } else if (localChanged) { + await this.syncLocalPath(relPath); + } else { + await this.syncRemotePath(relPath); + } + } catch (error) { + failedPaths.push(relPath); + logError(`Incremental sync failed for ${relPath}:`, error); + } + } + }); + } finally { + this.syncStateBatchDepth -= 1; + } + + if (failedPaths.length!==0) { + await this.persistSyncState(); + notifyError( + `Overleaf sync failed for ${failedPaths.length} path(s). The previous sync checkpoint was kept.`, + undefined, + 'local-replica-incremental-failed' + ); + return false; + } + + // Checkpoint only the version used to calculate this diff. Changes + // arriving during initialization must remain visible next time. + state.remoteVersion = currentRemoteVersion; + await this.persistSyncState(); + log(`Local replica incremental sync completed: ${changedPaths.length} changed path(s), version ${state.remoteVersion}.`); + return true; + } + + private canOverwriteWithoutLocalLoss(state: LocalReplicaSyncState | undefined, localHashes: Map): boolean { + if (localHashes.size===0) { return true; } + if (state===undefined) { return false; } + const knownPaths = new Set([...Object.keys(state.files), ...localHashes.keys()]); + for (const path of knownPaths) { + if (state.files[path]!==localHashes.get(path)) { return false; } + } + return true; + } + + private pauseUnsafeFullSync(reason: string): false { + notifyError( + 'Overleaf sync was paused because the remote history is unavailable and local files may have changed. No files were overwritten.', + reason, + 'local-replica-unsafe-full-sync' + ); + return false; + } + + private async initializeSync() { + const currentRemoteVersion = await this.vfs.getCurrentVersion(); + if (currentRemoteVersion===undefined) { + notifyError( + 'Overleaf sync was paused because the current remote version could not be determined.', + undefined, + 'local-replica-version-unavailable' + ); + return false; + } + const state = await this.loadSyncState(); + const localHashes = await this.scanLocalFileHashes(); + if (state===undefined || state.remoteVersion>currentRemoteVersion) { + if (!this.canOverwriteWithoutLocalLoss(state, localHashes)) { + return this.pauseUnsafeFullSync('The local replica has files that are not identical to the last checkpoint.'); + } + log('Local replica sync state unavailable or invalid; using full sync because no uncheckpointed local files were found.'); + return this.overwrite(currentRemoteVersion, localHashes); + } + + let remoteDiff: ProjectFileTreeDiffResponseSchema | undefined; + if (state.remoteVersion { + try { await (async () => { + const localUri = action==='push' ? fromUri : toUri; + if (await this.ignoreLocalSymbolicLink(localUri, action==='push' && type==='delete')) { + this.updateSyncStateFile(relPath, undefined); + return; + } if (type==='delete') { const newContent = undefined; if (this.bypassSync(action, type, relPath, newContent)) { return; } delete this.baseCache[relPath]; - await vscode.workspace.fs.delete(toUri, {recursive:true}); + if (await this.statOrUndefined(toUri)!==undefined) { + await vscode.workspace.fs.delete(toUri, {recursive:true}); + } + this.updateSyncStateFile(relPath, undefined); } else { const stat = await vscode.workspace.fs.stat(fromUri); - if (stat.type===vscode.FileType.Directory) { + if ((stat.type & vscode.FileType.SymbolicLink)!==0) { + this.ignoredLocalSymbolicLinks.add(this.normalizeRelPath(relPath)); + } + else if (stat.type===vscode.FileType.Directory) { const newContent = new Uint8Array(); if (this.bypassSync(action, type, relPath, newContent)) { return; } await vscode.workspace.fs.createDirectory(toUri); @@ -309,64 +845,110 @@ export class LocalReplicaSCMProvider extends BaseSCM { await vscode.workspace.fs.writeFile(toUri, newContent); this.baseCache[relPath] = newContent; if (action==='push') { await vscode.workspace.fs.readFile(toUri); } // update remote cache + this.updateSyncStateFile(relPath, newContent); } catch (error) { - console.error(error); + const errorMessage = error instanceof Error ? error.message : (error as any)?.message || String(error); + if (errorMessage.includes('remote document changed while the local document was being updated')) { + succeeded = true; + this.markConflict(relPath, errorMessage); + } else { + succeeded = false; + notifyError(`Failed to ${action} "${relPath}" during live sync.`, error, `local-replica:${action}:${relPath}`); + } } } else { - console.error(`Unknown file type: ${stat.type}`); + notifyError(`Overleaf sync encountered an unsupported file type at "${relPath}".`, undefined, `local-replica:unknown-type:${relPath}`); } } - })(); + })(); } catch (error) { + succeeded = false; + notifyError(`Failed to ${action} "${relPath}" during live sync.`, error, `local-replica:${action}:${relPath}`); + } this.status = {status: 'idle', message: ''}; + return succeeded; } private async syncFromVFS(vfsUri: vscode.Uri, type: 'update'|'delete') { + if (!this.syncReady) { return; } const {pathParts} = parseUri(vfsUri); pathParts.at(-1)==='' && pathParts.pop(); // remove the last empty string const relPath = ('/' + pathParts.join('/')); + if (this.matchIgnorePatterns(relPath)) { return; } + if (this.isConflictPath(relPath)) { return; } const localUri = vscode.Uri.joinPath(this.baseUri, relPath); - this.applySync('pull', type, relPath, vfsUri, localUri); + if (await this.hasUncheckpointedLocalChange(relPath, type)) { + this.markConflict(relPath, 'A remote event arrived while the local file differed from the last synchronized checkpoint.'); + return; + } + await this.applySync('pull', type, relPath, vfsUri, localUri); + } + + private scheduleLivePushRetry(localUri: vscode.Uri, relPath: string) { + if (this.livePushRetryTimers.has(relPath)) { return; } + const attempt = this.livePushRetryAttempts.get(relPath) || 0; + const delayMs = Math.min(5000 * Math.pow(2, attempt), 60000); + this.livePushRetryAttempts.set(relPath, attempt + 1); + log(`Live push for "${relPath}" will retry in ${delayMs}ms.`); + const timer = setTimeout(() => { + this.livePushRetryTimers.delete(relPath); + this.enqueueSync(() => this.syncToVFS(localUri, 'update')).catch(logError); + }, delayMs); + this.livePushRetryTimers.set(relPath, timer); + } + + private clearLivePushRetry(relPath: string) { + const timer = this.livePushRetryTimers.get(relPath); + if (timer!==undefined) { clearTimeout(timer); } + this.livePushRetryTimers.delete(relPath); + this.livePushRetryAttempts.delete(relPath); } private async syncToVFS(localUri: vscode.Uri, type: 'update'|'delete') { + if (!this.syncReady) { return; } // get relative path to baseUri const basePath = this.baseUri.path; const relPath = localUri.path.slice(basePath.length); + if (this.matchIgnorePatterns(relPath)) { + this.clearLivePushRetry(relPath); + return; + } + if (this.isConflictPath(relPath)) { return; } const vfsUri = this.vfs.pathToUri(relPath); - this.applySync('push', type, relPath, localUri, vfsUri); - } - - /** - * Push a saved document to the VFS. - * Only fires for explicit user saves in the editor, not for external - * file modifications (git, compilation tools, etc.). - * This is the general fix for issues #299 and #323. - */ - private onDocumentSaved(doc: vscode.TextDocument) { - const docUri = doc.uri; - // Only sync files within our baseUri (ensure path separator boundary) - const basePath = this.baseUri.path.endsWith('/') ? this.baseUri.path : this.baseUri.path + '/'; - if (!docUri.path.startsWith(basePath)) { return; } - this.syncToVFS(docUri, 'update'); + const succeeded = await this.applySync('push', type, relPath, localUri, vfsUri); + if (succeeded) { + this.clearLivePushRetry(relPath); + } else if (type==='update') { + this.scheduleLivePushRetry(localUri, relPath); + } else { + this.clearLivePushRetry(relPath); + } } private async initWatch() { + log('LocalReplica: initializing watchers and startup sync', { + projectId: this.vfs.projectId, + projectName: this.vfs.projectName, + baseUri: this.baseUri.toString(), + }); // write ".overleaf/settings.json" if not exist const settingUri = vscode.Uri.joinPath(this.baseUri, '.overleaf/settings.json'); try { await vscode.workspace.fs.stat(settingUri); } catch (error) { + const scmPersist = this.vfs.getProjectSCMPersist(this.scmKey); await vscode.workspace.fs.writeFile(settingUri, Buffer.from( JSON.stringify({ 'uri': this.vfs.origin.toString(), 'serverName': this.vfs.serverName, 'enableCompileNPreview': false, 'projectName': this.vfs.projectName, + 'localReplica': {settings: scmPersist?.settings ?? {}}, }, null, 4) )); } + await this.persistLocalSettingsMetadata(); this.vfsWatcher = vscode.workspace.createFileSystemWatcher( new vscode.RelativePattern( this.vfs.origin, '**/*' ) @@ -374,29 +956,53 @@ export class LocalReplicaSCMProvider extends BaseSCM { this.localWatcher = vscode.workspace.createFileSystemWatcher( new vscode.RelativePattern( this.baseUri.path, '**/*' ) ); - await this.overwrite(); - - // Listen for explicit user saves (not file system changes) to push local edits. - // File system watchers would also fire for git operations, compilation outputs, - // and other external modifications, causing unwanted sync (issues #299, #323). - this.saveListener = vscode.workspace.onDidSaveTextDocument( - doc => this.onDocumentSaved(doc) - ); + this.syncReady = (await this.initializeSync())===true; + if (!this.syncReady) { + log('Local replica watchers are paused until the synchronization conflict is resolved.'); + } else { + log('LocalReplica: watchers are active', {projectId: this.vfs.projectId, baseUri: this.baseUri.toString()}); + } + const syncStateDisposable = new vscode.Disposable(() => { + if (this.syncStateWriteTimer!==undefined) { + clearTimeout(this.syncStateWriteTimer); + this.syncStateWriteTimer = undefined; + } + for (const timer of this.localEventTimers.values()) { clearTimeout(timer); } + this.localEventTimers.clear(); + for (const timer of this.livePushRetryTimers.values()) { clearTimeout(timer); } + this.livePushRetryTimers.clear(); + this.persistSyncState().catch(logError); + }); return [ // sync from vfs to local - this.vfsWatcher.onDidChange(async uri => await this.syncFromVFS(uri, 'update')), - this.vfsWatcher.onDidCreate(async uri => await this.syncFromVFS(uri, 'update')), - this.vfsWatcher.onDidDelete(async uri => await this.syncFromVFS(uri, 'delete')), - // sync from local to vfs: file updates via editor saves (onDidSaveTextDocument above), - // file creation and deletion still via watcher (these are explicit user actions) - this.localWatcher.onDidCreate(async uri => await this.syncToVFS(uri, 'update')), - this.localWatcher.onDidDelete(async uri => await this.syncToVFS(uri, 'delete')), - // include save listener for proper disposal - this.saveListener, + this.vfsWatcher.onDidChange(uri => this.enqueueSync(() => this.syncFromVFS(uri, 'update')).catch(logError)), + this.vfsWatcher.onDidCreate(uri => this.enqueueSync(() => this.syncFromVFS(uri, 'update')).catch(logError)), + this.vfsWatcher.onDidDelete(uri => this.enqueueSync(() => this.syncFromVFS(uri, 'delete')).catch(logError)), + // sync from local to vfs, including changes made outside VS Code + this.localWatcher.onDidChange(uri => this.scheduleLocalSync(uri)), + this.localWatcher.onDidCreate(uri => this.scheduleLocalSync(uri)), + this.localWatcher.onDidDelete(uri => this.scheduleLocalSync(uri)), + syncStateDisposable, ]; } + private scheduleLocalSync(localUri: vscode.Uri) { + if (!this.syncReady) { return; } + const relPath = this.normalizeRelPath(localUri.path.slice(this.baseUri.path.length)); + if (this.matchIgnorePatterns(relPath) || this.isConflictPath(relPath)) { return; } + const previousTimer = this.localEventTimers.get(relPath); + if (previousTimer!==undefined) { clearTimeout(previousTimer); } + const timer = setTimeout(() => { + this.localEventTimers.delete(relPath); + this.enqueueSync(async () => { + const type = await this.statOrUndefined(localUri)===undefined ? 'delete' : 'update'; + return this.syncToVFS(localUri, type); + }).catch(logError); + }, 250); + this.localEventTimers.set(relPath, timer); + } + writeFile(relPath: string, content: Uint8Array): Thenable { const uri = vscode.Uri.joinPath(this.baseUri, relPath); return vscode.workspace.fs.writeFile(uri, content); @@ -485,6 +1091,7 @@ export class LocalReplicaSCMProvider extends BaseSCM { const index = ignorePatterns.indexOf(item.label); ignorePatterns.splice(index, 1); await this.setSetting(IGNORE_SETTING_KEY, ignorePatterns); + await this.persistLocalSettingsMetadata(); quickPick.items = ignorePatterns.map(pattern => ({ label: pattern, buttons: [{iconPath: new vscode.ThemeIcon('trash')}], @@ -497,6 +1104,7 @@ export class LocalReplicaSCMProvider extends BaseSCM { if (pattern!=='') { ignorePatterns.push(pattern); await this.setSetting(IGNORE_SETTING_KEY, ignorePatterns); + await this.persistLocalSettingsMetadata(); quickPick.items = ignorePatterns.map(pattern => ({ label: pattern, buttons: [{iconPath: new vscode.ThemeIcon('trash')}], diff --git a/src/scm/scmCollectionProvider.ts b/src/scm/scmCollectionProvider.ts index c03391af..e45f0a35 100644 --- a/src/scm/scmCollectionProvider.ts +++ b/src/scm/scmCollectionProvider.ts @@ -8,6 +8,7 @@ import { HistoryViewProvider } from './historyViewProvider'; import { GlobalStateManager } from '../utils/globalStateManager'; import { EventBus } from '../utils/eventBus'; import { ROOT_NAME } from '../consts'; +import { error as logError, log, notifyError } from '../utils/outputChannel'; const supportedSCMs = [ LocalReplicaSCMProvider, @@ -133,6 +134,13 @@ export class SCMCollectionProvider extends vscode.Disposable { } private async createSCM(scmProto: SupportedSCM, baseUri: vscode.Uri, newSCM=false, enabled=true) { + log('SCMCollection: creating SCM', { + label: scmProto.label, + baseUri: baseUri.toString(), + projectId: this.vfs.projectId, + newSCM, + enabled, + }); const scm = new scmProto(this.vfs, baseUri); // insert into global state if (newSCM) { @@ -148,10 +156,18 @@ export class SCMCollectionProvider extends vscode.Disposable { const triggers = enabled ? await scm.triggers : []; this.scms.push({scm,enabled,triggers}); this.updateStatus(); + log('SCMCollection: SCM created', {label: scmProto.label, baseUri: baseUri.toString(), triggerCount: triggers.length}); return scm; } catch (error) { - // permanently remove failed scm - // this.vfs.setProjectSCMPersist(scm.scmKey, undefined); + if (newSCM) { + this.vfs.setProjectSCMPersist(scm.scmKey, undefined); + } + logError('SCMCollection: SCM creation failed', { + label: scmProto.label, + baseUri: baseUri.toString(), + projectId: this.vfs.projectId, + error, + }); vscode.window.showErrorMessage( vscode.l10n.t('"{scm}" creation failed.', {scm:scmProto.label}) ); return undefined; } @@ -188,7 +204,10 @@ export class SCMCollectionProvider extends vscode.Disposable { } }); }) - .then((uri) => scmProto.validateBaseUri(uri as string || '', this.vfs.projectName)) + .then((uri) => { + log('SCMCollection: validating new SCM path', {label: scmProto.label, uri}); + return scmProto.validateBaseUri(uri as string || '', this.vfs.projectName); + }) .then(async (baseUri) => { if (baseUri) { const scm = await this.createSCM(scmProto, baseUri, true); @@ -198,6 +217,9 @@ export class SCMCollectionProvider extends vscode.Disposable { vscode.window.showErrorMessage( vscode.l10n.t('"{scm}" creation failed.', {scm:scmProto.label}) ); } } + }) + .catch(error => { + notifyError(`Could not create ${scmProto.label}. See the Overleaf Workshop output for details.`, error, 'scm-create-failed'); }); } @@ -310,4 +332,4 @@ export class SCMCollectionProvider extends vscode.Disposable { ]; } -} \ No newline at end of file +} diff --git a/src/utils/globalStateManager.ts b/src/utils/globalStateManager.ts index 5af1d427..5643fe61 100644 --- a/src/utils/globalStateManager.ts +++ b/src/utils/globalStateManager.ts @@ -198,6 +198,42 @@ export class GlobalStateManager { } } + /** + * Recreates the in-memory project entry needed by SCMCollectionProvider + * from a local replica's .overleaf/settings.json after a re-login. + */ + static async restoreLocalReplicaSCM( + context:vscode.ExtensionContext, + serverName:string, + projectId:string, + projectName:string, + userId:string|undefined, + scmKey:string, + scmPersist:ProjectSCMPersist, + ): Promise { + const persists = context.globalState.get(keyServerPersists, {}); + const server = persists[serverName]; + if (server?.login===undefined) { return false; } + server.login.projects ??= []; + let project = server.login.projects.find(item => item.id===projectId); + if (project===undefined) { + project = { + id: projectId, + userId: userId || server.login.userId, + name: projectName, + source: 'owner', + accessLevel: 'owner', + scm: {}, + }; + server.login.projects.push(project); + } + const scmPersists = (project.scm ?? {}) as ProjectSCMPersistMap; + scmPersists[scmKey] = scmPersist; + project.scm = scmPersists; + await context.globalState.update(keyServerPersists, persists); + return true; + } + static getPdfViewPersist(context:vscode.ExtensionContext, uri:string): any { return context.globalState.get(keyPdfViewPersists, {})[uri]?.state; } diff --git a/src/utils/outputChannel.ts b/src/utils/outputChannel.ts new file mode 100644 index 00000000..996ab800 --- /dev/null +++ b/src/utils/outputChannel.ts @@ -0,0 +1,78 @@ +import * as vscode from 'vscode'; + +let outputChannel: vscode.LogOutputChannel | undefined; +const notificationTimes = new Map(); +const NOTIFICATION_COOLDOWN_MS = 30000; + +type LogLevel = 'INFO' | 'WARN' | 'ERROR'; + +function formatArgument(argument: unknown): string { + if (argument instanceof Error) { + return argument.stack || argument.message; + } + if (typeof argument === 'string') { + return argument; + } + try { + const seen = new WeakSet(); + return JSON.stringify(argument, (_key, value) => { + if (value instanceof Error) { + return Object.fromEntries(Object.getOwnPropertyNames(value).map(key => [key, (value as any)[key]])); + } + if (typeof value==='object' && value!==null) { + if (seen.has(value)) { return '[Circular]'; } + seen.add(value); + } + return value; + }); + } catch { + return String(argument); + } +} + +export function initOutputChannel(context: vscode.ExtensionContext): vscode.LogOutputChannel { + if (outputChannel===undefined) { + outputChannel = vscode.window.createOutputChannel('Overleaf Workshop', {log:true}); + context.subscriptions.push(outputChannel); + } + return outputChannel; +} + +function writeLog(level: LogLevel, args: unknown[]) { + const message = args.map(formatArgument).join(' '); + if (level==='ERROR') { + outputChannel?.error(message); + console.error(...args); + } else if (level==='WARN') { + outputChannel?.warn(message); + console.warn(...args); + } else { + outputChannel?.info(message); + console.log(...args); + } +} + +export function log(...args: unknown[]) { + writeLog('INFO', args); +} + +export function warn(...args: unknown[]) { + writeLog('WARN', args); +} + +export function error(...args: unknown[]) { + writeLog('ERROR', args); +} + +export function notifyError(message: string, detail?: unknown, key: string = message) { + writeLog('ERROR', detail===undefined ? [message] : [message, detail]); + const now = Date.now(); + const previous = notificationTimes.get(key) || 0; + if (now-previous { + if (choice==='Show Logs') { + outputChannel?.show(true); + } + }, () => {}); +}