diff --git a/packages/boxel-cli/package.json b/packages/boxel-cli/package.json index 3030d5dd59..3225cf8a97 100644 --- a/packages/boxel-cli/package.json +++ b/packages/boxel-cli/package.json @@ -60,7 +60,7 @@ "lint:js:fix": "eslint . --report-unused-disable-directives --fix", "lint:types": "tsc --noEmit", "test": "vitest run", - "test:unit": "vitest run --exclude tests/integration/**", + "test:unit": "vitest run --exclude 'tests/integration/**'", "test:integration": "./tests/scripts/run-integration-with-test-pg.sh", "test:watch": "vitest", "version:patch": "npm version patch", diff --git a/packages/boxel-cli/src/commands/realm/create.ts b/packages/boxel-cli/src/commands/realm/create.ts index 2ecf80685f..0ce9943fb7 100644 --- a/packages/boxel-cli/src/commands/realm/create.ts +++ b/packages/boxel-cli/src/commands/realm/create.ts @@ -68,13 +68,16 @@ export async function createRealm( let response: Response; try { - response = await pm.authedFetch(`${realmServerUrl}/_create-realm`, { - method: 'POST', - headers: { 'Content-Type': 'application/vnd.api+json' }, - body: JSON.stringify({ - data: { type: 'realm', attributes }, - }), - }); + response = await pm.authedRealmServerFetch( + `${realmServerUrl}/_create-realm`, + { + method: 'POST', + headers: { 'Content-Type': 'application/vnd.api+json' }, + body: JSON.stringify({ + data: { type: 'realm', attributes }, + }), + }, + ); } catch (e: unknown) { console.error(`Error: failed to connect to realm server`); console.error(e instanceof Error ? e.message : String(e)); diff --git a/packages/boxel-cli/src/commands/realm/index.ts b/packages/boxel-cli/src/commands/realm/index.ts index a435659a41..41eaaab914 100644 --- a/packages/boxel-cli/src/commands/realm/index.ts +++ b/packages/boxel-cli/src/commands/realm/index.ts @@ -1,5 +1,6 @@ import type { Command } from 'commander'; import { registerCreateCommand } from './create'; +import { registerPullCommand } from './pull'; export function registerRealmCommand(program: Command): void { let realm = program @@ -7,4 +8,5 @@ export function registerRealmCommand(program: Command): void { .description('Manage realms on the realm server'); registerCreateCommand(realm); + registerPullCommand(realm); } diff --git a/packages/boxel-cli/src/commands/realm/pull.ts b/packages/boxel-cli/src/commands/realm/pull.ts new file mode 100644 index 0000000000..6b50a61ae0 --- /dev/null +++ b/packages/boxel-cli/src/commands/realm/pull.ts @@ -0,0 +1,228 @@ +import type { Command } from 'commander'; +import { RealmSyncBase, type SyncOptions } from '../../lib/realm-sync-base'; +import { + CheckpointManager, + type CheckpointChange, +} from '../../lib/checkpoint-manager'; +import { + getProfileManager, + type ProfileManager, +} from '../../lib/profile-manager'; +import * as fs from 'fs'; +import * as path from 'path'; + +interface PullOptions extends SyncOptions { + deleteLocal?: boolean; +} + +class RealmPuller extends RealmSyncBase { + hasError = false; + + constructor( + private pullOptions: PullOptions, + profileManager: ProfileManager, + ) { + super(pullOptions, profileManager); + } + + async sync(): Promise { + console.log( + `Starting pull from ${this.options.workspaceUrl} to ${this.options.localDir}`, + ); + + console.log('Testing workspace access...'); + try { + await this.getRemoteFileList(''); + } catch (error) { + console.error('Failed to access workspace:', error); + throw new Error( + 'Cannot proceed with pull: Authentication or access failed. ' + + 'Please check your Matrix credentials and workspace permissions.', + ); + } + console.log('Workspace access verified'); + + const remoteFiles = await this.getRemoteFileList(); + console.log(`Found ${remoteFiles.size} files in remote workspace`); + + const localFiles = await this.getLocalFileList(); + console.log(`Found ${localFiles.size} files in local directory`); + + if (!fs.existsSync(this.options.localDir)) { + if (this.options.dryRun) { + console.log( + `[DRY RUN] Would create directory: ${this.options.localDir}`, + ); + } else { + fs.mkdirSync(this.options.localDir, { recursive: true }); + console.log(`Created directory: ${this.options.localDir}`); + } + } + + const downloadedFiles: string[] = []; + for (const [relativePath] of remoteFiles) { + try { + const localPath = path.join(this.options.localDir, relativePath); + await this.downloadFile(relativePath, localPath); + downloadedFiles.push(relativePath); + } catch (error) { + this.hasError = true; + console.error(`Error downloading ${relativePath}:`, error); + } + } + + const deletedFiles: string[] = []; + if (this.pullOptions.deleteLocal) { + const filesToDelete = new Set(localFiles.keys()); + for (const relativePath of remoteFiles.keys()) { + filesToDelete.delete(relativePath); + } + + if (filesToDelete.size > 0) { + const checkpointManager = new CheckpointManager(this.options.localDir); + const deleteChanges: CheckpointChange[] = Array.from(filesToDelete).map( + (f) => ({ + file: f, + status: 'deleted' as const, + }), + ); + const preDeleteCheckpoint = checkpointManager.createCheckpoint( + 'remote', + deleteChanges, + `Pre-delete checkpoint: ${filesToDelete.size} files not on server`, + ); + if (preDeleteCheckpoint) { + console.log( + `\nCheckpoint created before deletion: ${preDeleteCheckpoint.shortHash}`, + ); + } + + console.log( + `\nDeleting ${filesToDelete.size} local files that don't exist in workspace...`, + ); + + for (const relativePath of filesToDelete) { + try { + const localPath = localFiles.get(relativePath); + if (localPath) { + await this.deleteLocalFile(localPath); + deletedFiles.push(relativePath); + console.log(` Deleted: ${relativePath}`); + } + } catch (error) { + this.hasError = true; + console.error(`Error deleting local file ${relativePath}:`, error); + } + } + } + } + + if (!this.options.dryRun && downloadedFiles.length > 0) { + const checkpointManager = new CheckpointManager(this.options.localDir); + const pullChanges: CheckpointChange[] = downloadedFiles.map((f) => ({ + file: f, + status: 'modified' as const, + })); + if (deletedFiles.length > 0) { + for (const f of deletedFiles) { + pullChanges.push({ file: f, status: 'deleted' as const }); + } + } + const checkpoint = checkpointManager.createCheckpoint( + 'remote', + pullChanges, + ); + if (checkpoint) { + const tag = checkpoint.isMajor ? '[MAJOR]' : '[minor]'; + console.log( + `\nCheckpoint created: ${checkpoint.shortHash} ${tag} ${checkpoint.message}`, + ); + } + } else if (!this.options.dryRun && deletedFiles.length > 0) { + const checkpointManager = new CheckpointManager(this.options.localDir); + const deleteChanges: CheckpointChange[] = deletedFiles.map((f) => ({ + file: f, + status: 'deleted' as const, + })); + const checkpoint = checkpointManager.createCheckpoint( + 'remote', + deleteChanges, + ); + if (checkpoint) { + const tag = checkpoint.isMajor ? '[MAJOR]' : '[minor]'; + console.log( + `\nCheckpoint created: ${checkpoint.shortHash} ${tag} ${checkpoint.message}`, + ); + } + } + + console.log('Pull completed'); + } +} + +export interface PullCommandOptions { + delete?: boolean; + dryRun?: boolean; + profileManager?: ProfileManager; +} + +export function registerPullCommand(realm: Command): void { + realm + .command('pull') + .description('Pull files from a Boxel realm to a local directory') + .argument( + '', + 'The URL of the source workspace (e.g., https://app.boxel.ai/demo/)', + ) + .argument('', 'The local directory to sync files to') + .option('--delete', 'Delete local files that do not exist in the workspace') + .option('--dry-run', 'Show what would be done without making changes') + .action( + async ( + workspaceUrl: string, + localDir: string, + options: { delete?: boolean; dryRun?: boolean }, + ) => { + await pullCommand(workspaceUrl, localDir, options); + }, + ); +} + +export async function pullCommand( + workspaceUrl: string, + localDir: string, + options: PullCommandOptions, +): Promise { + let pm = options.profileManager ?? getProfileManager(); + let active = pm.getActiveProfile(); + if (!active) { + console.error( + 'Error: no active profile. Run `boxel profile add` to create one.', + ); + process.exit(1); + } + + try { + const puller = new RealmPuller( + { + workspaceUrl, + localDir, + deleteLocal: options.delete, + dryRun: options.dryRun, + }, + pm, + ); + + await puller.sync(); + + if (puller.hasError) { + console.log('Pull did not complete successfully. View logs for details'); + process.exit(2); + } else { + console.log('Pull completed successfully'); + } + } catch (error) { + console.error('Pull failed:', error); + process.exit(1); + } +} diff --git a/packages/boxel-cli/src/lib/checkpoint-manager.ts b/packages/boxel-cli/src/lib/checkpoint-manager.ts new file mode 100644 index 0000000000..0a8377c177 --- /dev/null +++ b/packages/boxel-cli/src/lib/checkpoint-manager.ts @@ -0,0 +1,550 @@ +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { isProtectedFile } from './realm-sync-base'; + +export interface Checkpoint { + hash: string; + shortHash: string; + message: string; + description: string; + date: Date; + isMajor: boolean; + filesChanged: number; + insertions: number; + deletions: number; + source: 'local' | 'remote' | 'manual'; + isMilestone: boolean; + milestoneName?: string; +} + +export interface CheckpointChange { + file: string; + status: 'added' | 'modified' | 'deleted'; +} + +export class CheckpointManager { + private workspaceDir: string; + private gitDir: string; + + constructor(workspaceDir: string) { + this.workspaceDir = path.resolve(workspaceDir); + this.gitDir = path.join(this.workspaceDir, '.boxel-history'); + } + + init(): void { + if (!fs.existsSync(this.gitDir)) { + fs.mkdirSync(this.gitDir, { recursive: true }); + } + + const gitPath = path.join(this.gitDir, '.git'); + if (!fs.existsSync(gitPath)) { + this.git('init'); + this.git('config', 'user.email', 'boxel-cli@local'); + this.git('config', 'user.name', 'Boxel CLI'); + this.git( + 'commit', + '--allow-empty', + '-m', + '[init] Initialize checkpoint history', + ); + } + } + + isInitialized(): boolean { + return fs.existsSync(path.join(this.gitDir, '.git')); + } + + private syncFilesToHistory(): void { + const files = this.getWorkspaceFiles(); + + const historyFiles = this.getHistoryFiles(); + for (const file of historyFiles) { + if (!files.includes(file)) { + const historyPath = path.join(this.gitDir, file); + if (fs.existsSync(historyPath)) { + fs.unlinkSync(historyPath); + } + } + } + + for (const file of files) { + const srcPath = path.join(this.workspaceDir, file); + const destPath = path.join(this.gitDir, file); + + const destDir = path.dirname(destPath); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + + fs.copyFileSync(srcPath, destPath); + } + } + + private getWorkspaceFiles(): string[] { + const files: string[] = []; + + const scan = (dir: string, prefix = '') => { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if ( + entry.name === '.boxel-history' || + entry.name === '.boxel-sync.json' || + entry.name === 'node_modules' + ) { + continue; + } + if (entry.name.startsWith('.')) { + continue; + } + + const relPath = prefix ? `${prefix}/${entry.name}` : entry.name; + + if (entry.isDirectory()) { + scan(path.join(dir, entry.name), relPath); + } else { + files.push(relPath); + } + } + }; + + scan(this.workspaceDir); + return files; + } + + private getHistoryFiles(): string[] { + const files: string[] = []; + + const scan = (dir: string, prefix = '') => { + if (!fs.existsSync(dir)) return; + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + if (entry.name === '.git') continue; + + const relPath = prefix ? `${prefix}/${entry.name}` : entry.name; + + if (entry.isDirectory()) { + scan(path.join(dir, entry.name), relPath); + } else { + files.push(relPath); + } + } + }; + + scan(this.gitDir); + return files; + } + + detectCurrentChanges(): CheckpointChange[] { + if (!this.isInitialized()) { + const files = this.getWorkspaceFiles(); + return files.map((file) => ({ file, status: 'added' as const })); + } + + this.syncFilesToHistory(); + + const status = spawnSync('git', ['status', '--porcelain'], { + cwd: this.gitDir, + encoding: 'utf-8', + }); + + const statusOutput = status.stdout.trim(); + if (!statusOutput) { + return []; + } + + const changes: CheckpointChange[] = []; + for (const line of statusOutput.split('\n')) { + if (!line) continue; + + const statusCode = line.substring(0, 2); + const file = line.substring(3); + + if (statusCode.includes('R')) { + const arrowIndex = file.indexOf(' -> '); + if (arrowIndex !== -1) { + const oldFile = file.substring(0, arrowIndex); + const newFile = file.substring(arrowIndex + 4); + changes.push({ file: oldFile, status: 'deleted' }); + changes.push({ file: newFile, status: 'added' }); + continue; + } + } + + if (statusCode.includes('D')) { + changes.push({ file, status: 'deleted' }); + } else if ( + statusCode.includes('A') || + statusCode.includes('C') || + statusCode === '??' + ) { + changes.push({ file, status: 'added' }); + } else if ( + statusCode.includes('M') || + statusCode.includes('U') || + statusCode.includes('T') + ) { + changes.push({ file, status: 'modified' }); + } + } + + return changes; + } + + createCheckpoint( + source: 'local' | 'remote' | 'manual', + changes: CheckpointChange[], + customMessage?: string, + ): Checkpoint | null { + if (!this.isInitialized()) { + this.init(); + } + + this.syncFilesToHistory(); + + this.git('add', '-A'); + + const status = spawnSync('git', ['status', '--porcelain'], { + cwd: this.gitDir, + encoding: 'utf-8', + }); + + if (!status.stdout.trim()) { + return null; + } + + const isMajor = this.classifyChanges(changes); + + const { message, description } = customMessage + ? { message: customMessage, description: '' } + : this.generateCommitMessage(source, changes, isMajor); + + const prefix = isMajor ? '[MAJOR]' : '[minor]'; + const sourceTag = `[${source}]`; + const fullMessage = `${prefix} ${sourceTag} ${message}${description ? '\n\n' + description : ''}`; + + this.git('commit', '-m', fullMessage); + + const hash = this.git('rev-parse', 'HEAD').trim(); + const shortHash = hash.substring(0, 7); + + return { + hash, + shortHash, + message, + description, + date: new Date(), + isMajor, + filesChanged: changes.length, + insertions: 0, + deletions: 0, + source, + isMilestone: false, + }; + } + + private classifyChanges(changes: CheckpointChange[]): boolean { + if (changes.length > 3) return true; + + for (const change of changes) { + if (change.status === 'added' || change.status === 'deleted') return true; + if (change.file.endsWith('.gts')) return true; + } + + return false; + } + + private generateCommitMessage( + source: 'local' | 'remote' | 'manual', + changes: CheckpointChange[], + _isMajor: boolean, + ): { message: string; description: string } { + const sourceLabel = + source === 'local' ? 'Push' : source === 'remote' ? 'Pull' : 'Manual'; + + if (changes.length === 0) { + return { + message: `${sourceLabel}: No changes detected`, + description: '', + }; + } + + if (changes.length === 1) { + const change = changes[0]; + const action = + change.status === 'added' + ? 'Add' + : change.status === 'deleted' + ? 'Delete' + : 'Update'; + return { + message: `${sourceLabel}: ${action} ${change.file}`, + description: '', + }; + } + + const added = changes.filter((c) => c.status === 'added'); + const modified = changes.filter((c) => c.status === 'modified'); + const deleted = changes.filter((c) => c.status === 'deleted'); + + const parts: string[] = []; + if (added.length > 0) parts.push(`+${added.length}`); + if (modified.length > 0) parts.push(`~${modified.length}`); + if (deleted.length > 0) parts.push(`-${deleted.length}`); + + const message = `${sourceLabel}: ${changes.length} files (${parts.join(', ')})`; + + const lines: string[] = []; + if (added.length > 0) { + lines.push('Added:'); + added.forEach((c) => lines.push(` + ${c.file}`)); + } + if (modified.length > 0) { + lines.push('Modified:'); + modified.forEach((c) => lines.push(` ~ ${c.file}`)); + } + if (deleted.length > 0) { + lines.push('Deleted:'); + deleted.forEach((c) => lines.push(` - ${c.file}`)); + } + + return { message, description: lines.join('\n') }; + } + + getCheckpoints(limit = 50): Checkpoint[] { + if (!this.isInitialized()) { + return []; + } + + const format = '%H|%h|%s|%aI|%an'; + const log = this.git('log', `--format=${format}`, `-${limit}`); + + if (!log.trim()) { + return []; + } + + const milestones = this.getAllMilestones(); + + return log + .trim() + .split('\n') + .map((line) => { + const [hash, shortHash, subject, dateStr] = line.split('|'); + + const isMajor = subject.includes('[MAJOR]'); + const source = subject.includes('[local]') + ? ('local' as const) + : subject.includes('[remote]') + ? ('remote' as const) + : ('manual' as const); + + const message = subject + .replace(/\[(MAJOR|minor)\]\s*/i, '') + .replace(/\[(local|remote|manual)\]\s*/i, ''); + + const stats = this.getCommitStats(hash); + + const milestoneName = milestones.get(hash); + const isMilestone = !!milestoneName; + + return { + hash, + shortHash, + message, + description: '', + date: new Date(dateStr), + isMajor, + source, + isMilestone, + milestoneName, + ...stats, + }; + }); + } + + private getCommitStats(hash: string): { + filesChanged: number; + insertions: number; + deletions: number; + } { + try { + const stat = this.git('show', '--stat', '--format=', hash); + const lines = stat.trim().split('\n'); + const summaryLine = lines[lines.length - 1] || ''; + + const filesMatch = summaryLine.match(/(\d+) files? changed/); + const insertMatch = summaryLine.match(/(\d+) insertions?/); + const deleteMatch = summaryLine.match(/(\d+) deletions?/); + + return { + filesChanged: filesMatch ? parseInt(filesMatch[1]) : 0, + insertions: insertMatch ? parseInt(insertMatch[1]) : 0, + deletions: deleteMatch ? parseInt(deleteMatch[1]) : 0, + }; + } catch { + return { filesChanged: 0, insertions: 0, deletions: 0 }; + } + } + + getChangedFiles(hash: string): string[] { + const output = this.git('show', '--name-only', '--format=', hash); + return output.trim().split('\n').filter(Boolean); + } + + getDiff(hash: string): string { + return this.git('show', '--format=', hash); + } + + restore(hash: string): void { + const currentFiles = this.getHistoryFiles(); + for (const file of currentFiles) { + const filePath = path.join(this.gitDir, file); + if (fs.existsSync(filePath)) { + fs.unlinkSync(filePath); + } + } + + this.git('checkout', hash, '--', '.'); + + const historyFiles = this.getHistoryFiles(); + const workspaceFiles = this.getWorkspaceFiles(); + + for (const file of workspaceFiles) { + if (isProtectedFile(file)) continue; + if (!historyFiles.includes(file)) { + const filePath = path.join(this.workspaceDir, file); + fs.unlinkSync(filePath); + } + } + + for (const file of historyFiles) { + if (isProtectedFile(file)) continue; + const srcPath = path.join(this.gitDir, file); + const destPath = path.join(this.workspaceDir, file); + + const destDir = path.dirname(destPath); + if (!fs.existsSync(destDir)) { + fs.mkdirSync(destDir, { recursive: true }); + } + + fs.copyFileSync(srcPath, destPath); + } + + this.git('checkout', 'HEAD', '--', '.'); + } + + markMilestone( + hashOrIndex: string | number, + name: string, + ): { hash: string; name: string } | null { + if (!this.isInitialized()) { + return null; + } + + let hash: string; + if (typeof hashOrIndex === 'number') { + const checkpoints = this.getCheckpoints(hashOrIndex + 1); + if (hashOrIndex < 1 || hashOrIndex > checkpoints.length) { + return null; + } + hash = checkpoints[hashOrIndex - 1].hash; + } else { + hash = hashOrIndex; + } + + const tagName = `milestone/${name.replace(/\s+/g, '-').replace(/[^a-zA-Z0-9\-_.]/g, '')}`; + + try { + this.git('tag', '-a', tagName, hash, '-m', `Milestone: ${name}`); + return { hash, name }; + } catch { + return null; + } + } + + unmarkMilestone(hashOrIndex: string | number): boolean { + if (!this.isInitialized()) { + return false; + } + + let hash: string; + if (typeof hashOrIndex === 'number') { + const checkpoints = this.getCheckpoints(hashOrIndex + 1); + if (hashOrIndex < 1 || hashOrIndex > checkpoints.length) { + return false; + } + hash = checkpoints[hashOrIndex - 1].hash; + } else { + hash = hashOrIndex; + } + + const tags = this.getMilestoneTags(hash); + if (tags.length === 0) { + return false; + } + + for (const tag of tags) { + try { + this.git('tag', '-d', tag); + } catch { + // Ignore errors + } + } + + return true; + } + + private getMilestoneTags(hash: string): string[] { + try { + const output = this.git('tag', '--points-at', hash); + return output + .trim() + .split('\n') + .filter((tag) => tag.startsWith('milestone/')) + .filter(Boolean); + } catch { + return []; + } + } + + private getAllMilestones(): Map { + const milestones = new Map(); + try { + const tags = this.git('tag', '-l', 'milestone/*'); + for (const tag of tags.trim().split('\n').filter(Boolean)) { + try { + const hash = this.git('rev-list', '-1', tag).trim(); + const name = tag.replace('milestone/', '').replace(/-/g, ' '); + milestones.set(hash, name); + } catch { + // Ignore invalid tags + } + } + } catch { + // No tags + } + return milestones; + } + + getMilestones(): Checkpoint[] { + const all = this.getCheckpoints(100); + return all.filter((cp) => cp.isMilestone); + } + + private git(...args: string[]): string { + const result = spawnSync('git', args, { + cwd: this.gitDir, + encoding: 'utf-8', + }); + + if (result.error) { + throw result.error; + } + + if (result.status !== 0 && !args.includes('status')) { + throw new Error(`git ${args.join(' ')} failed: ${result.stderr}`); + } + + return result.stdout; + } +} diff --git a/packages/boxel-cli/src/lib/profile-manager.ts b/packages/boxel-cli/src/lib/profile-manager.ts index 2f5bebb58e..1938cb89ec 100644 --- a/packages/boxel-cli/src/lib/profile-manager.ts +++ b/packages/boxel-cli/src/lib/profile-manager.ts @@ -353,11 +353,50 @@ export class ProfileManager { return token; } - async authedFetch( + private findRealmTokenForUrl(url: string): string | undefined { + let active = this.getActiveProfile(); + let realmTokens = active?.profile.realmTokens; + if (!realmTokens) { + return undefined; + } + for (let [realmUrl, token] of Object.entries(realmTokens)) { + if (url.startsWith(realmUrl) && token) { + return token; + } + } + return undefined; + } + + private async fetchAndStoreAllRealmTokens(): Promise { + let serverToken = await this.getOrRefreshServerToken(); + let active = this.getActiveProfile()!; + let realmServerUrl = active.profile.realmServerUrl.replace(/\/$/, ''); + let tokens = await getRealmTokens(realmServerUrl, serverToken); + for (let [realmUrl, token] of Object.entries(tokens)) { + this.setRealmToken(realmUrl, token); + } + } + + private async getRealmTokenForUrl(url: string): Promise { + let realmToken = this.findRealmTokenForUrl(url); + if (realmToken) { + return realmToken; + } + + try { + await this.fetchAndStoreAllRealmTokens(); + } catch { + // Token prefetch failed (e.g. expired server token) — caller will handle 401 retry + return undefined; + } + return this.findRealmTokenForUrl(url); + } + + private buildHeaders( input: string | URL | Request, - init?: RequestInit, - ): Promise { - let token = await this.getOrRefreshServerToken(); + init: RequestInit | undefined, + token: string, + ): Headers { let baseHeaders = input instanceof Request ? new Headers(input.headers) : new Headers(); let initHeaders = new Headers(init?.headers); @@ -367,13 +406,63 @@ export class ProfileManager { if (!baseHeaders.has('Authorization')) { baseHeaders.set('Authorization', token); } + return baseHeaders; + } + + async authedRealmFetch( + input: string | URL | Request, + init?: RequestInit, + ): Promise { + let url = + input instanceof Request + ? input.url + : input instanceof URL + ? input.href + : input; + + let token = await this.getRealmTokenForUrl(url); + if (token) { + let headers = this.buildHeaders(input, init, token); + let response = await fetch(input, { ...init, headers }); + + if (response.status !== 401) { + return response; + } + } + + // Either no cached realm token (e.g. server token was expired during + // prefetch) or the request got a 401. Refresh everything and retry. + let active = this.getActiveProfile(); + if (active) { + active.profile.realmTokens = {}; + active.profile.realmServerToken = undefined; + this.saveConfig(); + } + await this.fetchAndStoreAllRealmTokens(); + token = this.findRealmTokenForUrl(url); + if (!token) { + throw new Error( + `No realm token available for ${url}. The realm may not be accessible.`, + ); + } + let headers = this.buildHeaders(input, init, token); + let response = await fetch(input, { ...init, headers }); + + return response; + } - let response = await fetch(input, { ...init, headers: baseHeaders }); + async authedRealmServerFetch( + input: string | URL | Request, + init?: RequestInit, + ): Promise { + let token = await this.getOrRefreshServerToken(); + let headers = this.buildHeaders(input, init, token); + let response = await fetch(input, { ...init, headers }); if (response.status === 401) { token = await this.refreshServerToken(); - baseHeaders.set('Authorization', token); - response = await fetch(input, { ...init, headers: baseHeaders }); + headers = this.buildHeaders(input, init, token); + response = await fetch(input, { ...init, headers }); } return response; diff --git a/packages/boxel-cli/src/lib/realm-sync-base.ts b/packages/boxel-cli/src/lib/realm-sync-base.ts new file mode 100644 index 0000000000..4e6d51db3d --- /dev/null +++ b/packages/boxel-cli/src/lib/realm-sync-base.ts @@ -0,0 +1,471 @@ +import type { ProfileManager } from './profile-manager'; +import * as fs from 'fs'; +import * as path from 'path'; +import ignoreModule from 'ignore'; + +const ignore = (ignoreModule as any).default || ignoreModule; +type Ignore = ReturnType; + +// Files that must never be pushed, deleted, or overwritten on the server via CLI. +export const PROTECTED_FILES = new Set(['.realm.json']); + +export function isProtectedFile(relativePath: string): boolean { + const normalizedPath = relativePath.replace(/\\/g, '/').replace(/^\/+/, ''); + return PROTECTED_FILES.has(normalizedPath); +} + +export const SupportedMimeType = { + CardSource: 'application/vnd.card+source', + DirectoryListing: 'application/vnd.api+json', + Mtimes: 'application/vnd.api+json', +} as const; + +export interface SyncOptions { + workspaceUrl: string; + localDir: string; + dryRun?: boolean; +} + +export abstract class RealmSyncBase { + protected normalizedRealmUrl: string; + private ignoreCache = new Map(); + + constructor( + protected options: SyncOptions, + protected profileManager: ProfileManager, + ) { + this.normalizedRealmUrl = this.normalizeRealmUrl(options.workspaceUrl); + } + + private normalizeRealmUrl(url: string): string { + try { + const urlObj = new URL(url); + + const pathPart = urlObj.pathname; + const lastSegment = pathPart.split('/').filter(Boolean).pop() || ''; + + if (lastSegment.includes('.')) { + console.warn( + `Warning: "${url}" looks like a file URL, not a realm URL.` + + `\n Realm URLs should point to a directory (e.g., ${urlObj.origin}${pathPart.replace(/\/[^/]*\.[^/]*$/, '/')})`, + ); + } else if (!url.endsWith('/')) { + console.warn( + `Warning: Realm URL should end with a trailing slash.` + + `\n Did you mean "${url}/"?`, + ); + } + + return urlObj.href.replace(/\/+$/, '') + '/'; + } catch { + throw new Error(`Invalid workspace URL: ${url}`); + } + } + + protected buildDirectoryUrl(dir = ''): string { + if (!dir) { + return this.normalizedRealmUrl; + } + const cleanDir = dir.replace(/^\/+|\/+$/g, ''); + return `${this.normalizedRealmUrl}${cleanDir}/`; + } + + protected buildFileUrl(relativePath: string): string { + const cleanPath = relativePath.replace(/^\/+/, ''); + return `${this.normalizedRealmUrl}${cleanPath}`; + } + + protected async getRemoteFileList(dir = ''): Promise> { + const files = new Map(); + + try { + const url = this.buildDirectoryUrl(dir); + + const response = await this.profileManager.authedRealmFetch(url, { + headers: { + Accept: 'application/vnd.api+json', + }, + }); + + if (!response.ok) { + if (response.status === 404) { + return files; + } + if (response.status === 401 || response.status === 403) { + throw new Error( + `Authentication failed (${response.status}): Cannot access workspace. Check your Matrix credentials and workspace permissions.`, + ); + } + throw new Error( + `Failed to get directory listing: ${response.status} ${response.statusText}`, + ); + } + + const data = (await response.json()) as { + data?: { + relationships?: Record; + }; + }; + + if (data.data && data.data.relationships) { + for (const [name, info] of Object.entries(data.data.relationships)) { + const entry = info as { meta: { kind: string } }; + const isFile = entry.meta.kind === 'file'; + const entryPath = dir ? path.posix.join(dir, name) : name; + + if (isFile) { + if (!this.shouldIgnoreRemoteFile(entryPath)) { + files.set(entryPath, true); + } + } else { + const subdirFiles = await this.getRemoteFileList(entryPath); + for (const [subPath, isFileEntry] of subdirFiles) { + files.set(subPath, isFileEntry); + } + } + } + } + } catch (error) { + if (error instanceof Error) { + if ( + error.message.includes('Authentication failed') || + error.message.includes('Cannot access workspace') || + error.message.includes('401') || + error.message.includes('403') + ) { + throw error; + } + } + console.error(`Error reading remote directory ${dir}:`, error); + throw error; + } + + return files; + } + + protected async getRemoteMtimes(): Promise> { + const mtimes = new Map(); + + try { + const url = `${this.normalizedRealmUrl}_mtimes`; + + const response = await this.profileManager.authedRealmFetch(url, { + headers: { + Accept: SupportedMimeType.Mtimes, + }, + }); + + if (!response.ok) { + if (response.status === 404) { + console.log( + 'Note: _mtimes endpoint not available, will upload all files', + ); + return mtimes; + } + throw new Error( + `Failed to get mtimes: ${response.status} ${response.statusText}`, + ); + } + + const data = (await response.json()) as { + data?: { + attributes?: { + mtimes?: Record; + }; + }; + }; + + if (data.data?.attributes?.mtimes) { + const remoteMtimeEntries = Object.entries(data.data.attributes.mtimes); + if (process.env.DEBUG) { + console.log( + `Remote mtimes received: ${remoteMtimeEntries.length} entries`, + ); + if (remoteMtimeEntries.length > 0) { + console.log( + `Sample: ${remoteMtimeEntries[0][0]} = ${remoteMtimeEntries[0][1]}`, + ); + } + } + for (const [fileUrl, mtime] of remoteMtimeEntries) { + const relativePath = fileUrl.replace(this.normalizedRealmUrl, ''); + if (!this.shouldIgnoreRemoteFile(relativePath)) { + mtimes.set(relativePath, mtime); + } + } + } else if (process.env.DEBUG) { + console.log( + 'No mtimes in response:', + JSON.stringify(data).slice(0, 200), + ); + } + } catch (error) { + console.warn( + 'Could not fetch remote mtimes, will upload all files:', + error, + ); + } + + return mtimes; + } + + protected async getLocalFileListWithMtimes( + dir = '', + ): Promise> { + const files = new Map(); + const fullDir = path.join(this.options.localDir, dir); + + if (!fs.existsSync(fullDir)) { + return files; + } + + const entries = fs.readdirSync(fullDir); + + for (const entry of entries) { + const fullPath = path.join(fullDir, entry); + const relativePath = dir ? path.posix.join(dir, entry) : entry; + const stats = fs.statSync(fullPath); + + if (this.shouldIgnoreFile(relativePath, fullPath)) { + continue; + } + + if (stats.isFile()) { + files.set(relativePath, { + path: fullPath, + mtime: stats.mtimeMs, + }); + } else if (stats.isDirectory()) { + const subdirFiles = await this.getLocalFileListWithMtimes(relativePath); + for (const [subPath, fileInfo] of subdirFiles) { + files.set(subPath, fileInfo); + } + } + } + + return files; + } + + protected async getLocalFileList(dir = ''): Promise> { + const files = new Map(); + const fullDir = path.join(this.options.localDir, dir); + + if (!fs.existsSync(fullDir)) { + return files; + } + + const entries = fs.readdirSync(fullDir); + + for (const entry of entries) { + const fullPath = path.join(fullDir, entry); + const relativePath = dir ? path.posix.join(dir, entry) : entry; + const stats = fs.statSync(fullPath); + + if (this.shouldIgnoreFile(relativePath, fullPath)) { + continue; + } + + if (stats.isFile()) { + files.set(relativePath, fullPath); + } else if (stats.isDirectory()) { + const subdirFiles = await this.getLocalFileList(relativePath); + for (const [subPath, fullSubPath] of subdirFiles) { + files.set(subPath, fullSubPath); + } + } + } + + return files; + } + + protected async uploadFile( + relativePath: string, + localPath: string, + ): Promise { + if (isProtectedFile(relativePath)) { + console.log(` Skipped (protected): ${relativePath}`); + return; + } + + console.log(`Uploading: ${relativePath}`); + + if (this.options.dryRun) { + console.log(`[DRY RUN] Would upload ${relativePath}`); + return; + } + + const content = fs.readFileSync(localPath, 'utf8'); + const url = this.buildFileUrl(relativePath); + + const response = await this.profileManager.authedRealmFetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'text/plain;charset=UTF-8', + Accept: SupportedMimeType.CardSource, + }, + body: content, + }); + + if (!response.ok) { + throw new Error( + `Failed to upload: ${response.status} ${response.statusText}`, + ); + } + + console.log(` Uploaded: ${relativePath}`); + } + + protected async downloadFile( + relativePath: string, + localPath: string, + ): Promise { + console.log(`Downloading: ${relativePath}`); + + if (this.options.dryRun) { + console.log(`[DRY RUN] Would download ${relativePath}`); + return; + } + + const url = this.buildFileUrl(relativePath); + + const response = await this.profileManager.authedRealmFetch(url, { + headers: { + Accept: SupportedMimeType.CardSource, + }, + }); + + if (!response.ok) { + throw new Error( + `Failed to download: ${response.status} ${response.statusText}`, + ); + } + + const content = await response.text(); + + const localDir = path.dirname(localPath); + if (!fs.existsSync(localDir)) { + fs.mkdirSync(localDir, { recursive: true }); + } + + fs.writeFileSync(localPath, content, 'utf8'); + console.log(` Downloaded: ${relativePath}`); + } + + protected async deleteFile(relativePath: string): Promise { + if (isProtectedFile(relativePath)) { + console.log(` Skipped (protected): ${relativePath}`); + return; + } + + console.log(`Deleting remote: ${relativePath}`); + + if (this.options.dryRun) { + console.log(`[DRY RUN] Would delete ${relativePath}`); + return; + } + + const url = this.buildFileUrl(relativePath); + + const response = await this.profileManager.authedRealmFetch(url, { + method: 'DELETE', + headers: { + Accept: SupportedMimeType.CardSource, + }, + }); + + if (!response.ok && response.status !== 404) { + throw new Error( + `Failed to delete: ${response.status} ${response.statusText}`, + ); + } + + console.log(` Deleted: ${relativePath}`); + } + + protected async deleteLocalFile(localPath: string): Promise { + console.log(`Deleting local: ${localPath}`); + + if (this.options.dryRun) { + console.log(`[DRY RUN] Would delete local file ${localPath}`); + return; + } + + if (fs.existsSync(localPath)) { + fs.unlinkSync(localPath); + console.log(` Deleted: ${localPath}`); + } + } + + private getIgnoreInstance(dirPath: string): Ignore { + if (this.ignoreCache.has(dirPath)) { + return this.ignoreCache.get(dirPath)!; + } + + const ig = ignore(); + let currentPath = dirPath; + const rootPath = this.options.localDir; + + while (currentPath.startsWith(rootPath)) { + const gitignorePath = path.join(currentPath, '.gitignore'); + if (fs.existsSync(gitignorePath)) { + try { + const gitignoreContent = fs.readFileSync(gitignorePath, 'utf8'); + ig.add(gitignoreContent); + } catch (error) { + console.warn( + `Warning: Could not read .gitignore file at ${gitignorePath}:`, + error, + ); + } + } + + const boxelignorePath = path.join(currentPath, '.boxelignore'); + if (fs.existsSync(boxelignorePath)) { + try { + const boxelignoreContent = fs.readFileSync(boxelignorePath, 'utf8'); + ig.add(boxelignoreContent); + } catch (error) { + console.warn( + `Warning: Could not read .boxelignore file at ${boxelignorePath}:`, + error, + ); + } + } + + const parentPath = path.dirname(currentPath); + if (parentPath === currentPath) break; + currentPath = parentPath; + } + + this.ignoreCache.set(dirPath, ig); + return ig; + } + + private shouldIgnoreFile(relativePath: string, fullPath: string): boolean { + const fileName = path.basename(relativePath); + + if (fileName === '.boxel-sync.json') { + return true; + } + + if (fileName.startsWith('.')) { + return true; + } + + const dirPath = path.dirname(fullPath); + const ig = this.getIgnoreInstance(dirPath); + const normalizedPath = relativePath.replace(/\\/g, '/'); + + return ig.ignores(normalizedPath); + } + + private shouldIgnoreRemoteFile(relativePath: string): boolean { + const fileName = path.basename(relativePath); + if (fileName.startsWith('.')) { + return true; + } + return false; + } + + abstract sync(): Promise; +} diff --git a/packages/boxel-cli/tests/helpers/integration.ts b/packages/boxel-cli/tests/helpers/integration.ts index 99249af38a..3c7d242670 100644 --- a/packages/boxel-cli/tests/helpers/integration.ts +++ b/packages/boxel-cli/tests/helpers/integration.ts @@ -17,13 +17,36 @@ import { PgQueueRunner, type PgAdapter, } from '@cardstack/postgres'; -import type { Prerenderer } from '@cardstack/runtime-common'; +import type { + Prerenderer, + LooseSingleCardDocument, +} from '@cardstack/runtime-common'; import type { Server } from 'http'; // CLI tests don't need card rendering — stub out the prerenderer // so we don't launch Chrome. const noopPrerenderer: Prerenderer = { - prerenderCard: async () => ({ html: '', status: 200 }) as any, + prerenderCard: async () => ({ + serialized: null, + searchDoc: null, + displayNames: null, + deps: null, + types: null, + isolatedHTML: null, + headHTML: null, + atomHTML: null, + embeddedHTML: null, + fittedHTML: null, + iconHTML: null, + error: { + type: 'instance-error' as const, + error: { + message: 'Prerendering disabled in CLI tests', + status: 500, + additionalErrors: null, + }, + }, + }), prerenderModule: async () => ({ html: '', status: 200 }) as any, prerenderFileExtract: async () => ({ html: '', status: 200 }) as any, prerenderFileRender: async () => ({ html: '', status: 200 }) as any, @@ -40,7 +63,9 @@ let dbAdapter: PgAdapter | undefined; let publisher: PgQueuePublisher | undefined; let runner: PgQueueRunner | undefined; -export async function startTestRealmServer(): Promise { +export async function startTestRealmServer(options?: { + fileSystem?: Record; +}): Promise { prepareTestDB(); dbAdapter = await createTestPgAdapter(); publisher = new PgQueuePublisher(dbAdapter); @@ -57,6 +82,7 @@ export async function startTestRealmServer(): Promise { realmsRootPath: fs.mkdtempSync( path.join(os.tmpdir(), 'boxel-cli-realms-root-'), ), + fileSystem: options?.fileSystem, realmURL, virtualNetwork, publisher, diff --git a/packages/boxel-cli/tests/integration/realm-pull.test.ts b/packages/boxel-cli/tests/integration/realm-pull.test.ts new file mode 100644 index 0000000000..76a583295e --- /dev/null +++ b/packages/boxel-cli/tests/integration/realm-pull.test.ts @@ -0,0 +1,131 @@ +import '../helpers/setup-realm-server'; +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import { pullCommand } from '../../src/commands/realm/pull'; +import { + startTestRealmServer, + stopTestRealmServer, + createTestProfileDir, + setupTestProfile, + TEST_REALM_SERVER_URL, +} from '../helpers/integration'; +import type { ProfileManager } from '../../src/lib/profile-manager'; + +let profileManager: ProfileManager; +let cleanupProfile: () => void; +let realmUrl: string; +let localDirs: string[] = []; + +function makeLocalDir(): string { + let dir = fs.mkdtempSync(path.join(os.tmpdir(), 'boxel-pull-int-')); + localDirs.push(dir); + return dir; +} + +beforeAll(async () => { + // Seed files into the realm at creation time, matching the realm-server + // test pattern of passing a fileSystem to the realm setup helpers. + await startTestRealmServer({ + fileSystem: { + 'hello.gts': 'export const hello = "world";\n', + 'nested/card.gts': 'export const nested = true;\n', + 'nested/deep/inner.gts': 'export const inner = "deep";\n', + }, + }); + + realmUrl = `${TEST_REALM_SERVER_URL}/test/`; + + let testProfile = createTestProfileDir(); + profileManager = testProfile.profileManager; + cleanupProfile = testProfile.cleanup; + await setupTestProfile(profileManager); +}); + +afterAll(async () => { + for (let dir of localDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } + cleanupProfile?.(); + await stopTestRealmServer(); +}); + +describe('realm pull (integration)', () => { + it('pulls seeded files into an empty local directory', async () => { + let localDir = makeLocalDir(); + + await pullCommand(realmUrl, localDir, { profileManager }); + + let helloPath = path.join(localDir, 'hello.gts'); + let nestedPath = path.join(localDir, 'nested', 'card.gts'); + expect(fs.existsSync(helloPath)).toBe(true); + expect(fs.existsSync(nestedPath)).toBe(true); + expect(fs.readFileSync(helloPath, 'utf8')).toContain('hello = "world"'); + expect(fs.readFileSync(nestedPath, 'utf8')).toContain('nested = true'); + + // Checkpoint history is written under .boxel-history/ + expect(fs.existsSync(path.join(localDir, '.boxel-history'))).toBe(true); + }); + + it('writes nothing when invoked with --dry-run', async () => { + let localDir = makeLocalDir(); + + await pullCommand(realmUrl, localDir, { dryRun: true, profileManager }); + + let entries = fs + .readdirSync(localDir) + .filter((e) => e !== '.boxel-history'); + expect(entries).toEqual([]); + }); + + it('preserves local-only files without --delete', async () => { + let localDir = makeLocalDir(); + let localOnlyDir = path.join(localDir, 'Notes'); + fs.mkdirSync(localOnlyDir, { recursive: true }); + fs.writeFileSync( + path.join(localOnlyDir, 'local-only.json'), + '{"local":"only"}', + ); + + await pullCommand(realmUrl, localDir, { profileManager }); + + expect(fs.existsSync(path.join(localDir, 'hello.gts'))).toBe(true); + expect(fs.existsSync(path.join(localOnlyDir, 'local-only.json'))).toBe( + true, + ); + }); + + it('removes local files missing from the realm when --delete is set', async () => { + let localDir = makeLocalDir(); + let staleRel = 'stale-only-local.gts'; + let stalePath = path.join(localDir, staleRel); + fs.writeFileSync(stalePath, 'export const stale = true;\n', 'utf8'); + + await pullCommand(realmUrl, localDir, { + delete: true, + profileManager, + }); + + expect(fs.existsSync(stalePath)).toBe(false); + expect(fs.existsSync(path.join(localDir, 'hello.gts'))).toBe(true); + }); + + it('pulls subdirectories recursively', async () => { + let localDir = makeLocalDir(); + + await pullCommand(realmUrl, localDir, { profileManager }); + + expect(fs.existsSync(path.join(localDir, 'hello.gts'))).toBe(true); + expect(fs.existsSync(path.join(localDir, 'nested', 'card.gts'))).toBe(true); + expect( + fs.existsSync(path.join(localDir, 'nested', 'deep', 'inner.gts')), + ).toBe(true); + expect( + fs.readFileSync( + path.join(localDir, 'nested', 'deep', 'inner.gts'), + 'utf8', + ), + ).toContain('inner = "deep"'); + }); +});