From cad4546732d1502fc5d9ec5e9273641b50270a8f Mon Sep 17 00:00:00 2001 From: Diego Barreto Date: Tue, 25 Aug 2026 00:01:35 -0300 Subject: [PATCH 1/2] feat(sync): send and sync projects between environments Adds client-orchestrated project sync across environments: the client reads a manifest (path/size/sha256) from the source environment and streams changed files to the destination over signed short-lived HTTP URLs, so it works over every existing connection mode (local, LAN, Tailscale, T3 Connect, SSH) without any server-to-server trails. - contracts: projectSync RPCs, capability flag, tagged errors - shared: binary framing codec for batched file transfer - server: manifest walker, signed export/import routes, path-safety guards, atomic writes, mirror deletions with empty-dir pruning - client-runtime: pure diff/batching plan + runProjectSync controller with progress and cancellation - web: SyncProjectDialog (send/sync modes, delete confirmation, include-.git toggle), command palette entries, Project Settings section; destination folder defaults from the environment's add project base directory - docs: user guide, internals architecture page, glossary terms Implemented by Claude Fable 5 subagents (Sonnet/Opus) via Claude Code. --- apps/server/src/auth/RpcAuthorization.ts | 7 + .../src/environment/ServerEnvironment.ts | 1 + apps/server/src/http.ts | 110 ++ apps/server/src/server.ts | 4 + .../src/workspace/ProjectSyncApply.test.ts | 291 +++++ apps/server/src/workspace/ProjectSyncApply.ts | 324 ++++++ .../src/workspace/ProjectSyncManifest.test.ts | 169 +++ .../src/workspace/ProjectSyncManifest.ts | 196 ++++ .../src/workspace/ProjectSyncTransfer.test.ts | 215 ++++ .../src/workspace/ProjectSyncTransfer.ts | 342 ++++++ apps/server/src/workspace/projectSyncErrno.ts | 22 + apps/server/src/ws.ts | 78 ++ apps/web/src/components/CommandPalette.tsx | 49 +- .../SyncProjectDialog.logic.test.ts | 396 +++++++ .../src/components/SyncProjectDialog.logic.ts | 310 ++++++ apps/web/src/components/SyncProjectDialog.tsx | 995 ++++++++++++++++++ .../settings/ProjectSettingsPanel.tsx | 58 +- apps/web/src/state/projectSync.ts | 122 +++ docs/README.md | 2 + docs/internals/glossary.md | 32 + docs/internals/project-sync.md | 184 ++++ docs/user/project-sync.md | 77 ++ packages/client-runtime/package.json | 8 + .../client-runtime/src/operations/index.ts | 1 + .../src/operations/projectSync.test.ts | 200 ++++ .../src/operations/projectSync.ts | 141 +++ .../src/state/projectSync.test.ts | 278 +++++ .../client-runtime/src/state/projectSync.ts | 283 +++++ packages/contracts/src/environment.ts | 5 + packages/contracts/src/index.ts | 1 + packages/contracts/src/projectSync.ts | 175 +++ packages/contracts/src/rpc.ts | 44 + packages/shared/package.json | 4 + .../shared/src/projectSyncFraming.test.ts | 199 ++++ packages/shared/src/projectSyncFraming.ts | 297 ++++++ 35 files changed, 5618 insertions(+), 2 deletions(-) create mode 100644 apps/server/src/workspace/ProjectSyncApply.test.ts create mode 100644 apps/server/src/workspace/ProjectSyncApply.ts create mode 100644 apps/server/src/workspace/ProjectSyncManifest.test.ts create mode 100644 apps/server/src/workspace/ProjectSyncManifest.ts create mode 100644 apps/server/src/workspace/ProjectSyncTransfer.test.ts create mode 100644 apps/server/src/workspace/ProjectSyncTransfer.ts create mode 100644 apps/server/src/workspace/projectSyncErrno.ts create mode 100644 apps/web/src/components/SyncProjectDialog.logic.test.ts create mode 100644 apps/web/src/components/SyncProjectDialog.logic.ts create mode 100644 apps/web/src/components/SyncProjectDialog.tsx create mode 100644 apps/web/src/state/projectSync.ts create mode 100644 docs/internals/project-sync.md create mode 100644 docs/user/project-sync.md create mode 100644 packages/client-runtime/src/operations/projectSync.test.ts create mode 100644 packages/client-runtime/src/operations/projectSync.ts create mode 100644 packages/client-runtime/src/state/projectSync.test.ts create mode 100644 packages/client-runtime/src/state/projectSync.ts create mode 100644 packages/contracts/src/projectSync.ts create mode 100644 packages/shared/src/projectSyncFraming.test.ts create mode 100644 packages/shared/src/projectSyncFraming.ts diff --git a/apps/server/src/auth/RpcAuthorization.ts b/apps/server/src/auth/RpcAuthorization.ts index 41305b920f33..79a340f82113 100644 --- a/apps/server/src/auth/RpcAuthorization.ts +++ b/apps/server/src/auth/RpcAuthorization.ts @@ -82,6 +82,13 @@ export const RPC_REQUIRED_SCOPES = { [WS_METHODS.projectsSearchContents]: AuthOrchestrationReadScope, [WS_METHODS.projectsSearchEntries]: AuthOrchestrationReadScope, [WS_METHODS.projectsWriteFile]: AuthOrchestrationOperateScope, + // A manifest and an export URL only ever read the workspace, so they sit + // with projects.readFile. Import URLs and deletions write it, so they sit + // with projects.writeFile. + [WS_METHODS.projectSyncManifest]: AuthOrchestrationReadScope, + [WS_METHODS.projectSyncCreateExportUrl]: AuthOrchestrationReadScope, + [WS_METHODS.projectSyncCreateImportUrl]: AuthOrchestrationOperateScope, + [WS_METHODS.projectSyncApplyDeletions]: AuthOrchestrationOperateScope, [WS_METHODS.shellOpenInEditor]: AuthOrchestrationOperateScope, [WS_METHODS.filesystemBrowse]: AuthOrchestrationReadScope, [WS_METHODS.assetsCreateUrl]: AuthOrchestrationReadScope, diff --git a/apps/server/src/environment/ServerEnvironment.ts b/apps/server/src/environment/ServerEnvironment.ts index e55639ce659c..861527a9c68a 100644 --- a/apps/server/src/environment/ServerEnvironment.ts +++ b/apps/server/src/environment/ServerEnvironment.ts @@ -153,6 +153,7 @@ export const make = Effect.gen(function* () { threadPinning: true, threadPinReorder: true, threadTitleRegeneration: true, + projectSync: true, ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }), ...(serverSelfUpdate === "boot-service" ? { serverSelfUpdateProgress: true } : {}), }, diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index c3104e7bc420..d231283899ea 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -3,15 +3,22 @@ import { AuthOrchestrationOperateScope, AuthOrchestrationReadScope, EnvironmentHttpApi, + PROJECT_SYNC_EXPORT_ROUTE_PREFIX, + PROJECT_SYNC_IMPORT_ROUTE_PREFIX, } from "@t3tools/contracts"; import { isDevProxiedPath } from "@t3tools/shared/devProxy"; import { decodeOtlpTraceRecords } from "@t3tools/shared/observability"; +import { + createProjectSyncFrameDecoder, + encodeProjectSyncRecords, +} from "@t3tools/shared/projectSyncFraming"; import * as Data from "effect/Data"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as Stream from "effect/Stream"; import { cast } from "effect/Function"; import { HttpBody, @@ -44,6 +51,12 @@ import { } from "./auth/http.ts"; import * as ServerEnvironment from "./environment/ServerEnvironment.ts"; import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts"; +import { applyProjectSyncRecords } from "./workspace/ProjectSyncApply.ts"; +import { + projectSyncExportRecords, + resolveProjectSyncExportToken, + resolveProjectSyncImportToken, +} from "./workspace/ProjectSyncTransfer.ts"; const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces"; const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]); @@ -280,6 +293,103 @@ export const attachmentUploadRouteLayer = HttpRouter.add( }), ); +class ProjectSyncExportStreamError extends Data.TaggedError("ProjectSyncExportStreamError")<{ + readonly cause: unknown; +}> {} + +function projectSyncToken(pathname: string, prefix: string): string | null { + const token = pathname.slice(`${prefix}/`.length); + return token.length > 0 && !token.includes("/") ? token : null; +} + +/** + * Streams the origin side of a project sync transfer. + * + * Authorization lives entirely in the signed token: it names the workspace + * root and the request id whose path list the issuing RPC parked, both bound + * to the same 10-minute TTL. That mirrors the asset download route beside it, + * and keeps the route free of the projection read model. + */ +export const projectSyncExportRouteLayer = HttpRouter.add( + "GET", + `${PROJECT_SYNC_EXPORT_ROUTE_PREFIX}/*`, + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = HttpServerRequest.toURL(request); + if (Option.isNone(url)) { + return HttpServerResponse.text("Bad Request", { status: 400 }); + } + + const token = projectSyncToken(url.value.pathname, PROJECT_SYNC_EXPORT_ROUTE_PREFIX); + if (!token) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + + const resolved = yield* resolveProjectSyncExportToken(token); + if (!resolved) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + + return HttpServerResponse.stream( + Stream.fromAsyncIterable( + encodeProjectSyncRecords( + projectSyncExportRecords(resolved.claims.workspaceRoot, resolved.paths), + ), + (cause) => new ProjectSyncExportStreamError({ cause }), + ), + { status: 200, contentType: "application/octet-stream" }, + ); + }), +); + +/** + * Applies the destination side of a project sync transfer, decoding the + * request body as it arrives so a multi-gigabyte project never lands in + * memory. The signed byte budget is enforced while decoding, not from + * `Content-Length`, because the body also carries framing headers. + */ +export const projectSyncImportRouteLayer = HttpRouter.add( + "POST", + `${PROJECT_SYNC_IMPORT_ROUTE_PREFIX}/*`, + Effect.gen(function* () { + const request = yield* HttpServerRequest.HttpServerRequest; + const url = HttpServerRequest.toURL(request); + if (Option.isNone(url)) { + return HttpServerResponse.text("Bad Request", { status: 400 }); + } + + const token = projectSyncToken(url.value.pathname, PROJECT_SYNC_IMPORT_ROUTE_PREFIX); + if (!token) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + + const claims = yield* resolveProjectSyncImportToken(token); + if (!claims) { + return HttpServerResponse.text("Not Found", { status: 404 }); + } + + return yield* applyProjectSyncRecords({ + workspaceRoot: claims.workspaceRoot, + records: createProjectSyncFrameDecoder(Stream.toAsyncIterable(request.stream)), + maxContentBytes: claims.totalBytes, + }).pipe( + Effect.map((result) => + HttpServerResponse.jsonUnsafe({ applied: result.applied, bytes: result.bytes }), + ), + Effect.catchTags({ + ProjectSyncPathViolationError: (error) => + Effect.succeed(HttpServerResponse.text(error.message, { status: 400 })), + ProjectSyncImportLimitError: (error) => + Effect.succeed(HttpServerResponse.text(error.message, { status: 413 })), + ProjectSyncIoError: (error) => + Effect.logError("Project sync import failed.", { error }).pipe( + Effect.as(HttpServerResponse.text("Failed to apply the import.", { status: 500 })), + ), + }), + ); + }), +); + export const staticAndDevRouteLayer = HttpRouter.add( "GET", "*", diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 8e068508ca3d..bfedd1bbc9b5 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -14,6 +14,8 @@ import { otlpTracesProxyRouteLayer, assetRouteLayer, attachmentUploadRouteLayer, + projectSyncExportRouteLayer, + projectSyncImportRouteLayer, serverEnvironmentHttpApiLayer, staticAndDevRouteLayer, browserApiCorsLayer, @@ -460,6 +462,8 @@ export const makeRoutesLayer = Layer.mergeAll( otlpTracesProxyRouteLayer, assetRouteLayer, attachmentUploadRouteLayer, + projectSyncExportRouteLayer, + projectSyncImportRouteLayer, staticAndDevRouteLayer, websocketRpcRouteLayer, ), diff --git a/apps/server/src/workspace/ProjectSyncApply.test.ts b/apps/server/src/workspace/ProjectSyncApply.test.ts new file mode 100644 index 000000000000..298bc3c39883 --- /dev/null +++ b/apps/server/src/workspace/ProjectSyncApply.test.ts @@ -0,0 +1,291 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { + createProjectSyncFrameDecoder, + encodeProjectSyncRecords, + type ProjectSyncFrameRecord, +} from "@t3tools/shared/projectSyncFraming"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; + +import { + applyProjectSyncDeletions, + applyProjectSyncRecords, + resolveProjectSyncRelativePath, +} from "./ProjectSyncApply.ts"; + +const makeTempDir = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-project-sync-apply-" }); +}); + +const encoder = new TextEncoder(); + +function fileRecord(path: string, contents: string, mode?: number): ProjectSyncFrameRecord { + const content = encoder.encode(contents); + return { + header: { path, size: content.length, kind: "file", ...(mode === undefined ? {} : { mode }) }, + content, + }; +} + +function dirRecord(path: string): ProjectSyncFrameRecord { + return { header: { path, size: 0, kind: "dir" }, content: new Uint8Array(0) }; +} + +function symlinkRecord(path: string, linkTarget: string): ProjectSyncFrameRecord { + return { header: { path, size: 0, kind: "symlink", linkTarget }, content: new Uint8Array(0) }; +} + +async function* iterate(records: ReadonlyArray) { + for (const record of records) yield record; +} + +/** Runs the real wire path: encode to bytes, decode them back, then apply. */ +const applyThroughFraming = (input: { + readonly workspaceRoot: string; + readonly records: ReadonlyArray; + readonly maxContentBytes?: number; +}) => + applyProjectSyncRecords({ + workspaceRoot: input.workspaceRoot, + records: createProjectSyncFrameDecoder(encodeProjectSyncRecords(iterate(input.records))), + ...(input.maxContentBytes === undefined ? {} : { maxContentBytes: input.maxContentBytes }), + }); + +const readText = (root: string, relativePath: string) => + Effect.promise(() => NodeFSP.readFile(NodePath.join(root, relativePath), "utf8")); + +const lstat = (root: string, relativePath: string) => + Effect.promise(() => NodeFSP.lstat(NodePath.join(root, relativePath))); + +const exists = (root: string, relativePath: string) => + Effect.promise(() => + NodeFSP.lstat(NodePath.join(root, relativePath)).then( + () => true, + () => false, + ), + ); + +const writeFile = (root: string, relativePath: string, contents: string) => + Effect.promise(async () => { + const absolutePath = NodePath.join(root, relativePath); + await NodeFSP.mkdir(NodePath.dirname(absolutePath), { recursive: true }); + await NodeFSP.writeFile(absolutePath, contents); + }); + +it.layer(NodeServices.layer, { excludeTestServices: true })("ProjectSyncApply", (it) => { + describe("resolveProjectSyncRelativePath", () => { + it.effect("rejects traversal, absolute, and empty paths", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + for (const relativePath of [ + "../escape.txt", + "a/../../escape.txt", + "/etc/passwd", + "", + " ", + ".", + "a/./b", + "a//b", + "a/\0b", + ]) { + expect(resolveProjectSyncRelativePath({ workspaceRoot, relativePath })).toBeNull(); + } + + expect(resolveProjectSyncRelativePath({ workspaceRoot, relativePath: "a/b.txt" })).toEqual({ + absolutePath: NodePath.join(workspaceRoot, "a", "b.txt"), + relativePath: "a/b.txt", + }); + }), + ); + }); + + describe("import", () => { + it.effect("round-trips files, modes, symlinks, and empty directories", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + + const result = yield* applyThroughFraming({ + workspaceRoot, + records: [ + fileRecord("README.md", "# hello\n"), + fileRecord("bin/run.sh", "#!/bin/sh\necho hi\n", 0o755), + fileRecord("src/nested/deep/index.ts", "export {};\n", 0o644), + dirRecord("empty/dir"), + symlinkRecord("link.md", "README.md"), + ], + }); + + expect(result.applied).toBe(5); + expect(yield* readText(workspaceRoot, "README.md")).toBe("# hello\n"); + expect(yield* readText(workspaceRoot, "src/nested/deep/index.ts")).toBe("export {};\n"); + expect((yield* lstat(workspaceRoot, "bin/run.sh")).mode & 0o777).toBe(0o755); + expect((yield* lstat(workspaceRoot, "empty/dir")).isDirectory()).toBe(true); + + const link = yield* lstat(workspaceRoot, "link.md"); + expect(link.isSymbolicLink()).toBe(true); + expect( + yield* Effect.promise(() => NodeFSP.readlink(NodePath.join(workspaceRoot, "link.md"))), + ).toBe("README.md"); + }), + ); + + it.effect("replaces an existing file without following a symlink at the target", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const outside = yield* makeTempDir; + yield* writeFile(outside, "victim.txt", "original"); + yield* Effect.promise(() => + NodeFSP.symlink( + NodePath.join(outside, "victim.txt"), + NodePath.join(workspaceRoot, "victim.txt"), + ), + ); + + yield* applyThroughFraming({ + workspaceRoot, + records: [fileRecord("victim.txt", "replaced")], + }); + + expect(yield* readText(workspaceRoot, "victim.txt")).toBe("replaced"); + expect(yield* readText(outside, "victim.txt")).toBe("original"); + }), + ); + + it.effect("refuses a record whose path escapes the workspace root", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const error = yield* Effect.flip( + applyThroughFraming({ + workspaceRoot, + records: [fileRecord("../escape.txt", "nope")], + }), + ); + expect(error._tag).toBe("ProjectSyncPathViolationError"); + expect(yield* exists(workspaceRoot, "../escape.txt")).toBe(false); + }), + ); + + it.effect("refuses a record whose ancestor directory is a symlink", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const outside = yield* makeTempDir; + yield* Effect.promise(() => + NodeFSP.symlink(outside, NodePath.join(workspaceRoot, "linked")), + ); + + const error = yield* Effect.flip( + applyThroughFraming({ + workspaceRoot, + records: [fileRecord("linked/planted.txt", "nope")], + }), + ); + expect(error._tag).toBe("ProjectSyncPathViolationError"); + expect(yield* exists(outside, "planted.txt")).toBe(false); + }), + ); + + it.effect("stops once the signed byte budget is exhausted", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const error = yield* Effect.flip( + applyThroughFraming({ + workspaceRoot, + records: [fileRecord("a.txt", "0123456789"), fileRecord("b.txt", "0123456789")], + maxContentBytes: 10, + }), + ); + expect(error._tag).toBe("ProjectSyncImportLimitError"); + expect(yield* exists(workspaceRoot, "b.txt")).toBe(false); + }), + ); + }); + + describe("deletions", () => { + it.effect("removes entries and prunes the directories they emptied", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* writeFile(workspaceRoot, "a/b/c.txt", "gone"); + yield* writeFile(workspaceRoot, "a/keep.txt", "kept"); + + const result = yield* applyProjectSyncDeletions({ + workspaceRoot, + paths: ["a/b/c.txt"], + }); + + expect(result.deleted).toBe(1); + expect(yield* exists(workspaceRoot, "a/b")).toBe(false); + expect(yield* exists(workspaceRoot, "a/keep.txt")).toBe(true); + }), + ); + + it.effect("prunes up to but never past the workspace root", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* writeFile(workspaceRoot, "only/one.txt", "gone"); + + yield* applyProjectSyncDeletions({ workspaceRoot, paths: ["only/one.txt"] }); + + expect(yield* exists(workspaceRoot, "only")).toBe(false); + expect((yield* Effect.promise(() => NodeFSP.lstat(workspaceRoot))).isDirectory()).toBe( + true, + ); + }), + ); + + it.effect("is idempotent: an already-missing path is not an error", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* writeFile(workspaceRoot, "gone.txt", "x"); + + expect( + (yield* applyProjectSyncDeletions({ workspaceRoot, paths: ["gone.txt"] })).deleted, + ).toBe(1); + expect( + (yield* applyProjectSyncDeletions({ workspaceRoot, paths: ["gone.txt"] })).deleted, + ).toBe(0); + }), + ); + + it.effect("refuses to delete through a symlinked ancestor", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const outside = yield* makeTempDir; + yield* writeFile(outside, "victim.txt", "keep me"); + yield* Effect.promise(() => + NodeFSP.symlink(outside, NodePath.join(workspaceRoot, "linked")), + ); + + const error = yield* Effect.flip( + applyProjectSyncDeletions({ workspaceRoot, paths: ["linked/victim.txt"] }), + ); + expect(error._tag).toBe("ProjectSyncPathViolationError"); + expect(yield* exists(outside, "victim.txt")).toBe(true); + }), + ); + + it.effect("removes an empty directory entry but leaves a populated one", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* Effect.promise(() => + NodeFSP.mkdir(NodePath.join(workspaceRoot, "empty"), { recursive: true }), + ); + yield* writeFile(workspaceRoot, "full/child.txt", "x"); + + const result = yield* applyProjectSyncDeletions({ + workspaceRoot, + paths: ["empty", "full"], + }); + + expect(result.deleted).toBe(1); + expect(yield* exists(workspaceRoot, "empty")).toBe(false); + expect(yield* exists(workspaceRoot, "full/child.txt")).toBe(true); + }), + ); + }); +}); diff --git a/apps/server/src/workspace/ProjectSyncApply.ts b/apps/server/src/workspace/ProjectSyncApply.ts new file mode 100644 index 000000000000..918b2aa2bb4c --- /dev/null +++ b/apps/server/src/workspace/ProjectSyncApply.ts @@ -0,0 +1,324 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * ProjectSyncApply - Writes and deletes workspace entries on behalf of a + * project sync transfer. + * + * Every path that arrives here came off the wire, so nothing is trusted: paths + * are validated segment by segment, and no ancestor of a write target is ever + * allowed to be a symlink. A symlink *entry* is copied faithfully (its target + * may legitimately point anywhere), but a symlink *ancestor* would let a + * crafted manifest write outside the workspace root, so it is refused. + * + * @module ProjectSyncApply + */ +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; +import * as NodeStreamPromises from "node:stream/promises"; + +import { ProjectSyncIoError, ProjectSyncPathViolationError } from "@t3tools/contracts"; +import type { ProjectSyncFrameDecodedRecord } from "@t3tools/shared/projectSyncFraming"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import { errnoCode } from "./projectSyncErrno.ts"; + +/** Raised when an import body carries more content than the signed URL + authorized. Never crosses the WebSocket contract — the HTTP import route + turns it into a 413. */ +export class ProjectSyncImportLimitError extends Schema.TaggedErrorClass()( + "ProjectSyncImportLimitError", + { + limitBytes: Schema.Number, + message: Schema.String, + }, +) {} + +export interface ResolvedProjectSyncPath { + readonly absolutePath: string; + readonly relativePath: string; +} + +/** + * Validates a wire path and resolves it inside `workspaceRoot`, returning + * `null` for anything that is absolute, empty, traversing, or otherwise not a + * plain relative path. The root itself is compared unresolved, so a workspace + * that lives behind a symlink (every macOS temp directory, for one) still + * works. + */ +export function resolveProjectSyncRelativePath(input: { + readonly workspaceRoot: string; + readonly relativePath: string; +}): ResolvedProjectSyncPath | null { + const raw = input.relativePath.trim().replaceAll("\\", "/"); + if (raw.length === 0 || raw.startsWith("/") || raw.includes("\0")) return null; + + const segments = raw.split("/"); + for (const segment of segments) { + if (segment.length === 0 || segment === "." || segment === "..") return null; + } + + const workspaceRoot = NodePath.resolve(input.workspaceRoot); + const absolutePath = NodePath.resolve(workspaceRoot, ...segments); + if (!absolutePath.startsWith(`${workspaceRoot}${NodePath.sep}`)) return null; + + return { absolutePath, relativePath: segments.join("/") }; +} + +function requireResolvedPath(input: { + readonly workspaceRoot: string; + readonly relativePath: string; +}): ResolvedProjectSyncPath { + const resolved = resolveProjectSyncRelativePath(input); + if (resolved === null) { + throw new ProjectSyncPathViolationError({ path: input.relativePath }); + } + return resolved; +} + +async function lstatOrNull(path: string) { + try { + return await NodeFSP.lstat(path); + } catch (cause) { + const code = errnoCode(cause); + if (code === "ENOENT" || code === "ENOTDIR") return null; + throw cause; + } +} + +/** + * Walks the ancestors of `relativePath` from the workspace root down, refusing + * to traverse a symlink and (optionally) materializing the directories that do + * not exist yet. Checking before creating matters: `mkdir -p` would happily + * follow a symlinked ancestor and create directories outside the root. + */ +async function prepareAncestors( + workspaceRoot: string, + relativePath: string, + options: { readonly create: boolean }, +): Promise { + const segments = relativePath.split("/"); + segments.pop(); + + let current = workspaceRoot; + for (const segment of segments) { + current = NodePath.join(current, segment); + const stats = await lstatOrNull(current); + if (stats === null) { + if (!options.create) return; + await NodeFSP.mkdir(current).catch((cause: unknown) => { + if (errnoCode(cause) !== "EEXIST") throw cause; + }); + continue; + } + if (stats.isSymbolicLink()) { + throw new ProjectSyncPathViolationError({ path: relativePath }); + } + if (!stats.isDirectory()) { + throw new ProjectSyncIoError({ + message: `Cannot sync '${relativePath}': '${segment}' exists and is not a directory.`, + }); + } + } +} + +/** Clears whatever currently occupies a target path so the incoming entry can + take it, including a directory being replaced by a file. */ +async function clearTarget(absolutePath: string): Promise { + const stats = await lstatOrNull(absolutePath); + if (stats === null) return; + await NodeFSP.rm(absolutePath, { force: true, recursive: stats.isDirectory() }); +} + +async function writeFileRecord( + absolutePath: string, + content: AsyncIterable, + mode: number | undefined, +): Promise { + const directory = NodePath.dirname(absolutePath); + const partPath = NodePath.join( + directory, + `.${NodePath.basename(absolutePath)}.${NodeCrypto.randomUUID()}.t3sync-part`, + ); + + try { + await NodeStreamPromises.pipeline(content, NodeFS.createWriteStream(partPath)); + if (mode !== undefined) { + await NodeFSP.chmod(partPath, mode & 0o777); + } + const existing = await lstatOrNull(absolutePath); + if (existing?.isDirectory()) { + await NodeFSP.rm(absolutePath, { force: true, recursive: true }); + } + // Rename replaces the entry itself, so an existing symlink at the target is + // swapped out rather than written through. + await NodeFSP.rename(partPath, absolutePath); + } catch (cause) { + await NodeFSP.rm(partPath, { force: true }).catch(() => {}); + throw cause; + } +} + +async function applyRecord( + workspaceRoot: string, + record: ProjectSyncFrameDecodedRecord, +): Promise { + const { absolutePath, relativePath } = requireResolvedPath({ + workspaceRoot, + relativePath: record.header.path, + }); + await prepareAncestors(workspaceRoot, relativePath, { create: true }); + + if (record.header.kind === "dir") { + const stats = await lstatOrNull(absolutePath); + if (stats?.isDirectory()) return; + if (stats !== null) await clearTarget(absolutePath); + await NodeFSP.mkdir(absolutePath, { recursive: true }); + return; + } + + if (record.header.kind === "symlink") { + const linkTarget = record.header.linkTarget; + if (linkTarget === undefined || linkTarget.length === 0) { + throw new ProjectSyncIoError({ + message: `Symlink record '${relativePath}' arrived without a link target.`, + }); + } + await clearTarget(absolutePath); + await NodeFSP.symlink(linkTarget, absolutePath); + return; + } + + await writeFileRecord(absolutePath, record.content, record.header.mode); +} + +const isPathViolationError = Schema.is(ProjectSyncPathViolationError); +const isIoError = Schema.is(ProjectSyncIoError); +const isImportLimitError = Schema.is(ProjectSyncImportLimitError); + +function toFilesystemFailure(cause: unknown) { + if (isPathViolationError(cause) || isIoError(cause)) { + return cause; + } + return new ProjectSyncIoError({ + message: `Project sync failed to apply an entry${ + errnoCode(cause) ? ` (${errnoCode(cause)})` : "" + }.`, + cause, + }); +} + +function toImportFailure(cause: unknown) { + return isImportLimitError(cause) ? cause : toFilesystemFailure(cause); +} + +export interface ProjectSyncApplyResult { + readonly applied: number; + readonly bytes: number; +} + +/** + * Applies a decoded frame stream into `workspaceRoot`, one record at a time. + * + * `maxContentBytes` mirrors the byte budget the signed import URL was issued + * for; exceeding it aborts mid-stream instead of letting a token authorize an + * unbounded write. + */ +export const applyProjectSyncRecords = Effect.fn("ProjectSyncApply.applyRecords")( + function* (input: { + readonly workspaceRoot: string; + readonly records: AsyncIterable; + readonly maxContentBytes?: number | undefined; + }) { + const workspaceRoot = NodePath.resolve(input.workspaceRoot); + return yield* Effect.tryPromise({ + try: async (): Promise => { + await NodeFSP.mkdir(workspaceRoot, { recursive: true }); + + let applied = 0; + let bytes = 0; + for await (const record of input.records) { + bytes += record.header.size; + if (input.maxContentBytes !== undefined && bytes > input.maxContentBytes) { + throw new ProjectSyncImportLimitError({ + limitBytes: input.maxContentBytes, + message: `Import body exceeded the ${input.maxContentBytes} byte budget the URL was signed for.`, + }); + } + await applyRecord(workspaceRoot, record); + applied += 1; + } + return { applied, bytes }; + }, + catch: toImportFailure, + }); + }, +); + +/** + * Removes the given paths and then prunes the directories they emptied, up to + * (but never including) the workspace root. A path that is already gone is not + * an error — the destination simply agreed with the diff early. + */ +export const applyProjectSyncDeletions = Effect.fn("ProjectSyncApply.applyDeletions")( + function* (input: { readonly workspaceRoot: string; readonly paths: ReadonlyArray }) { + const workspaceRoot = NodePath.resolve(input.workspaceRoot); + return yield* Effect.tryPromise({ + try: async () => { + let deleted = 0; + const emptiedDirectories = new Set(); + + for (const relativePath of input.paths) { + const resolved = requireResolvedPath({ workspaceRoot, relativePath }); + await prepareAncestors(workspaceRoot, resolved.relativePath, { create: false }); + + const stats = await lstatOrNull(resolved.absolutePath); + if (stats === null) continue; + + if (stats.isDirectory()) { + // Only empty directories are ever published as their own entry, so a + // non-empty one here still owns children the diff kept. + const removed = await NodeFSP.rmdir(resolved.absolutePath).then( + () => true, + (cause: unknown) => { + const code = errnoCode(cause); + if (code === "ENOTEMPTY" || code === "EEXIST" || code === "ENOENT") return false; + throw cause; + }, + ); + if (!removed) continue; + } else { + await NodeFSP.rm(resolved.absolutePath, { force: true }); + } + + deleted += 1; + emptiedDirectories.add(NodePath.dirname(resolved.absolutePath)); + } + + // Deepest first, so a chain of newly emptied directories collapses in one + // pass instead of leaving the parents behind. + for (const directory of [...emptiedDirectories].sort( + (left, right) => right.length - left.length, + )) { + await pruneEmptyDirectories(workspaceRoot, directory); + } + + return { deleted }; + }, + catch: toFilesystemFailure, + }); + }, +); + +async function pruneEmptyDirectories(workspaceRoot: string, startDirectory: string): Promise { + let current = startDirectory; + while (current.startsWith(`${workspaceRoot}${NodePath.sep}`)) { + const removed = await NodeFSP.rmdir(current).then( + () => true, + () => false, + ); + if (!removed) return; + current = NodePath.dirname(current); + } +} diff --git a/apps/server/src/workspace/ProjectSyncManifest.test.ts b/apps/server/src/workspace/ProjectSyncManifest.test.ts new file mode 100644 index 000000000000..5a96eb322cf1 --- /dev/null +++ b/apps/server/src/workspace/ProjectSyncManifest.test.ts @@ -0,0 +1,169 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; + +import { buildProjectSyncManifest } from "./ProjectSyncManifest.ts"; + +const makeTempDir = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-project-sync-manifest-" }); +}); + +const writeFile = (root: string, relativePath: string, contents: string) => + Effect.promise(async () => { + const absolutePath = NodePath.join(root, relativePath); + await NodeFSP.mkdir(NodePath.dirname(absolutePath), { recursive: true }); + await NodeFSP.writeFile(absolutePath, contents); + return absolutePath; + }); + +const makeDirectory = (root: string, relativePath: string) => + Effect.promise(async () => { + const absolutePath = NodePath.join(root, relativePath); + await NodeFSP.mkdir(absolutePath, { recursive: true }); + return absolutePath; + }); + +const makeSymlink = (root: string, relativePath: string, target: string) => + Effect.promise(async () => { + const absolutePath = NodePath.join(root, relativePath); + await NodeFSP.mkdir(NodePath.dirname(absolutePath), { recursive: true }); + await NodeFSP.symlink(target, absolutePath); + }); + +const paths = (entries: ReadonlyArray<{ readonly path: string }>) => + entries.map((entry) => entry.path); + +it.layer(NodeServices.layer, { excludeTestServices: true })("buildProjectSyncManifest", (it) => { + describe("ignores", () => { + it.effect("always drops node_modules, .t3, and .DS_Store", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* writeFile(workspaceRoot, "src/index.ts", "export {};\n"); + yield* writeFile(workspaceRoot, "node_modules/pkg/index.js", "module.exports = 1;"); + yield* writeFile(workspaceRoot, ".t3/state.json", "{}"); + yield* writeFile(workspaceRoot, ".DS_Store", "junk"); + yield* writeFile(workspaceRoot, "src/.DS_Store", "junk"); + + const entries = yield* buildProjectSyncManifest({ workspaceRoot, includeGit: true }); + expect(paths(entries)).toEqual(["src/index.ts"]); + }), + ); + + it.effect("layers extraIgnores on top of the defaults", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* writeFile(workspaceRoot, "src/index.ts", "export {};\n"); + yield* writeFile(workspaceRoot, "dist/bundle.js", "1"); + yield* writeFile(workspaceRoot, "packages/app/dist/bundle.js", "1"); + + const entries = yield* buildProjectSyncManifest({ + workspaceRoot, + includeGit: true, + extraIgnores: ["dist"], + }); + // `packages/app` survives as an empty-directory entry: everything + // inside it was ignored, so the destination still rebuilds the shape. + expect(paths(entries)).toEqual(["packages/app", "src/index.ts"]); + }), + ); + + it.effect("keeps .git when includeGit is true and drops it when false", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* writeFile(workspaceRoot, ".git/HEAD", "ref: refs/heads/main\n"); + yield* writeFile(workspaceRoot, "README.md", "# hi\n"); + + const withGit = yield* buildProjectSyncManifest({ workspaceRoot, includeGit: true }); + expect(paths(withGit)).toEqual([".git/HEAD", "README.md"]); + + const withoutGit = yield* buildProjectSyncManifest({ workspaceRoot, includeGit: false }); + expect(paths(withoutGit)).toEqual(["README.md"]); + }), + ); + }); + + describe("entry kinds", () => { + it.effect("records symlinks by target without following them", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const outside = yield* makeTempDir; + yield* writeFile(outside, "secret.txt", "nope"); + yield* writeFile(workspaceRoot, "target.txt", "hi"); + yield* makeSymlink(workspaceRoot, "link.txt", "target.txt"); + yield* makeSymlink(workspaceRoot, "outside", outside); + + const entries = yield* buildProjectSyncManifest({ workspaceRoot, includeGit: true }); + expect(paths(entries)).toEqual(["link.txt", "outside", "target.txt"]); + expect(entries.find((entry) => entry.path === "link.txt")).toEqual({ + path: "link.txt", + kind: "symlink", + size: 0, + linkTarget: "target.txt", + }); + expect(entries.find((entry) => entry.path === "outside")?.kind).toBe("symlink"); + }), + ); + + it.effect("emits a dir entry only for directories that contribute nothing else", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* makeDirectory(workspaceRoot, "empty"); + yield* makeDirectory(workspaceRoot, "nested/deep"); + yield* writeFile(workspaceRoot, "kept/file.txt", "x"); + yield* writeFile(workspaceRoot, "only-ignored/node_modules/pkg.js", "x"); + + const entries = yield* buildProjectSyncManifest({ workspaceRoot, includeGit: true }); + expect(paths(entries)).toEqual(["empty", "kept/file.txt", "nested/deep", "only-ignored"]); + expect(entries.find((entry) => entry.path === "nested/deep")).toEqual({ + path: "nested/deep", + kind: "dir", + size: 0, + }); + }), + ); + + it.effect("hashes file contents with sha256 and reports size and mode", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const absolutePath = yield* writeFile(workspaceRoot, "bin/run.sh", "hello world"); + yield* Effect.promise(() => NodeFSP.chmod(absolutePath, 0o755)); + yield* writeFile(workspaceRoot, "empty.txt", ""); + + const entries = yield* buildProjectSyncManifest({ workspaceRoot, includeGit: true }); + expect(entries.find((entry) => entry.path === "bin/run.sh")).toEqual({ + path: "bin/run.sh", + kind: "file", + size: 11, + mode: 0o755, + hash: "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9", + }); + expect(entries.find((entry) => entry.path === "empty.txt")?.hash).toBe( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + ); + }), + ); + }); + + it.effect("returns path-sorted entries and repeats itself exactly", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* writeFile(workspaceRoot, "b.txt", "b"); + yield* writeFile(workspaceRoot, "a/z.txt", "z"); + yield* writeFile(workspaceRoot, "a/a.txt", "a"); + yield* writeFile(workspaceRoot, "a.txt", "a"); + yield* makeDirectory(workspaceRoot, "c"); + + const first = yield* buildProjectSyncManifest({ workspaceRoot, includeGit: true }); + const second = yield* buildProjectSyncManifest({ workspaceRoot, includeGit: true }); + + expect(paths(first)).toEqual(["a.txt", "a/a.txt", "a/z.txt", "b.txt", "c"]); + expect(second).toEqual(first); + }), + ); +}); diff --git a/apps/server/src/workspace/ProjectSyncManifest.ts b/apps/server/src/workspace/ProjectSyncManifest.ts new file mode 100644 index 000000000000..9318d51bd8cd --- /dev/null +++ b/apps/server/src/workspace/ProjectSyncManifest.ts @@ -0,0 +1,196 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * ProjectSyncManifest - Walks a project's workspace root and describes it as a + * flat, hash-addressed manifest. + * + * The manifest is the input to the client-orchestrated diff between two + * environments: the client asks both sides for one, compares them, and then + * only exports/imports/deletes what actually differs. Nothing here touches the + * event store — it is a read-only filesystem walk. + * + * @module ProjectSyncManifest + */ +import * as NodeCrypto from "node:crypto"; +import * as NodeFS from "node:fs"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import { ProjectSyncIoError, type ProjectSyncManifestEntry } from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; + +import { errnoCode, isMissingOrUnreadable } from "./projectSyncErrno.ts"; + +/** + * Segments that never round-trip between environments: installable state, T3's + * own runtime directory, and macOS folder metadata. Mirrors the guarantee the + * `ProjectSyncManifestInput` contract makes to clients. + */ +export const PROJECT_SYNC_ALWAYS_IGNORED_SEGMENTS = ["node_modules", ".t3", ".DS_Store"] as const; + +/** Bounded so a manifest of a large repository does not saturate the disk + queue and starve every other request the server is serving. */ +const MANIFEST_FILE_CONCURRENCY = 8; + +type ManifestDraft = + | { readonly kind: "dir"; readonly path: string } + | { readonly kind: "symlink"; readonly path: string; readonly linkTarget: string } + | { readonly kind: "file"; readonly path: string; readonly absolutePath: string }; + +export interface ProjectSyncManifestInputs { + readonly workspaceRoot: string; + readonly includeGit: boolean; + readonly extraIgnores?: ReadonlyArray | undefined; +} + +function compareByPath(left: { readonly path: string }, right: { readonly path: string }): number { + if (left.path === right.path) return 0; + return left.path < right.path ? -1 : 1; +} + +function resolveIgnoredSegments(input: ProjectSyncManifestInputs): ReadonlySet { + const ignored = new Set(PROJECT_SYNC_ALWAYS_IGNORED_SEGMENTS); + if (!input.includeGit) { + ignored.add(".git"); + } + for (const extra of input.extraIgnores ?? []) { + const trimmed = extra.trim(); + if (trimmed.length > 0) { + ignored.add(trimmed); + } + } + return ignored; +} + +/** + * Depth-first walk producing one draft per entry we intend to publish. + * + * A directory only earns its own `"dir"` draft when it contributes nothing + * else to the manifest — either because it is genuinely empty or because + * everything inside it was ignored. Destinations rebuild every other directory + * implicitly from the paths of the entries inside it. + */ +async function collectDrafts( + absoluteDir: string, + relativeDir: string, + ignoredSegments: ReadonlySet, +): Promise> { + let dirents; + try { + dirents = await NodeFSP.readdir(absoluteDir, { withFileTypes: true }); + } catch (cause) { + if (isMissingOrUnreadable(cause)) return []; + throw cause; + } + + const drafts: Array = []; + for (const dirent of dirents.sort((left, right) => (left.name < right.name ? -1 : 1))) { + if (ignoredSegments.has(dirent.name)) continue; + + const path = relativeDir === "" ? dirent.name : `${relativeDir}/${dirent.name}`; + const absolutePath = NodePath.join(absoluteDir, dirent.name); + + if (dirent.isSymbolicLink()) { + // Never followed: a symlink is copied as a link, so its target stays + // whatever it means on the destination machine. + const linkTarget = await NodeFSP.readlink(absolutePath).catch((cause: unknown) => { + if (isMissingOrUnreadable(cause)) return null; + throw cause; + }); + if (linkTarget !== null && linkTarget.length > 0) { + drafts.push({ kind: "symlink", path, linkTarget }); + } + continue; + } + + if (dirent.isDirectory()) { + const nested = await collectDrafts(absolutePath, path, ignoredSegments); + if (nested.length === 0) drafts.push({ kind: "dir", path }); + else drafts.push(...nested); + continue; + } + + if (dirent.isFile()) { + drafts.push({ kind: "file", path, absolutePath }); + } + } + + return drafts; +} + +/** Streams the file through sha256 so a multi-gigabyte artifact costs one + buffer, not its own size in resident memory. */ +async function describeFile(draft: { + readonly path: string; + readonly absolutePath: string; +}): Promise { + let stats; + try { + stats = await NodeFSP.lstat(draft.absolutePath); + } catch (cause) { + if (isMissingOrUnreadable(cause)) return null; + throw cause; + } + if (!stats.isFile()) return null; + + const hash = NodeCrypto.createHash("sha256"); + let size = 0; + try { + for await (const chunk of NodeFS.createReadStream(draft.absolutePath)) { + hash.update(chunk as Uint8Array); + size += (chunk as Uint8Array).length; + } + } catch (cause) { + if (isMissingOrUnreadable(cause)) return null; + throw cause; + } + + return { + path: draft.path, + kind: "file", + size, + mode: stats.mode & 0o777, + hash: hash.digest("hex"), + }; +} + +export const buildProjectSyncManifest = Effect.fn("ProjectSyncManifest.build")(function* ( + input: ProjectSyncManifestInputs, +) { + const workspaceRoot = NodePath.resolve(input.workspaceRoot); + const ignoredSegments = resolveIgnoredSegments(input); + + const drafts = yield* Effect.tryPromise({ + try: () => collectDrafts(workspaceRoot, "", ignoredSegments), + catch: (cause) => + new ProjectSyncIoError({ + message: `Failed to walk '${workspaceRoot}' for a project sync manifest${ + errnoCode(cause) ? ` (${errnoCode(cause)})` : "" + }.`, + cause, + }), + }); + + const entries = yield* Effect.forEach( + drafts, + (draft) => + draft.kind === "file" + ? Effect.tryPromise({ + try: () => describeFile(draft), + catch: (cause) => + new ProjectSyncIoError({ + message: `Failed to hash '${draft.path}' for a project sync manifest.`, + cause, + }), + }) + : Effect.succeed( + draft.kind === "symlink" + ? { path: draft.path, kind: "symlink", size: 0, linkTarget: draft.linkTarget } + : { path: draft.path, kind: "dir", size: 0 }, + ), + { concurrency: MANIFEST_FILE_CONCURRENCY }, + ); + + // Globally sorted by path so two manifests can be diffed with a linear + // merge-join instead of building a map of every entry on the client. + return entries.filter((entry) => entry !== null).sort(compareByPath); +}); diff --git a/apps/server/src/workspace/ProjectSyncTransfer.test.ts b/apps/server/src/workspace/ProjectSyncTransfer.test.ts new file mode 100644 index 000000000000..09f029736959 --- /dev/null +++ b/apps/server/src/workspace/ProjectSyncTransfer.test.ts @@ -0,0 +1,215 @@ +// @effect-diagnostics nodeBuiltinImport:off +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { + PROJECT_SYNC_EXPORT_ROUTE_PREFIX, + PROJECT_SYNC_IMPORT_ROUTE_PREFIX, +} from "@t3tools/contracts"; +import { + createProjectSyncFrameDecoder, + encodeProjectSyncRecords, +} from "@t3tools/shared/projectSyncFraming"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import * as TestClock from "effect/testing/TestClock"; + +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import * as ServerConfig from "../config.ts"; +import { applyProjectSyncRecords } from "./ProjectSyncApply.ts"; +import { buildProjectSyncManifest } from "./ProjectSyncManifest.ts"; +import { + issueProjectSyncExportUrl, + issueProjectSyncImportUrl, + projectSyncExportRecords, + resolveProjectSyncExportToken, + resolveProjectSyncImportToken, +} from "./ProjectSyncTransfer.ts"; + +const testLayer = ServerSecretStore.layer.pipe( + Layer.provideMerge( + ServerConfig.layerTest(process.cwd(), { prefix: "t3-project-sync-transfer-" }), + ), + Layer.provideMerge(NodeServices.layer), +); + +const makeTempDir = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-project-sync-transfer-" }); +}); + +const writeFile = (root: string, relativePath: string, contents: string) => + Effect.promise(async () => { + const absolutePath = NodePath.join(root, relativePath); + await NodeFSP.mkdir(NodePath.dirname(absolutePath), { recursive: true }); + await NodeFSP.writeFile(absolutePath, contents); + return absolutePath; + }); + +const tokenOf = (url: string, prefix: string) => url.slice(`${prefix}/`.length); + +describe("ProjectSyncTransfer", () => { + it.effect("signs an export URL and resolves it back to its workspace and paths", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const issued = yield* issueProjectSyncExportUrl({ + projectId: "project-1", + workspaceRoot, + paths: ["src/index.ts", "docs/readme.md"], + }); + + expect(issued.url.startsWith(`${PROJECT_SYNC_EXPORT_ROUTE_PREFIX}/`)).toBe(true); + + const resolved = yield* resolveProjectSyncExportToken( + tokenOf(issued.url, PROJECT_SYNC_EXPORT_ROUTE_PREFIX), + ); + expect(resolved?.claims.workspaceRoot).toBe(workspaceRoot); + expect(resolved?.claims.projectId).toBe("project-1"); + expect(resolved?.paths).toEqual(["src/index.ts", "docs/readme.md"]); + }).pipe(Effect.provide(testLayer)), + ); + + it.effect("refuses to sign an export URL for a path outside the workspace", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const error = yield* Effect.flip( + issueProjectSyncExportUrl({ + projectId: "project-1", + workspaceRoot, + paths: ["src/index.ts", "../../etc/passwd"], + }), + ); + expect(error._tag).toBe("ProjectSyncPathViolationError"); + }).pipe(Effect.provide(testLayer)), + ); + + it.effect("rejects tampered, malformed, and expired export tokens", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const issued = yield* issueProjectSyncExportUrl({ + projectId: "project-1", + workspaceRoot, + paths: ["a.txt"], + }); + const token = tokenOf(issued.url, PROJECT_SYNC_EXPORT_ROUTE_PREFIX); + const [payload, signature] = token.split("."); + + expect(yield* resolveProjectSyncExportToken(`${payload}x.${signature}`)).toBeNull(); + expect(yield* resolveProjectSyncExportToken(`${token}.extra`)).toBeNull(); + expect(yield* resolveProjectSyncExportToken("garbage")).toBeNull(); + + yield* TestClock.adjust("11 minutes"); + expect(yield* resolveProjectSyncExportToken(token)).toBeNull(); + }).pipe(Effect.provide(testLayer)), + ); + + it.effect("signs an import URL carrying its workspace and byte budget", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const issued = yield* issueProjectSyncImportUrl({ + projectId: "project-2", + workspaceRoot, + fileCount: 3, + totalBytes: 4096, + }); + + expect(issued.url.startsWith(`${PROJECT_SYNC_IMPORT_ROUTE_PREFIX}/`)).toBe(true); + const claims = yield* resolveProjectSyncImportToken( + tokenOf(issued.url, PROJECT_SYNC_IMPORT_ROUTE_PREFIX), + ); + expect(claims).toMatchObject({ + kind: "project-sync-import", + projectId: "project-2", + workspaceRoot, + fileCount: 3, + totalBytes: 4096, + }); + + yield* TestClock.adjust("11 minutes"); + expect( + yield* resolveProjectSyncImportToken(tokenOf(issued.url, PROJECT_SYNC_IMPORT_ROUTE_PREFIX)), + ).toBeNull(); + }).pipe(Effect.provide(testLayer)), + ); + + it.effect("exports and re-imports a workspace into an identical manifest", () => + Effect.gen(function* () { + const origin = yield* makeTempDir; + const destination = yield* makeTempDir; + + yield* writeFile(origin, "README.md", "# hello\n"); + yield* writeFile(origin, "src/nested/index.ts", "export const answer = 42;\n"); + const script = yield* writeFile(origin, "bin/run.sh", "#!/bin/sh\n"); + yield* Effect.promise(() => NodeFSP.chmod(script, 0o755)); + yield* Effect.promise(() => + NodeFSP.mkdir(NodePath.join(origin, "empty"), { recursive: true }), + ); + yield* Effect.promise(() => NodeFSP.symlink("README.md", NodePath.join(origin, "link.md"))); + + const manifest = yield* buildProjectSyncManifest({ workspaceRoot: origin, includeGit: true }); + const issued = yield* issueProjectSyncExportUrl({ + projectId: "project-3", + workspaceRoot: origin, + paths: manifest.map((entry) => entry.path), + }); + const resolved = yield* resolveProjectSyncExportToken( + tokenOf(issued.url, PROJECT_SYNC_EXPORT_ROUTE_PREFIX), + ); + expect(resolved).not.toBeNull(); + + const applied = yield* applyProjectSyncRecords({ + workspaceRoot: destination, + records: createProjectSyncFrameDecoder( + encodeProjectSyncRecords( + projectSyncExportRecords(resolved!.claims.workspaceRoot, resolved!.paths), + ), + ), + }); + + expect(applied.applied).toBe(manifest.length); + const destinationManifest = yield* buildProjectSyncManifest({ + workspaceRoot: destination, + includeGit: true, + }); + expect(destinationManifest).toEqual(manifest); + }).pipe(Effect.provide(testLayer)), + ); + + it.effect("skips entries that vanished between the manifest and the export", () => + Effect.gen(function* () { + const origin = yield* makeTempDir; + const destination = yield* makeTempDir; + yield* writeFile(origin, "kept.txt", "kept"); + yield* writeFile(origin, "vanishing.txt", "gone"); + + const issued = yield* issueProjectSyncExportUrl({ + projectId: "project-4", + workspaceRoot: origin, + paths: ["kept.txt", "vanishing.txt"], + }); + yield* Effect.promise(() => NodeFSP.rm(NodePath.join(origin, "vanishing.txt"))); + + const resolved = yield* resolveProjectSyncExportToken( + tokenOf(issued.url, PROJECT_SYNC_EXPORT_ROUTE_PREFIX), + ); + const applied = yield* applyProjectSyncRecords({ + workspaceRoot: destination, + records: createProjectSyncFrameDecoder( + encodeProjectSyncRecords( + projectSyncExportRecords(resolved!.claims.workspaceRoot, resolved!.paths), + ), + ), + }); + + expect(applied.applied).toBe(1); + expect( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(destination, "kept.txt"), "utf8"), + ), + ).toBe("kept"); + }).pipe(Effect.provide(testLayer)), + ); +}); diff --git a/apps/server/src/workspace/ProjectSyncTransfer.ts b/apps/server/src/workspace/ProjectSyncTransfer.ts new file mode 100644 index 000000000000..ff2f861268d5 --- /dev/null +++ b/apps/server/src/workspace/ProjectSyncTransfer.ts @@ -0,0 +1,342 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * ProjectSyncTransfer - Issues and validates the signed URLs a project sync + * transfer runs over, and streams the origin side of an export. + * + * Signing follows the asset/attachment precedent exactly: HMAC-SHA256 over a + * base64url claims blob, keyed by the shared `asset-access-signing-key` + * secret, with the claim `kind` keeping the token families apart. + * + * Two things differ from an attachment upload, both forced by scale: + * + * - The set of paths an export covers can run to thousands of entries, which + * would blow far past the 4096-character URL the contract allows. The token + * therefore carries a random request id and the path list stays in a + * short-lived in-process registry, expiring with the URL itself. + * - The claims carry the resolved `workspaceRoot`, so the HTTP routes never + * need the projection read model. That mirrors how workspace-file asset + * claims already work. + * + * @module ProjectSyncTransfer + */ +import * as NodeCrypto from "node:crypto"; +import * as NodeFSP from "node:fs/promises"; + +import { + PROJECT_SYNC_EXPORT_ROUTE_PREFIX, + PROJECT_SYNC_IMPORT_ROUTE_PREFIX, + PROJECT_SYNC_URL_TTL_MS, + ProjectSyncIoError, + ProjectSyncPathViolationError, +} from "@t3tools/contracts"; +import type { ProjectSyncFrameRecord } from "@t3tools/shared/projectSyncFraming"; +import * as Clock from "effect/Clock"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; +import * as Schema from "effect/Schema"; + +import { + base64UrlDecodeUtf8, + base64UrlEncode, + signPayload, + timingSafeEqualBase64Url, +} from "../auth/utils.ts"; +import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; +import { isMissingOrUnreadable } from "./projectSyncErrno.ts"; +import { resolveProjectSyncRelativePath } from "./ProjectSyncApply.ts"; + +/** Shared with asset download and attachment upload tokens; the signed `kind` + is what keeps a token from being replayed against another route. */ +const SIGNING_SECRET_NAME = "asset-access-signing-key"; + +/** A pending export holds its whole path list in memory, so the registry is + bounded on both ends: entries expire with their URL, and the oldest is + evicted once too many pile up. */ +const MAX_PENDING_EXPORTS = 64; + +const ProjectSyncExportClaims = Schema.Struct({ + version: Schema.Literal(1), + kind: Schema.Literal("project-sync-export"), + requestId: Schema.String, + projectId: Schema.String, + workspaceRoot: Schema.String, + expiresAt: Schema.Number, +}); +type ProjectSyncExportClaims = typeof ProjectSyncExportClaims.Type; + +const ProjectSyncImportClaims = Schema.Struct({ + version: Schema.Literal(1), + kind: Schema.Literal("project-sync-import"), + projectId: Schema.String, + workspaceRoot: Schema.String, + fileCount: Schema.Number, + totalBytes: Schema.Number, + expiresAt: Schema.Number, +}); +export type ProjectSyncImportClaims = typeof ProjectSyncImportClaims.Type; + +const exportClaimsJson = Schema.fromJsonString(ProjectSyncExportClaims); +const decodeExportClaims = Schema.decodeUnknownOption(exportClaimsJson); +const encodeExportClaims = Schema.encodeSync(exportClaimsJson); + +const importClaimsJson = Schema.fromJsonString(ProjectSyncImportClaims); +const decodeImportClaims = Schema.decodeUnknownOption(importClaimsJson); +const encodeImportClaims = Schema.encodeSync(importClaimsJson); + +interface PendingExport { + readonly paths: ReadonlyArray; + readonly expiresAt: number; +} + +const pendingExports = new Map(); + +function sweepPendingExports(nowMs: number): void { + for (const [requestId, pending] of pendingExports) { + if (pending.expiresAt <= nowMs) pendingExports.delete(requestId); + } + while (pendingExports.size >= MAX_PENDING_EXPORTS) { + const oldest = pendingExports.keys().next(); + if (oldest.done) break; + pendingExports.delete(oldest.value); + } +} + +const loadSigningSecret = Effect.gen(function* () { + const secretStore = yield* ServerSecretStore.ServerSecretStore; + return yield* secretStore.getOrCreateRandom(SIGNING_SECRET_NAME, 32); +}).pipe( + Effect.mapError( + (cause) => + new ProjectSyncIoError({ message: "Failed to load the project sync signing key.", cause }), + ), +); + +function splitToken(token: string): { payload: string; signature: string } | null { + const [payload, signature, unexpected] = token.split("."); + if (!payload || !signature || unexpected) return null; + return { payload, signature }; +} + +const verifyToken = Effect.fn("ProjectSyncTransfer.verifyToken")(function* (token: string) { + const parts = splitToken(token); + if (!parts) return null; + + const secret = yield* loadSigningSecret.pipe( + Effect.tapError((cause) => + Effect.logError("Failed to load the project sync signing key.", { cause }), + ), + Effect.orElseSucceed(() => null), + ); + if (!secret || !timingSafeEqualBase64Url(parts.signature, signPayload(parts.payload, secret))) { + return null; + } + return parts.payload; +}); + +function decodeClaimsPayload( + payload: string, + decode: (input: unknown) => Option.Option, +): A | null { + try { + return Option.getOrNull(decode(base64UrlDecodeUtf8(payload))); + } catch { + return null; + } +} + +export const issueProjectSyncExportUrl = Effect.fn("ProjectSyncTransfer.issueExportUrl")( + function* (input: { + readonly projectId: string; + readonly workspaceRoot: string; + readonly paths: ReadonlyArray; + }) { + const secret = yield* loadSigningSecret; + const nowMs = yield* Clock.currentTimeMillis; + + const paths: Array = []; + for (const relativePath of input.paths) { + const resolved = resolveProjectSyncRelativePath({ + workspaceRoot: input.workspaceRoot, + relativePath, + }); + if (resolved === null) { + return yield* new ProjectSyncPathViolationError({ path: relativePath }); + } + paths.push(resolved.relativePath); + } + + sweepPendingExports(nowMs); + const requestId = NodeCrypto.randomUUID(); + const expiresAt = nowMs + PROJECT_SYNC_URL_TTL_MS; + pendingExports.set(requestId, { paths, expiresAt }); + + const payload = base64UrlEncode( + encodeExportClaims({ + version: 1, + kind: "project-sync-export", + requestId, + projectId: input.projectId, + workspaceRoot: input.workspaceRoot, + expiresAt, + }), + ); + + return { + url: `${PROJECT_SYNC_EXPORT_ROUTE_PREFIX}/${payload}.${signPayload(payload, secret)}`, + expiresAt, + }; + }, +); + +export interface ResolvedProjectSyncExport { + readonly claims: ProjectSyncExportClaims; + readonly paths: ReadonlyArray; +} + +export const resolveProjectSyncExportToken = Effect.fn("ProjectSyncTransfer.resolveExportToken")( + function* (token: string) { + const payload = yield* verifyToken(token); + if (!payload) return null; + + const claims = decodeClaimsPayload(payload, decodeExportClaims); + const nowMs = yield* Clock.currentTimeMillis; + if (!claims || claims.expiresAt <= nowMs) return null; + + const pending = pendingExports.get(claims.requestId); + if (!pending || pending.expiresAt <= nowMs) { + pendingExports.delete(claims.requestId); + return null; + } + + return { claims, paths: pending.paths } satisfies ResolvedProjectSyncExport; + }, +); + +export const issueProjectSyncImportUrl = Effect.fn("ProjectSyncTransfer.issueImportUrl")( + function* (input: { + readonly projectId: string; + readonly workspaceRoot: string; + readonly fileCount: number; + readonly totalBytes: number; + }) { + const secret = yield* loadSigningSecret; + const nowMs = yield* Clock.currentTimeMillis; + const expiresAt = nowMs + PROJECT_SYNC_URL_TTL_MS; + + const payload = base64UrlEncode( + encodeImportClaims({ + version: 1, + kind: "project-sync-import", + projectId: input.projectId, + workspaceRoot: input.workspaceRoot, + fileCount: input.fileCount, + totalBytes: input.totalBytes, + expiresAt, + }), + ); + + return { + url: `${PROJECT_SYNC_IMPORT_ROUTE_PREFIX}/${payload}.${signPayload(payload, secret)}`, + expiresAt, + }; + }, +); + +export const resolveProjectSyncImportToken = Effect.fn("ProjectSyncTransfer.resolveImportToken")( + function* (token: string) { + const payload = yield* verifyToken(token); + if (!payload) return null; + + const claims = decodeClaimsPayload(payload, decodeImportClaims); + if (!claims || claims.expiresAt <= (yield* Clock.currentTimeMillis)) return null; + return claims; + }, +); + +const EMPTY_CONTENT = new Uint8Array(0); + +/** + * Streams the requested entries as framing records. + * + * The manifest the client diffed is a snapshot; by the time it asks for these + * bytes an agent may have deleted or replaced any of them. A vanished entry is + * skipped rather than failing the transfer — the next sync reconciles it — but + * a file's size is taken from the handle we are about to read so the declared + * frame length and the bytes we emit come from the same open file. + */ +export async function* projectSyncExportRecords( + workspaceRoot: string, + paths: ReadonlyArray, +): AsyncGenerator { + for (const relativePath of paths) { + const resolved = resolveProjectSyncRelativePath({ workspaceRoot, relativePath }); + if (resolved === null) continue; + + let stats; + try { + stats = await NodeFSP.lstat(resolved.absolutePath); + } catch (cause) { + if (isMissingOrUnreadable(cause)) continue; + throw cause; + } + + if (stats.isSymbolicLink()) { + const linkTarget = await NodeFSP.readlink(resolved.absolutePath).catch((cause: unknown) => { + if (isMissingOrUnreadable(cause)) return null; + throw cause; + }); + if (linkTarget === null || linkTarget.length === 0) continue; + yield { + header: { path: resolved.relativePath, size: 0, kind: "symlink", linkTarget }, + content: EMPTY_CONTENT, + }; + continue; + } + + if (stats.isDirectory()) { + yield { + header: { path: resolved.relativePath, size: 0, kind: "dir" }, + content: EMPTY_CONTENT, + }; + continue; + } + + if (!stats.isFile()) continue; + + let handle; + try { + handle = await NodeFSP.open(resolved.absolutePath, "r"); + } catch (cause) { + if (isMissingOrUnreadable(cause)) continue; + throw cause; + } + + try { + const fileStats = await handle.stat(); + const size = fileStats.size; + yield { + header: { + path: resolved.relativePath, + size, + kind: "file", + mode: fileStats.mode & 0o777, + }, + content: size === 0 ? EMPTY_CONTENT : readHandle(handle, size), + }; + } finally { + await handle.close().catch(() => {}); + } + } +} + +/** Reads at most `size` bytes so a file that grew since it was stat'd cannot + desynchronize the frame it is being written into. */ +async function* readHandle(handle: NodeFSP.FileHandle, size: number): AsyncGenerator { + for await (const chunk of handle.createReadStream({ + autoClose: false, + start: 0, + end: size - 1, + })) { + yield chunk as Uint8Array; + } +} diff --git a/apps/server/src/workspace/projectSyncErrno.ts b/apps/server/src/workspace/projectSyncErrno.ts new file mode 100644 index 000000000000..2d501fe1547e --- /dev/null +++ b/apps/server/src/workspace/projectSyncErrno.ts @@ -0,0 +1,22 @@ +/** + * Small errno helpers shared by the project sync filesystem paths. + * + * A sync walks a live working tree while an agent may be editing it, so + * "the entry vanished" and "we may not look at it" are normal outcomes to skip + * past rather than failures to abort the whole transfer on. + * + * @module projectSyncErrno + */ +import * as Predicate from "effect/Predicate"; + +const SKIPPABLE_CODES = new Set(["ENOENT", "ENOTDIR", "EACCES", "EPERM", "ELOOP"]); + +export function errnoCode(cause: unknown): string | undefined { + if (!Predicate.isObject(cause) || !("code" in cause)) return undefined; + return Predicate.isString(cause.code) ? cause.code : undefined; +} + +export function isMissingOrUnreadable(cause: unknown): boolean { + const code = errnoCode(cause); + return code !== undefined && SKIPPABLE_CODES.has(code); +} diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 5eb08c0807e2..9f4fe280fc3d 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -41,6 +41,8 @@ import { ProjectReadFileError, ProjectSearchContentsError, ProjectSearchEntriesError, + ProjectSyncIoError, + ProjectSyncProjectNotFoundError, ProjectWriteFileError, ProviderUploadFeedbackError, ServerProviderRateLimitsRefreshError, @@ -100,6 +102,12 @@ import { deletePendingAttachment, issueAttachmentUploadUrl } from "./assets/Atta import * as PortScanner from "./preview/PortScanner.ts"; import * as WorkspaceEntries from "./workspace/WorkspaceEntries.ts"; import * as WorkspaceFileSystem from "./workspace/WorkspaceFileSystem.ts"; +import { applyProjectSyncDeletions } from "./workspace/ProjectSyncApply.ts"; +import { buildProjectSyncManifest } from "./workspace/ProjectSyncManifest.ts"; +import { + issueProjectSyncExportUrl, + issueProjectSyncImportUrl, +} from "./workspace/ProjectSyncTransfer.ts"; import { readWorkflowScript } from "./orchestration/workflowScriptQuery.ts"; import * as WorkspacePaths from "./workspace/WorkspacePaths.ts"; import * as VcsStatusBroadcaster from "./vcs/VcsStatusBroadcaster.ts"; @@ -541,6 +549,26 @@ const makeWsRpcLayer = ( authorizeEffect(requiredScopeForRpcMethod(method), effect), traceAttributes, ); + // Project sync always addresses a project by id and works from the + // workspace root the projection holds, so a deleted or unknown project + // fails before any filesystem work starts. + const resolveProjectSyncWorkspaceRoot = Effect.fn("ws.resolveProjectSyncWorkspaceRoot")( + function* (projectId: ProjectId) { + const project = yield* projectionSnapshotQuery.getProjectShellById(projectId).pipe( + Effect.mapError( + (cause) => + new ProjectSyncIoError({ + message: `Failed to look up project '${projectId}'.`, + cause, + }), + ), + ); + if (Option.isNone(project)) { + return yield* new ProjectSyncProjectNotFoundError({ projectId }); + } + return project.value.workspaceRoot; + }, + ); const toDispatchCommandError = (cause: unknown, fallbackMessage: string) => isOrchestrationDispatchCommandError(cause) ? cause @@ -1961,6 +1989,56 @@ const makeWsRpcLayer = ( ), { "rpc.aggregate": "workspace" }, ), + [WS_METHODS.projectSyncManifest]: (input) => + observeRpcEffect( + WS_METHODS.projectSyncManifest, + Effect.gen(function* () { + const workspaceRoot = yield* resolveProjectSyncWorkspaceRoot(input.projectId); + const entries = yield* buildProjectSyncManifest({ + workspaceRoot, + includeGit: input.includeGit, + extraIgnores: input.extraIgnores, + }); + return { workspaceRoot, entries, generatedAt: yield* nowIso }; + }), + { "rpc.aggregate": "workspace" }, + ), + [WS_METHODS.projectSyncCreateExportUrl]: (input) => + observeRpcEffect( + WS_METHODS.projectSyncCreateExportUrl, + Effect.gen(function* () { + const workspaceRoot = yield* resolveProjectSyncWorkspaceRoot(input.projectId); + return yield* issueProjectSyncExportUrl({ + projectId: input.projectId, + workspaceRoot, + paths: input.paths, + }); + }), + { "rpc.aggregate": "workspace" }, + ), + [WS_METHODS.projectSyncCreateImportUrl]: (input) => + observeRpcEffect( + WS_METHODS.projectSyncCreateImportUrl, + Effect.gen(function* () { + const workspaceRoot = yield* resolveProjectSyncWorkspaceRoot(input.projectId); + return yield* issueProjectSyncImportUrl({ + projectId: input.projectId, + workspaceRoot, + fileCount: input.fileCount, + totalBytes: input.totalBytes, + }); + }), + { "rpc.aggregate": "workspace" }, + ), + [WS_METHODS.projectSyncApplyDeletions]: (input) => + observeRpcEffect( + WS_METHODS.projectSyncApplyDeletions, + Effect.gen(function* () { + const workspaceRoot = yield* resolveProjectSyncWorkspaceRoot(input.projectId); + return yield* applyProjectSyncDeletions({ workspaceRoot, paths: input.paths }); + }), + { "rpc.aggregate": "workspace" }, + ), [WS_METHODS.shellOpenInEditor]: (input) => observeRpcEffect(WS_METHODS.shellOpenInEditor, externalLauncher.launchEditor(input), { "rpc.aggregate": "workspace", diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 0a75d486fe05..9c07a3586322 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -40,6 +40,7 @@ import { FileSearchIcon, FolderIcon, FolderPlusIcon, + FolderSyncIcon, LinkIcon, MessageSquareIcon, PaletteIcon, @@ -93,6 +94,7 @@ import { import { onOpenCommandPalette } from "../commandPaletteBus"; import { isPreviewFocused } from "../lib/previewFocus"; import { isTerminalFocused } from "../lib/terminalFocus"; +import { SyncProjectDialog, type SyncProjectDialogSource } from "./SyncProjectDialog"; import { selectActiveRightPanel, useRightPanelStore } from "../rightPanelStore"; import { getLatestThreadForProject, sortThreads } from "../lib/threadSort"; import { @@ -412,6 +414,16 @@ export function CommandPalette({ children }: { children: ReactNode }) { ); const openNewThreadIn = useCallback(() => dispatch({ _tag: "OpenNewThreadIn" }), []); const clearOpenIntent = useCallback(() => dispatch({ _tag: "ClearOpenIntent" }), []); + // The sync dialog's state lives on this always-mounted wrapper, not on + // `OpenCommandPaletteDialog` below — that component (and any state inside + // it) unmounts the moment the palette closes, which is exactly what + // happens when one of its actions opens this dialog. + const [syncDialogOpen, setSyncDialogOpen] = useState(false); + const [syncDialogSource, setSyncDialogSource] = useState(null); + const openSyncDialog = useCallback((source: SyncProjectDialogSource | null) => { + setSyncDialogSource(source); + setSyncDialogOpen(true); + }, []); const keybindings = useAtomValue(primaryServerKeybindingsAtom); const { theme, themeHalves, resolvedTheme } = useTheme(); const composerHandleRef = useRef(null); @@ -512,8 +524,14 @@ export function CommandPalette({ children }: { children: ReactNode }) { setOpen={setOpen} openOverlayMode={toggleMode} clearOpenIntent={clearOpenIntent} + openSyncDialog={openSyncDialog} /> + ); } @@ -524,6 +542,7 @@ function CommandPaletteDialog(props: { readonly setOpen: (open: boolean) => void; readonly openOverlayMode: (mode: SearchOverlayMode) => void; readonly clearOpenIntent: () => void; + readonly openSyncDialog: (source: SyncProjectDialogSource | null) => void; }) { const composerHandleRef = useComposerHandleContext(); @@ -558,6 +577,7 @@ function CommandPaletteDialog(props: { setOpen={props.setOpen} openOverlayMode={props.openOverlayMode} clearOpenIntent={props.clearOpenIntent} + openSyncDialog={props.openSyncDialog} /> )} @@ -569,9 +589,10 @@ function OpenCommandPaletteDialog(props: { readonly setOpen: (open: boolean) => void; readonly openOverlayMode: (mode: SearchOverlayMode) => void; readonly clearOpenIntent: () => void; + readonly openSyncDialog: (source: SyncProjectDialogSource | null) => void; }) { const navigate = useNavigate(); - const { clearOpenIntent, openIntent, openOverlayMode, setOpen } = props; + const { clearOpenIntent, openIntent, openOverlayMode, openSyncDialog, setOpen } = props; const [query, setQuery] = useState(""); const deferredQuery = useDeferredValue(query); const isActionsOnly = deferredQuery.startsWith(">"); @@ -1640,6 +1661,17 @@ function OpenCommandPaletteDialog(props: { }, }); + actionItems.push({ + kind: "action", + value: "action:sync-project", + searchTerms: ["sync", "copy", "transfer", "environment", "send project"], + title: "Sync project between environments", + icon: , + run: async () => { + openSyncDialog(null); + }, + }); + // There is no projects listing page; the action targets the contextual // project (active thread/draft, falling back to the first sidebar group). const contextualProjectGroup = @@ -1665,6 +1697,21 @@ function OpenCommandPaletteDialog(props: { }); }, }); + + actionItems.push({ + kind: "action", + value: "action:sync-project-contextual", + searchTerms: ["sync", "copy", "transfer", "environment", "send project"], + title: "Sync this project to another environment", + description: contextualProjectGroup.displayName, + icon: , + run: async () => { + openSyncDialog({ + environmentId: contextualProjectGroup.environmentId, + projectId: contextualProjectGroup.id, + }); + }, + }); } const rootGroups = buildRootGroups({ actionItems, recentThreadItems }); diff --git a/apps/web/src/components/SyncProjectDialog.logic.test.ts b/apps/web/src/components/SyncProjectDialog.logic.test.ts new file mode 100644 index 000000000000..4bac8cb06e81 --- /dev/null +++ b/apps/web/src/components/SyncProjectDialog.logic.test.ts @@ -0,0 +1,396 @@ +import { + ProjectSyncAbortedError, + ProjectSyncTransferError, + ProjectSyncUrlResolutionError, + type ProjectSyncProgress, +} from "@t3tools/client-runtime/state/project-sync"; +import { EnvironmentId, ProjectId } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + buildDefaultSendDestinationPath, + buildSyncProjectDialogSteps, + canStartProjectSync, + describeProjectSyncError, + describeProjectSyncPlanSummary, + deriveProjectSyncProgressPercent, + formatProjectSyncBytes, + projectSyncPlanNeedsDeleteConfirmation, + selectDestinationProjectCandidates, + selectProjectSyncEnvironmentOptions, + slugifyProjectFolderName, + sortDestinationProjectCandidatesBySourceMatch, + type SyncEnvironmentCandidate, + type SyncProjectCandidate, +} from "./SyncProjectDialog.logic"; + +const ENV_A = EnvironmentId.make("env-a"); +const ENV_B = EnvironmentId.make("env-b"); +const PROJECT_1 = ProjectId.make("project-1"); +const PROJECT_2 = ProjectId.make("project-2"); +const PROJECT_3 = ProjectId.make("project-3"); + +function envCandidate(overrides: Partial = {}): SyncEnvironmentCandidate { + return { + environmentId: ENV_A, + label: "Env A", + connected: true, + projectSyncCapable: true, + ...overrides, + }; +} + +function projectCandidate(overrides: Partial = {}): SyncProjectCandidate { + return { + environmentId: ENV_B, + projectId: PROJECT_2, + title: "Project", + workspaceRoot: "/workspace/project", + repositoryCanonicalKey: null, + ...overrides, + }; +} + +function progress(overrides: Partial = {}): ProjectSyncProgress { + return { + stage: "transferring", + transferredBytes: 0, + totalBytes: 0, + transferredFiles: 0, + totalFiles: 0, + ...overrides, + }; +} + +describe("selectProjectSyncEnvironmentOptions", () => { + it("keeps only connected, capability-flagged environments", () => { + const candidates = [ + envCandidate({ environmentId: ENV_A, connected: true, projectSyncCapable: true }), + envCandidate({ environmentId: ENV_B, connected: false, projectSyncCapable: true }), + envCandidate({ + environmentId: EnvironmentId.make("env-c"), + connected: true, + projectSyncCapable: false, + }), + ]; + expect(selectProjectSyncEnvironmentOptions(candidates)).toEqual([candidates[0]]); + }); +}); + +describe("selectDestinationProjectCandidates", () => { + it("filters to the destination environment and excludes the literal source project", () => { + const source = { environmentId: ENV_A, projectId: PROJECT_1 }; + const candidates = [ + projectCandidate({ environmentId: ENV_A, projectId: PROJECT_1 }), // same env+project as source + projectCandidate({ environmentId: ENV_A, projectId: PROJECT_2 }), // same env, different project + projectCandidate({ environmentId: ENV_B, projectId: PROJECT_3 }), // different env entirely + ]; + + const result = selectDestinationProjectCandidates({ + allProjects: candidates, + destinationEnvironmentId: ENV_A, + source, + }); + + expect(result).toEqual([candidates[1]]); + }); +}); + +describe("sortDestinationProjectCandidatesBySourceMatch", () => { + it("moves repository-matching candidates first, preserving relative order otherwise", () => { + const candidates = [ + projectCandidate({ projectId: PROJECT_1, repositoryCanonicalKey: "other" }), + projectCandidate({ projectId: PROJECT_2, repositoryCanonicalKey: "match" }), + projectCandidate({ projectId: PROJECT_3, repositoryCanonicalKey: null }), + ]; + + const result = sortDestinationProjectCandidatesBySourceMatch({ + candidates, + sourceRepositoryCanonicalKey: "match", + }); + + expect(result.map((candidate) => candidate.projectId)).toEqual([ + PROJECT_2, + PROJECT_1, + PROJECT_3, + ]); + }); + + it("returns candidates unchanged when the source has no repository identity", () => { + const candidates = [ + projectCandidate({ projectId: PROJECT_1 }), + projectCandidate({ projectId: PROJECT_2 }), + ]; + expect( + sortDestinationProjectCandidatesBySourceMatch({ + candidates, + sourceRepositoryCanonicalKey: null, + }), + ).toEqual(candidates); + }); +}); + +describe("slugifyProjectFolderName", () => { + it("replaces path-unsafe characters and whitespace with dashes", () => { + expect(slugifyProjectFolderName("My App: v2/final")).toBe("My-App-v2-final"); + }); + + it("collapses repeated separators and trims leading/trailing dashes", () => { + expect(slugifyProjectFolderName(" ///weird/// ")).toBe("weird"); + }); + + it("falls back to 'project' when nothing survives", () => { + expect(slugifyProjectFolderName("///")).toBe("project"); + }); +}); + +describe("buildDefaultSendDestinationPath", () => { + it("joins the configured base directory with a slug of the source title", () => { + expect( + buildDefaultSendDestinationPath({ + baseDirectory: "/home/user/code", + sourceProjectTitle: "My App", + }), + ).toBe("/home/user/code/My-App"); + }); + + it("falls back to ~/ when no base directory is configured", () => { + expect( + buildDefaultSendDestinationPath({ baseDirectory: "", sourceProjectTitle: "My App" }), + ).toBe("~/My-App"); + }); + + it("does not duplicate an existing trailing separator", () => { + expect( + buildDefaultSendDestinationPath({ + baseDirectory: "/home/user/code/", + sourceProjectTitle: "App", + }), + ).toBe("/home/user/code/App"); + }); +}); + +describe("deriveProjectSyncProgressPercent", () => { + it("reports 0 while reading manifests or planning", () => { + expect(deriveProjectSyncProgressPercent(progress({ stage: "manifest" }))).toBe(0); + expect(deriveProjectSyncProgressPercent(progress({ stage: "planning" }))).toBe(0); + }); + + it("reports 100 once deleting or done", () => { + expect(deriveProjectSyncProgressPercent(progress({ stage: "deleting" }))).toBe(100); + expect(deriveProjectSyncProgressPercent(progress({ stage: "done" }))).toBe(100); + }); + + it("derives a percentage from bytes while transferring", () => { + expect( + deriveProjectSyncProgressPercent( + progress({ stage: "transferring", transferredBytes: 25, totalBytes: 100 }), + ), + ).toBe(25); + }); + + it("falls back to file counts when there are no bytes to copy", () => { + expect( + deriveProjectSyncProgressPercent( + progress({ stage: "transferring", totalBytes: 0, transferredFiles: 1, totalFiles: 4 }), + ), + ).toBe(25); + }); + + it("reports complete immediately when the plan has neither bytes nor files", () => { + expect(deriveProjectSyncProgressPercent(progress({ stage: "transferring" }))).toBe(100); + }); +}); + +describe("describeProjectSyncError", () => { + it("describes an aborted sync as retryable", () => { + const result = describeProjectSyncError(new ProjectSyncAbortedError()); + expect(result.canRetry).toBe(true); + expect(result.title).toBe("Sync cancelled"); + }); + + it("names the unreachable environment for a URL resolution failure", () => { + const result = describeProjectSyncError(new ProjectSyncUrlResolutionError(ENV_A)); + expect(result.message).toContain(ENV_A); + expect(result.canRetry).toBe(true); + }); + + it("translates a 404 transfer failure into an expired-link message", () => { + const result = describeProjectSyncError( + new ProjectSyncTransferError("Import request to environment 'env-b' failed with status 404."), + ); + expect(result.message).toBe("The transfer link expired before the batch finished. Try again."); + }); + + it("translates a 413 transfer failure into a too-large message", () => { + const result = describeProjectSyncError( + new ProjectSyncTransferError("Import request to environment 'env-b' failed with status 413."), + ); + expect(result.message).toContain("too large"); + }); + + it("maps a tagged ProjectSyncPathViolationError to a non-retryable description", () => { + const result = describeProjectSyncError({ + _tag: "ProjectSyncPathViolationError", + message: "Path '../etc' escapes the project workspace root.", + }); + expect(result.canRetry).toBe(false); + expect(result.message).toBe("Path '../etc' escapes the project workspace root."); + }); + + it("maps a tagged ProjectSyncIoError to a retryable description", () => { + const result = describeProjectSyncError({ _tag: "ProjectSyncIoError", message: "disk full" }); + expect(result.canRetry).toBe(true); + expect(result.message).toBe("disk full"); + }); + + it("falls back to the error's own message for a plain Error", () => { + const result = describeProjectSyncError(new Error("boom")); + expect(result.message).toBe("boom"); + }); + + it("falls back to a generic message for a non-error value", () => { + const result = describeProjectSyncError("nope"); + expect(result.message).toBe("Sync failed for an unknown reason."); + }); +}); + +describe("formatProjectSyncBytes", () => { + it("formats zero and sub-unit values", () => { + expect(formatProjectSyncBytes(0)).toBe("0 B"); + expect(formatProjectSyncBytes(512)).toBe("512 B"); + }); + + it("formats larger values with the appropriate unit", () => { + expect(formatProjectSyncBytes(5 * 1024 * 1024)).toBe("5 MB"); + expect(formatProjectSyncBytes(1536)).toBe("1.5 KB"); + }); +}); + +describe("describeProjectSyncPlanSummary", () => { + it("describes a copy-only plan", () => { + expect(describeProjectSyncPlanSummary({ copyCount: 3, deleteCount: 0, copyBytes: 2048 })).toBe( + "3 files to copy (2 KB)", + ); + }); + + it("appends the delete count when the plan removes files", () => { + expect(describeProjectSyncPlanSummary({ copyCount: 1, deleteCount: 1, copyBytes: 10 })).toBe( + "1 file to copy (10 B), 1 file removed", + ); + }); +}); + +describe("projectSyncPlanNeedsDeleteConfirmation", () => { + it("requires confirmation only when the plan deletes files", () => { + expect(projectSyncPlanNeedsDeleteConfirmation({ deleteCount: 0 })).toBe(false); + expect(projectSyncPlanNeedsDeleteConfirmation({ deleteCount: 2 })).toBe(true); + }); +}); + +describe("buildSyncProjectDialogSteps", () => { + it("includes the origin step only when the source is not fixed", () => { + expect(buildSyncProjectDialogSteps({ hasFixedSource: false })).toEqual([ + "origin", + "destination", + "mode", + "review", + "progress", + ]); + expect(buildSyncProjectDialogSteps({ hasFixedSource: true })).toEqual([ + "destination", + "mode", + "review", + "progress", + ]); + }); +}); + +describe("canStartProjectSync", () => { + it("blocks when no mode is chosen", () => { + expect( + canStartProjectSync({ + mode: null, + sendDestinationPath: "/x", + existingDestinationProjectId: null, + planSummary: null, + deleteConfirmed: false, + }), + ).toBe(false); + }); + + it("requires a non-empty destination path for send", () => { + expect( + canStartProjectSync({ + mode: "send", + sendDestinationPath: " ", + existingDestinationProjectId: null, + planSummary: null, + deleteConfirmed: false, + }), + ).toBe(false); + expect( + canStartProjectSync({ + mode: "send", + sendDestinationPath: "/home/user/app", + existingDestinationProjectId: null, + planSummary: null, + deleteConfirmed: false, + }), + ).toBe(true); + }); + + it("requires an existing project and a computed plan for sync", () => { + expect( + canStartProjectSync({ + mode: "sync", + sendDestinationPath: "", + existingDestinationProjectId: null, + planSummary: { deleteCount: 0 }, + deleteConfirmed: false, + }), + ).toBe(false); + expect( + canStartProjectSync({ + mode: "sync", + sendDestinationPath: "", + existingDestinationProjectId: PROJECT_1, + planSummary: null, + deleteConfirmed: false, + }), + ).toBe(false); + }); + + it("blocks a deleting sync plan until deletions are explicitly confirmed", () => { + expect( + canStartProjectSync({ + mode: "sync", + sendDestinationPath: "", + existingDestinationProjectId: PROJECT_1, + planSummary: { deleteCount: 2 }, + deleteConfirmed: false, + }), + ).toBe(false); + expect( + canStartProjectSync({ + mode: "sync", + sendDestinationPath: "", + existingDestinationProjectId: PROJECT_1, + planSummary: { deleteCount: 2 }, + deleteConfirmed: true, + }), + ).toBe(true); + }); + + it("allows a non-deleting sync plan without explicit confirmation", () => { + expect( + canStartProjectSync({ + mode: "sync", + sendDestinationPath: "", + existingDestinationProjectId: PROJECT_1, + planSummary: { deleteCount: 0 }, + deleteConfirmed: false, + }), + ).toBe(true); + }); +}); diff --git a/apps/web/src/components/SyncProjectDialog.logic.ts b/apps/web/src/components/SyncProjectDialog.logic.ts new file mode 100644 index 000000000000..b8d26ead7be4 --- /dev/null +++ b/apps/web/src/components/SyncProjectDialog.logic.ts @@ -0,0 +1,310 @@ +import { + ProjectSyncAbortedError, + ProjectSyncTransferError, + ProjectSyncUrlResolutionError, + type ProjectSyncProgress, +} from "@t3tools/client-runtime/state/project-sync"; +import type { ProjectSyncPlanSummary } from "@t3tools/client-runtime/operations/project-sync"; +import { ensureBrowseDirectoryPath } from "@t3tools/client-runtime/state/projects"; +import type { EnvironmentId, ProjectId } from "@t3tools/contracts"; + +/** + * One connected environment considered as a sync source/destination + * candidate. Kept intentionally narrow (rather than the full presentation + * type) so this module's pure functions are easy to unit test without + * constructing a real `EnvironmentPresentation`. + */ +export interface SyncEnvironmentCandidate { + readonly environmentId: EnvironmentId; + readonly label: string; + readonly connected: boolean; + readonly projectSyncCapable: boolean; +} + +/** + * Environments eligible as either end of a sync: connected, and reporting the + * `projectSync` capability. Older servers (or ones mid-reconnect) never show + * up, since neither can safely serve the manifest/export/import RPCs. + */ +export function selectProjectSyncEnvironmentOptions( + candidates: ReadonlyArray, +): SyncEnvironmentCandidate[] { + return candidates.filter((candidate) => candidate.connected && candidate.projectSyncCapable); +} + +export interface SyncProjectCandidate { + readonly environmentId: EnvironmentId; + readonly projectId: ProjectId; + readonly title: string; + readonly workspaceRoot: string; + readonly repositoryCanonicalKey: string | null; +} + +/** + * Existing projects on the destination environment eligible as a "sync" + * target, i.e. every project on that environment except the literal source + * project (relevant when source and destination happen to be the same + * environment). + */ +export function selectDestinationProjectCandidates(input: { + readonly allProjects: ReadonlyArray; + readonly destinationEnvironmentId: EnvironmentId; + readonly source: { readonly environmentId: EnvironmentId; readonly projectId: ProjectId }; +}): SyncProjectCandidate[] { + return input.allProjects.filter( + (project) => + project.environmentId === input.destinationEnvironmentId && + !( + project.environmentId === input.source.environmentId && + project.projectId === input.source.projectId + ), + ); +} + +/** + * Stable-sorts destination candidates so projects sharing the source's + * repository identity are suggested first. Array#sort is a stable sort in + * every engine this app targets, so ties preserve the caller's original + * (e.g. alphabetical) order. + */ +export function sortDestinationProjectCandidatesBySourceMatch(input: { + readonly candidates: ReadonlyArray; + readonly sourceRepositoryCanonicalKey: string | null; +}): SyncProjectCandidate[] { + const key = input.sourceRepositoryCanonicalKey; + if (key === null) { + return [...input.candidates]; + } + return [...input.candidates].sort((a, b) => { + const aRank = a.repositoryCanonicalKey === key ? 0 : 1; + const bRank = b.repositoryCanonicalKey === key ? 0 : 1; + return aRank - bRank; + }); +} + +/** Turns a project title into a filesystem-safe folder name for the default + "send" destination path. Falls back to "project" if nothing survives. */ +export function slugifyProjectFolderName(title: string): string { + const cleaned = title + .trim() + .replace(/[\\/:*?"<>|]+/g, "-") + .replace(/\s+/g, "-") + .replace(/-{2,}/g, "-") + .replace(/^-+|-+$/g, ""); + return cleaned.length > 0 ? cleaned : "project"; +} + +/** + * Default destination path for a "send" (new project) sync: the + * destination environment's configured `addProjectBaseDirectory` (or `~/` + * when unset), plus a slug of the source project's title. + */ +export function buildDefaultSendDestinationPath(input: { + readonly baseDirectory: string; + readonly sourceProjectTitle: string; +}): string { + const trimmedBase = input.baseDirectory.trim(); + const base = trimmedBase.length > 0 ? trimmedBase : "~/"; + const directory = ensureBrowseDirectoryPath(base); + return `${directory}${slugifyProjectFolderName(input.sourceProjectTitle)}`; +} + +function clampPercent(value: number): number { + if (!Number.isFinite(value)) { + return 0; + } + return Math.max(0, Math.min(100, Math.round(value))); +} + +/** + * Maps a `ProjectSyncProgress` snapshot to a 0-100 percentage for a progress + * bar. Bytes drive the percentage whenever the plan has any; a delete-only + * plan (no bytes to copy) falls back to file counts, and a plan with neither + * is already complete the moment the copy stage starts. + */ +export function deriveProjectSyncProgressPercent(progress: ProjectSyncProgress): number { + switch (progress.stage) { + case "manifest": + case "planning": + return 0; + case "done": + case "deleting": + return 100; + case "transferring": + if (progress.totalBytes > 0) { + return clampPercent((progress.transferredBytes / progress.totalBytes) * 100); + } + if (progress.totalFiles > 0) { + return clampPercent((progress.transferredFiles / progress.totalFiles) * 100); + } + return 100; + } +} + +export function describeProjectSyncStage(stage: ProjectSyncProgress["stage"]): string { + switch (stage) { + case "manifest": + return "Reading file lists…"; + case "planning": + return "Comparing projects…"; + case "transferring": + return "Copying files…"; + case "deleting": + return "Removing files no longer in the source…"; + case "done": + return "Done"; + } +} + +export interface SyncProjectErrorDescription { + readonly title: string; + readonly message: string; + readonly canRetry: boolean; +} + +function describeTransferFailureMessage(message: string): string { + const statusMatch = /status (\d+)/.exec(message); + const status = statusMatch ? Number(statusMatch[1]) : null; + if (status === 404) { + return "The transfer link expired before the batch finished. Try again."; + } + if (status === 413) { + return "The destination rejected the transfer because a batch was too large."; + } + if (status === 400) { + return "The destination rejected one or more file paths."; + } + return message; +} + +/** Maps any error `runProjectSync` can throw (or reject with) to a + human-readable title/message and whether a retry is worth offering. */ +export function describeProjectSyncError(error: unknown): SyncProjectErrorDescription { + if (error instanceof ProjectSyncAbortedError) { + return { + title: "Sync cancelled", + message: "The sync was cancelled before it finished.", + canRetry: true, + }; + } + if (error instanceof ProjectSyncUrlResolutionError) { + return { + title: "Connection lost", + message: `Could not reach environment "${error.environmentId}". Check its connection and try again.`, + canRetry: true, + }; + } + if (error instanceof ProjectSyncTransferError) { + return { + title: "Transfer failed", + message: describeTransferFailureMessage(error.message), + canRetry: true, + }; + } + + const tagged = error as { readonly _tag?: unknown; readonly message?: unknown } | null; + if (tagged && typeof tagged === "object") { + const message = typeof tagged.message === "string" ? tagged.message : null; + if (tagged._tag === "ProjectSyncProjectNotFoundError") { + return { + title: "Project not found", + message: message ?? "The project was not found.", + canRetry: false, + }; + } + if (tagged._tag === "ProjectSyncPathViolationError") { + return { + title: "Blocked file path", + message: message ?? "A file path escaped the project workspace.", + canRetry: false, + }; + } + if (tagged._tag === "ProjectSyncIoError") { + return { + title: "File operation failed", + message: message ?? "A file operation failed on one of the environments.", + canRetry: true, + }; + } + } + + if (error instanceof Error) { + return { title: "Sync failed", message: error.message, canRetry: true }; + } + return { title: "Sync failed", message: "Sync failed for an unknown reason.", canRetry: true }; +} + +/** Whether a sync plan requires the user to explicitly acknowledge deletions + before starting — deletions mirror the destination onto the source, so + they are the one part of a sync that is not simply additive. */ +export function projectSyncPlanNeedsDeleteConfirmation(summary: { + readonly deleteCount: number; +}): boolean { + return summary.deleteCount > 0; +} + +export function formatProjectSyncBytes(bytes: number): string { + if (bytes <= 0) { + return "0 B"; + } + const units = ["B", "KB", "MB", "GB", "TB"] as const; + let value = bytes; + let unitIndex = 0; + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024; + unitIndex += 1; + } + const precision = unitIndex === 0 || Number.isInteger(value) ? 0 : 1; + return `${value.toFixed(precision)} ${units[unitIndex]}`; +} + +export function describeProjectSyncPlanSummary(summary: ProjectSyncPlanSummary): string { + const copyPart = summary.copyCount === 1 ? "1 file" : `${summary.copyCount} files`; + const bytesPart = formatProjectSyncBytes(summary.copyBytes); + const deletePart = + summary.deleteCount > 0 + ? `, ${summary.deleteCount === 1 ? "1 file" : `${summary.deleteCount} files`} removed` + : ""; + return `${copyPart} to copy (${bytesPart})${deletePart}`; +} + +export type SyncProjectDialogStepId = "origin" | "destination" | "mode" | "review" | "progress"; + +/** + * The dialog skips the origin step entirely when it was opened with a fixed + * source project (from Project Settings, or the command palette's + * project-scoped entry) — asking again would ignore where the request came + * from, the same rule the add-project flow already follows. + */ +export function buildSyncProjectDialogSteps(input: { + readonly hasFixedSource: boolean; +}): SyncProjectDialogStepId[] { + return input.hasFixedSource + ? ["destination", "mode", "review", "progress"] + : ["origin", "destination", "mode", "review", "progress"]; +} + +export type SyncProjectDialogMode = "send" | "sync"; + +/** Gates the review step's "Start sync" action. */ +export function canStartProjectSync(input: { + readonly mode: SyncProjectDialogMode | null; + readonly sendDestinationPath: string; + readonly existingDestinationProjectId: ProjectId | null; + readonly planSummary: { readonly deleteCount: number } | null; + readonly deleteConfirmed: boolean; +}): boolean { + if (input.mode === null) { + return false; + } + if (input.mode === "send") { + return input.sendDestinationPath.trim().length > 0; + } + if (input.existingDestinationProjectId === null || input.planSummary === null) { + return false; + } + if (input.planSummary.deleteCount > 0 && !input.deleteConfirmed) { + return false; + } + return true; +} diff --git a/apps/web/src/components/SyncProjectDialog.tsx b/apps/web/src/components/SyncProjectDialog.tsx new file mode 100644 index 000000000000..53a907cd47bf --- /dev/null +++ b/apps/web/src/components/SyncProjectDialog.tsx @@ -0,0 +1,995 @@ +import { + computeProjectSyncPlan, + summarizeProjectSyncPlan, +} from "@t3tools/client-runtime/operations/project-sync"; +import { getBrowseDirectoryPath } from "@t3tools/client-runtime/state/projects"; +import { + runProjectSync, + type ProjectSyncProgress, +} from "@t3tools/client-runtime/state/project-sync"; +import { + isAtomCommandInterrupted, + squashAtomCommandFailure, +} from "@t3tools/client-runtime/state/runtime"; +import { EnvironmentId, type ProjectId } from "@t3tools/contracts"; +import { + AlertTriangleIcon, + ArrowRightIcon, + CircleCheckIcon, + CircleXIcon, + FolderSyncIcon, +} from "lucide-react"; +import { useCallback, useEffect, useMemo, useRef, useState } from "react"; + +import { useUpdateEnvironmentSettings } from "../hooks/useSettings"; +import { cn, newProjectId } from "../lib/utils"; +import { useEnvironments } from "../state/environments"; +import { useProjects } from "../state/entities"; +import { projectEnvironment } from "../state/projects"; +import { useProjectSyncDeps } from "../state/projectSync"; +import { useAtomCommand } from "../state/use-atom-command"; +import { AnimatedHeight } from "./AnimatedHeight"; +import { + buildDefaultSendDestinationPath, + buildSyncProjectDialogSteps, + canStartProjectSync, + describeProjectSyncError, + describeProjectSyncPlanSummary, + deriveProjectSyncProgressPercent, + describeProjectSyncStage, + projectSyncPlanNeedsDeleteConfirmation, + selectDestinationProjectCandidates, + selectProjectSyncEnvironmentOptions, + sortDestinationProjectCandidatesBySourceMatch, + type SyncEnvironmentCandidate, + type SyncProjectCandidate, + type SyncProjectDialogMode, + type SyncProjectErrorDescription, +} from "./SyncProjectDialog.logic"; +import { Button } from "./ui/button"; +import { Checkbox } from "./ui/checkbox"; +import { + Dialog, + DialogDescription, + DialogFooter, + DialogHeader, + DialogPopup, + DialogTitle, +} from "./ui/dialog"; +import { Input } from "./ui/input"; +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "./ui/select"; +import { Spinner } from "./ui/spinner"; + +// Used only to satisfy `useUpdateEnvironmentSettings`'s non-null signature +// before a destination environment is chosen; the update it drives is only +// ever invoked once a real destination is selected, so this value is never +// actually persisted against. +const UNSET_ENVIRONMENT_ID = EnvironmentId.make("__sync-dialog-unset__"); + +export interface SyncProjectDialogSource { + readonly environmentId: EnvironmentId; + readonly projectId: ProjectId; +} + +export interface SyncProjectDialogProps { + readonly open: boolean; + readonly onOpenChange: (open: boolean) => void; + /** Preselected source project. When set, the origin step is skipped — + the caller already knows the answer (a project's own settings page, or + the command palette's project-scoped entry). */ + readonly initialSource?: SyncProjectDialogSource; +} + +type RunState = "idle" | "running" | "done" | "error" | "cancelled"; + +export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncProjectDialogProps) { + const { environments } = useEnvironments(); + const allProjects = useProjects(); + const deps = useProjectSyncDeps(); + const createProject = useAtomCommand(projectEnvironment.create, { reportFailure: false }); + + const hasFixedSource = initialSource !== undefined; + const steps = useMemo(() => buildSyncProjectDialogSteps({ hasFixedSource }), [hasFixedSource]); + + const [stepIndex, setStepIndex] = useState(0); + const [originEnvironmentId, setOriginEnvironmentId] = useState(null); + const [originProjectId, setOriginProjectId] = useState(null); + const [destEnvironmentId, setDestEnvironmentId] = useState(null); + const [mode, setMode] = useState(null); + const [sendDestinationPath, setSendDestinationPath] = useState(""); + const [sendDestinationPathTouched, setSendDestinationPathTouched] = useState(false); + const [setAsDefaultFolder, setSetAsDefaultFolder] = useState(false); + const [existingDestProjectId, setExistingDestProjectId] = useState(null); + const [includeGit, setIncludeGit] = useState(true); + const [planSummary, setPlanSummary] = useState<{ + readonly copyCount: number; + readonly deleteCount: number; + readonly copyBytes: number; + } | null>(null); + const [isLoadingPlan, setIsLoadingPlan] = useState(false); + const [planError, setPlanError] = useState(null); + const [planRetryToken, setPlanRetryToken] = useState(0); + const [deleteConfirmed, setDeleteConfirmed] = useState(false); + const [progress, setProgress] = useState(null); + const [runState, setRunState] = useState("idle"); + const [runError, setRunError] = useState(null); + const [finalSummary, setFinalSummary] = useState<{ + readonly copyCount: number; + readonly deleteCount: number; + readonly copyBytes: number; + } | null>(null); + const abortControllerRef = useRef(null); + + // Reset every field whenever the dialog opens, so a previous run's choices + // never leak into the next one. + useEffect(() => { + if (!open) { + return; + } + setStepIndex(0); + setOriginEnvironmentId(null); + setOriginProjectId(null); + setDestEnvironmentId(null); + setMode(null); + setSendDestinationPath(""); + setSendDestinationPathTouched(false); + setSetAsDefaultFolder(false); + setExistingDestProjectId(null); + setIncludeGit(true); + setPlanSummary(null); + setIsLoadingPlan(false); + setPlanError(null); + setPlanRetryToken(0); + setDeleteConfirmed(false); + setProgress(null); + setRunState("idle"); + setRunError(null); + setFinalSummary(null); + abortControllerRef.current = null; + }, [open]); + + // Derived from primitive fields (not the `initialSource` object itself) so + // its identity stays stable across renders even when a caller passes a + // freshly-literal `initialSource` prop every render (as + // `ProjectSettingsPanel` does) — otherwise every downstream effect keyed on + // `source` would re-run on every unrelated parent re-render. + const fixedSourceEnvironmentId = initialSource?.environmentId ?? null; + const fixedSourceProjectId = initialSource?.projectId ?? null; + const source: SyncProjectDialogSource | null = useMemo(() => { + if (hasFixedSource) { + return fixedSourceEnvironmentId !== null && fixedSourceProjectId !== null + ? { environmentId: fixedSourceEnvironmentId, projectId: fixedSourceProjectId } + : null; + } + return originEnvironmentId !== null && originProjectId !== null + ? { environmentId: originEnvironmentId, projectId: originProjectId } + : null; + }, [ + hasFixedSource, + fixedSourceEnvironmentId, + fixedSourceProjectId, + originEnvironmentId, + originProjectId, + ]); + + const sourceProject = useMemo( + () => + source + ? (allProjects.find( + (project) => + project.environmentId === source.environmentId && project.id === source.projectId, + ) ?? null) + : null, + [allProjects, source], + ); + + const environmentCandidates: SyncEnvironmentCandidate[] = useMemo( + () => + environments.map((environment) => ({ + environmentId: environment.environmentId, + label: environment.label, + connected: environment.connection.phase === "connected", + projectSyncCapable: environment.serverConfig?.environment.capabilities.projectSync === true, + })), + [environments], + ); + const eligibleEnvironments = useMemo( + () => selectProjectSyncEnvironmentOptions(environmentCandidates), + [environmentCandidates], + ); + + const originEnvironmentProjects = useMemo( + () => + originEnvironmentId === null + ? [] + : allProjects.filter((project) => project.environmentId === originEnvironmentId), + [allProjects, originEnvironmentId], + ); + + const destEnvironment = useMemo( + () => + environments.find((environment) => environment.environmentId === destEnvironmentId) ?? null, + [environments, destEnvironmentId], + ); + const destBaseDirectory = destEnvironment?.serverConfig?.settings.addProjectBaseDirectory ?? ""; + + const defaultSendDestinationPath = useMemo( + () => + sourceProject + ? buildDefaultSendDestinationPath({ + baseDirectory: destBaseDirectory, + sourceProjectTitle: sourceProject.title, + }) + : "", + [destBaseDirectory, sourceProject], + ); + + useEffect(() => { + if (mode === "send" && !sendDestinationPathTouched) { + setSendDestinationPath(defaultSendDestinationPath); + } + }, [mode, defaultSendDestinationPath, sendDestinationPathTouched]); + + // Any change to the destination environment or mode invalidates a + // previously picked existing project and any plan computed against it. + useEffect(() => { + setExistingDestProjectId(null); + setPlanSummary(null); + setPlanError(null); + setDeleteConfirmed(false); + }, [destEnvironmentId, mode]); + + const destProjectCandidates: SyncProjectCandidate[] = useMemo(() => { + if (source === null || destEnvironmentId === null) { + return []; + } + const candidates: SyncProjectCandidate[] = allProjects.map((project) => ({ + environmentId: project.environmentId, + projectId: project.id, + title: project.title, + workspaceRoot: project.workspaceRoot, + repositoryCanonicalKey: project.repositoryIdentity?.canonicalKey ?? null, + })); + const filtered = selectDestinationProjectCandidates({ + allProjects: candidates, + destinationEnvironmentId: destEnvironmentId, + source, + }); + return sortDestinationProjectCandidatesBySourceMatch({ + candidates: filtered, + sourceRepositoryCanonicalKey: sourceProject?.repositoryIdentity?.canonicalKey ?? null, + }); + }, [allProjects, destEnvironmentId, source, sourceProject]); + + const currentStep = steps[Math.min(stepIndex, steps.length - 1)]!; + + // Fetch both manifests and compute the plan once the user reaches the + // review step in "sync" mode — this is what surfaces the delete warning + // before anything on the destination is touched. + useEffect(() => { + if ( + currentStep !== "review" || + mode !== "sync" || + source === null || + destEnvironmentId === null + ) { + return; + } + if (existingDestProjectId === null) { + return; + } + let cancelled = false; + setIsLoadingPlan(true); + setPlanError(null); + setPlanSummary(null); + (async () => { + try { + const [sourceManifest, destManifest] = await Promise.all([ + deps.getManifest( + { environmentId: source.environmentId, projectId: source.projectId }, + includeGit, + ), + deps.getManifest( + { environmentId: destEnvironmentId, projectId: existingDestProjectId }, + includeGit, + ), + ]); + if (cancelled) return; + const plan = computeProjectSyncPlan(sourceManifest.entries, destManifest.entries); + setPlanSummary(summarizeProjectSyncPlan(plan)); + } catch (error) { + if (cancelled) return; + setPlanError(describeProjectSyncError(error).message); + } finally { + if (!cancelled) setIsLoadingPlan(false); + } + })(); + return () => { + cancelled = true; + }; + // `planRetryToken` has no direct use inside the effect body — bumping it + // is how the review step's "Retry" button re-triggers this same fetch. + }, [ + currentStep, + mode, + source, + destEnvironmentId, + existingDestProjectId, + includeGit, + deps, + planRetryToken, + ]); + + const updateDestSettings = useUpdateEnvironmentSettings( + destEnvironmentId ?? UNSET_ENVIRONMENT_ID, + ); + + const handleStart = useCallback(async () => { + if (source === null || destEnvironmentId === null || mode === null) { + return; + } + setStepIndex(steps.length - 1); + setRunState("running"); + setRunError(null); + setProgress(null); + const controller = new AbortController(); + abortControllerRef.current = controller; + + let destProjectId: ProjectId; + if (mode === "send") { + const trimmedPath = sendDestinationPath.trim(); + if (setAsDefaultFolder) { + updateDestSettings({ addProjectBaseDirectory: getBrowseDirectoryPath(trimmedPath) }); + } + const createdProjectId = newProjectId(); + const createResult = await createProject({ + environmentId: destEnvironmentId, + input: { + projectId: createdProjectId, + title: sourceProject?.title ?? "Synced project", + workspaceRoot: trimmedPath, + createWorkspaceRootIfMissing: true, + }, + }); + if (createResult._tag !== "Success") { + abortControllerRef.current = null; + if (isAtomCommandInterrupted(createResult)) { + setRunState("idle"); + return; + } + setRunState("error"); + setRunError(describeProjectSyncError(squashAtomCommandFailure(createResult))); + return; + } + destProjectId = createdProjectId; + setExistingDestProjectId(createdProjectId); + } else { + if (existingDestProjectId === null) { + abortControllerRef.current = null; + return; + } + destProjectId = existingDestProjectId; + } + + try { + const summary = await runProjectSync(deps, { + source: { environmentId: source.environmentId, projectId: source.projectId }, + dest: { environmentId: destEnvironmentId, projectId: destProjectId }, + mode, + includeGit, + signal: controller.signal, + onProgress: setProgress, + }); + setFinalSummary(summary); + setRunState("done"); + } catch (error) { + setRunState(controller.signal.aborted ? "cancelled" : "error"); + setRunError(describeProjectSyncError(error)); + } finally { + abortControllerRef.current = null; + } + }, [ + createProject, + deps, + destEnvironmentId, + existingDestProjectId, + includeGit, + mode, + sendDestinationPath, + setAsDefaultFolder, + source, + sourceProject, + steps.length, + updateDestSettings, + ]); + + const handleCancel = useCallback(() => { + abortControllerRef.current?.abort(); + }, []); + + const handleRetry = useCallback(() => { + void handleStart(); + }, [handleStart]); + + const goBack = useCallback(() => setStepIndex((index) => Math.max(0, index - 1)), []); + const goNext = useCallback( + () => setStepIndex((index) => Math.min(steps.length - 1, index + 1)), + [steps.length], + ); + + const canProceedOrigin = source !== null; + const canProceedDestination = destEnvironmentId !== null; + const canProceedMode = + mode !== null && + (mode === "send" ? sendDestinationPath.trim().length > 0 : existingDestProjectId !== null); + const needsDeleteConfirmation = + planSummary !== null && projectSyncPlanNeedsDeleteConfirmation(planSummary); + const canStart = + mode !== null && + canStartProjectSync({ + mode, + sendDestinationPath, + existingDestinationProjectId: existingDestProjectId, + planSummary: mode === "sync" ? planSummary : { deleteCount: 0 }, + deleteConfirmed, + }); + + const busy = runState === "running"; + + return ( + { + if (!next && busy) { + return; + } + onOpenChange(next); + }} + > + +
+ + + + Sync project between environments + + + Copy a project's files from one environment to another, or keep an existing project in + sync with its source. + + + +
+ + {currentStep === "origin" ? ( + { + setOriginEnvironmentId(environmentId); + setOriginProjectId(null); + }} + originProjects={originEnvironmentProjects} + originProjectId={originProjectId} + onOriginProjectChange={setOriginProjectId} + /> + ) : null} + + {currentStep === "destination" ? ( + { + setDestEnvironmentId(environmentId); + setSendDestinationPathTouched(false); + }} + /> + ) : null} + + {currentStep === "mode" ? ( + { + setMode(nextMode); + if (nextMode === "send") { + setSendDestinationPathTouched(false); + } + }} + sendDestinationPath={sendDestinationPath} + onSendDestinationPathChange={(value) => { + setSendDestinationPath(value); + setSendDestinationPathTouched(true); + }} + setAsDefaultFolder={setAsDefaultFolder} + onSetAsDefaultFolderChange={setSetAsDefaultFolder} + destProjectCandidates={destProjectCandidates} + existingDestProjectId={existingDestProjectId} + onExistingDestProjectChange={setExistingDestProjectId} + includeGit={includeGit} + onIncludeGitChange={setIncludeGit} + sourceRepositoryCanonicalKey={ + sourceProject?.repositoryIdentity?.canonicalKey ?? null + } + /> + ) : null} + + {currentStep === "review" ? ( + candidate.projectId === existingDestProjectId, + ) ?? null + } + isLoadingPlan={isLoadingPlan} + planSummary={planSummary} + planError={planError} + onRetryPlan={() => setPlanRetryToken((token) => token + 1)} + needsDeleteConfirmation={needsDeleteConfirmation} + deleteConfirmed={deleteConfirmed} + onDeleteConfirmedChange={setDeleteConfirmed} + /> + ) : null} + + {currentStep === "progress" ? ( + + ) : null} + +
+ + + {currentStep === "progress" ? ( + busy ? ( + + ) : ( + <> + {runState === "error" && runError?.canRetry ? ( + + ) : null} + + + ) + ) : ( + <> + + {currentStep === "review" ? ( + + ) : ( + + )} + + )} + +
+ + + ); +} + +function OriginStep(props: { + readonly eligibleEnvironments: ReadonlyArray; + readonly originEnvironmentId: EnvironmentId | null; + readonly onOriginEnvironmentChange: (environmentId: EnvironmentId) => void; + readonly originProjects: ReadonlyArray<{ readonly id: ProjectId; readonly title: string }>; + readonly originProjectId: ProjectId | null; + readonly onOriginProjectChange: (projectId: ProjectId) => void; +}) { + return ( +
+ + + +
+ ); +} + +function DestinationStep(props: { + readonly eligibleEnvironments: ReadonlyArray; + readonly destEnvironmentId: EnvironmentId | null; + readonly onDestEnvironmentChange: (environmentId: EnvironmentId) => void; +}) { + return ( +
+ +
+ ); +} + +function ModeStep(props: { + readonly mode: SyncProjectDialogMode | null; + readonly onModeChange: (mode: SyncProjectDialogMode) => void; + readonly sendDestinationPath: string; + readonly onSendDestinationPathChange: (value: string) => void; + readonly setAsDefaultFolder: boolean; + readonly onSetAsDefaultFolderChange: (value: boolean) => void; + readonly destProjectCandidates: ReadonlyArray; + readonly existingDestProjectId: ProjectId | null; + readonly onExistingDestProjectChange: (projectId: ProjectId) => void; + readonly includeGit: boolean; + readonly onIncludeGitChange: (value: boolean) => void; + readonly sourceRepositoryCanonicalKey: string | null; +}) { + return ( +
+
+ + +
+ + {props.mode === "send" ? ( + + ) : null} + + {props.mode === "sync" ? ( + + ) : null} + +
+ props.onIncludeGitChange(checked === true)} + id="sync-project-include-git" + /> + +
+
+ ); +} + +function ReviewStep(props: { + readonly mode: SyncProjectDialogMode | null; + readonly source: SyncProjectDialogSource | null; + readonly sourceProjectTitle: string | null; + readonly destEnvironmentLabel: string | null; + readonly sendDestinationPath: string; + readonly existingDestProject: SyncProjectCandidate | null; + readonly isLoadingPlan: boolean; + readonly planSummary: { + readonly copyCount: number; + readonly deleteCount: number; + readonly copyBytes: number; + } | null; + readonly planError: string | null; + readonly onRetryPlan: () => void; + readonly needsDeleteConfirmation: boolean; + readonly deleteConfirmed: boolean; + readonly onDeleteConfirmedChange: (value: boolean) => void; +}) { + return ( +
+
+ + {props.sourceProjectTitle ?? "Source project"} + + + + {props.mode === "send" + ? `${props.destEnvironmentLabel ?? "destination"} · ${props.sendDestinationPath}` + : `${props.destEnvironmentLabel ?? "destination"} · ${props.existingDestProject?.title ?? ""}`} + +
+ + {props.mode === "sync" ? ( + props.isLoadingPlan ? ( +
+ + Comparing projects… +
+ ) : props.planError !== null ? ( +
+
+ + {props.planError} +
+ +
+ ) : props.planSummary !== null ? ( +
+

+ {describeProjectSyncPlanSummary(props.planSummary)} +

+ {props.needsDeleteConfirmation ? ( +
+
+ + + This deletes{" "} + {props.planSummary.deleteCount === 1 + ? "1 file" + : `${props.planSummary.deleteCount} files`}{" "} + on the destination to mirror the source. This cannot be undone. + +
+
+ props.onDeleteConfirmedChange(checked === true)} + id="sync-project-confirm-delete" + /> + +
+
+ ) : null} +
+ ) : null + ) : ( +

+ A new project will be created at this path, then the source's files will be copied over. +

+ )} +
+ ); +} + +function ProgressStep(props: { + readonly runState: RunState; + readonly progress: ProjectSyncProgress | null; + readonly runError: SyncProjectErrorDescription | null; + readonly finalSummary: { + readonly copyCount: number; + readonly deleteCount: number; + readonly copyBytes: number; + } | null; +}) { + if (props.runState === "done") { + return ( +
+
+ + Sync complete +
+ {props.finalSummary ? ( +

+ {describeProjectSyncPlanSummary(props.finalSummary)} +

+ ) : null} +
+ ); + } + + if (props.runState === "error" || props.runState === "cancelled") { + return ( +
+
+ + {props.runError?.title ?? "Sync failed"} +
+

{props.runError?.message}

+
+ ); + } + + const percent = props.progress ? deriveProjectSyncProgressPercent(props.progress) : 0; + const stageLabel = props.progress ? describeProjectSyncStage(props.progress.stage) : "Starting…"; + + return ( +
+
+ + {stageLabel} +
+
+
+
+ {props.progress && props.progress.totalFiles > 0 ? ( +

+ {props.progress.transferredFiles} of {props.progress.totalFiles} files +

+ ) : null} +
+ ); +} diff --git a/apps/web/src/components/settings/ProjectSettingsPanel.tsx b/apps/web/src/components/settings/ProjectSettingsPanel.tsx index 6047b8fc48dc..58180a3dcd6a 100644 --- a/apps/web/src/components/settings/ProjectSettingsPanel.tsx +++ b/apps/web/src/components/settings/ProjectSettingsPanel.tsx @@ -25,7 +25,14 @@ import { createModelSelection } from "@t3tools/shared/model"; import { DEFAULT_RESOLVED_KEYBINDINGS } from "@t3tools/shared/keybindings"; import { useCanGoBack, useNavigate } from "@tanstack/react-router"; import * as Cause from "effect/Cause"; -import { ChevronDownIcon, CopyIcon, PlusIcon, SettingsIcon, Trash2Icon } from "lucide-react"; +import { + ChevronDownIcon, + CopyIcon, + FolderSyncIcon, + PlusIcon, + SettingsIcon, + Trash2Icon, +} from "lucide-react"; import { useCallback, useEffect, @@ -74,6 +81,8 @@ import { useAtomCommand } from "../../state/use-atom-command"; import { ProviderModelPicker } from "../chat/ProviderModelPicker"; import { TraitsPicker } from "../chat/TraitsPicker"; import { ProjectFavicon } from "../ProjectFavicon"; +import { SyncProjectDialog } from "../SyncProjectDialog"; +import { selectProjectSyncEnvironmentOptions } from "../SyncProjectDialog.logic"; import { EMPTY_PROJECT_SCRIPT_INPUT, editorRequestForScript, @@ -291,6 +300,7 @@ export function ProjectSettingsPanel({ projectKey }: { projectKey: string }) { function ProjectDetail({ group }: { group: SidebarProjectSnapshot }) { const navigate = useNavigate(); const primaryEnvironmentId = usePrimaryEnvironmentId(); + const { environments } = useEnvironments(); const settings = usePrimarySettings(); const updateClientSettings = useUpdateClientSettings(); const projectGroupingSettings = useClientSettings(selectProjectGroupingSettings); @@ -439,6 +449,22 @@ function ProjectDetail({ group }: { group: SidebarProjectSnapshot }) { [updateAllMembers], ); + // ----- sync ----- + const [syncDialogOpen, setSyncDialogOpen] = useState(false); + const projectSyncCapableEnvironments = useMemo( + () => + selectProjectSyncEnvironmentOptions( + environments.map((environment) => ({ + environmentId: environment.environmentId, + label: environment.label, + connected: environment.connection.phase === "connected", + projectSyncCapable: + environment.serverConfig?.environment.capabilities.projectSync === true, + })), + ), + [environments], + ); + // ----- favicon ----- const [faviconPickerOpen, setFaviconPickerOpen] = useState(false); const [isSavingFavicon, setIsSavingFavicon] = useState(false); @@ -1146,6 +1172,28 @@ function ProjectDetail({ group }: { group: SidebarProjectSnapshot }) { ) : null} + + 0 + ? "Send this checkout's files to another environment as a new project, or keep an existing project there in sync with it." + : "No other connected environment supports project sync yet. Connect one that supports it to send or sync this checkout." + } + control={ + + } + /> + + + ); } diff --git a/apps/web/src/state/projectSync.ts b/apps/web/src/state/projectSync.ts new file mode 100644 index 000000000000..a289a2642cb3 --- /dev/null +++ b/apps/web/src/state/projectSync.ts @@ -0,0 +1,122 @@ +import { createProjectSyncEnvironmentAtoms } from "@t3tools/client-runtime/state/project-sync"; +import { resolveAssetUrl } from "@t3tools/client-runtime/state/assets"; +import type { + ProjectSyncDeps, + ProjectSyncTarget, +} from "@t3tools/client-runtime/state/project-sync"; +import { useCallback, useMemo } from "react"; + +import { connectionAtomRuntime } from "../connection/runtime"; +import { useAtomCommand } from "./use-atom-command"; +import { useAtomQueryRunner } from "./use-atom-query-runner"; +import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; +import { readPreparedConnection } from "./session"; + +/** Per-environment RPC atoms for the four project-sync methods. Instantiated + once against the shared connection runtime, same as every other + environment-scoped atom family in `state/`. */ +export const projectSyncEnvironment = createProjectSyncEnvironmentAtoms(connectionAtomRuntime); + +/** + * Builds the full `ProjectSyncDeps` the `runProjectSync` controller (from + * `@t3tools/client-runtime/state/project-sync`) needs to drive a sync: the + * four RPCs, wired through this environment's atom registry, plus `fetch` + * and `resolveUrl` wired exactly the way attachment uploads resolve a signed + * URL (`readPreparedConnection` + `resolveAssetUrl`) — see + * `apps/web/src/lib/attachmentUploadQueue.ts`. + */ +export function useProjectSyncDeps(): ProjectSyncDeps { + const getManifestRpc = useAtomQueryRunner(projectSyncEnvironment.manifest, { + reportFailure: false, + }); + const createExportUrlRpc = useAtomCommand(projectSyncEnvironment.createExportUrl, { + reportFailure: false, + }); + const createImportUrlRpc = useAtomCommand(projectSyncEnvironment.createImportUrl, { + reportFailure: false, + }); + const applyDeletionsRpc = useAtomCommand(projectSyncEnvironment.applyDeletions, { + reportFailure: false, + }); + + const getManifest = useCallback( + async (target: ProjectSyncTarget, includeGit: boolean) => { + const result = await getManifestRpc({ + environmentId: target.environmentId, + input: { projectId: target.projectId, includeGit }, + }); + if (result._tag !== "Success") { + throw squashAtomCommandFailure(result); + } + return result.value; + }, + [getManifestRpc], + ); + + const createExportUrl = useCallback( + async (target: ProjectSyncTarget, paths: ReadonlyArray) => { + const result = await createExportUrlRpc({ + environmentId: target.environmentId, + input: { projectId: target.projectId, paths }, + }); + if (result._tag !== "Success") { + throw squashAtomCommandFailure(result); + } + return result.value; + }, + [createExportUrlRpc], + ); + + const createImportUrl = useCallback( + async (target: ProjectSyncTarget, fileCount: number, totalBytes: number) => { + const result = await createImportUrlRpc({ + environmentId: target.environmentId, + input: { projectId: target.projectId, fileCount, totalBytes }, + }); + if (result._tag !== "Success") { + throw squashAtomCommandFailure(result); + } + return result.value; + }, + [createImportUrlRpc], + ); + + const applyDeletions = useCallback( + async (target: ProjectSyncTarget, paths: ReadonlyArray) => { + const result = await applyDeletionsRpc({ + environmentId: target.environmentId, + input: { projectId: target.projectId, paths }, + }); + if (result._tag !== "Success") { + throw squashAtomCommandFailure(result); + } + return result.value; + }, + [applyDeletionsRpc], + ); + + const resolveUrl = useCallback( + (environmentId: ProjectSyncTarget["environmentId"], relativeUrl: string) => { + const connection = readPreparedConnection(environmentId); + return connection ? resolveAssetUrl(connection.httpBaseUrl, relativeUrl) : null; + }, + [], + ); + + // Memoized so callers can safely put the returned object in an effect's + // dependency array without it changing identity every render. + return useMemo( + () => ({ + fetch: globalFetch, + resolveUrl, + getManifest, + createExportUrl, + createImportUrl, + applyDeletions, + }), + [applyDeletions, createExportUrl, createImportUrl, getManifest, resolveUrl], + ); +} + +// Stable reference so `deps.fetch` never changes identity across renders. +const globalFetch: typeof fetch = (...args) => fetch(...args); diff --git a/docs/README.md b/docs/README.md index 63b87e55ca83..feda2a162704 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ - [Working with multiple threads](./user/thread-workspace.md) - [Review usage](./user/usage.md) - [Customize a project icon](./user/project-settings.md) +- [Sync projects between environments](./user/project-sync.md) - [Mobile appearance](./user/mobile-appearance.md) - [Remote access](./user/remote-access.md) - [Keeping app and server in sync](./user/updating.md) @@ -31,6 +32,7 @@ policy in [CONTRIBUTING.md](../CONTRIBUTING.md); agent rules in [AGENTS.md](../A - [Scripts](./internals/scripts.md) - [Connection runtime](./internals/connection-runtime.md) - [Providers](./internals/providers.md) +- [Project sync](./internals/project-sync.md) - [Remote environments](./internals/remote.md) - [Server updates](./internals/server-updates.md) - [Resource telemetry](./internals/resource-telemetry.md) diff --git a/docs/internals/glossary.md b/docs/internals/glossary.md index f0168ec20777..26be4e213335 100644 --- a/docs/internals/glossary.md +++ b/docs/internals/glossary.md @@ -7,6 +7,7 @@ This is a living glossary for T3 Code. It explains what common terms mean in thi ## Table of contents - [Project and workspace](#project-and-workspace) +- [Project sync](#project-sync) - [Thread timeline](#thread-timeline) - [Orchestration](#orchestration) - [Provider runtime](#provider-runtime) @@ -28,6 +29,32 @@ The root filesystem path for a project. In [the orchestration model][1], it is t A Git worktree used as an isolated workspace for a thread. If a thread has a `worktreePath` in [the contracts][1], it runs there instead of in the main working tree. Git operations live behind the VCS driver contract in `apps/server/src/vcs/VcsDriver.ts`, implemented by [GitVcsDriverCore.ts][3]. +### Project sync + +Copying a project's workspace between two environments, driven entirely by the client: there is no +server-to-server communication. See [project-sync.md][27]. + +#### Project sync manifest + +A flat, sorted, hash-addressed description of a project's workspace tree — files with a sha256 +digest, empty directories, and symlinks with their raw target. Built by [`ProjectSyncManifest.ts`][28] +and defined on the wire in [the project sync contracts][29]. The client diffs the origin and +destination manifests to decide what a sync needs to copy or delete; it is never used for anything +else. + +#### Send mode + +The project sync mode that creates a new project on the destination environment and copies +everything into it. Never deletes destination entries, since the destination did not exist before +the sync started. See [`operations/projectSync.ts`][30]. + +#### Sync mode + +The project sync mode that mirrors an existing destination project against the source: entries +missing or changed on the destination are copied, and destination entries absent from the source +manifest are deleted. Unlike Send, Sync can delete destination files — the client surfaces that as +an explicit confirmation before it runs. See [`operations/projectSync.ts`][30]. + ### Thread timeline #### Thread @@ -162,6 +189,7 @@ The file patch and changed-file summary for one turn. It is usually computed in - [Provider architecture][16] - [Permission modes][18] - [Workspace layout][2] +- [Project sync architecture][27] [1]: ../../packages/contracts/src/orchestration.ts [2]: ./workspace-layout.md @@ -189,3 +217,7 @@ The file patch and changed-file summary for one turn. It is usually computed in [24]: ./overview.md [25]: ../../apps/server/src/provider/providerRateLimits.ts [26]: ../../apps/server/src/orchestration/Layers/ProviderRateLimitsReactor.ts +[27]: ./project-sync.md +[28]: ../../apps/server/src/workspace/ProjectSyncManifest.ts +[29]: ../../packages/contracts/src/projectSync.ts +[30]: ../../packages/client-runtime/src/operations/projectSync.ts diff --git a/docs/internals/project-sync.md b/docs/internals/project-sync.md new file mode 100644 index 000000000000..6d4a606ebd65 --- /dev/null +++ b/docs/internals/project-sync.md @@ -0,0 +1,184 @@ +# Project Sync Architecture + +> For maintainers. Using T3 Code? See [docs/user](../user/). + +Project sync copies a project's workspace between two environments. For the user-facing flow see +[Sync Projects Between Environments](../user/project-sync.md). + +## The model + +There is no server-to-server communication. Every environment is just a T3 server that already +knows how to walk its own workspace, describe files, and stream bytes over HTTP; a sync is the +client asking one environment for a manifest and some file bytes, then handing those bytes to the +other environment. The client is the only thing that ever talks to both sides. + +```text +┌───────────────────────────────┐ ┌───────────────────────────────┐ +│ Origin environment │ │ Destination environment │ +│ workspace root, files │ │ workspace root, files │ +└───────────────┬───────────────┘ └───────────────┬───────────────┘ + │ manifest (WS) │ manifest (WS) + │ export bytes (HTTP, signed URL) │ import bytes (HTTP, signed URL) + │ │ deletions (WS) + └──────────────────┬───────────────────────┘ + │ + ┌──────────▼──────────┐ + │ Client │ + │ diff, batch, plan │ + │ drive both sides │ + └──────────────────────┘ +``` + +Because the client owns the whole flow, project sync works over any connection mode T3 already +supports for each environment individually — local, LAN, Tailscale, T3 Connect relay, or an +SSH-launched environment. Nothing new had to be built at the transport layer, and nothing pierces +the one-server-owns-its-workspace boundary the rest of the app relies on. This also rules out +`scp`/`rsync`-style approaches: those would require either shell access to both hosts from wherever +the sync runs, or a direct network path between the two environments, neither of which T3's +connection model guarantees. Reusing the client's existing connection to each environment sidesteps +both requirements. + +## Flow + +1. **Manifest.** The client calls `projectSync.manifest` against both the origin and destination + environments. Each server walks its project's workspace root and returns a flat, sorted list of + entries — files with a sha256 hash, empty directories, and symlinks with their raw target — built + in [`ProjectSyncManifest.ts`][manifest]. `node_modules`, `.t3`, and `.DS_Store` are always + excluded; `.git` is included unless the caller opts out; `extraIgnores` layers exact-segment + exclusions on top. +2. **Plan.** The client diffs the two manifests with `computeProjectSyncPlan` in + [`operations/projectSync.ts`][client-ops]: entries missing or changed on the destination go into + `toCopy`, destination-only paths go into `toDelete` (deepest-first, so deletion never removes a + directory before what was inside it). `batchProjectSyncEntries` then groups `toCopy` into batches + bounded by ~32MB or 500 files, so neither side ever buffers an entire project in memory. +3. **Transfer.** For each batch, the client calls `projectSync.createExportUrl` on the origin and + `projectSync.createImportUrl` on the destination, then streams the export response body directly + into the import request body. Content moves over HTTP, not the WebSocket, using the same signed + short-lived URL mechanism as attachment/asset uploads (HMAC-SHA256 over a claims blob, 10-minute + TTL) — see [`ProjectSyncTransfer.ts`][transfer]. The body itself is a small purpose-built binary + framing (`[uint32 header length][JSON header][raw content bytes]`, repeated) defined in + [`projectSyncFraming.ts`][framing], decoded and applied on the destination by + [`ProjectSyncApply.ts`][apply]. +4. **Deletions.** In Sync mode, once every batch has landed, the client calls + `projectSync.applyDeletions` on the destination with `plan.toDelete`. The destination removes + those paths and prunes any directory that deletion left empty, up to (never including) the + workspace root. Send mode never calls this — a project it just created cannot have destination-only + cruft that matters. + +`runProjectSync` in [`operations/projectSync.ts`][client-ops] and the progress/target types in +[`state/projectSync.ts`][client-state] are what web and desktop wire up to a UI: a batch failure +aborts the rest of the sync, leaving the destination with whatever batches already landed — +always a subset of the plan, never partial destination-only leftovers. + +## Contracts + +[`packages/contracts/src/projectSync.ts`][contracts] defines the manifest entry shape and the four +RPC payload/result/error types. Four `WS_METHODS` entries and their `Rpc.make` definitions live in +[`rpc.ts`][rpc]: + +- `projectSync.manifest` — walk a project's workspace and return its manifest. +- `projectSync.createExportUrl` — mint a signed URL to stream a set of paths out of a project. +- `projectSync.createImportUrl` — mint a signed URL to stream content into a project, bounded to a + declared file count and byte total. +- `projectSync.applyDeletions` — remove a set of paths from a project's workspace. + +## Security guarantees + +- **Path handling.** Every wire path is validated and resolved with + `resolveProjectSyncRelativePath` before it touches the filesystem: absolute paths, `..` segments, + and null bytes are rejected outright. Ancestor directories are walked and checked before any + `mkdir`, and a symlinked ancestor aborts the write — `mkdir -p` semantics would otherwise happily + follow a crafted symlink out of the workspace root. A symlink _entry itself_ is still copied + faithfully; only ancestors are restricted. +- **Atomic writes.** Files land through a temp-file-plus-rename in the same directory + (`writeFileRecord` in [`ProjectSyncApply.ts`][apply]), so a partial write is never observable at + the final path and an existing symlink at the target is swapped rather than written through. +- **Import byte budget.** `projectSync.createImportUrl` declares a `totalBytes` ceiling up front; + the decode loop in `applyProjectSyncRecords` aborts mid-stream if the running total exceeds it, + and the HTTP route turns that into a 413. The budget is enforced from the framed content as it + decodes, not from `Content-Length`, since the body also carries header bytes. +- **Export tolerates drift.** The manifest a client diffed is a snapshot; a vanished entry between + manifest and export time is skipped rather than failing the whole transfer — the next sync + reconciles it — and each frame's declared size is always taken from the still-open file handle + being read, so a file that grew mid-export cannot desynchronize the stream. +- **Deletions stay inside the workspace.** `applyProjectSyncDeletions` re-resolves every path the + same way writes do, so a deletion request cannot escape the project root either. +- **Signed URLs are single-purpose.** Export/import tokens follow the asset/attachment upload + precedent exactly: HMAC-SHA256 over a base64url claims blob keyed by the `asset-access-signing-key` + secret, a `kind` claim that keeps export and import tokens from being replayed against each + other's routes, and a 10-minute TTL (`PROJECT_SYNC_URL_TTL_MS`). Export claims carry a random + `requestId` whose path list is kept server-side in a small in-process, TTL-bounded registry + (`pendingExports` in [`ProjectSyncTransfer.ts`][transfer]) rather than in the URL itself, since an + export can cover thousands of paths — far more than would fit in the 4096-character URL the + contract allows. + +## Capability flag and version skew + +`ExecutionEnvironmentCapabilities.projectSync` (added in [`environment.ts`][environment-contract]) +is set unconditionally by [`ServerEnvironment.ts`][server-environment] for any server that ships +this feature. It is `Schema.optionalKey`, so an older server simply omits it. Clients gate the +sync entry points in the command palette and Project Settings on this flag being `true`, rather +than probing the RPCs and handling a method-not-found error — the same pattern other +version-gated capabilities use. + +## Authorization scopes + +`RPC_REQUIRED_SCOPES` in [`RpcAuthorization.ts`][rpc-auth] assigns scopes by read/write shape, the +same way `projects.readFile`/`projects.writeFile` already split: + +| Method | Scope | +| ----------------------------- | ----------------------- | +| `projectSync.manifest` | `orchestration:read` | +| `projectSync.createExportUrl` | `orchestration:read` | +| `projectSync.createImportUrl` | `orchestration:operate` | +| `projectSync.applyDeletions` | `orchestration:operate` | + +Manifests and export URLs only ever read a workspace; import URLs and deletions write it. See +[environment-auth.md](./environment-auth.md) for the scope model these fit into. + +## HTTP routes + +The signed URLs resolve against two routes registered in [`http.ts`][http]: +`GET /api/projectSync/export/*` streams `encodeProjectSyncRecords` over the requested paths; +`POST /api/projectSync/import/*` decodes the request body with +`createProjectSyncFrameDecoder` and applies it through `applyProjectSyncRecords`. Both routes +authorize purely from the signed token — the token already carries the resolved workspace root and +either the request's path list (export) or its byte/file budget (import) — so neither route needs +the orchestration read model. + +## Not event-sourced + +Project sync is a filesystem operation, not a domain one: it never dispatches an orchestration +command and never produces a domain event. Copying files into a project's workspace this way is +invisible to that project's thread history and checkpoints, the same way editing a file outside T3 +would be. + +## Limitations + +- **No glob support in ignores.** `extraIgnores` matches exact path segments only. +- **No concurrency lock.** Sync does not coordinate with a running turn's checkpoint or filesystem + activity in the same workspace. See the user-facing limitation note for the recommended + workaround. +- **Mobile is out of scope for v1.** The capability flag and UI entry points are web/desktop only; + nothing in the mobile client offers project sync. +- **No resume.** If a signed URL expires mid-transfer, the client has to reissue export/import URLs + and restart the affected batch rather than resuming a partial one. + +## Possible evolutions + +- Glob-style `extraIgnores` patterns instead of exact-segment matching. +- Mobile support, once the UI has a place for a two-environment picker on that surface. +- Resumable transfers that can pick up a batch after a URL expires instead of restarting it. + +[manifest]: ../../apps/server/src/workspace/ProjectSyncManifest.ts +[apply]: ../../apps/server/src/workspace/ProjectSyncApply.ts +[transfer]: ../../apps/server/src/workspace/ProjectSyncTransfer.ts +[framing]: ../../packages/shared/src/projectSyncFraming.ts +[contracts]: ../../packages/contracts/src/projectSync.ts +[rpc]: ../../packages/contracts/src/rpc.ts +[rpc-auth]: ../../apps/server/src/auth/RpcAuthorization.ts +[environment-contract]: ../../packages/contracts/src/environment.ts +[server-environment]: ../../apps/server/src/environment/ServerEnvironment.ts +[http]: ../../apps/server/src/http.ts +[client-ops]: ../../packages/client-runtime/src/operations/projectSync.ts +[client-state]: ../../packages/client-runtime/src/state/projectSync.ts diff --git a/docs/user/project-sync.md b/docs/user/project-sync.md new file mode 100644 index 000000000000..acad6c6e08ab --- /dev/null +++ b/docs/user/project-sync.md @@ -0,0 +1,77 @@ +# Sync Projects Between Environments + +Copy a project from one environment to another — for example, from your desktop to a homelab +server, or from a laptop to a remote box you pair over Tailscale. T3 Code reads the files from +wherever the project already lives and writes them to the destination, using whatever connection +you already have to each environment. There is no separate transfer setup: if you can pair with +both environments, you can sync between them. + +## When to Use It + +- Starting the same project on a second machine, such as moving work from a laptop onto a + homelab or cloud box for a long-running task. +- Keeping a project's working tree up to date on a remote environment as you keep working on it + locally (or the other way around). +- Standing up a fresh environment with a project you already have elsewhere, including its Git + history. + +This is a filesystem copy between two environments you already control. It does not create +threads, run an agent, or touch either environment's conversation history — only project files +move. + +## Starting a Sync + +You can start a sync from either environment involved, in two places: + +- **Command Palette** (`Cmd/Ctrl + K`) → **Sync project between environments** +- **Project Settings** → **Sync**, from the project you want to copy + +Either entry point walks you through picking the other environment and the project (or +destination folder) on that side. + +## Send vs. Sync + +Two modes cover the two things you actually want to do: + +- **Send** creates a new project on the destination environment and copies everything into it. + The destination folder defaults to that environment's own "Add project starts in" location, and + you can change it before sending. Use Send the first time a project needs to exist somewhere + else. +- **Sync** updates a project that already exists on both sides. T3 Code compares the two projects + file by file and copies only what changed. + +Before either mode starts moving files, T3 Code shows you a summary of the plan — how many files +will be copied and how much data that is — so you know what you are about to do. + +## Sync Mirrors the Source + +**Sync is a mirror, not a merge.** If a file exists on the destination but not on the source, Sync +deletes it from the destination so the two projects match. When a plan includes deletions, T3 Code +calls them out and asks you to confirm before continuing. If you are not sure both sides agree on +which one is the "real" copy, use Send into a fresh location instead of Sync, or review the plan +carefully before confirming. + +## Including Git History + +Sync includes your project's `.git` history by default, so the destination ends up as a complete +working copy you can commit and push from independently. Turn off **Include .git** if you only +want the working files without history — useful for a quick copy where you do not need the +destination to be its own Git checkout. + +## Progress and Cancelling + +While a sync runs, T3 Code shows progress by bytes and files transferred. You can cancel at any +time; anything already copied stays on the destination, and nothing partially written is left in a +broken state. + +## Limitations + +- **Mobile is not supported yet.** Start and manage a sync from the web or desktop app. +- **Ignoring extra folders** (beyond the always-skipped `node_modules`, T3's own state, and + `.DS_Store`) matches exact folder names only — you cannot use wildcard patterns. +- **Avoid syncing while a turn is actively running** in the project you're copying from or to. + Sync does not lock the workspace against concurrent writes, so files an agent is mid-edit on can + end up copied in an inconsistent state. Wait for the agent to finish, or pause it, before you + sync. +- Sync transfers are time-limited in flight. If a transfer sits idle for a long time and the link + it was using expires, start the sync again rather than trying to resume it. diff --git a/packages/client-runtime/package.json b/packages/client-runtime/package.json index abed33998966..f4d139f11a40 100644 --- a/packages/client-runtime/package.json +++ b/packages/client-runtime/package.json @@ -35,6 +35,10 @@ "types": "./src/operations/projects.ts", "default": "./src/operations/projects.ts" }, + "./operations/project-sync": { + "types": "./src/operations/projectSync.ts", + "default": "./src/operations/projectSync.ts" + }, "./platform": { "types": "./src/platform/index.ts", "default": "./src/platform/index.ts" @@ -91,6 +95,10 @@ "types": "./src/state/projects.ts", "default": "./src/state/projects.ts" }, + "./state/project-sync": { + "types": "./src/state/projectSync.ts", + "default": "./src/state/projectSync.ts" + }, "./state/pull-requests": { "types": "./src/state/pullRequests.ts", "default": "./src/state/pullRequests.ts" diff --git a/packages/client-runtime/src/operations/index.ts b/packages/client-runtime/src/operations/index.ts index b7307fbb81fe..df33744ece78 100644 --- a/packages/client-runtime/src/operations/index.ts +++ b/packages/client-runtime/src/operations/index.ts @@ -1,2 +1,3 @@ export * from "./commands.ts"; export * from "./projects.ts"; +export * from "./projectSync.ts"; diff --git a/packages/client-runtime/src/operations/projectSync.test.ts b/packages/client-runtime/src/operations/projectSync.test.ts new file mode 100644 index 000000000000..23397e491130 --- /dev/null +++ b/packages/client-runtime/src/operations/projectSync.test.ts @@ -0,0 +1,200 @@ +import type { ProjectSyncManifestEntry } from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + batchProjectSyncEntries, + computeProjectSyncPlan, + summarizeProjectSyncPlan, +} from "./projectSync.ts"; + +function file( + path: string, + overrides: Partial = {}, +): ProjectSyncManifestEntry { + return { + path, + kind: "file", + size: 10, + hash: "a".repeat(64), + ...overrides, + }; +} + +function dir(path: string): ProjectSyncManifestEntry { + return { path, kind: "dir", size: 0 }; +} + +function symlink(path: string, linkTarget: string): ProjectSyncManifestEntry { + return { path, kind: "symlink", size: 0, linkTarget }; +} + +describe("computeProjectSyncPlan", () => { + it("copies entries missing from the destination", () => { + const plan = computeProjectSyncPlan([file("a.txt"), file("b.txt")], [file("a.txt")]); + + expect(plan.toCopy.map((entry) => entry.path)).toEqual(["b.txt"]); + expect(plan.toDelete).toEqual([]); + expect(plan.copyBytes).toBe(10); + }); + + it("copies files whose hash differs even when size and mode match", () => { + const source = [file("a.txt", { hash: "b".repeat(64), size: 10, mode: 0o644 })]; + const dest = [file("a.txt", { hash: "a".repeat(64), size: 10, mode: 0o644 })]; + + expect(computeProjectSyncPlan(source, dest).toCopy.map((entry) => entry.path)).toEqual([ + "a.txt", + ]); + }); + + it("copies files whose size differs even when hash matches", () => { + const hash = "a".repeat(64); + const source = [file("a.txt", { hash, size: 20 })]; + const dest = [file("a.txt", { hash, size: 10 })]; + + expect(computeProjectSyncPlan(source, dest).toCopy.map((entry) => entry.path)).toEqual([ + "a.txt", + ]); + }); + + it("copies files whose mode differs even when hash and size match", () => { + const hash = "a".repeat(64); + const source = [file("a.txt", { hash, size: 10, mode: 0o755 })]; + const dest = [file("a.txt", { hash, size: 10, mode: 0o644 })]; + + expect(computeProjectSyncPlan(source, dest).toCopy.map((entry) => entry.path)).toEqual([ + "a.txt", + ]); + }); + + it("does not copy an unchanged file", () => { + const hash = "a".repeat(64); + const source = [file("a.txt", { hash, size: 10, mode: 0o644 })]; + const dest = [file("a.txt", { hash, size: 10, mode: 0o644 })]; + + expect(computeProjectSyncPlan(source, dest).toCopy).toEqual([]); + }); + + it("copies when kind changes between source and destination", () => { + const source = [symlink("link", "target-a")]; + const dest = [file("link")]; + + expect(computeProjectSyncPlan(source, dest).toCopy.map((entry) => entry.path)).toEqual([ + "link", + ]); + }); + + it("copies symlinks whose target differs", () => { + const source = [symlink("link", "target-a")]; + const dest = [symlink("link", "target-b")]; + + expect(computeProjectSyncPlan(source, dest).toCopy.map((entry) => entry.path)).toEqual([ + "link", + ]); + }); + + it("does not copy an unchanged symlink", () => { + const source = [symlink("link", "target-a")]; + const dest = [symlink("link", "target-a")]; + + expect(computeProjectSyncPlan(source, dest).toCopy).toEqual([]); + }); + + it("copies an empty directory only when it is missing from the destination", () => { + const source = [dir("empty")]; + + expect(computeProjectSyncPlan(source, []).toCopy.map((entry) => entry.path)).toEqual(["empty"]); + expect(computeProjectSyncPlan(source, [dir("empty")]).toCopy).toEqual([]); + }); + + it("deletes destination-only entries, deepest paths first", () => { + const dest = [ + file("a.txt"), + file("dir/nested/deep.txt"), + file("dir/shallow.txt"), + dir("empty"), + ]; + + const plan = computeProjectSyncPlan([], dest); + + expect(plan.toDelete).toEqual(["dir/nested/deep.txt", "dir/shallow.txt", "a.txt", "empty"]); + }); + + it("leaves entries present on both sides out of the deletion list", () => { + const source = [file("keep.txt")]; + const dest = [file("keep.txt"), file("remove.txt")]; + + expect(computeProjectSyncPlan(source, dest).toDelete).toEqual(["remove.txt"]); + }); + + it("sums copyBytes across every entry marked for copy", () => { + const plan = computeProjectSyncPlan( + [file("a.txt", { size: 100 }), file("b.txt", { size: 250 }), dir("empty")], + [], + ); + + expect(plan.copyBytes).toBe(350); + }); +}); + +describe("batchProjectSyncEntries", () => { + it("groups entries under both the byte and file ceilings", () => { + const entries = [ + file("a.txt", { size: 10 }), + file("b.txt", { size: 10 }), + file("c.txt", { size: 10 }), + ]; + + const batches = batchProjectSyncEntries(entries, { maxBytes: 25, maxFiles: 500 }); + + expect(batches).toEqual([[entries[0], entries[1]], [entries[2]]]); + }); + + it("splits on the file-count ceiling even when bytes have headroom", () => { + const entries = [ + file("a.txt", { size: 1 }), + file("b.txt", { size: 1 }), + file("c.txt", { size: 1 }), + ]; + + const batches = batchProjectSyncEntries(entries, { maxBytes: 1024, maxFiles: 2 }); + + expect(batches).toEqual([[entries[0], entries[1]], [entries[2]]]); + }); + + it("gives an entry larger than maxBytes its own batch instead of dropping or splitting it", () => { + const entries = [ + file("small.txt", { size: 10 }), + file("huge.bin", { size: 1000 }), + file("also-small.txt", { size: 10 }), + ]; + + const batches = batchProjectSyncEntries(entries, { maxBytes: 100, maxFiles: 500 }); + + expect(batches).toEqual([[entries[0]], [entries[1]], [entries[2]]]); + }); + + it("returns no batches for an empty entry list", () => { + expect(batchProjectSyncEntries([])).toEqual([]); + }); + + it("uses sane defaults when no options are given", () => { + const entries = [file("a.txt", { size: 10 }), file("b.txt", { size: 10 })]; + + expect(batchProjectSyncEntries(entries)).toEqual([entries]); + }); +}); + +describe("summarizeProjectSyncPlan", () => { + it("reports copy/delete counts and total copy bytes", () => { + const plan = computeProjectSyncPlan( + [file("a.txt", { size: 30 }), file("b.txt", { size: 20 })], + [file("b.txt", { size: 20 }), file("stale.txt")], + ); + + expect(summarizeProjectSyncPlan(plan)).toEqual({ + copyCount: 1, + deleteCount: 1, + copyBytes: 30, + }); + }); +}); diff --git a/packages/client-runtime/src/operations/projectSync.ts b/packages/client-runtime/src/operations/projectSync.ts new file mode 100644 index 000000000000..914c2fd5b09e --- /dev/null +++ b/packages/client-runtime/src/operations/projectSync.ts @@ -0,0 +1,141 @@ +import type { ProjectSyncManifestEntry } from "@t3tools/contracts"; + +/** Bytes-per-batch ceiling used when the caller does not override it. Bounds + peak memory for a single transfer to a small, predictable amount, since a + project can otherwise be arbitrarily large. */ +export const DEFAULT_PROJECT_SYNC_BATCH_MAX_BYTES = 32 * 1024 * 1024; + +/** Files-per-batch ceiling used when the caller does not override it. Keeps + the export/import URL request payload (one path per file) small even for + projects with many tiny files. */ +export const DEFAULT_PROJECT_SYNC_BATCH_MAX_FILES = 500; + +export interface ProjectSyncPlan { + /** Manifest entries present in the source that are missing, or changed, + relative to the destination. Order follows the source manifest. */ + readonly toCopy: ReadonlyArray; + /** Destination paths absent from the source, ordered deepest-first so a + sequential delete never removes a directory before its contents. */ + readonly toDelete: ReadonlyArray; + /** Sum of `size` across every entry in `toCopy`. */ + readonly copyBytes: number; +} + +export interface ProjectSyncPlanSummary { + readonly copyCount: number; + readonly deleteCount: number; + readonly copyBytes: number; +} + +export interface ProjectSyncBatchOptions { + readonly maxBytes?: number; + readonly maxFiles?: number; +} + +function pathDepth(path: string): number { + return path.split("/").length; +} + +function projectSyncEntryNeedsCopy( + source: ProjectSyncManifestEntry, + dest: ProjectSyncManifestEntry, +): boolean { + if (source.kind !== dest.kind) { + return true; + } + if (source.kind === "file") { + return source.hash !== dest.hash || source.size !== dest.size || source.mode !== dest.mode; + } + if (source.kind === "symlink") { + return source.linkTarget !== dest.linkTarget; + } + // "dir" entries only mark otherwise-empty directories: same path + same + // kind means the destination already has that (empty) directory, and a + // directory carries no further content to diff. + return false; +} + +/** + * Diffs a source manifest against a destination manifest to decide what a + * client-orchestrated sync needs to copy and delete. Pure and side-effect + * free: the caller is responsible for fetching both manifests first and for + * executing the resulting plan. + */ +export function computeProjectSyncPlan( + source: ReadonlyArray, + dest: ReadonlyArray, +): ProjectSyncPlan { + const destByPath = new Map(dest.map((entry) => [entry.path, entry] as const)); + const sourcePaths = new Set(source.map((entry) => entry.path)); + + const toCopy: ProjectSyncManifestEntry[] = []; + for (const entry of source) { + const existing = destByPath.get(entry.path); + if (existing === undefined || projectSyncEntryNeedsCopy(entry, existing)) { + toCopy.push(entry); + } + } + + const toDelete = dest + .filter((entry) => !sourcePaths.has(entry.path)) + .map((entry) => entry.path) + .sort((a, b) => pathDepth(b) - pathDepth(a) || (a < b ? -1 : a > b ? 1 : 0)); + + const copyBytes = toCopy.reduce((total, entry) => total + entry.size, 0); + + return { toCopy, toDelete, copyBytes }; +} + +/** + * Groups manifest entries to copy into batches bounded by both total bytes + * and file count, so a single request/response pair never has to hold an + * entire (potentially huge) project in memory. A single entry larger than + * `maxBytes` still gets its own one-entry batch rather than being split or + * dropped. + */ +export function batchProjectSyncEntries( + entries: ReadonlyArray, + options: ProjectSyncBatchOptions = {}, +): ProjectSyncManifestEntry[][] { + const maxBytes = options.maxBytes ?? DEFAULT_PROJECT_SYNC_BATCH_MAX_BYTES; + const maxFiles = options.maxFiles ?? DEFAULT_PROJECT_SYNC_BATCH_MAX_FILES; + + const batches: ProjectSyncManifestEntry[][] = []; + let current: ProjectSyncManifestEntry[] = []; + let currentBytes = 0; + + const flush = () => { + if (current.length > 0) { + batches.push(current); + current = []; + currentBytes = 0; + } + }; + + for (const entry of entries) { + if (entry.size > maxBytes) { + flush(); + batches.push([entry]); + continue; + } + if ( + current.length > 0 && + (current.length + 1 > maxFiles || currentBytes + entry.size > maxBytes) + ) { + flush(); + } + current.push(entry); + currentBytes += entry.size; + } + flush(); + + return batches; +} + +export function summarizeProjectSyncPlan(plan: ProjectSyncPlan): ProjectSyncPlanSummary { + return { + copyCount: plan.toCopy.length, + deleteCount: plan.toDelete.length, + copyBytes: plan.copyBytes, + }; +} diff --git a/packages/client-runtime/src/state/projectSync.test.ts b/packages/client-runtime/src/state/projectSync.test.ts new file mode 100644 index 000000000000..f4330998485a --- /dev/null +++ b/packages/client-runtime/src/state/projectSync.test.ts @@ -0,0 +1,278 @@ +import { + EnvironmentId, + ProjectId, + type ProjectSyncApplyDeletionsResult, + type ProjectSyncCreateUrlResult, + type ProjectSyncManifestEntry, + type ProjectSyncManifestResult, +} from "@t3tools/contracts"; +import { describe, expect, it } from "vite-plus/test"; + +import { + ProjectSyncAbortedError, + runProjectSync, + type ProjectSyncDeps, + type ProjectSyncProgress, + type ProjectSyncTarget, +} from "./projectSync.ts"; + +const SOURCE: ProjectSyncTarget = { + environmentId: EnvironmentId.make("environment-source"), + projectId: ProjectId.make("project-source"), +}; +const DEST: ProjectSyncTarget = { + environmentId: EnvironmentId.make("environment-dest"), + projectId: ProjectId.make("project-dest"), +}; + +function manifest(entries: ProjectSyncManifestEntry[]): ProjectSyncManifestResult { + return { workspaceRoot: "/workspace/project", entries, generatedAt: "2026-01-01T00:00:00.000Z" }; +} + +function file(path: string, size = 10): ProjectSyncManifestEntry { + return { path, kind: "file", size, hash: "a".repeat(64) }; +} + +function fakeResponse(status: number, body: ArrayBuffer = new ArrayBuffer(0)): Response { + return { + ok: status >= 200 && status < 300, + status, + arrayBuffer: () => Promise.resolve(body), + } as unknown as Response; +} + +interface FakeDepsOptions { + readonly sourceManifest: ProjectSyncManifestResult; + readonly destManifest: ProjectSyncManifestResult; + readonly maxFilesPerBatch?: number; + readonly onFetch?: (url: string, init: RequestInit | undefined) => void; +} + +interface FakeDepsHandle { + readonly deps: ProjectSyncDeps; + readonly calls: { + readonly getManifest: ProjectSyncTarget[]; + readonly createExportUrl: { target: ProjectSyncTarget; paths: ReadonlyArray }[]; + readonly createImportUrl: { + target: ProjectSyncTarget; + fileCount: number; + totalBytes: number; + }[]; + readonly applyDeletions: { target: ProjectSyncTarget; paths: ReadonlyArray }[]; + readonly fetchUrls: string[]; + }; +} + +function makeFakeDeps(options: FakeDepsOptions): FakeDepsHandle { + const calls: FakeDepsHandle["calls"] = { + getManifest: [], + createExportUrl: [], + createImportUrl: [], + applyDeletions: [], + fetchUrls: [], + }; + + const deps: ProjectSyncDeps = { + fetch: (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + calls.fetchUrls.push(url); + options.onFetch?.(url, init); + return fakeResponse(200); + }) as unknown as typeof fetch, + resolveUrl: (environmentId, relativeUrl) => `https://${environmentId}${relativeUrl}`, + getManifest: (target) => { + calls.getManifest.push(target); + return Promise.resolve( + target.environmentId === SOURCE.environmentId + ? options.sourceManifest + : options.destManifest, + ); + }, + createExportUrl: (target, paths) => { + calls.createExportUrl.push({ target, paths }); + const result: ProjectSyncCreateUrlResult = { + url: `/api/projectSync/export/token-${calls.createExportUrl.length}`, + expiresAt: 1_700_000_000_000, + }; + return Promise.resolve(result); + }, + createImportUrl: (target, fileCount, totalBytes) => { + calls.createImportUrl.push({ target, fileCount, totalBytes }); + const result: ProjectSyncCreateUrlResult = { + url: `/api/projectSync/import/token-${calls.createImportUrl.length}`, + expiresAt: 1_700_000_000_000, + }; + return Promise.resolve(result); + }, + applyDeletions: (target, paths) => { + calls.applyDeletions.push({ target, paths }); + const result: ProjectSyncApplyDeletionsResult = { deleted: paths.length }; + return Promise.resolve(result); + }, + ...(options.maxFilesPerBatch === undefined + ? {} + : { maxFilesPerBatch: options.maxFilesPerBatch }), + }; + + return { deps, calls }; +} + +function expectMonotonicProgress(progress: ProjectSyncProgress[]): void { + let lastBytes = -1; + let lastFiles = -1; + for (const entry of progress) { + expect(entry.transferredBytes).toBeGreaterThanOrEqual(lastBytes); + expect(entry.transferredFiles).toBeGreaterThanOrEqual(lastFiles); + lastBytes = entry.transferredBytes; + lastFiles = entry.transferredFiles; + } +} + +describe("runProjectSync", () => { + it("sends every changed file in 'send' mode and never deletes", async () => { + const sourceManifest = manifest([file("a.txt", 10), file("b.txt", 10)]); + // The destination happens to report a stray file too; "send" must leave + // it alone since deletions are a "sync"-only behavior. + const destManifest = manifest([file("stale.txt", 5)]); + const { deps, calls } = makeFakeDeps({ sourceManifest, destManifest, maxFilesPerBatch: 1 }); + + const progress: ProjectSyncProgress[] = []; + const summary = await runProjectSync(deps, { + source: SOURCE, + dest: DEST, + mode: "send", + includeGit: false, + onProgress: (p) => progress.push(p), + }); + + expect(summary).toEqual({ copyCount: 2, deleteCount: 1, copyBytes: 20 }); + expect(calls.getManifest).toEqual([SOURCE, DEST]); + // maxFilesPerBatch: 1 forces two batches, one export/import round-trip each. + expect(calls.createExportUrl.map((call) => call.paths)).toEqual([["a.txt"], ["b.txt"]]); + expect(calls.createImportUrl.map((call) => call.fileCount)).toEqual([1, 1]); + expect(calls.applyDeletions).toEqual([]); + expect(calls.fetchUrls).toEqual([ + "https://environment-source/api/projectSync/export/token-1", + "https://environment-dest/api/projectSync/import/token-1", + "https://environment-source/api/projectSync/export/token-2", + "https://environment-dest/api/projectSync/import/token-2", + ]); + + expect(progress.map((entry) => entry.stage)).toEqual([ + "manifest", + "planning", + "transferring", + "transferring", + "transferring", + "done", + ]); + expectMonotonicProgress(progress); + expect(progress.at(-1)).toEqual({ + stage: "done", + transferredBytes: 20, + totalBytes: 20, + transferredFiles: 2, + totalFiles: 2, + }); + }); + + it("applies destination deletions in 'sync' mode", async () => { + const sourceManifest = manifest([file("keep.txt", 10)]); + const destManifest = manifest([file("keep.txt", 10), file("stale.txt", 5)]); + const { deps, calls } = makeFakeDeps({ sourceManifest, destManifest }); + + const progress: ProjectSyncProgress[] = []; + const summary = await runProjectSync(deps, { + source: SOURCE, + dest: DEST, + mode: "sync", + includeGit: true, + onProgress: (p) => progress.push(p), + }); + + expect(summary).toEqual({ copyCount: 0, deleteCount: 1, copyBytes: 0 }); + expect(calls.createExportUrl).toEqual([]); + expect(calls.applyDeletions).toEqual([{ target: DEST, paths: ["stale.txt"] }]); + expect(progress.map((entry) => entry.stage)).toEqual([ + "manifest", + "planning", + "transferring", + "deleting", + "done", + ]); + }); + + it("does not delete in 'sync' mode when nothing is stale", async () => { + const sourceManifest = manifest([file("keep.txt", 10)]); + const destManifest = manifest([file("keep.txt", 10)]); + const { deps, calls } = makeFakeDeps({ sourceManifest, destManifest }); + + await runProjectSync(deps, { + source: SOURCE, + dest: DEST, + mode: "sync", + includeGit: true, + onProgress: () => {}, + }); + + expect(calls.applyDeletions).toEqual([]); + }); + + it("aborts between batches without starting the next one", async () => { + const sourceManifest = manifest([file("a.txt", 10), file("b.txt", 10)]); + const destManifest = manifest([]); + const { deps, calls } = makeFakeDeps({ sourceManifest, destManifest, maxFilesPerBatch: 1 }); + + const controller = new AbortController(); + let batchesSeen = 0; + const progress: ProjectSyncProgress[] = []; + + await expect( + runProjectSync(deps, { + source: SOURCE, + dest: DEST, + mode: "sync", + includeGit: false, + signal: controller.signal, + onProgress: (p) => { + progress.push(p); + if (p.stage === "transferring" && p.transferredFiles > 0) { + batchesSeen += 1; + // Abort right after the first batch reports progress, before the + // loop moves on to the second batch. + if (batchesSeen === 1) { + controller.abort(); + } + } + }, + }), + ).rejects.toBeInstanceOf(ProjectSyncAbortedError); + + // Only the first batch's export/import pair ran. + expect(calls.createExportUrl).toHaveLength(1); + expect(calls.createImportUrl).toHaveLength(1); + expect(calls.applyDeletions).toEqual([]); + expectMonotonicProgress(progress); + }); + + it("rejects immediately when the signal is already aborted", async () => { + const sourceManifest = manifest([file("a.txt", 10)]); + const destManifest = manifest([]); + const { deps, calls } = makeFakeDeps({ sourceManifest, destManifest }); + const controller = new AbortController(); + controller.abort(); + + await expect( + runProjectSync(deps, { + source: SOURCE, + dest: DEST, + mode: "send", + includeGit: false, + signal: controller.signal, + onProgress: () => {}, + }), + ).rejects.toBeInstanceOf(ProjectSyncAbortedError); + + expect(calls.getManifest).toEqual([]); + }); +}); diff --git a/packages/client-runtime/src/state/projectSync.ts b/packages/client-runtime/src/state/projectSync.ts new file mode 100644 index 000000000000..b8ea6412386c --- /dev/null +++ b/packages/client-runtime/src/state/projectSync.ts @@ -0,0 +1,283 @@ +import type { + EnvironmentId, + ProjectId, + ProjectSyncApplyDeletionsResult, + ProjectSyncCreateUrlResult, + ProjectSyncManifestEntry, + ProjectSyncManifestResult, +} from "@t3tools/contracts"; +import { WS_METHODS } from "@t3tools/contracts"; +import { Atom } from "effect/unstable/reactivity"; + +import type { EnvironmentRegistry } from "../connection/registry.ts"; +import { + batchProjectSyncEntries, + computeProjectSyncPlan, + summarizeProjectSyncPlan, + type ProjectSyncPlanSummary, +} from "../operations/projectSync.ts"; +import { createEnvironmentRpcCommand, createEnvironmentRpcQueryAtomFamily } from "./runtime.ts"; + +/** + * Per-environment RPC atoms for the four project-sync methods. Follows the + * same factory shape as `createFilesystemEnvironmentAtoms` / + * `createAssetEnvironmentAtoms`: the concrete atom runtime is owned by each + * app (web/mobile/desktop), not by client-runtime. + */ +export function createProjectSyncEnvironmentAtoms( + runtime: Atom.AtomRuntime, +) { + return { + manifest: createEnvironmentRpcQueryAtomFamily(runtime, { + label: "environment-data:project-sync:manifest", + tag: WS_METHODS.projectSyncManifest, + }), + createExportUrl: createEnvironmentRpcCommand(runtime, { + label: "environment-command:project-sync:create-export-url", + tag: WS_METHODS.projectSyncCreateExportUrl, + }), + createImportUrl: createEnvironmentRpcCommand(runtime, { + label: "environment-command:project-sync:create-import-url", + tag: WS_METHODS.projectSyncCreateImportUrl, + }), + applyDeletions: createEnvironmentRpcCommand(runtime, { + label: "environment-command:project-sync:apply-deletions", + tag: WS_METHODS.projectSyncApplyDeletions, + }), + }; +} + +export type ProjectSyncMode = "send" | "sync"; + +export interface ProjectSyncTarget { + readonly environmentId: EnvironmentId; + readonly projectId: ProjectId; +} + +export type ProjectSyncStage = "manifest" | "planning" | "transferring" | "deleting" | "done"; + +export interface ProjectSyncProgress { + readonly stage: ProjectSyncStage; + readonly transferredBytes: number; + readonly totalBytes: number; + readonly transferredFiles: number; + readonly totalFiles: number; +} + +export interface RunProjectSyncParams { + readonly source: ProjectSyncTarget; + readonly dest: ProjectSyncTarget; + readonly mode: ProjectSyncMode; + readonly includeGit: boolean; + readonly signal?: AbortSignal; + readonly onProgress: (progress: ProjectSyncProgress) => void; +} + +/** + * Everything `runProjectSync` needs from the outside world, injected so the + * controller can be exercised in tests without a socket, an `AtomRegistry`, + * or a real `fetch`. A real caller (the web/desktop/mobile app) wires + * `resolveUrl` the same way `resolveAssetUrl` is used today: look up the + * environment's prepared connection, then resolve the RPC-returned relative + * URL against its `httpBaseUrl`. The signed URL itself carries the auth + * token, so no extra header is needed for the export/import fetches. + */ +export interface ProjectSyncDeps { + readonly fetch: typeof fetch; + readonly resolveUrl: (environmentId: EnvironmentId, relativeUrl: string) => string | null; + readonly getManifest: ( + target: ProjectSyncTarget, + includeGit: boolean, + ) => Promise; + readonly createExportUrl: ( + target: ProjectSyncTarget, + paths: ReadonlyArray, + ) => Promise; + readonly createImportUrl: ( + target: ProjectSyncTarget, + fileCount: number, + totalBytes: number, + ) => Promise; + readonly applyDeletions: ( + target: ProjectSyncTarget, + paths: ReadonlyArray, + ) => Promise; + /** Overrides for `batchProjectSyncEntries`; mainly useful in tests. */ + readonly maxBytesPerBatch?: number; + readonly maxFilesPerBatch?: number; +} + +export class ProjectSyncAbortedError extends Error { + readonly _tag = "ProjectSyncAbortedError"; + constructor() { + super("Project sync was aborted."); + this.name = "ProjectSyncAbortedError"; + } +} + +export class ProjectSyncUrlResolutionError extends Error { + readonly _tag = "ProjectSyncUrlResolutionError"; + readonly environmentId: EnvironmentId; + constructor(environmentId: EnvironmentId) { + super(`Could not resolve a signed project sync URL against environment '${environmentId}'.`); + this.name = "ProjectSyncUrlResolutionError"; + this.environmentId = environmentId; + } +} + +export class ProjectSyncTransferError extends Error { + readonly _tag = "ProjectSyncTransferError"; + override readonly cause?: unknown; + constructor(message: string, cause?: unknown) { + super(message); + this.name = "ProjectSyncTransferError"; + this.cause = cause; + } +} + +function checkProjectSyncAborted(signal: AbortSignal | undefined): void { + if (signal?.aborted) { + throw new ProjectSyncAbortedError(); + } +} + +async function transferProjectSyncBatch( + deps: ProjectSyncDeps, + source: ProjectSyncTarget, + dest: ProjectSyncTarget, + batch: ReadonlyArray, + signal: AbortSignal | undefined, +): Promise { + const paths = batch.map((entry) => entry.path); + const totalBytes = batch.reduce((total, entry) => total + entry.size, 0); + + checkProjectSyncAborted(signal); + const exportUrlResult = await deps.createExportUrl(source, paths); + const exportUrl = deps.resolveUrl(source.environmentId, exportUrlResult.url); + if (exportUrl === null) { + throw new ProjectSyncUrlResolutionError(source.environmentId); + } + + checkProjectSyncAborted(signal); + let exportResponse: Response; + try { + exportResponse = await deps.fetch(exportUrl, { + method: "GET", + ...(signal ? { signal } : {}), + }); + } catch (cause) { + throw new ProjectSyncTransferError( + `Could not reach environment '${source.environmentId}' to export project files.`, + cause, + ); + } + if (!exportResponse.ok) { + throw new ProjectSyncTransferError( + `Export request to environment '${source.environmentId}' failed with status ${exportResponse.status}.`, + ); + } + const bytes = await exportResponse.arrayBuffer(); + + checkProjectSyncAborted(signal); + const importUrlResult = await deps.createImportUrl(dest, batch.length, totalBytes); + const importUrl = deps.resolveUrl(dest.environmentId, importUrlResult.url); + if (importUrl === null) { + throw new ProjectSyncUrlResolutionError(dest.environmentId); + } + + checkProjectSyncAborted(signal); + let importResponse: Response; + try { + importResponse = await deps.fetch(importUrl, { + method: "POST", + headers: { "content-type": "application/octet-stream" }, + body: bytes, + ...(signal ? { signal } : {}), + }); + } catch (cause) { + throw new ProjectSyncTransferError( + `Could not reach environment '${dest.environmentId}' to import project files.`, + cause, + ); + } + if (!importResponse.ok) { + throw new ProjectSyncTransferError( + `Import request to environment '${dest.environmentId}' failed with status ${importResponse.status}.`, + ); + } +} + +/** + * Orchestrates a client-driven project sync between two environments: reads + * both projects' manifests, diffs them, and streams changed files from the + * source into the destination in bounded batches (never buffering the whole + * project in memory at once), then applies deletions on the destination when + * running in `"sync"` mode. `"send"` never deletes, even if the destination + * (a project the caller just created) happens to report existing entries. + * + * A batch failure aborts the rest of the sync; the destination is left with + * whatever batches completed before the failure, which is always a subset of + * `toCopy` and never partially-written destination-only cruft. + */ +export async function runProjectSync( + deps: ProjectSyncDeps, + params: RunProjectSyncParams, +): Promise { + const { source, dest, mode, includeGit, signal, onProgress } = params; + + const idleProgress = (stage: ProjectSyncStage): ProjectSyncProgress => ({ + stage, + transferredBytes: 0, + totalBytes: 0, + transferredFiles: 0, + totalFiles: 0, + }); + + checkProjectSyncAborted(signal); + onProgress(idleProgress("manifest")); + const [sourceManifest, destManifest] = await Promise.all([ + deps.getManifest(source, includeGit), + deps.getManifest(dest, includeGit), + ]); + + checkProjectSyncAborted(signal); + onProgress(idleProgress("planning")); + const plan = computeProjectSyncPlan(sourceManifest.entries, destManifest.entries); + const batches = batchProjectSyncEntries(plan.toCopy, { + ...(deps.maxBytesPerBatch === undefined ? {} : { maxBytes: deps.maxBytesPerBatch }), + ...(deps.maxFilesPerBatch === undefined ? {} : { maxFiles: deps.maxFilesPerBatch }), + }); + + const totalBytes = plan.copyBytes; + const totalFiles = plan.toCopy.length; + let transferredBytes = 0; + let transferredFiles = 0; + + const reportTransferring = () => + onProgress({ + stage: "transferring", + transferredBytes, + totalBytes, + transferredFiles, + totalFiles, + }); + + reportTransferring(); + for (const batch of batches) { + checkProjectSyncAborted(signal); + await transferProjectSyncBatch(deps, source, dest, batch, signal); + transferredBytes += batch.reduce((total, entry) => total + entry.size, 0); + transferredFiles += batch.length; + reportTransferring(); + } + + if (mode === "sync" && plan.toDelete.length > 0) { + checkProjectSyncAborted(signal); + onProgress({ stage: "deleting", transferredBytes, totalBytes, transferredFiles, totalFiles }); + await deps.applyDeletions(dest, plan.toDelete); + } + + onProgress({ stage: "done", transferredBytes, totalBytes, transferredFiles, totalFiles }); + + return summarizeProjectSyncPlan(plan); +} diff --git a/packages/contracts/src/environment.ts b/packages/contracts/src/environment.ts index 1468fe9ef3d3..e1b729891ebe 100644 --- a/packages/contracts/src/environment.ts +++ b/packages/contracts/src/environment.ts @@ -82,6 +82,11 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({ this is false — no update would ever repaint it. Absent on older servers, which may still publish, so only an explicit false skips. */ agentActivityPublishing: Schema.optionalKey(Schema.Boolean), + /** Server exposes the project sync manifest/export/import/deletion RPCs and + HTTP routes used to copy a project between environments. Absent on + servers from before project sync shipped, so clients must not offer the + sync flow or probe these RPCs under version skew. */ + projectSync: Schema.optionalKey(Schema.Boolean), }); export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 3f0f410b1ee1..fbf54ddb410b 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -24,6 +24,7 @@ export * from "./orchestration.ts"; export * from "./t3ProjectFile.ts"; export * from "./editor.ts"; export * from "./project.ts"; +export * from "./projectSync.ts"; export * from "./filesystem.ts"; export * from "./assets.ts"; export * from "./review.ts"; diff --git a/packages/contracts/src/projectSync.ts b/packages/contracts/src/projectSync.ts new file mode 100644 index 000000000000..daed919a45ab --- /dev/null +++ b/packages/contracts/src/projectSync.ts @@ -0,0 +1,175 @@ +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import { IsoDateTime, NonNegativeInt, ProjectId, TrimmedNonEmptyString } from "./baseSchemas.ts"; + +const PROJECT_SYNC_PATH_MAX_LENGTH = 1024; +const PROJECT_SYNC_URL_MAX_LENGTH = 4096; +const PROJECT_SYNC_MAX_PATHS_PER_REQUEST = 5000; + +/** A lowercase-hex sha256 digest, or empty when the entry has no content + (directories and symlinks). */ +const SHA256_HEX_PATTERN = /^(?:[0-9a-f]{64})?$/i; + +export const ProjectSyncEntryKind = Schema.Literals(["file", "dir", "symlink"]); +export type ProjectSyncEntryKind = typeof ProjectSyncEntryKind.Type; + +/** + * One entry in a project sync manifest. `path` is always relative to the + * project's workspace root and always uses `/` as the separator, regardless + * of the origin environment's OS, so destination environments on a different + * platform can rebuild the tree without re-deriving path semantics. + * + * `kind: "dir"` entries only exist for otherwise-empty directories: a + * directory that contains any file/symlink entry is implied by those + * entries' paths and does not get its own "dir" entry. + */ +export const ProjectSyncManifestEntry = Schema.Struct({ + path: TrimmedNonEmptyString.check(Schema.isMaxLength(PROJECT_SYNC_PATH_MAX_LENGTH)), + kind: ProjectSyncEntryKind, + size: NonNegativeInt, + // POSIX permission bits. Absent when the origin platform has no concept of + // file mode (or the caller chooses not to report it). + mode: Schema.optional(NonNegativeInt), + // Empty or absent for "dir" and "symlink" entries. A 64-character + // lowercase-hex sha256 digest of the file contents for "file" entries. + hash: Schema.optional(Schema.String.check(Schema.isPattern(SHA256_HEX_PATTERN))), + // Only present for "symlink" entries: the raw link target, unresolved. + linkTarget: Schema.optional( + TrimmedNonEmptyString.check(Schema.isMaxLength(PROJECT_SYNC_PATH_MAX_LENGTH)), + ), +}); +export type ProjectSyncManifestEntry = typeof ProjectSyncManifestEntry.Type; + +/** + * Requests a manifest (file tree + hashes) for a project so the client can + * diff it against the destination environment's tree before deciding what to + * copy or delete. + * + * The server always ignores `node_modules`, `.t3`, and `.DS_Store` regardless + * of `extraIgnores` — those are noise/local-state paths that must never + * round-trip between environments. `extraIgnores` layers additional + * project-specific exclusions (e.g. build output) on top of that default + * set; it never narrows it. + */ +export const ProjectSyncManifestInput = Schema.Struct({ + projectId: ProjectId, + // Whether the manifest should include the project's `.git` directory. + // Defaults to true so omitted/legacy payloads keep syncing history; callers + // that only want the working tree can opt out explicitly. + includeGit: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))), + extraIgnores: Schema.optional( + Schema.Array(TrimmedNonEmptyString.check(Schema.isMaxLength(PROJECT_SYNC_PATH_MAX_LENGTH))), + ), +}); +export type ProjectSyncManifestInput = typeof ProjectSyncManifestInput.Type; + +export const ProjectSyncManifestResult = Schema.Struct({ + workspaceRoot: TrimmedNonEmptyString, + entries: Schema.Array(ProjectSyncManifestEntry), + generatedAt: IsoDateTime, +}); +export type ProjectSyncManifestResult = typeof ProjectSyncManifestResult.Type; + +/** How long a signed project-sync export/import URL stays valid. Mirrors the + attachment upload URL's TTL: long enough to cover a slow transfer, short + enough that a leaked URL is not a standing liability. */ +export const PROJECT_SYNC_URL_TTL_MS = 10 * 60 * 1000; + +/** HTTP route prefix a client resolves a signed export URL against to stream + project file contents out of the origin environment. */ +export const PROJECT_SYNC_EXPORT_ROUTE_PREFIX = "/api/projectSync/export"; + +/** HTTP route prefix a client resolves a signed import URL against to stream + project file contents into the destination environment. */ +export const PROJECT_SYNC_IMPORT_ROUTE_PREFIX = "/api/projectSync/import"; + +export const ProjectSyncCreateExportUrlInput = Schema.Struct({ + projectId: ProjectId, + paths: Schema.Array( + TrimmedNonEmptyString.check(Schema.isMaxLength(PROJECT_SYNC_PATH_MAX_LENGTH)), + ).check(Schema.isMaxLength(PROJECT_SYNC_MAX_PATHS_PER_REQUEST)), +}); +export type ProjectSyncCreateExportUrlInput = typeof ProjectSyncCreateExportUrlInput.Type; + +export const ProjectSyncCreateImportUrlInput = Schema.Struct({ + projectId: ProjectId, + fileCount: NonNegativeInt, + totalBytes: NonNegativeInt, +}); +export type ProjectSyncCreateImportUrlInput = typeof ProjectSyncCreateImportUrlInput.Type; + +export const ProjectSyncCreateUrlResult = Schema.Struct({ + url: TrimmedNonEmptyString.check(Schema.isMaxLength(PROJECT_SYNC_URL_MAX_LENGTH)), + expiresAt: Schema.Number, +}); +export type ProjectSyncCreateUrlResult = typeof ProjectSyncCreateUrlResult.Type; + +export const ProjectSyncApplyDeletionsInput = Schema.Struct({ + projectId: ProjectId, + paths: Schema.Array( + TrimmedNonEmptyString.check(Schema.isMaxLength(PROJECT_SYNC_PATH_MAX_LENGTH)), + ).check(Schema.isMaxLength(PROJECT_SYNC_MAX_PATHS_PER_REQUEST)), +}); +export type ProjectSyncApplyDeletionsInput = typeof ProjectSyncApplyDeletionsInput.Type; + +export const ProjectSyncApplyDeletionsResult = Schema.Struct({ + deleted: NonNegativeInt, +}); +export type ProjectSyncApplyDeletionsResult = typeof ProjectSyncApplyDeletionsResult.Type; + +type ProjectSyncErrorContext = { + readonly message?: string; + readonly cause?: unknown; +}; + +export class ProjectSyncProjectNotFoundError extends Schema.TaggedErrorClass()( + "ProjectSyncProjectNotFoundError", + { + projectId: Schema.optional(TrimmedNonEmptyString), + message: TrimmedNonEmptyString, + cause: Schema.optional(Schema.Defect()), + }, +) { + // The structured field is optional on the wire so newer peers can decode legacy + // message-only failures. New application code must provide it through this constructor. + // @effect-diagnostics-next-line overriddenSchemaConstructor:off + constructor(props: ProjectSyncErrorContext & { readonly projectId: string }) { + super({ + ...props, + message: props.message ?? `Project '${props.projectId}' was not found.`, + } as any); + } +} + +export class ProjectSyncPathViolationError extends Schema.TaggedErrorClass()( + "ProjectSyncPathViolationError", + { + path: Schema.optional(TrimmedNonEmptyString), + message: TrimmedNonEmptyString, + cause: Schema.optional(Schema.Defect()), + }, +) { + // @effect-diagnostics-next-line overriddenSchemaConstructor:off + constructor(props: ProjectSyncErrorContext & { readonly path: string }) { + super({ + ...props, + message: props.message ?? `Path '${props.path}' escapes the project workspace root.`, + } as any); + } +} + +export class ProjectSyncIoError extends Schema.TaggedErrorClass()( + "ProjectSyncIoError", + { + message: TrimmedNonEmptyString, + cause: Schema.optional(Schema.Defect()), + }, +) {} + +export const ProjectSyncError = Schema.Union([ + ProjectSyncProjectNotFoundError, + ProjectSyncPathViolationError, + ProjectSyncIoError, +]); +export type ProjectSyncError = typeof ProjectSyncError.Type; diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 3dccc16ae561..044a1d8477a7 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -128,6 +128,16 @@ import { ProjectWriteFileInput, ProjectWriteFileResult, } from "./project.ts"; +import { + ProjectSyncApplyDeletionsInput, + ProjectSyncApplyDeletionsResult, + ProjectSyncCreateExportUrlInput, + ProjectSyncCreateImportUrlInput, + ProjectSyncCreateUrlResult, + ProjectSyncError, + ProjectSyncManifestInput, + ProjectSyncManifestResult, +} from "./projectSync.ts"; import { TerminalAttachInput, TerminalAttachStreamEvent, @@ -218,6 +228,12 @@ export const WS_METHODS = { projectsSearchEntries: "projects.searchEntries", projectsWriteFile: "projects.writeFile", + // Project sync methods + projectSyncManifest: "projectSync.manifest", + projectSyncCreateExportUrl: "projectSync.createExportUrl", + projectSyncCreateImportUrl: "projectSync.createImportUrl", + projectSyncApplyDeletions: "projectSync.applyDeletions", + // Shell methods shellOpenInEditor: "shell.openInEditor", @@ -679,6 +695,30 @@ export const WsProjectsWriteFileRpc = Rpc.make(WS_METHODS.projectsWriteFile, { error: Schema.Union([ProjectWriteFileError, EnvironmentAuthorizationError]), }); +export const WsProjectSyncManifestRpc = Rpc.make(WS_METHODS.projectSyncManifest, { + payload: ProjectSyncManifestInput, + success: ProjectSyncManifestResult, + error: Schema.Union([ProjectSyncError, EnvironmentAuthorizationError]), +}); + +export const WsProjectSyncCreateExportUrlRpc = Rpc.make(WS_METHODS.projectSyncCreateExportUrl, { + payload: ProjectSyncCreateExportUrlInput, + success: ProjectSyncCreateUrlResult, + error: Schema.Union([ProjectSyncError, EnvironmentAuthorizationError]), +}); + +export const WsProjectSyncCreateImportUrlRpc = Rpc.make(WS_METHODS.projectSyncCreateImportUrl, { + payload: ProjectSyncCreateImportUrlInput, + success: ProjectSyncCreateUrlResult, + error: Schema.Union([ProjectSyncError, EnvironmentAuthorizationError]), +}); + +export const WsProjectSyncApplyDeletionsRpc = Rpc.make(WS_METHODS.projectSyncApplyDeletions, { + payload: ProjectSyncApplyDeletionsInput, + success: ProjectSyncApplyDeletionsResult, + error: Schema.Union([ProjectSyncError, EnvironmentAuthorizationError]), +}); + export const WsShellOpenInEditorRpc = Rpc.make(WS_METHODS.shellOpenInEditor, { payload: LaunchEditorInput, error: Schema.Union([ExternalLauncherError, EnvironmentAuthorizationError]), @@ -1080,6 +1120,10 @@ export const WsRpcGroup = RpcGroup.make( WsProjectsSearchContentsRpc, WsProjectsSearchEntriesRpc, WsProjectsWriteFileRpc, + WsProjectSyncManifestRpc, + WsProjectSyncCreateExportUrlRpc, + WsProjectSyncCreateImportUrlRpc, + WsProjectSyncApplyDeletionsRpc, WsShellOpenInEditorRpc, WsFilesystemBrowseRpc, WsAssetsCreateUrlRpc, diff --git a/packages/shared/package.json b/packages/shared/package.json index 698a16905fd5..43091d5d0586 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -230,6 +230,10 @@ "./providerRateLimits": { "types": "./src/providerRateLimits.ts", "import": "./src/providerRateLimits.ts" + }, + "./projectSyncFraming": { + "types": "./src/projectSyncFraming.ts", + "import": "./src/projectSyncFraming.ts" } }, "scripts": { diff --git a/packages/shared/src/projectSyncFraming.test.ts b/packages/shared/src/projectSyncFraming.test.ts new file mode 100644 index 000000000000..839ef2b027b4 --- /dev/null +++ b/packages/shared/src/projectSyncFraming.test.ts @@ -0,0 +1,199 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + createProjectSyncFrameDecoder, + encodeProjectSyncRecords, + type ProjectSyncFrameDecodedRecord, + type ProjectSyncFrameRecord, +} from "./projectSyncFraming.ts"; + +async function collectBytes(chunks: AsyncIterable): Promise { + const parts: Uint8Array[] = []; + let total = 0; + for await (const chunk of chunks) { + parts.push(chunk); + total += chunk.length; + } + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +} + +async function* asChunksOfSize(bytes: Uint8Array, size: number): AsyncIterable { + for (let offset = 0; offset < bytes.length; offset += size) { + yield bytes.subarray(offset, Math.min(offset + size, bytes.length)); + } +} + +async function* asOne(records: ProjectSyncFrameRecord[]): AsyncIterable { + for (const record of records) yield record; +} + +async function encodeToBytes(records: ProjectSyncFrameRecord[]): Promise { + return collectBytes(encodeProjectSyncRecords(asOne(records))); +} + +async function decodeAll( + bytes: Uint8Array, + chunkSize: number, +): Promise> { + const decoded: Array<{ header: ProjectSyncFrameDecodedRecord["header"]; content: Uint8Array }> = + []; + for await (const record of createProjectSyncFrameDecoder(asChunksOfSize(bytes, chunkSize))) { + const content = await collectBytes(record.content); + decoded.push({ header: record.header, content }); + } + return decoded; +} + +describe("projectSyncFraming", () => { + it("round-trips multiple files through a single-byte-chunked stream", async () => { + const fileA = new TextEncoder().encode("hello world"); + const fileB = new TextEncoder().encode("a".repeat(300)); + + const records: ProjectSyncFrameRecord[] = [ + { + header: { path: "src/a.txt", size: fileA.length, kind: "file", mode: 0o644 }, + content: fileA, + }, + { + header: { path: "src/nested/b.txt", size: fileB.length, kind: "file" }, + content: fileB, + }, + ]; + + const encoded = await encodeToBytes(records); + const decoded = await decodeAll(encoded, 1); + + expect(decoded).toHaveLength(2); + expect(decoded[0]!.header).toEqual({ + path: "src/a.txt", + size: fileA.length, + kind: "file", + mode: 0o644, + }); + expect(new TextDecoder().decode(decoded[0]!.content)).toBe("hello world"); + expect(decoded[1]!.header.path).toBe("src/nested/b.txt"); + expect(decoded[1]!.content).toEqual(fileB); + }); + + it("round-trips an empty file", async () => { + const records: ProjectSyncFrameRecord[] = [ + { header: { path: "empty.txt", size: 0, kind: "file" }, content: new Uint8Array(0) }, + ]; + + const encoded = await encodeToBytes(records); + const decoded = await decodeAll(encoded, 7); + + expect(decoded).toHaveLength(1); + expect(decoded[0]!.header.size).toBe(0); + expect(decoded[0]!.content.length).toBe(0); + }); + + it("round-trips dir and symlink entries with no content", async () => { + const records: ProjectSyncFrameRecord[] = [ + { header: { path: "src/empty-dir", size: 0, kind: "dir" }, content: new Uint8Array(0) }, + { + header: { + path: "src/link", + size: 0, + kind: "symlink", + linkTarget: "../target.txt", + }, + content: new Uint8Array(0), + }, + ]; + + const encoded = await encodeToBytes(records); + const decoded = await decodeAll(encoded, 3); + + expect(decoded).toHaveLength(2); + expect(decoded[0]!.header.kind).toBe("dir"); + expect(decoded[1]!.header.kind).toBe("symlink"); + expect(decoded[1]!.header.linkTarget).toBe("../target.txt"); + }); + + it("round-trips a record with a large header", async () => { + const longPath = `src/${"segment/".repeat(400)}file.txt`; + const content = new TextEncoder().encode("payload after a large header"); + + const records: ProjectSyncFrameRecord[] = [ + { header: { path: longPath, size: content.length, kind: "file" }, content }, + ]; + + const encoded = await encodeToBytes(records); + // Chunk right around the header-length boundary in a few different ways. + for (const chunkSize of [1, 5, 64, 4096]) { + const decoded = await decodeAll(encoded, chunkSize); + expect(decoded).toHaveLength(1); + expect(decoded[0]!.header.path).toBe(longPath); + expect(new TextDecoder().decode(decoded[0]!.content)).toBe("payload after a large header"); + } + }); + + it("accepts async-iterable content and streams it back out", async () => { + const total = new TextEncoder().encode("streamed-content-body"); + async function* chunked(): AsyncIterable { + yield total.subarray(0, 5); + yield total.subarray(5, 12); + yield total.subarray(12); + } + + const records: ProjectSyncFrameRecord[] = [ + { header: { path: "stream.bin", size: total.length, kind: "file" }, content: chunked() }, + ]; + + const encoded = await encodeToBytes(records); + const decoded = await decodeAll(encoded, 4); + + expect(decoded).toHaveLength(1); + expect(decoded[0]!.content).toEqual(total); + }); + + it("drains an unread record so the next header still parses correctly", async () => { + const fileA = new TextEncoder().encode("first-file-content"); + const fileB = new TextEncoder().encode("second-file-content"); + + const records: ProjectSyncFrameRecord[] = [ + { header: { path: "a.txt", size: fileA.length, kind: "file" }, content: fileA }, + { header: { path: "b.txt", size: fileB.length, kind: "file" }, content: fileB }, + ]; + + const encoded = await encodeToBytes(records); + + const paths: string[] = []; + for await (const record of createProjectSyncFrameDecoder(asChunksOfSize(encoded, 6))) { + // Deliberately never touch `record.content` for the first record. + paths.push(record.header.path); + } + + expect(paths).toEqual(["a.txt", "b.txt"]); + }); + + it("throws when the content stream ends before the declared size", async () => { + const encoded = await encodeToBytes([ + { header: { path: "truncated.txt", size: 10, kind: "file" }, content: new Uint8Array(10) }, + ]); + const truncated = encoded.subarray(0, encoded.length - 5); + + await expect( + (async () => { + for await (const record of createProjectSyncFrameDecoder(asChunksOfSize(truncated, 3))) { + await collectBytes(record.content); + } + })(), + ).rejects.toThrow(/mid-content/); + }); + + it("throws when a record's declared size does not match the actual content length", async () => { + await expect( + encodeToBytes([ + { header: { path: "mismatch.txt", size: 5, kind: "file" }, content: new Uint8Array(3) }, + ]), + ).rejects.toThrow(/length mismatch/); + }); +}); diff --git a/packages/shared/src/projectSyncFraming.ts b/packages/shared/src/projectSyncFraming.ts new file mode 100644 index 000000000000..ba13e4f3555b --- /dev/null +++ b/packages/shared/src/projectSyncFraming.ts @@ -0,0 +1,297 @@ +/** + * Binary framing for project sync file transfers. + * + * Both ends of a project sync transfer are our own code (the sync client + * streaming a request body, our HTTP handler streaming a response body), so + * this is a purpose-built framing format rather than a general interchange + * format. Each record on the wire is: + * + * [ uint32 BE header length ][ UTF-8 JSON header ][ raw content bytes ] + * + * `content` is exactly `header.size` bytes, unencoded. Directories and + * symlinks carry `size: 0` and therefore no content bytes at all. A stream is + * simply a concatenation of records with no trailing marker; the reader + * knows it has reached the end when the underlying byte source is exhausted + * cleanly on a record boundary. + * + * @module projectSyncFraming + */ +import type { ProjectSyncEntryKind } from "@t3tools/contracts"; + +const HEADER_LENGTH_BYTES = 4; + +export interface ProjectSyncFrameHeader { + readonly path: string; + readonly size: number; + readonly kind: ProjectSyncEntryKind; + readonly mode?: number; + readonly linkTarget?: string; +} + +/** One record to encode: a header plus its content, given either as a single + buffer or as a stream of chunks summing to exactly `header.size` bytes. */ +export interface ProjectSyncFrameRecord { + readonly header: ProjectSyncFrameHeader; + readonly content: AsyncIterable | Uint8Array; +} + +/** One record as decoded off the wire. `content` must be iterated to + completion (or left untouched) before requesting the next record from the + same decoder — the decoder shares a single cursor across records and + drains any unread bytes automatically once the caller moves on. */ +export interface ProjectSyncFrameDecodedRecord { + readonly header: ProjectSyncFrameHeader; + readonly content: AsyncIterable; +} + +function encodeHeaderLength(length: number): Uint8Array { + const bytes = new Uint8Array(HEADER_LENGTH_BYTES); + new DataView(bytes.buffer).setUint32(0, length, false); + return bytes; +} + +function isAsyncIterableContent( + value: ProjectSyncFrameRecord["content"], +): value is AsyncIterable { + return typeof value === "object" && value !== null && Symbol.asyncIterator in value; +} + +/** + * Encodes an async iterable of records into the wire framing described above. + * + * Throws if a record's actual content length does not match its declared + * `header.size` — a mismatch means the caller computed the manifest entry + * wrong, and the alternative (silently sending a corrupt frame) is worse. + */ +export async function* encodeProjectSyncRecords( + records: AsyncIterable, +): AsyncGenerator { + const textEncoder = new TextEncoder(); + + for await (const record of records) { + const headerJson = textEncoder.encode(JSON.stringify(record.header)); + yield encodeHeaderLength(headerJson.length); + if (headerJson.length > 0) yield headerJson; + + if (isAsyncIterableContent(record.content)) { + let written = 0; + for await (const chunk of record.content) { + if (chunk.length === 0) continue; + written += chunk.length; + yield chunk; + } + if (written !== record.header.size) { + throw new Error( + `Project sync frame content length mismatch for '${record.header.path}': ` + + `declared ${record.header.size} bytes, wrote ${written}.`, + ); + } + } else { + if (record.content.length !== record.header.size) { + throw new Error( + `Project sync frame content length mismatch for '${record.header.path}': ` + + `declared ${record.header.size} bytes, wrote ${record.content.length}.`, + ); + } + if (record.content.length > 0) yield record.content; + } + } +} + +/** + * Incrementally buffers chunks pulled from an underlying async iterator so + * callers can ask for "at least N bytes" without caring how the upstream + * source happened to chunk the data (one byte at a time, one giant buffer, + * anything in between). + * + * Kept as a small chunk queue with an amortized-O(1) `take`/`takeUpTo` + * instead of repeatedly concatenating into one growing buffer, since project + * sync payloads can be large and framing is on the hot path of every sync. + */ +class ChunkCursor { + private readonly source: AsyncIterator; + private readonly pending: Uint8Array[] = []; + private pendingLength = 0; + private sourceExhausted = false; + + constructor(source: AsyncIterator) { + this.source = source; + } + + private async pullOne(): Promise { + if (this.sourceExhausted) return false; + const { value, done } = await this.source.next(); + if (done) { + this.sourceExhausted = true; + return false; + } + if (value.length > 0) { + this.pending.push(value); + this.pendingLength += value.length; + } + return true; + } + + /** Pulls from the source until at least `n` bytes are buffered, or the + source is exhausted. Returns whether `n` bytes are now available. */ + async ensure(n: number): Promise { + while (this.pendingLength < n) { + if (!(await this.pullOne())) break; + } + return this.pendingLength >= n; + } + + get bufferedLength(): number { + return this.pendingLength; + } + + /** Consumes exactly `n` buffered bytes. Callers must `ensure(n)` first. */ + take(n: number): Uint8Array { + if (n === 0) return new Uint8Array(0); + const first = this.pending[0]; + if (first && first.length >= n) { + const out = first.subarray(0, n); + if (first.length === n) this.pending.shift(); + else this.pending[0] = first.subarray(n); + this.pendingLength -= n; + return out; + } + const out = new Uint8Array(n); + let offset = 0; + while (offset < n) { + const chunk = this.pending[0]; + if (!chunk) throw new Error("ChunkCursor.take: fewer than n bytes were buffered."); + const wanted = n - offset; + if (chunk.length <= wanted) { + out.set(chunk, offset); + offset += chunk.length; + this.pending.shift(); + } else { + out.set(chunk.subarray(0, wanted), offset); + this.pending[0] = chunk.subarray(wanted); + offset += wanted; + } + } + this.pendingLength -= n; + return out; + } + + /** Consumes up to `n` bytes without merging across chunk boundaries — the + returned view is a subarray of a single already-buffered chunk. Used + for streaming content out without forcing extra copies. */ + takeUpTo(n: number): Uint8Array { + const first = this.pending[0]; + if (!first || n <= 0) return new Uint8Array(0); + const want = Math.min(n, first.length); + const out = first.subarray(0, want); + if (first.length === want) this.pending.shift(); + else this.pending[0] = first.subarray(want); + this.pendingLength -= want; + return out; + } +} + +function createContentIterable( + cursor: ChunkCursor, + size: number, + path: string, + state: { consumed: number }, +): AsyncIterable { + const iterator: AsyncIterator = { + async next(): Promise> { + if (state.consumed >= size) { + return { done: true, value: undefined }; + } + const remaining = size - state.consumed; + const haveByte = await cursor.ensure(1); + if (!haveByte) { + throw new Error( + `Project sync frame stream ended mid-content for '${path}' ` + + `(${state.consumed}/${size} bytes read).`, + ); + } + const chunk = cursor.takeUpTo(remaining); + state.consumed += chunk.length; + return { done: false, value: chunk }; + }, + async return(value?: unknown): Promise> { + // `for await...of` calls `return()` when the consumer stops early + // (e.g. `break`). The decoder shares one cursor across every record in + // the stream, so an early stop must still drain this record's + // remaining bytes — otherwise the next record would be parsed starting + // from the middle of this one's leftover content. + while (state.consumed < size) { + const haveByte = await cursor.ensure(1); + if (!haveByte) break; + const chunk = cursor.takeUpTo(size - state.consumed); + state.consumed += chunk.length; + } + return { done: true, value: value as Uint8Array | undefined }; + }, + }; + + return { + [Symbol.asyncIterator]() { + return iterator; + }, + }; +} + +/** + * Parses a stream of raw chunks (of any, possibly tiny, granularity) into + * project sync frame records. Yields one `{ header, content }` pair per + * record; `content` streams exactly `header.size` bytes. + * + * If the caller moves on to the next record (or stops iterating the decoder + * entirely) before fully draining a record's `content`, the decoder drains + * the remainder itself so the shared byte cursor stays aligned on the next + * record's header. + */ +export async function* createProjectSyncFrameDecoder( + chunks: AsyncIterable, +): AsyncGenerator { + const cursor = new ChunkCursor(chunks[Symbol.asyncIterator]()); + const textDecoder = new TextDecoder(); + + while (true) { + const haveLength = await cursor.ensure(HEADER_LENGTH_BYTES); + if (!haveLength) { + if (cursor.bufferedLength > 0) { + throw new Error("Project sync frame stream ended mid-header-length."); + } + return; + } + const lengthBytes = cursor.take(HEADER_LENGTH_BYTES); + const headerLength = new DataView( + lengthBytes.buffer, + lengthBytes.byteOffset, + lengthBytes.byteLength, + ).getUint32(0, false); + + const haveHeader = await cursor.ensure(headerLength); + if (!haveHeader) { + throw new Error("Project sync frame stream ended mid-header."); + } + const headerBytes = cursor.take(headerLength); + const header = JSON.parse(textDecoder.decode(headerBytes)) as ProjectSyncFrameHeader; + + const state = { consumed: 0 }; + const content = createContentIterable(cursor, header.size, header.path, state); + + yield { header, content }; + + // Defensive drain: covers the case where the caller never touched + // `content` at all (so the iterator's `return()` was never invoked). + while (state.consumed < header.size) { + const haveByte = await cursor.ensure(1); + if (!haveByte) { + throw new Error( + `Project sync frame stream ended mid-content for '${header.path}' ` + + `(${state.consumed}/${header.size} bytes read).`, + ); + } + const chunk = cursor.takeUpTo(header.size - state.consumed); + state.consumed += chunk.length; + } + } +} From 970712e858ba44454d0048cce22161a8af611bc1 Mon Sep 17 00:00:00 2001 From: Diego Barreto Date: Tue, 25 Aug 2026 09:10:12 -0300 Subject: [PATCH 2/2] fix(sync): harden project sync and honor the confirmed plan Review pass over the project sync feature. Security and correctness: - validate framing headers on decode; a negative size decremented the signed byte budget and let a small token write unlimited bytes - enforce the signed fileCount as a record cap (413), closing an inode-exhaustion path via zero-byte records - guard symlinked ancestors on export, which could read outside the workspace root with only read scope - stop transforming paths (no trim, no backslash rewrite) so names that are legal on POSIX round-trip instead of being silently skipped - skip files whose size changed since the manifest instead of blowing the batch budget and aborting the whole sync - release export registrations after use and expire by TTL, so a busy server no longer evicts valid in-flight tokens - tolerate a non-directory ancestor, replacing it as the mirror intends - re-assert source empty directories after the deletion pass - chunk deletions to the contract's per-request cap - check the capability on the origin environment, not just the destination, and reuse the created project when a send is retried Behavior and structure: - the plan the user confirms is the plan that runs: planProjectSync is computed once for review and handed to runProjectSync, so deletions cannot drift between confirmation and execution - stream batch bodies through when the runtime supports it, buffering only as a fallback - carry an HTTP status on transfer errors instead of scraping messages - move the dialog's pure logic to client-runtime so mobile can reuse it - share the signed-token verification and default-folder helpers with their existing counterparts Docs corrected against the implementation. Implemented by Claude Fable 5 subagents (Sonnet/Opus) via Claude Code. --- apps/server/src/assets/AssetAccess.ts | 4 + apps/server/src/assets/AttachmentUpload.ts | 35 +- apps/server/src/auth/utils.ts | 35 ++ apps/server/src/http.projectSync.test.ts | 365 ++++++++++++++++++ apps/server/src/http.ts | 7 +- .../src/workspace/ProjectSyncApply.test.ts | 131 ++++++- apps/server/src/workspace/ProjectSyncApply.ts | 158 ++++++-- .../workspace/ProjectSyncRoundTrip.test.ts | 265 +++++++++++++ .../src/workspace/ProjectSyncTransfer.test.ts | 163 +++++++- .../src/workspace/ProjectSyncTransfer.ts | 168 ++++---- apps/server/src/ws.ts | 2 +- apps/web/src/components/CommandPalette.tsx | 11 +- apps/web/src/components/SyncProjectDialog.tsx | 239 +++++++----- .../settings/ProjectSettingsPanel.tsx | 37 +- apps/web/src/state/projectSync.ts | 5 +- docs/internals/project-sync.md | 82 ++-- docs/user/project-sync.md | 19 +- packages/client-runtime/package.json | 4 + .../src/operations/projectSyncFlow.test.ts | 128 +++++- .../src/operations/projectSyncFlow.ts | 166 +++++--- .../src/state/projectSync.test.ts | 111 +++++- .../client-runtime/src/state/projectSync.ts | 223 +++++++++-- packages/contracts/src/projectSync.test.ts | 119 ++++++ packages/contracts/src/projectSync.ts | 58 ++- .../shared/src/projectSyncFraming.test.ts | 135 +++++++ packages/shared/src/projectSyncFraming.ts | 77 +++- 26 files changed, 2341 insertions(+), 406 deletions(-) create mode 100644 apps/server/src/http.projectSync.test.ts create mode 100644 apps/server/src/workspace/ProjectSyncRoundTrip.test.ts rename apps/web/src/components/SyncProjectDialog.logic.test.ts => packages/client-runtime/src/operations/projectSyncFlow.test.ts (73%) rename apps/web/src/components/SyncProjectDialog.logic.ts => packages/client-runtime/src/operations/projectSyncFlow.ts (65%) create mode 100644 packages/contracts/src/projectSync.test.ts diff --git a/apps/server/src/assets/AssetAccess.ts b/apps/server/src/assets/AssetAccess.ts index 232a41e5a9c8..4334945469b2 100644 --- a/apps/server/src/assets/AssetAccess.ts +++ b/apps/server/src/assets/AssetAccess.ts @@ -431,6 +431,10 @@ export const resolveAsset = Effect.fn("AssetAccess.resolveAsset")(function* ( token: string, relativePath: string, ) { + // Deliberately not `verifySignedClaims` (auth/utils.ts), which the attachment + // upload and project sync tokens share: this one tolerates extra + // dot-separated segments and rewrites `expiresAt` for favicon claims, so + // adopting the shared check would change what it accepts. const [encodedPayload, signature] = token.split("."); if (!encodedPayload || !signature) return null; diff --git a/apps/server/src/assets/AttachmentUpload.ts b/apps/server/src/assets/AttachmentUpload.ts index 6142b69d7342..500cbf7a36f1 100644 --- a/apps/server/src/assets/AttachmentUpload.ts +++ b/apps/server/src/assets/AttachmentUpload.ts @@ -9,7 +9,6 @@ import { import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; -import * as Option from "effect/Option"; import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; @@ -21,12 +20,7 @@ import { sweepStalePendingAttachments, } from "../attachmentStore.ts"; import { resolveAttachmentRelativePath } from "../attachmentPaths.ts"; -import { - base64UrlDecodeUtf8, - base64UrlEncode, - signPayload, - timingSafeEqualBase64Url, -} from "../auth/utils.ts"; +import { base64UrlEncode, signPayload, verifySignedClaims } from "../auth/utils.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import * as ServerConfig from "../config.ts"; import { inferImageExtension } from "../imageMime.ts"; @@ -53,14 +47,6 @@ const attachmentUploadClaimsJson = Schema.fromJsonString(AttachmentUploadClaims) const decodeAttachmentUploadClaims = Schema.decodeUnknownOption(attachmentUploadClaimsJson); const encodeAttachmentUploadClaims = Schema.encodeSync(attachmentUploadClaimsJson); -function decodeClaims(encodedPayload: string): AttachmentUploadClaims | null { - try { - return Option.getOrNull(decodeAttachmentUploadClaims(base64UrlDecodeUtf8(encodedPayload))); - } catch { - return null; - } -} - const loadSigningSecret = Effect.gen(function* () { const secretStore = yield* ServerSecretStore.ServerSecretStore; return yield* secretStore.getOrCreateRandom(SIGNING_SECRET_NAME, 32); @@ -113,26 +99,19 @@ export const issueAttachmentUploadUrl = Effect.fn("AttachmentUpload.issueUrl")(f export const validateAttachmentUploadToken = Effect.fn("AttachmentUpload.validateToken")(function* ( token: string, ) { - const [encodedPayload, signature, unexpectedSegment] = token.split("."); - if (!encodedPayload || !signature || unexpectedSegment) { - return null; - } - const secret = yield* loadSigningSecret.pipe( Effect.tapError((cause) => Effect.logError("Failed to load the attachment upload signing key.", { cause }), ), Effect.orElseSucceed(() => null), ); - if (!secret || !timingSafeEqualBase64Url(signature, signPayload(encodedPayload, secret))) { - return null; - } - const claims = decodeClaims(encodedPayload); - if (!claims || claims.expiresAt <= (yield* Clock.currentTimeMillis)) { - return null; - } - return claims; + return verifySignedClaims({ + token, + secret, + nowMs: yield* Clock.currentTimeMillis, + decode: decodeAttachmentUploadClaims, + }); }); export type StoreAttachmentUploadResult = diff --git a/apps/server/src/auth/utils.ts b/apps/server/src/auth/utils.ts index 32a6799b01f4..931c6ea00d37 100644 --- a/apps/server/src/auth/utils.ts +++ b/apps/server/src/auth/utils.ts @@ -6,6 +6,7 @@ import type { import type * as HttpServerRequest from "effect/unstable/http/HttpServerRequest"; import * as NodeCrypto from "node:crypto"; import * as Encoding from "effect/Encoding"; +import * as Option from "effect/Option"; import * as Result from "effect/Result"; const SESSION_COOKIE_NAME = "t3_session"; @@ -89,6 +90,40 @@ export function timingSafeEqualBase64Url(left: string, right: string): boolean { return NodeCrypto.timingSafeEqual(leftBuffer, rightBuffer); } +/** + * Verifies one `.` bearer token and returns + * its claims, or `null` for any token that is malformed, unsigned, signed with + * the wrong key, undecodable, or expired. + * + * Every family of URL-borne token the server issues (attachment uploads, + * project sync export/import) shares this shape, and each hand-rolled copy of + * the check is one place for the families to drift apart on what counts as + * valid. `secret` is nullable so callers can pass the result of a signing-key + * load that failed: no key means no valid token. + */ +export function verifySignedClaims(input: { + readonly token: string; + readonly secret: Uint8Array | null; + readonly nowMs: number; + readonly decode: (encoded: unknown) => Option.Option; +}): A | null { + const [payload, signature, unexpectedSegment] = input.token.split("."); + if (!payload || !signature || unexpectedSegment !== undefined) { + return null; + } + if (!input.secret || !timingSafeEqualBase64Url(signature, signPayload(payload, input.secret))) { + return null; + } + + let claims: A | null; + try { + claims = Option.getOrNull(input.decode(base64UrlDecodeUtf8(payload))); + } catch { + return null; + } + return claims && claims.expiresAt > input.nowMs ? claims : null; +} + function normalizeNonEmptyString(value: string | null | undefined): string | undefined { if (typeof value !== "string") { return undefined; diff --git a/apps/server/src/http.projectSync.test.ts b/apps/server/src/http.projectSync.test.ts new file mode 100644 index 000000000000..091033c95eb7 --- /dev/null +++ b/apps/server/src/http.projectSync.test.ts @@ -0,0 +1,365 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * Route-level tests for the project sync export/import HTTP pair. + * + * The transfer only works if the bytes one route writes are exactly the bytes + * the other route reads, so these drive the real handlers over a real HTTP + * server rather than calling the underlying functions directly. + */ +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import { NodeHttpServer } from "@effect/platform-node"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { describe, expect, it } from "@effect/vitest"; +import { + PROJECT_SYNC_EXPORT_ROUTE_PREFIX, + PROJECT_SYNC_IMPORT_ROUTE_PREFIX, +} from "@t3tools/contracts"; +import { + encodeProjectSyncRecords, + type ProjectSyncFrameRecord, +} from "@t3tools/shared/projectSyncFraming"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Layer from "effect/Layer"; +import { HttpBody, HttpClient, HttpRouter } from "effect/unstable/http"; + +import * as ServerSecretStore from "./auth/ServerSecretStore.ts"; +import * as ServerConfig from "./config.ts"; +import { projectSyncExportRouteLayer, projectSyncImportRouteLayer } from "./http.ts"; +import { buildProjectSyncManifest } from "./workspace/ProjectSyncManifest.ts"; +import { + issueProjectSyncExportUrl, + issueProjectSyncImportUrl, +} from "./workspace/ProjectSyncTransfer.ts"; + +const testLayer = ServerSecretStore.layer.pipe( + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-http-project-sync-" })), + Layer.provideMerge(NodeServices.layer), + Layer.provideMerge(NodeHttpServer.layerTest), +); + +const serveProjectSyncRoutes = HttpRouter.serve( + Layer.mergeAll(projectSyncExportRouteLayer, projectSyncImportRouteLayer), + { disableListenLog: true, disableLogger: true }, +).pipe(Layer.build, Effect.asVoid); + +const makeTempDir = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-http-project-sync-" }); +}); + +const writeFile = (root: string, relativePath: string, contents: string) => + Effect.promise(async () => { + const absolutePath = NodePath.join(root, relativePath); + await NodeFSP.mkdir(NodePath.dirname(absolutePath), { recursive: true }); + await NodeFSP.writeFile(absolutePath, contents); + return absolutePath; + }); + +const encoder = new TextEncoder(); + +function fileRecord(path: string, contents: string): ProjectSyncFrameRecord { + const content = encoder.encode(contents); + return { header: { path, size: content.length, kind: "file" }, content }; +} + +async function encodeRecords(records: ReadonlyArray): Promise { + async function* iterate() { + for (const record of records) yield record; + } + const parts: Uint8Array[] = []; + let total = 0; + for await (const chunk of encodeProjectSyncRecords(iterate())) { + parts.push(chunk); + total += chunk.length; + } + const out = new Uint8Array(total); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.length; + } + return out; +} + +const exists = (root: string, relativePath: string) => + Effect.promise(() => + NodeFSP.lstat(NodePath.join(root, relativePath)).then( + () => true, + () => false, + ), + ); + +describe("project sync HTTP routes", () => { + it.effect("streams an export and applies the identical bytes back through the import", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveProjectSyncRoutes; + const client = yield* HttpClient.HttpClient; + + const origin = yield* makeTempDir; + const destination = yield* makeTempDir; + yield* writeFile(origin, "README.md", "# hello\n"); + yield* writeFile(origin, "src/nested/index.ts", "export const answer = 42;\n"); + const script = yield* writeFile(origin, "bin/run.sh", "#!/bin/sh\n"); + yield* Effect.promise(() => NodeFSP.chmod(script, 0o755)); + yield* Effect.promise(() => NodeFSP.mkdir(NodePath.join(origin, "empty"))); + yield* Effect.promise(() => NodeFSP.symlink("README.md", NodePath.join(origin, "link.md"))); + + const manifest = yield* buildProjectSyncManifest({ + workspaceRoot: origin, + includeGit: true, + }); + const exportUrl = yield* issueProjectSyncExportUrl({ + projectId: "project-http-1", + workspaceRoot: origin, + entries: manifest.map((entry) => ({ path: entry.path, size: entry.size })), + }); + + const exportResponse = yield* client.get(exportUrl.url); + expect(exportResponse.status).toBe(200); + const body = new Uint8Array(yield* exportResponse.arrayBuffer); + expect(body.length).toBeGreaterThan(0); + + // Exactly what the client signs for: file content bytes only, no framing. + const totalBytes = manifest.reduce((total, entry) => total + entry.size, 0); + const importUrl = yield* issueProjectSyncImportUrl({ + projectId: "project-http-2", + workspaceRoot: destination, + fileCount: manifest.length, + totalBytes, + }); + + const importResponse = yield* client.post(importUrl.url, { + body: HttpBody.uint8Array(body), + }); + expect(importResponse.status).toBe(200); + expect(yield* importResponse.json).toEqual({ + applied: manifest.length, + bytes: totalBytes, + }); + + const destinationManifest = yield* buildProjectSyncManifest({ + workspaceRoot: destination, + includeGit: true, + }); + expect(destinationManifest).toEqual(manifest); + expect( + (yield* Effect.promise(() => + NodeFSP.lstat(NodePath.join(destination, "link.md")), + )).isSymbolicLink(), + ).toBe(true); + }), + ).pipe(Effect.provide(testLayer)), + ); + + it.effect("404s an export URL that was already fetched", () => + Effect.scoped( + Effect.gen(function* () { + // The registration is released the moment the stream completes, so a + // leaked URL is worth one fetch rather than ten minutes of them. A + // client that needs the bytes again just asks for a new URL. + yield* serveProjectSyncRoutes; + const client = yield* HttpClient.HttpClient; + + const origin = yield* makeTempDir; + yield* writeFile(origin, "once.txt", "once\n"); + const exportUrl = yield* issueProjectSyncExportUrl({ + projectId: "project-http-single-use", + workspaceRoot: origin, + entries: [{ path: "once.txt", size: 5 }], + }); + + const first = yield* client.get(exportUrl.url); + expect(first.status).toBe(200); + expect(new Uint8Array(yield* first.arrayBuffer).length).toBeGreaterThan(0); + + expect((yield* client.get(exportUrl.url)).status).toBe(404); + }), + ).pipe(Effect.provide(testLayer)), + ); + + it.effect("404s an export token that was never issued", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveProjectSyncRoutes; + const client = yield* HttpClient.HttpClient; + + expect((yield* client.get(`${PROJECT_SYNC_EXPORT_ROUTE_PREFIX}/not-a-token`)).status).toBe( + 404, + ); + expect( + (yield* client.post(`${PROJECT_SYNC_IMPORT_ROUTE_PREFIX}/not-a-token`, { + body: HttpBody.uint8Array(new Uint8Array(0)), + })).status, + ).toBe(404); + }), + ).pipe(Effect.provide(testLayer)), + ); + + it.effect("400s an import whose record escapes the workspace mid-stream", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveProjectSyncRoutes; + const client = yield* HttpClient.HttpClient; + + const destination = yield* makeTempDir; + const outside = yield* makeTempDir; + const body = yield* Effect.promise(() => + encodeRecords([ + fileRecord("safe.txt", "safe"), + fileRecord("../planted.txt", "planted"), + fileRecord("never.txt", "never"), + ]), + ); + + const importUrl = yield* issueProjectSyncImportUrl({ + projectId: "project-http-3", + workspaceRoot: destination, + fileCount: 3, + totalBytes: 1024, + }); + const response = yield* client.post(importUrl.url, { body: HttpBody.uint8Array(body) }); + + expect(response.status).toBe(400); + // The valid record before the violation is allowed to land; what must + // never happen is a write outside the workspace root, or the stream + // continuing past the violation. + expect(yield* exists(destination, "safe.txt")).toBe(true); + expect(yield* exists(destination, "never.txt")).toBe(false); + expect(yield* exists(outside, "planted.txt")).toBe(false); + expect( + yield* Effect.promise(() => + NodeFSP.lstat(NodePath.join(destination, "..", "planted.txt")).then( + () => true, + () => false, + ), + ), + ).toBe(false); + }), + ).pipe(Effect.provide(testLayer)), + ); + + it.effect("413s an import body that outgrows the budget its URL was signed for", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveProjectSyncRoutes; + const client = yield* HttpClient.HttpClient; + + const destination = yield* makeTempDir; + const body = yield* Effect.promise(() => + encodeRecords([ + fileRecord("first.txt", "0123456789"), + fileRecord("second.txt", "0123456789"), + ]), + ); + + const importUrl = yield* issueProjectSyncImportUrl({ + projectId: "project-http-4", + workspaceRoot: destination, + fileCount: 2, + totalBytes: 10, + }); + const response = yield* client.post(importUrl.url, { body: HttpBody.uint8Array(body) }); + + expect(response.status).toBe(413); + expect(yield* exists(destination, "first.txt")).toBe(true); + expect(yield* exists(destination, "second.txt")).toBe(false); + }), + ).pipe(Effect.provide(testLayer)), + ); + + it.effect("413s an import body carrying more entries than its URL was signed for", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveProjectSyncRoutes; + const client = yield* HttpClient.HttpClient; + + // Zero-byte directory records cost nothing against the byte budget, so + // only the record cap stands between a one-entry token and as many + // inodes as the peer cares to send. + const destination = yield* makeTempDir; + const body = yield* Effect.promise(() => + encodeRecords( + ["one", "two", "three"].map((path) => ({ + header: { path, size: 0, kind: "dir" as const }, + content: new Uint8Array(0), + })), + ), + ); + + const importUrl = yield* issueProjectSyncImportUrl({ + projectId: "project-http-6", + workspaceRoot: destination, + fileCount: 1, + totalBytes: 1024, + }); + const response = yield* client.post(importUrl.url, { body: HttpBody.uint8Array(body) }); + + expect(response.status).toBe(413); + expect(yield* exists(destination, "one")).toBe(true); + expect(yield* exists(destination, "two")).toBe(false); + expect(yield* exists(destination, "three")).toBe(false); + }), + ).pipe(Effect.provide(testLayer)), + ); + + it.effect("refuses an import whose framing header lies about its size", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveProjectSyncRoutes; + const client = yield* HttpClient.HttpClient; + + // A negative size would otherwise *credit* the byte budget, letting a + // token signed for a kilobyte authorize an unbounded write. + const destination = yield* makeTempDir; + const headerBytes = encoder.encode( + '{"path":"planted.txt","size":-10000000000,"kind":"file"}', + ); + const body = new Uint8Array(4 + headerBytes.length); + new DataView(body.buffer).setUint32(0, headerBytes.length, false); + body.set(headerBytes, 4); + + const importUrl = yield* issueProjectSyncImportUrl({ + projectId: "project-http-7", + workspaceRoot: destination, + fileCount: 1, + totalBytes: 1024, + }); + const response = yield* client.post(importUrl.url, { body: HttpBody.uint8Array(body) }); + + expect(response.status).toBe(500); + expect(yield* exists(destination, "planted.txt")).toBe(false); + }), + ).pipe(Effect.provide(testLayer)), + ); + + it.effect("500s an import whose framing header over-reports its content", () => + Effect.scoped( + Effect.gen(function* () { + yield* serveProjectSyncRoutes; + const client = yield* HttpClient.HttpClient; + + const destination = yield* makeTempDir; + const truncated = (yield* Effect.promise(() => + encodeRecords([fileRecord("truncated.txt", "0123456789")]), + )).slice(0, -4); + + const importUrl = yield* issueProjectSyncImportUrl({ + projectId: "project-http-5", + workspaceRoot: destination, + fileCount: 1, + totalBytes: 10, + }); + const response = yield* client.post(importUrl.url, { + body: HttpBody.uint8Array(truncated), + }); + + expect(response.status).toBe(500); + expect(yield* exists(destination, "truncated.txt")).toBe(false); + }), + ).pipe(Effect.provide(testLayer)), + ); +}); diff --git a/apps/server/src/http.ts b/apps/server/src/http.ts index d231283899ea..f66d119b30df 100644 --- a/apps/server/src/http.ts +++ b/apps/server/src/http.ts @@ -333,7 +333,11 @@ export const projectSyncExportRouteLayer = HttpRouter.add( return HttpServerResponse.stream( Stream.fromAsyncIterable( encodeProjectSyncRecords( - projectSyncExportRecords(resolved.claims.workspaceRoot, resolved.paths), + projectSyncExportRecords({ + workspaceRoot: resolved.claims.workspaceRoot, + entries: resolved.entries, + requestId: resolved.claims.requestId, + }), ), (cause) => new ProjectSyncExportStreamError({ cause }), ), @@ -372,6 +376,7 @@ export const projectSyncImportRouteLayer = HttpRouter.add( workspaceRoot: claims.workspaceRoot, records: createProjectSyncFrameDecoder(Stream.toAsyncIterable(request.stream)), maxContentBytes: claims.totalBytes, + maxRecordCount: claims.fileCount, }).pipe( Effect.map((result) => HttpServerResponse.jsonUnsafe({ applied: result.applied, bytes: result.bytes }), diff --git a/apps/server/src/workspace/ProjectSyncApply.test.ts b/apps/server/src/workspace/ProjectSyncApply.test.ts index 298bc3c39883..7b7b2544539e 100644 --- a/apps/server/src/workspace/ProjectSyncApply.test.ts +++ b/apps/server/src/workspace/ProjectSyncApply.test.ts @@ -50,11 +50,13 @@ const applyThroughFraming = (input: { readonly workspaceRoot: string; readonly records: ReadonlyArray; readonly maxContentBytes?: number; + readonly maxRecordCount?: number; }) => applyProjectSyncRecords({ workspaceRoot: input.workspaceRoot, records: createProjectSyncFrameDecoder(encodeProjectSyncRecords(iterate(input.records))), ...(input.maxContentBytes === undefined ? {} : { maxContentBytes: input.maxContentBytes }), + ...(input.maxRecordCount === undefined ? {} : { maxRecordCount: input.maxRecordCount }), }); const readText = (root: string, relativePath: string) => @@ -88,11 +90,13 @@ it.layer(NodeServices.layer, { excludeTestServices: true })("ProjectSyncApply", "a/../../escape.txt", "/etc/passwd", "", - " ", ".", "a/./b", "a//b", "a/\0b", + "a/b/..", + "a/../b", + "..", ]) { expect(resolveProjectSyncRelativePath({ workspaceRoot, relativePath })).toBeNull(); } @@ -103,6 +107,37 @@ it.layer(NodeServices.layer, { excludeTestServices: true })("ProjectSyncApply", }); }), ); + + it.effect("never rewrites a legal path: no trimming, no separator swapping", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + // Whitespace at either edge and a backslash are all ordinary + // characters in a POSIX filename. Normalizing any of them would point + // the export at a file that does not exist, and the entry would be + // dropped from a sync that still reported success. + const onWindows = NodePath.sep === "\\"; + const preserved = onWindows + ? ["docs/notes .md", " leading.md"] + : ["docs/notes .md", " leading.md", "draft v2\\final.md", " "]; + + for (const relativePath of preserved) { + expect( + resolveProjectSyncRelativePath({ workspaceRoot, relativePath })?.relativePath, + ).toBe(relativePath); + } + + // A backslash is a separator on Windows, so a segment carrying one is + // refused there rather than silently split into a traversal. + if (onWindows) { + expect( + resolveProjectSyncRelativePath({ + workspaceRoot, + relativePath: "a\\..\\..\\escape.txt", + }), + ).toBeNull(); + } + }), + ); }); describe("import", () => { @@ -190,6 +225,21 @@ it.layer(NodeServices.layer, { excludeTestServices: true })("ProjectSyncApply", }), ); + it.effect("replaces an ancestor that is a plain file with the directory it must be", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + yield* writeFile(workspaceRoot, "thing", "was a file"); + + yield* applyThroughFraming({ + workspaceRoot, + records: [fileRecord("thing/inner.txt", "now a directory")], + }); + + expect((yield* lstat(workspaceRoot, "thing")).isDirectory()).toBe(true); + expect(yield* readText(workspaceRoot, "thing/inner.txt")).toBe("now a directory"); + }), + ); + it.effect("stops once the signed byte budget is exhausted", () => Effect.gen(function* () { const workspaceRoot = yield* makeTempDir; @@ -204,6 +254,64 @@ it.layer(NodeServices.layer, { excludeTestServices: true })("ProjectSyncApply", expect(yield* exists(workspaceRoot, "b.txt")).toBe(false); }), ); + + it.effect("stops once the signed record count is exhausted", () => + Effect.gen(function* () { + // Zero-byte records cost no budget bytes at all, so without a record + // cap a token signed for two entries would authorize a million empty + // directories and the inodes they take. + const workspaceRoot = yield* makeTempDir; + const error = yield* Effect.flip( + applyThroughFraming({ + workspaceRoot, + records: [dirRecord("one"), dirRecord("two"), dirRecord("three")], + maxContentBytes: 1024, + maxRecordCount: 2, + }), + ); + + expect(error._tag).toBe("ProjectSyncImportLimitError"); + expect(yield* exists(workspaceRoot, "one")).toBe(true); + expect(yield* exists(workspaceRoot, "two")).toBe(true); + expect(yield* exists(workspaceRoot, "three")).toBe(false); + }), + ); + + it.effect("applies exactly the signed record count without tripping the cap", () => + Effect.gen(function* () { + const workspaceRoot = yield* makeTempDir; + const result = yield* applyThroughFraming({ + workspaceRoot, + records: [fileRecord("a.txt", "a"), fileRecord("b.txt", "b")], + maxRecordCount: 2, + }); + expect(result.applied).toBe(2); + }), + ); + + it.effect("re-checks an ancestor the same batch replaced with a file", () => + Effect.gen(function* () { + // Ancestors verified earlier in the batch are cached to keep a large + // sync off a per-record lstat chain; a record that turns a cached + // directory into a file has to drop that knowledge or the next record + // under it would be written against a path that no longer exists. + const workspaceRoot = yield* makeTempDir; + + const result = yield* applyThroughFraming({ + workspaceRoot, + records: [ + fileRecord("thing/first.txt", "under a directory"), + fileRecord("thing", "now a file"), + fileRecord("thing/second.txt", "a directory again"), + ], + }); + + expect(result.applied).toBe(3); + expect((yield* lstat(workspaceRoot, "thing")).isDirectory()).toBe(true); + expect(yield* readText(workspaceRoot, "thing/second.txt")).toBe("a directory again"); + expect(yield* exists(workspaceRoot, "thing/first.txt")).toBe(false); + }), + ); }); describe("deletions", () => { @@ -252,6 +360,27 @@ it.layer(NodeServices.layer, { excludeTestServices: true })("ProjectSyncApply", }), ); + it.effect("treats a path under a non-directory ancestor as already deleted", () => + Effect.gen(function* () { + // The copy pass replaces a destination directory with a file of the + // same name, so the delete pass meets paths whose parent is that file. + // Nothing is left to remove there, and the rest of the batch must + // still run. + const workspaceRoot = yield* makeTempDir; + yield* writeFile(workspaceRoot, "thing", "now a file"); + yield* writeFile(workspaceRoot, "stale.txt", "remove me"); + + const result = yield* applyProjectSyncDeletions({ + workspaceRoot, + paths: ["thing/inner.txt", "stale.txt"], + }); + + expect(result.deleted).toBe(1); + expect(yield* readText(workspaceRoot, "thing")).toBe("now a file"); + expect(yield* exists(workspaceRoot, "stale.txt")).toBe(false); + }), + ); + it.effect("refuses to delete through a symlinked ancestor", () => Effect.gen(function* () { const workspaceRoot = yield* makeTempDir; diff --git a/apps/server/src/workspace/ProjectSyncApply.ts b/apps/server/src/workspace/ProjectSyncApply.ts index 918b2aa2bb4c..7c59fe043c91 100644 --- a/apps/server/src/workspace/ProjectSyncApply.ts +++ b/apps/server/src/workspace/ProjectSyncApply.ts @@ -24,13 +24,14 @@ import * as Schema from "effect/Schema"; import { errnoCode } from "./projectSyncErrno.ts"; -/** Raised when an import body carries more content than the signed URL - authorized. Never crosses the WebSocket contract — the HTTP import route - turns it into a 413. */ +/** Raised when an import body carries more than the signed URL authorized, + either in content bytes or in record count. Never crosses the WebSocket + contract — the HTTP import route turns it into a 413. */ export class ProjectSyncImportLimitError extends Schema.TaggedErrorClass()( "ProjectSyncImportLimitError", { - limitBytes: Schema.Number, + limit: Schema.Number, + unit: Schema.Literals(["bytes", "records"]), message: Schema.String, }, ) {} @@ -46,17 +47,25 @@ export interface ResolvedProjectSyncPath { * plain relative path. The root itself is compared unresolved, so a workspace * that lives behind a symlink (every macOS temp directory, for one) still * works. + * + * The path is never rewritten, only accepted or refused. Trailing spaces and + * backslashes are ordinary characters in a POSIX filename, and normalizing + * either one would turn a real file into a path the origin cannot export — + * a silent omission from a sync that reports itself complete. On Windows a + * backslash *is* a separator, so a segment carrying one is refused there + * rather than quietly split. */ export function resolveProjectSyncRelativePath(input: { readonly workspaceRoot: string; readonly relativePath: string; }): ResolvedProjectSyncPath | null { - const raw = input.relativePath.trim().replaceAll("\\", "/"); + const raw = input.relativePath; if (raw.length === 0 || raw.startsWith("/") || raw.includes("\0")) return null; const segments = raw.split("/"); for (const segment of segments) { if (segment.length === 0 || segment === "." || segment === "..") return null; + if (NodePath.sep === "\\" && segment.includes("\\")) return null; } const workspaceRoot = NodePath.resolve(input.workspaceRoot); @@ -87,47 +96,125 @@ async function lstatOrNull(path: string) { } } +/** + * Absolute directories already proven, during one apply/delete/export pass, to + * be real directories under the workspace root. + * + * A 20k-file sync shares a few thousand ancestors across its records, so + * without this every record re-`lstat`s its whole chain and the pass spends + * most of its syscalls re-answering the same question. Any directory removal + * invalidates the whole set rather than trying to reason about which cached + * descendants it took with it. + */ +export class ProjectSyncAncestorCache { + private readonly verified = new Set(); + + has(absolutePath: string): boolean { + return this.verified.has(absolutePath); + } + + add(absolutePath: string): void { + this.verified.add(absolutePath); + } + + /** A directory just went away, so nothing proven below it can be trusted. */ + invalidate(): void { + this.verified.clear(); + } +} + +type AncestorWalkResult = "ok" | "symlink" | "missing"; + /** * Walks the ancestors of `relativePath` from the workspace root down, refusing * to traverse a symlink and (optionally) materializing the directories that do * not exist yet. Checking before creating matters: `mkdir -p` would happily * follow a symlinked ancestor and create directories outside the root. */ -async function prepareAncestors( +async function walkAncestors( workspaceRoot: string, relativePath: string, - options: { readonly create: boolean }, -): Promise { + options: { readonly create: boolean; readonly cache?: ProjectSyncAncestorCache | undefined }, +): Promise { const segments = relativePath.split("/"); segments.pop(); + const mkdirIgnoringExisting = (path: string) => + NodeFSP.mkdir(path).catch((cause: unknown) => { + if (errnoCode(cause) !== "EEXIST") throw cause; + }); + + const cache = options.cache; let current = workspaceRoot; for (const segment of segments) { current = NodePath.join(current, segment); + if (cache?.has(current)) continue; + const stats = await lstatOrNull(current); if (stats === null) { - if (!options.create) return; - await NodeFSP.mkdir(current).catch((cause: unknown) => { - if (errnoCode(cause) !== "EEXIST") throw cause; - }); + if (!options.create) return "missing"; + await mkdirIgnoringExisting(current); + cache?.add(current); continue; } if (stats.isSymbolicLink()) { - throw new ProjectSyncPathViolationError({ path: relativePath }); + return "symlink"; } if (!stats.isDirectory()) { - throw new ProjectSyncIoError({ - message: `Cannot sync '${relativePath}': '${segment}' exists and is not a directory.`, - }); + // An ancestor that is a plain file cannot hold this entry. Writing means + // the origin replaced that file with a directory, so it is cleared the + // same way a target of the wrong kind is; deleting means the entry is + // already gone, so the caller has nothing left to do. + if (!options.create) return "missing"; + await NodeFSP.rm(current, { force: true }); + await mkdirIgnoringExisting(current); + cache?.add(current); + continue; } + cache?.add(current); } + return "ok"; +} + +async function prepareAncestors( + workspaceRoot: string, + relativePath: string, + options: { readonly create: boolean; readonly cache?: ProjectSyncAncestorCache | undefined }, +): Promise { + if ((await walkAncestors(workspaceRoot, relativePath, options)) === "symlink") { + throw new ProjectSyncPathViolationError({ path: relativePath }); + } +} + +/** + * Read-only version of the ancestor check, for the export side. + * + * The apply pass refuses to *write* through a symlinked ancestor; without the + * same check here a `link -> /Users/someone` planted inside a workspace would + * let a signed export URL *read* `link/.ssh/id_rsa` straight out of the home + * directory. Callers skip the entry the same way they skip one that vanished. + */ +export async function projectSyncAncestorsAreSafe(input: { + readonly workspaceRoot: string; + readonly relativePath: string; + readonly cache?: ProjectSyncAncestorCache | undefined; +}): Promise { + const walked = await walkAncestors(input.workspaceRoot, input.relativePath, { + create: false, + cache: input.cache, + }); + return walked !== "symlink"; } /** Clears whatever currently occupies a target path so the incoming entry can take it, including a directory being replaced by a file. */ -async function clearTarget(absolutePath: string): Promise { +async function clearTarget( + absolutePath: string, + cache?: ProjectSyncAncestorCache | undefined, +): Promise { const stats = await lstatOrNull(absolutePath); if (stats === null) return; + if (stats.isDirectory()) cache?.invalidate(); await NodeFSP.rm(absolutePath, { force: true, recursive: stats.isDirectory() }); } @@ -135,6 +222,7 @@ async function writeFileRecord( absolutePath: string, content: AsyncIterable, mode: number | undefined, + cache: ProjectSyncAncestorCache | undefined, ): Promise { const directory = NodePath.dirname(absolutePath); const partPath = NodePath.join( @@ -149,6 +237,7 @@ async function writeFileRecord( } const existing = await lstatOrNull(absolutePath); if (existing?.isDirectory()) { + cache?.invalidate(); await NodeFSP.rm(absolutePath, { force: true, recursive: true }); } // Rename replaces the entry itself, so an existing symlink at the target is @@ -163,17 +252,18 @@ async function writeFileRecord( async function applyRecord( workspaceRoot: string, record: ProjectSyncFrameDecodedRecord, + cache: ProjectSyncAncestorCache, ): Promise { const { absolutePath, relativePath } = requireResolvedPath({ workspaceRoot, relativePath: record.header.path, }); - await prepareAncestors(workspaceRoot, relativePath, { create: true }); + await prepareAncestors(workspaceRoot, relativePath, { create: true, cache }); if (record.header.kind === "dir") { const stats = await lstatOrNull(absolutePath); if (stats?.isDirectory()) return; - if (stats !== null) await clearTarget(absolutePath); + if (stats !== null) await clearTarget(absolutePath, cache); await NodeFSP.mkdir(absolutePath, { recursive: true }); return; } @@ -185,12 +275,12 @@ async function applyRecord( message: `Symlink record '${relativePath}' arrived without a link target.`, }); } - await clearTarget(absolutePath); + await clearTarget(absolutePath, cache); await NodeFSP.symlink(linkTarget, absolutePath); return; } - await writeFileRecord(absolutePath, record.content, record.header.mode); + await writeFileRecord(absolutePath, record.content, record.header.mode, cache); } const isPathViolationError = Schema.is(ProjectSyncPathViolationError); @@ -221,32 +311,44 @@ export interface ProjectSyncApplyResult { /** * Applies a decoded frame stream into `workspaceRoot`, one record at a time. * - * `maxContentBytes` mirrors the byte budget the signed import URL was issued - * for; exceeding it aborts mid-stream instead of letting a token authorize an - * unbounded write. + * `maxContentBytes` and `maxRecordCount` mirror the two budgets the signed + * import URL was issued for; exceeding either aborts mid-stream instead of + * letting a token authorize unbounded work. The record cap matters on its own: + * a body of a million zero-byte `"dir"` records costs no content bytes at all + * and would still exhaust the destination's inodes. */ export const applyProjectSyncRecords = Effect.fn("ProjectSyncApply.applyRecords")( function* (input: { readonly workspaceRoot: string; readonly records: AsyncIterable; readonly maxContentBytes?: number | undefined; + readonly maxRecordCount?: number | undefined; }) { const workspaceRoot = NodePath.resolve(input.workspaceRoot); return yield* Effect.tryPromise({ try: async (): Promise => { await NodeFSP.mkdir(workspaceRoot, { recursive: true }); + const cache = new ProjectSyncAncestorCache(); let applied = 0; let bytes = 0; for await (const record of input.records) { + if (input.maxRecordCount !== undefined && applied >= input.maxRecordCount) { + throw new ProjectSyncImportLimitError({ + limit: input.maxRecordCount, + unit: "records", + message: `Import body carried more than the ${input.maxRecordCount} entries the URL was signed for.`, + }); + } bytes += record.header.size; if (input.maxContentBytes !== undefined && bytes > input.maxContentBytes) { throw new ProjectSyncImportLimitError({ - limitBytes: input.maxContentBytes, + limit: input.maxContentBytes, + unit: "bytes", message: `Import body exceeded the ${input.maxContentBytes} byte budget the URL was signed for.`, }); } - await applyRecord(workspaceRoot, record); + await applyRecord(workspaceRoot, record, cache); applied += 1; } return { applied, bytes }; @@ -268,10 +370,11 @@ export const applyProjectSyncDeletions = Effect.fn("ProjectSyncApply.applyDeleti try: async () => { let deleted = 0; const emptiedDirectories = new Set(); + const cache = new ProjectSyncAncestorCache(); for (const relativePath of input.paths) { const resolved = requireResolvedPath({ workspaceRoot, relativePath }); - await prepareAncestors(workspaceRoot, resolved.relativePath, { create: false }); + await prepareAncestors(workspaceRoot, resolved.relativePath, { create: false, cache }); const stats = await lstatOrNull(resolved.absolutePath); if (stats === null) continue; @@ -288,6 +391,7 @@ export const applyProjectSyncDeletions = Effect.fn("ProjectSyncApply.applyDeleti }, ); if (!removed) continue; + cache.invalidate(); } else { await NodeFSP.rm(resolved.absolutePath, { force: true }); } diff --git a/apps/server/src/workspace/ProjectSyncRoundTrip.test.ts b/apps/server/src/workspace/ProjectSyncRoundTrip.test.ts new file mode 100644 index 000000000000..509efc5698aa --- /dev/null +++ b/apps/server/src/workspace/ProjectSyncRoundTrip.test.ts @@ -0,0 +1,265 @@ +// @effect-diagnostics nodeBuiltinImport:off +/** + * End-to-end project sync between two real directories. + * + * The three server pieces (manifest walk, export stream, import/delete apply) + * are only correct together: the diff the client computes has to name paths + * the export can serve, the framing has to survive the trip, and the delete + * pass has to leave the destination mirroring the origin. These tests drive + * the whole loop rather than any one piece. + * + * `computeProjectSyncPlan` is imported from its source file because the diff + * lives on the client (client-runtime is not, and should not become, a server + * dependency) while the thing under test here is that the client's plan and + * the server's filesystem work agree. + */ +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { expect, it } from "@effect/vitest"; +import { + createProjectSyncFrameDecoder, + encodeProjectSyncRecords, +} from "@t3tools/shared/projectSyncFraming"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; + +import { computeProjectSyncPlan } from "../../../../packages/client-runtime/src/operations/projectSync.ts"; +import { applyProjectSyncDeletions, applyProjectSyncRecords } from "./ProjectSyncApply.ts"; +import { buildProjectSyncManifest } from "./ProjectSyncManifest.ts"; +import { projectSyncExportRecords } from "./ProjectSyncTransfer.ts"; + +const makeTempDir = Effect.gen(function* () { + const fileSystem = yield* FileSystem.FileSystem; + return yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3code-project-sync-e2e-" }); +}); + +const writeFile = (root: string, relativePath: string, contents: string) => + Effect.promise(async () => { + const absolutePath = NodePath.join(root, relativePath); + await NodeFSP.mkdir(NodePath.dirname(absolutePath), { recursive: true }); + await NodeFSP.writeFile(absolutePath, contents); + return absolutePath; + }); + +const makeDir = (root: string, relativePath: string) => + Effect.promise(() => NodeFSP.mkdir(NodePath.join(root, relativePath), { recursive: true })); + +const makeSymlink = (root: string, relativePath: string, target: string) => + Effect.promise(async () => { + const absolutePath = NodePath.join(root, relativePath); + await NodeFSP.mkdir(NodePath.dirname(absolutePath), { recursive: true }); + await NodeFSP.symlink(target, absolutePath); + }); + +const exists = (root: string, relativePath: string) => + Effect.promise(() => + NodeFSP.lstat(NodePath.join(root, relativePath)).then( + () => true, + () => false, + ), + ); + +const manifestOf = (workspaceRoot: string) => + buildProjectSyncManifest({ workspaceRoot, includeGit: true }); + +const copyEntries = ( + origin: string, + destination: string, + entries: ReadonlyArray<{ readonly path: string; readonly size: number }>, +) => + applyProjectSyncRecords({ + workspaceRoot: destination, + records: createProjectSyncFrameDecoder( + encodeProjectSyncRecords( + projectSyncExportRecords({ + workspaceRoot: origin, + entries: entries.map((entry) => ({ path: entry.path, size: entry.size })), + }), + ), + ), + maxRecordCount: entries.length, + }); + +/** + * Runs one sync exactly the way `runProjectSync` drives it from the client: + * diff both manifests, copy what changed, delete what the origin no longer + * has, then re-assert the origin's empty directories (the delete pass prunes + * the directories it empties, which can take one of those with it). + */ +const syncOnce = (origin: string, destination: string) => + Effect.gen(function* () { + const plan = computeProjectSyncPlan(yield* manifestOf(origin), yield* manifestOf(destination)); + + yield* copyEntries(origin, destination, plan.toCopy); + if (plan.toDelete.length > 0) { + yield* applyProjectSyncDeletions({ workspaceRoot: destination, paths: plan.toDelete }); + yield* copyEntries( + origin, + destination, + plan.toCopy.filter((entry) => entry.kind === "dir"), + ); + } + return plan; + }); + +it.layer(NodeServices.layer, { excludeTestServices: true })("project sync round trip", (it) => { + it.effect("mirrors an origin onto a populated destination in a single pass", () => + Effect.gen(function* () { + const origin = yield* makeTempDir; + const destination = yield* makeTempDir; + + yield* writeFile(origin, "README.md", "# origin\n"); + yield* writeFile(origin, "src/index.ts", "export const answer = 42;\n"); + yield* writeFile(origin, "src/added.ts", "export const added = true;\n"); + const script = yield* writeFile(origin, "bin/run.sh", "#!/bin/sh\necho hi\n"); + yield* Effect.promise(() => NodeFSP.chmod(script, 0o755)); + yield* makeDir(origin, "empty"); + yield* makeSymlink(origin, "link.md", "README.md"); + + // The destination starts out stale in every way the diff has to notice: + // an unchanged file, a changed file, a file the origin dropped, a whole + // subtree the origin dropped, and a symlink pointing somewhere else. + yield* writeFile(destination, "README.md", "# origin\n"); + yield* writeFile(destination, "src/index.ts", "export const answer = 0;\n"); + yield* writeFile(destination, "stale.txt", "remove me\n"); + yield* writeFile(destination, "old/deep/nested.txt", "remove me too\n"); + yield* makeSymlink(destination, "link.md", "elsewhere.md"); + + const plan = yield* syncOnce(origin, destination); + + expect(plan.toCopy.map((entry) => entry.path)).toEqual([ + "bin/run.sh", + "empty", + "link.md", + "src/added.ts", + "src/index.ts", + ]); + // Deepest first, so a sequential delete never orphans a child. + expect(plan.toDelete).toEqual(["old/deep/nested.txt", "stale.txt"]); + + expect(yield* manifestOf(destination)).toEqual(yield* manifestOf(origin)); + // Directories the deleted files lived in are pruned, not left behind as + // empty husks the origin never had. + expect(yield* exists(destination, "old")).toBe(false); + expect(yield* exists(destination, "empty")).toBe(true); + expect( + (yield* Effect.promise(() => + NodeFSP.lstat(NodePath.join(destination, "link.md")), + )).isSymbolicLink(), + ).toBe(true); + expect( + yield* Effect.promise(() => NodeFSP.readlink(NodePath.join(destination, "link.md"))), + ).toBe("README.md"); + }), + ); + + it.effect("keeps an origin's empty directory that the destination filled", () => + Effect.gen(function* () { + const origin = yield* makeTempDir; + const destination = yield* makeTempDir; + + yield* makeDir(origin, "logs"); + yield* writeFile(origin, "keep.txt", "keep\n"); + yield* writeFile(destination, "keep.txt", "keep\n"); + yield* writeFile(destination, "logs/app.log", "noise\n"); + + yield* syncOnce(origin, destination); + + expect(yield* manifestOf(destination)).toEqual(yield* manifestOf(origin)); + expect(yield* exists(destination, "logs")).toBe(true); + expect(yield* exists(destination, "logs/app.log")).toBe(false); + }), + ); + + it.effect("converges: a second pass over a mirrored destination is a no-op", () => + Effect.gen(function* () { + const origin = yield* makeTempDir; + const destination = yield* makeTempDir; + + yield* writeFile(origin, "a/b/c.txt", "content\n"); + yield* makeDir(origin, "empty"); + yield* makeSymlink(origin, "a/link", "b/c.txt"); + yield* writeFile(destination, "gone.txt", "gone\n"); + + yield* syncOnce(origin, destination); + const secondPass = yield* syncOnce(origin, destination); + + expect(secondPass.toCopy).toEqual([]); + expect(secondPass.toDelete).toEqual([]); + expect(yield* manifestOf(destination)).toEqual(yield* manifestOf(origin)); + }), + ); + + it.effect("replaces a destination directory with the origin's file of the same name", () => + Effect.gen(function* () { + const origin = yield* makeTempDir; + const destination = yield* makeTempDir; + + yield* writeFile(origin, "thing", "now a file\n"); + yield* writeFile(destination, "thing/inner.txt", "was a directory\n"); + + yield* syncOnce(origin, destination); + + expect(yield* manifestOf(destination)).toEqual(yield* manifestOf(origin)); + expect( + yield* Effect.promise(() => NodeFSP.readFile(NodePath.join(destination, "thing"), "utf8")), + ).toBe("now a file\n"); + }), + ); + + it.effect("carries filenames with edge whitespace and backslashes through intact", () => + Effect.gen(function* () { + // Every one of these is a legal POSIX filename that an earlier trimming + // or `\`→`/` rewrite would have renamed on the wire: the export would + // then miss the file, and the sync would report itself complete while + // silently omitting it. + const awkward = + NodePath.sep === "\\" + ? ["docs/notes .md", " leading.md"] + : ["docs/notes .md", " leading.md", "draft v2\\final.md", "trailing space "]; + + const origin = yield* makeTempDir; + const destination = yield* makeTempDir; + for (const [index, path] of awkward.entries()) { + yield* writeFile(origin, path, `content ${index}\n`); + } + + const plan = yield* syncOnce(origin, destination); + + expect(plan.toCopy.map((entry) => entry.path).sort()).toEqual([...awkward].sort()); + expect(yield* manifestOf(destination)).toEqual(yield* manifestOf(origin)); + for (const [index, path] of awkward.entries()) { + expect( + yield* Effect.promise(() => NodeFSP.readFile(NodePath.join(destination, path), "utf8")), + ).toBe(`content ${index}\n`); + } + + // And the second pass sees nothing to do, which it could not if either + // side had renamed anything. + const secondPass = yield* syncOnce(origin, destination); + expect(secondPass.toCopy).toEqual([]); + expect(secondPass.toDelete).toEqual([]); + }), + ); + + it.effect("replaces a destination file with the origin's directory of the same name", () => + Effect.gen(function* () { + const origin = yield* makeTempDir; + const destination = yield* makeTempDir; + + yield* writeFile(origin, "thing/inner.txt", "now a directory\n"); + yield* writeFile(destination, "thing", "was a file\n"); + + yield* syncOnce(origin, destination); + + expect(yield* manifestOf(destination)).toEqual(yield* manifestOf(origin)); + expect( + yield* Effect.promise(() => + NodeFSP.readFile(NodePath.join(destination, "thing/inner.txt"), "utf8"), + ), + ).toBe("now a directory\n"); + }), + ); +}); diff --git a/apps/server/src/workspace/ProjectSyncTransfer.test.ts b/apps/server/src/workspace/ProjectSyncTransfer.test.ts index 09f029736959..8b1cd2ebb014 100644 --- a/apps/server/src/workspace/ProjectSyncTransfer.test.ts +++ b/apps/server/src/workspace/ProjectSyncTransfer.test.ts @@ -51,14 +51,46 @@ const writeFile = (root: string, relativePath: string, contents: string) => const tokenOf = (url: string, prefix: string) => url.slice(`${prefix}/`.length); +/** The `{ path, size }` pairs a client sends after diffing a manifest. */ +const exportEntries = (entries: ReadonlyArray<{ readonly path: string; readonly size: number }>) => + entries.map((entry) => ({ path: entry.path, size: entry.size })); + +const pathsOf = (records: ReadonlyArray<{ readonly header: { readonly path: string } }>) => + records.map((record) => record.header.path); + +/** Drains an export stream through the real framing round trip. */ +const collectExport = (input: { + readonly workspaceRoot: string; + readonly entries: ReadonlyArray<{ readonly path: string; readonly size: number }>; + readonly requestId?: string; +}) => + Effect.promise(async () => { + const collected: Array<{ header: { path: string }; content: string }> = []; + const decoder = createProjectSyncFrameDecoder( + encodeProjectSyncRecords(projectSyncExportRecords(input)), + ); + for await (const record of decoder) { + const parts: Uint8Array[] = []; + for await (const chunk of record.content) parts.push(chunk); + collected.push({ + header: record.header, + content: parts.map((part) => new TextDecoder().decode(part)).join(""), + }); + } + return collected; + }); + describe("ProjectSyncTransfer", () => { - it.effect("signs an export URL and resolves it back to its workspace and paths", () => + it.effect("signs an export URL and resolves it back to its workspace and entries", () => Effect.gen(function* () { const workspaceRoot = yield* makeTempDir; const issued = yield* issueProjectSyncExportUrl({ projectId: "project-1", workspaceRoot, - paths: ["src/index.ts", "docs/readme.md"], + entries: [ + { path: "src/index.ts", size: 12 }, + { path: "docs/readme.md", size: 34 }, + ], }); expect(issued.url.startsWith(`${PROJECT_SYNC_EXPORT_ROUTE_PREFIX}/`)).toBe(true); @@ -68,7 +100,10 @@ describe("ProjectSyncTransfer", () => { ); expect(resolved?.claims.workspaceRoot).toBe(workspaceRoot); expect(resolved?.claims.projectId).toBe("project-1"); - expect(resolved?.paths).toEqual(["src/index.ts", "docs/readme.md"]); + expect(resolved?.entries).toEqual([ + { path: "src/index.ts", size: 12 }, + { path: "docs/readme.md", size: 34 }, + ]); }).pipe(Effect.provide(testLayer)), ); @@ -79,7 +114,10 @@ describe("ProjectSyncTransfer", () => { issueProjectSyncExportUrl({ projectId: "project-1", workspaceRoot, - paths: ["src/index.ts", "../../etc/passwd"], + entries: [ + { path: "src/index.ts", size: 0 }, + { path: "../../etc/passwd", size: 0 }, + ], }), ); expect(error._tag).toBe("ProjectSyncPathViolationError"); @@ -92,7 +130,7 @@ describe("ProjectSyncTransfer", () => { const issued = yield* issueProjectSyncExportUrl({ projectId: "project-1", workspaceRoot, - paths: ["a.txt"], + entries: [{ path: "a.txt", size: 0 }], }); const token = tokenOf(issued.url, PROJECT_SYNC_EXPORT_ROUTE_PREFIX); const [payload, signature] = token.split("."); @@ -153,7 +191,7 @@ describe("ProjectSyncTransfer", () => { const issued = yield* issueProjectSyncExportUrl({ projectId: "project-3", workspaceRoot: origin, - paths: manifest.map((entry) => entry.path), + entries: exportEntries(manifest), }); const resolved = yield* resolveProjectSyncExportToken( tokenOf(issued.url, PROJECT_SYNC_EXPORT_ROUTE_PREFIX), @@ -164,7 +202,10 @@ describe("ProjectSyncTransfer", () => { workspaceRoot: destination, records: createProjectSyncFrameDecoder( encodeProjectSyncRecords( - projectSyncExportRecords(resolved!.claims.workspaceRoot, resolved!.paths), + projectSyncExportRecords({ + workspaceRoot: resolved!.claims.workspaceRoot, + entries: resolved!.entries, + }), ), ), }); @@ -188,7 +229,10 @@ describe("ProjectSyncTransfer", () => { const issued = yield* issueProjectSyncExportUrl({ projectId: "project-4", workspaceRoot: origin, - paths: ["kept.txt", "vanishing.txt"], + entries: [ + { path: "kept.txt", size: 4 }, + { path: "vanishing.txt", size: 4 }, + ], }); yield* Effect.promise(() => NodeFSP.rm(NodePath.join(origin, "vanishing.txt"))); @@ -199,7 +243,10 @@ describe("ProjectSyncTransfer", () => { workspaceRoot: destination, records: createProjectSyncFrameDecoder( encodeProjectSyncRecords( - projectSyncExportRecords(resolved!.claims.workspaceRoot, resolved!.paths), + projectSyncExportRecords({ + workspaceRoot: resolved!.claims.workspaceRoot, + entries: resolved!.entries, + }), ), ), }); @@ -212,4 +259,102 @@ describe("ProjectSyncTransfer", () => { ).toBe("kept"); }).pipe(Effect.provide(testLayer)), ); + + it.effect("never exports through a symlinked ancestor", () => + Effect.gen(function* () { + // The apply side refuses to *write* through a symlinked ancestor; without + // the same check here a `link -> $HOME` planted in a workspace would let + // a signed export URL *read* whatever lives under it. + const origin = yield* makeTempDir; + const outside = yield* makeTempDir; + yield* writeFile(outside, ".ssh/id_rsa", "PRIVATE KEY"); + yield* writeFile(origin, "legit.txt", "legit"); + yield* Effect.promise(() => NodeFSP.symlink(outside, NodePath.join(origin, "link"))); + + const records = yield* collectExport({ + workspaceRoot: origin, + entries: [ + { path: "link/.ssh/id_rsa", size: 11 }, + { path: "legit.txt", size: 5 }, + ], + }); + + expect(pathsOf(records)).toEqual(["legit.txt"]); + expect(records[0]!.content).toBe("legit"); + }).pipe(Effect.provide(testLayer)), + ); + + it.effect("skips a file that changed size between the manifest and the fetch", () => + Effect.gen(function* () { + // The destination's import token is signed for the manifest's byte total. + // Emitting the file's new, larger size would blow that budget and 413 the + // whole batch; the entry waits for the next sync instead. + const origin = yield* makeTempDir; + yield* writeFile(origin, "grows.txt", "small"); + yield* writeFile(origin, "shrinks.txt", "0123456789"); + yield* writeFile(origin, "steady.txt", "same"); + + const entries = [ + { path: "grows.txt", size: 5 }, + { path: "shrinks.txt", size: 10 }, + { path: "steady.txt", size: 4 }, + ]; + yield* writeFile(origin, "grows.txt", "small plus a lot more"); + yield* writeFile(origin, "shrinks.txt", "0"); + + const records = yield* collectExport({ workspaceRoot: origin, entries }); + + expect(pathsOf(records)).toEqual(["steady.txt"]); + expect(records[0]!.content).toBe("same"); + }).pipe(Effect.provide(testLayer)), + ); + + it.effect("releases a pending export once its stream completes", () => + Effect.gen(function* () { + const origin = yield* makeTempDir; + yield* writeFile(origin, "a.txt", "aaaa"); + + const issued = yield* issueProjectSyncExportUrl({ + projectId: "project-5", + workspaceRoot: origin, + entries: [{ path: "a.txt", size: 4 }], + }); + const token = tokenOf(issued.url, PROJECT_SYNC_EXPORT_ROUTE_PREFIX); + + const resolved = yield* resolveProjectSyncExportToken(token); + expect(resolved).not.toBeNull(); + + // `requestId` is what makes the URL single-use, so it travels with the + // stream exactly the way the HTTP route passes it. + const records = yield* collectExport({ + workspaceRoot: resolved!.claims.workspaceRoot, + entries: resolved!.entries, + requestId: resolved!.claims.requestId, + }); + expect(pathsOf(records)).toEqual(["a.txt"]); + + expect(yield* resolveProjectSyncExportToken(token)).toBeNull(); + }).pipe(Effect.provide(testLayer)), + ); + + it.effect("keeps unexpired tokens alive while many URLs are issued", () => + Effect.gen(function* () { + // The registry used to evict the oldest entry once 64 piled up, so a + // burst of concurrent syncs could 404 a token that was still in flight. + const workspaceRoot = yield* makeTempDir; + const tokens: Array = []; + for (let index = 0; index < 200; index += 1) { + const issued = yield* issueProjectSyncExportUrl({ + projectId: "project-6", + workspaceRoot, + entries: [{ path: `file-${index}.txt`, size: 0 }], + }); + tokens.push(tokenOf(issued.url, PROJECT_SYNC_EXPORT_ROUTE_PREFIX)); + } + + const firstResolved = yield* resolveProjectSyncExportToken(tokens[0]!); + expect(firstResolved?.entries).toEqual([{ path: "file-0.txt", size: 0 }]); + expect(yield* resolveProjectSyncExportToken(tokens.at(-1)!)).not.toBeNull(); + }).pipe(Effect.provide(testLayer)), + ); }); diff --git a/apps/server/src/workspace/ProjectSyncTransfer.ts b/apps/server/src/workspace/ProjectSyncTransfer.ts index ff2f861268d5..15be747bcd72 100644 --- a/apps/server/src/workspace/ProjectSyncTransfer.ts +++ b/apps/server/src/workspace/ProjectSyncTransfer.ts @@ -26,33 +26,38 @@ import { PROJECT_SYNC_EXPORT_ROUTE_PREFIX, PROJECT_SYNC_IMPORT_ROUTE_PREFIX, PROJECT_SYNC_URL_TTL_MS, + type ProjectSyncExportEntry, ProjectSyncIoError, ProjectSyncPathViolationError, } from "@t3tools/contracts"; import type { ProjectSyncFrameRecord } from "@t3tools/shared/projectSyncFraming"; import * as Clock from "effect/Clock"; import * as Effect from "effect/Effect"; -import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; -import { - base64UrlDecodeUtf8, - base64UrlEncode, - signPayload, - timingSafeEqualBase64Url, -} from "../auth/utils.ts"; +import { base64UrlEncode, signPayload, verifySignedClaims } from "../auth/utils.ts"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { isMissingOrUnreadable } from "./projectSyncErrno.ts"; -import { resolveProjectSyncRelativePath } from "./ProjectSyncApply.ts"; +import { + ProjectSyncAncestorCache, + projectSyncAncestorsAreSafe, + resolveProjectSyncRelativePath, +} from "./ProjectSyncApply.ts"; /** Shared with asset download and attachment upload tokens; the signed `kind` is what keeps a token from being replayed against another route. */ const SIGNING_SECRET_NAME = "asset-access-signing-key"; -/** A pending export holds its whole path list in memory, so the registry is - bounded on both ends: entries expire with their URL, and the oldest is - evicted once too many pile up. */ -const MAX_PENDING_EXPORTS = 64; +/** + * Backstop on the pending-export registry. + * + * A registration is normally short-lived: it is released the moment its export + * finishes streaming, and otherwise expires with its URL. The cap only exists + * so a peer that issues URLs it never fetches cannot grow the map without + * bound, and it is set high enough that legitimate concurrent syncs never + * evict each other's in-flight token. + */ +const MAX_PENDING_EXPORTS = 1024; const ProjectSyncExportClaims = Schema.Struct({ version: Schema.Literal(1), @@ -84,7 +89,7 @@ const decodeImportClaims = Schema.decodeUnknownOption(importClaimsJson); const encodeImportClaims = Schema.encodeSync(importClaimsJson); interface PendingExport { - readonly paths: ReadonlyArray; + readonly entries: ReadonlyArray; readonly expiresAt: number; } @@ -94,6 +99,7 @@ function sweepPendingExports(nowMs: number): void { for (const [requestId, pending] of pendingExports) { if (pending.expiresAt <= nowMs) pendingExports.delete(requestId); } + // Only reached if expiry alone did not bring the map back under the cap. while (pendingExports.size >= MAX_PENDING_EXPORTS) { const oldest = pendingExports.keys().next(); if (oldest.done) break; @@ -111,64 +117,40 @@ const loadSigningSecret = Effect.gen(function* () { ), ); -function splitToken(token: string): { payload: string; signature: string } | null { - const [payload, signature, unexpected] = token.split("."); - if (!payload || !signature || unexpected) return null; - return { payload, signature }; -} - -const verifyToken = Effect.fn("ProjectSyncTransfer.verifyToken")(function* (token: string) { - const parts = splitToken(token); - if (!parts) return null; - - const secret = yield* loadSigningSecret.pipe( - Effect.tapError((cause) => - Effect.logError("Failed to load the project sync signing key.", { cause }), - ), - Effect.orElseSucceed(() => null), - ); - if (!secret || !timingSafeEqualBase64Url(parts.signature, signPayload(parts.payload, secret))) { - return null; - } - return parts.payload; -}); - -function decodeClaimsPayload( - payload: string, - decode: (input: unknown) => Option.Option, -): A | null { - try { - return Option.getOrNull(decode(base64UrlDecodeUtf8(payload))); - } catch { - return null; - } -} +/** The signing key, or `null` when it cannot be loaded — in which case no + token can be considered valid. */ +const loadSigningSecretOrNull = loadSigningSecret.pipe( + Effect.tapError((cause) => + Effect.logError("Failed to load the project sync signing key.", { cause }), + ), + Effect.orElseSucceed(() => null), +); export const issueProjectSyncExportUrl = Effect.fn("ProjectSyncTransfer.issueExportUrl")( function* (input: { readonly projectId: string; readonly workspaceRoot: string; - readonly paths: ReadonlyArray; + readonly entries: ReadonlyArray; }) { const secret = yield* loadSigningSecret; const nowMs = yield* Clock.currentTimeMillis; - const paths: Array = []; - for (const relativePath of input.paths) { + const entries: Array = []; + for (const entry of input.entries) { const resolved = resolveProjectSyncRelativePath({ workspaceRoot: input.workspaceRoot, - relativePath, + relativePath: entry.path, }); if (resolved === null) { - return yield* new ProjectSyncPathViolationError({ path: relativePath }); + return yield* new ProjectSyncPathViolationError({ path: entry.path }); } - paths.push(resolved.relativePath); + entries.push({ path: resolved.relativePath, size: entry.size }); } sweepPendingExports(nowMs); const requestId = NodeCrypto.randomUUID(); const expiresAt = nowMs + PROJECT_SYNC_URL_TTL_MS; - pendingExports.set(requestId, { paths, expiresAt }); + pendingExports.set(requestId, { entries, expiresAt }); const payload = base64UrlEncode( encodeExportClaims({ @@ -190,17 +172,19 @@ export const issueProjectSyncExportUrl = Effect.fn("ProjectSyncTransfer.issueExp export interface ResolvedProjectSyncExport { readonly claims: ProjectSyncExportClaims; - readonly paths: ReadonlyArray; + readonly entries: ReadonlyArray; } export const resolveProjectSyncExportToken = Effect.fn("ProjectSyncTransfer.resolveExportToken")( function* (token: string) { - const payload = yield* verifyToken(token); - if (!payload) return null; - - const claims = decodeClaimsPayload(payload, decodeExportClaims); const nowMs = yield* Clock.currentTimeMillis; - if (!claims || claims.expiresAt <= nowMs) return null; + const claims = verifySignedClaims({ + token, + secret: yield* loadSigningSecretOrNull, + nowMs, + decode: decodeExportClaims, + }); + if (!claims) return null; const pending = pendingExports.get(claims.requestId); if (!pending || pending.expiresAt <= nowMs) { @@ -208,7 +192,7 @@ export const resolveProjectSyncExportToken = Effect.fn("ProjectSyncTransfer.reso return null; } - return { claims, paths: pending.paths } satisfies ResolvedProjectSyncExport; + return { claims, entries: pending.entries } satisfies ResolvedProjectSyncExport; }, ); @@ -244,12 +228,12 @@ export const issueProjectSyncImportUrl = Effect.fn("ProjectSyncTransfer.issueImp export const resolveProjectSyncImportToken = Effect.fn("ProjectSyncTransfer.resolveImportToken")( function* (token: string) { - const payload = yield* verifyToken(token); - if (!payload) return null; - - const claims = decodeClaimsPayload(payload, decodeImportClaims); - if (!claims || claims.expiresAt <= (yield* Clock.currentTimeMillis)) return null; - return claims; + return verifySignedClaims({ + token, + secret: yield* loadSigningSecretOrNull, + nowMs: yield* Clock.currentTimeMillis, + decode: decodeImportClaims, + }); }, ); @@ -259,19 +243,42 @@ const EMPTY_CONTENT = new Uint8Array(0); * Streams the requested entries as framing records. * * The manifest the client diffed is a snapshot; by the time it asks for these - * bytes an agent may have deleted or replaced any of them. A vanished entry is - * skipped rather than failing the transfer — the next sync reconciles it — but - * a file's size is taken from the handle we are about to read so the declared - * frame length and the bytes we emit come from the same open file. + * bytes an agent may have deleted or replaced any of them. Three things can + * therefore make an entry drop out here, all of them silent because the next + * sync reconciles them: + * + * - it vanished; + * - one of its ancestors is a symlink, which would read outside the root; + * - its size no longer matches the size the client signed a budget for. + * + * That last one is what keeps a growing file from failing the whole transfer: + * the destination's token authorizes exactly the manifest's bytes, so emitting + * the file's *current* larger size would blow the budget and 413 the batch. */ -export async function* projectSyncExportRecords( - workspaceRoot: string, - paths: ReadonlyArray, -): AsyncGenerator { - for (const relativePath of paths) { - const resolved = resolveProjectSyncRelativePath({ workspaceRoot, relativePath }); +export async function* projectSyncExportRecords(input: { + readonly workspaceRoot: string; + readonly entries: ReadonlyArray; + /** When set, the pending registration is released once the stream finishes, + making the signed URL single-use. */ + readonly requestId?: string | undefined; +}): AsyncGenerator { + const { workspaceRoot, entries } = input; + const ancestorCache = new ProjectSyncAncestorCache(); + + for (const entry of entries) { + const resolved = resolveProjectSyncRelativePath({ workspaceRoot, relativePath: entry.path }); if (resolved === null) continue; + if ( + !(await projectSyncAncestorsAreSafe({ + workspaceRoot, + relativePath: resolved.relativePath, + cache: ancestorCache, + })) + ) { + continue; + } + let stats; try { stats = await NodeFSP.lstat(resolved.absolutePath); @@ -314,6 +321,10 @@ export async function* projectSyncExportRecords( try { const fileStats = await handle.stat(); const size = fileStats.size; + // The client signed a byte budget built from `entry.size`; a file that + // changed size since then is a different file than the one this transfer + // was authorized for, so it waits for the next sync. + if (size !== entry.size) continue; yield { header: { path: resolved.relativePath, @@ -327,6 +338,13 @@ export async function* projectSyncExportRecords( await handle.close().catch(() => {}); } } + + // Reached only when every entry has been streamed: the URL is single-use, and + // an aborted transfer leaves the registration to expire so a retry of the + // same token still works. + if (input.requestId !== undefined) { + pendingExports.delete(input.requestId); + } } /** Reads at most `size` bytes so a file that grew since it was stat'd cannot diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 9f4fe280fc3d..dfc86c2d15a8 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -2011,7 +2011,7 @@ const makeWsRpcLayer = ( return yield* issueProjectSyncExportUrl({ projectId: input.projectId, workspaceRoot, - paths: input.paths, + entries: input.entries, }); }), { "rpc.aggregate": "workspace" }, diff --git a/apps/web/src/components/CommandPalette.tsx b/apps/web/src/components/CommandPalette.tsx index 9c07a3586322..c4402ee49059 100644 --- a/apps/web/src/components/CommandPalette.tsx +++ b/apps/web/src/components/CommandPalette.tsx @@ -3,6 +3,7 @@ import { scopeProjectRef, scopeThreadRef } from "@t3tools/client-runtime/environment"; import { canCreateProjectInEnvironment, + getAddProjectInitialQuery, getCloneDestinationBrowsePath, getCloneDestinationPath, getCloneDirectoryName, @@ -82,7 +83,6 @@ import { useThreadSearch } from "../state/queries"; import { resolveThreadActionProjectRef, startNewThreadFromContext } from "../lib/chatThreadActions"; import { appendBrowsePathSegment, - ensureBrowseDirectoryPath, findProjectByPath, getBrowseDirectoryPath, hasTrailingPathSeparator, @@ -882,12 +882,9 @@ function OpenCommandPaletteDialog(props: { const environment = environments.find( (candidate) => candidate.environmentId === environmentId, ); - const environmentSettings = environment?.serverConfig?.settings ?? null; - const baseDirectory = environmentSettings?.addProjectBaseDirectory?.trim() ?? ""; - if (baseDirectory.length === 0) { - return "~/"; - } - return ensureBrowseDirectoryPath(baseDirectory); + return getAddProjectInitialQuery( + environment?.serverConfig?.settings?.addProjectBaseDirectory, + ); }, [environments], ); diff --git a/apps/web/src/components/SyncProjectDialog.tsx b/apps/web/src/components/SyncProjectDialog.tsx index 53a907cd47bf..cc19432e45e5 100644 --- a/apps/web/src/components/SyncProjectDialog.tsx +++ b/apps/web/src/components/SyncProjectDialog.tsx @@ -1,16 +1,33 @@ -import { - computeProjectSyncPlan, - summarizeProjectSyncPlan, -} from "@t3tools/client-runtime/operations/project-sync"; +import type { ProjectSyncPlanSummary } from "@t3tools/client-runtime/operations/project-sync"; import { getBrowseDirectoryPath } from "@t3tools/client-runtime/state/projects"; import { + planProjectSync, runProjectSync, + type ProjectSyncPlanResult, type ProjectSyncProgress, } from "@t3tools/client-runtime/state/project-sync"; import { isAtomCommandInterrupted, squashAtomCommandFailure, } from "@t3tools/client-runtime/state/runtime"; +import { + buildDefaultSendDestinationPath, + buildSyncProjectDialogSteps, + canStartProjectSync, + describeProjectSyncError, + describeProjectSyncPlanSummary, + deriveProjectSyncProgressPercent, + describeProjectSyncStage, + isProjectSyncEnvironmentEligible, + projectSyncPlanNeedsDeleteConfirmation, + selectDestinationProjectCandidates, + selectProjectSyncEnvironmentOptions, + sortDestinationProjectCandidatesBySourceMatch, + type SyncEnvironmentCandidate, + type SyncProjectCandidate, + type SyncProjectDialogMode, + type SyncProjectErrorDescription, +} from "@t3tools/client-runtime/operations/project-sync-flow"; import { EnvironmentId, type ProjectId } from "@t3tools/contracts"; import { AlertTriangleIcon, @@ -29,23 +46,6 @@ import { projectEnvironment } from "../state/projects"; import { useProjectSyncDeps } from "../state/projectSync"; import { useAtomCommand } from "../state/use-atom-command"; import { AnimatedHeight } from "./AnimatedHeight"; -import { - buildDefaultSendDestinationPath, - buildSyncProjectDialogSteps, - canStartProjectSync, - describeProjectSyncError, - describeProjectSyncPlanSummary, - deriveProjectSyncProgressPercent, - describeProjectSyncStage, - projectSyncPlanNeedsDeleteConfirmation, - selectDestinationProjectCandidates, - selectProjectSyncEnvironmentOptions, - sortDestinationProjectCandidatesBySourceMatch, - type SyncEnvironmentCandidate, - type SyncProjectCandidate, - type SyncProjectDialogMode, - type SyncProjectErrorDescription, -} from "./SyncProjectDialog.logic"; import { Button } from "./ui/button"; import { Checkbox } from "./ui/checkbox"; import { @@ -80,7 +80,26 @@ export interface SyncProjectDialogProps { readonly initialSource?: SyncProjectDialogSource; } -type RunState = "idle" | "running" | "done" | "error" | "cancelled"; +/** + * The progress step's whole story in one value: a run is either untouched, + * in flight (with its latest progress snapshot), finished (with the summary + * it produced), or over with a described failure. Keeping these as separate + * `useState` slots lets combinations exist that never should — a "done" run + * still holding the previous attempt's error, say. + */ +type SyncRunState = + | { readonly status: "idle" } + | { readonly status: "running"; readonly progress: ProjectSyncProgress | null } + | { readonly status: "done"; readonly summary: ProjectSyncPlanSummary } + | { + readonly status: "failed"; + readonly error: SyncProjectErrorDescription; + /** The user aborted this run rather than it breaking on its own, so + there is nothing to retry — they can simply start again. */ + readonly cancelled: boolean; + }; + +const IDLE_RUN_STATE: SyncRunState = { status: "idle" }; export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncProjectDialogProps) { const { environments } = useEnvironments(); @@ -101,24 +120,18 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro const [setAsDefaultFolder, setSetAsDefaultFolder] = useState(false); const [existingDestProjectId, setExistingDestProjectId] = useState(null); const [includeGit, setIncludeGit] = useState(true); - const [planSummary, setPlanSummary] = useState<{ - readonly copyCount: number; - readonly deleteCount: number; - readonly copyBytes: number; - } | null>(null); + // The plan the review step showed and the user confirmed, kept whole (not + // just its summary) so `runProjectSync` executes exactly it instead of + // walking both workspaces a second time to recompute a possibly different + // one. + const [planResult, setPlanResult] = useState(null); const [isLoadingPlan, setIsLoadingPlan] = useState(false); const [planError, setPlanError] = useState(null); const [planRetryToken, setPlanRetryToken] = useState(0); const [deleteConfirmed, setDeleteConfirmed] = useState(false); - const [progress, setProgress] = useState(null); - const [runState, setRunState] = useState("idle"); - const [runError, setRunError] = useState(null); - const [finalSummary, setFinalSummary] = useState<{ - readonly copyCount: number; - readonly deleteCount: number; - readonly copyBytes: number; - } | null>(null); + const [runState, setRunState] = useState(IDLE_RUN_STATE); const abortControllerRef = useRef(null); + const planSummary = planResult?.summary ?? null; // Reset every field whenever the dialog opens, so a previous run's choices // never leak into the next one. @@ -136,15 +149,12 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro setSetAsDefaultFolder(false); setExistingDestProjectId(null); setIncludeGit(true); - setPlanSummary(null); + setPlanResult(null); setIsLoadingPlan(false); setPlanError(null); setPlanRetryToken(0); setDeleteConfirmed(false); - setProgress(null); - setRunState("idle"); - setRunError(null); - setFinalSummary(null); + setRunState(IDLE_RUN_STATE); abortControllerRef.current = null; }, [open]); @@ -198,6 +208,13 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro [environmentCandidates], ); + // A fixed source bypasses the origin picker, so its environment is the one + // end of the sync nothing else has vetted. + const sourceEnvironmentEligible = isProjectSyncEnvironmentEligible( + environmentCandidates, + source?.environmentId ?? null, + ); + const originEnvironmentProjects = useMemo( () => originEnvironmentId === null @@ -230,14 +247,20 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro } }, [mode, defaultSendDestinationPath, sendDestinationPathTouched]); - // Any change to the destination environment or mode invalidates a - // previously picked existing project and any plan computed against it. + // A previously picked existing project may not even exist on a different + // destination environment, and means nothing in "send" mode. useEffect(() => { setExistingDestProjectId(null); - setPlanSummary(null); + }, [destEnvironmentId, mode]); + + // Every input the plan was computed from. Change any of them and the stored + // plan no longer describes what would happen, so it is dropped (and the + // review step recomputes it) rather than being handed to `runProjectSync`. + useEffect(() => { + setPlanResult(null); setPlanError(null); setDeleteConfirmed(false); - }, [destEnvironmentId, mode]); + }, [destEnvironmentId, mode, existingDestProjectId, includeGit, source]); const destProjectCandidates: SyncProjectCandidate[] = useMemo(() => { if (source === null || destEnvironmentId === null) { @@ -261,7 +284,13 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro }); }, [allProjects, destEnvironmentId, source, sourceProject]); - const currentStep = steps[Math.min(stepIndex, steps.length - 1)]!; + // A fixed source whose environment cannot sync replaces the whole flow with + // an explanation: none of the steps below could do anything useful, and the + // manifest RPC would only fail later with a much worse message. + const sourceBlocked = hasFixedSource && !sourceEnvironmentEligible; + const currentStep = sourceBlocked + ? ("unavailable" as const) + : steps[Math.min(stepIndex, steps.length - 1)]!; // Fetch both manifests and compute the plan once the user reaches the // review step in "sync" mode — this is what surfaces the delete warning @@ -281,22 +310,16 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro let cancelled = false; setIsLoadingPlan(true); setPlanError(null); - setPlanSummary(null); + setPlanResult(null); (async () => { try { - const [sourceManifest, destManifest] = await Promise.all([ - deps.getManifest( - { environmentId: source.environmentId, projectId: source.projectId }, - includeGit, - ), - deps.getManifest( - { environmentId: destEnvironmentId, projectId: existingDestProjectId }, - includeGit, - ), - ]); + const result = await planProjectSync(deps, { + source: { environmentId: source.environmentId, projectId: source.projectId }, + dest: { environmentId: destEnvironmentId, projectId: existingDestProjectId }, + includeGit, + }); if (cancelled) return; - const plan = computeProjectSyncPlan(sourceManifest.entries, destManifest.entries); - setPlanSummary(summarizeProjectSyncPlan(plan)); + setPlanResult(result); } catch (error) { if (cancelled) return; setPlanError(describeProjectSyncError(error).message); @@ -329,14 +352,17 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro return; } setStepIndex(steps.length - 1); - setRunState("running"); - setRunError(null); - setProgress(null); + setRunState({ status: "running", progress: null }); const controller = new AbortController(); abortControllerRef.current = controller; let destProjectId: ProjectId; - if (mode === "send") { + if (mode === "send" && existingDestProjectId !== null) { + // Retrying a send that failed after its project was created must reuse + // that project: the destination refuses a second active project on the + // same workspace root, so creating again would fail every time. + destProjectId = existingDestProjectId; + } else if (mode === "send") { const trimmedPath = sendDestinationPath.trim(); if (setAsDefaultFolder) { updateDestSettings({ addProjectBaseDirectory: getBrowseDirectoryPath(trimmedPath) }); @@ -354,11 +380,14 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro if (createResult._tag !== "Success") { abortControllerRef.current = null; if (isAtomCommandInterrupted(createResult)) { - setRunState("idle"); + setRunState(IDLE_RUN_STATE); return; } - setRunState("error"); - setRunError(describeProjectSyncError(squashAtomCommandFailure(createResult))); + setRunState({ + status: "failed", + error: describeProjectSyncError(squashAtomCommandFailure(createResult)), + cancelled: false, + }); return; } destProjectId = createdProjectId; @@ -378,13 +407,19 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro mode, includeGit, signal: controller.signal, - onProgress: setProgress, + onProgress: (next) => setRunState({ status: "running", progress: next }), + // Only "sync" reaches the review step with a plan; a "send" targets a + // project that did not exist when the user confirmed, so there is + // nothing to reuse and `runProjectSync` plans it itself. + ...(mode === "sync" && planResult !== null ? { precomputedPlan: planResult } : {}), }); - setFinalSummary(summary); - setRunState("done"); + setRunState({ status: "done", summary }); } catch (error) { - setRunState(controller.signal.aborted ? "cancelled" : "error"); - setRunError(describeProjectSyncError(error)); + setRunState({ + status: "failed", + error: describeProjectSyncError(error), + cancelled: controller.signal.aborted, + }); } finally { abortControllerRef.current = null; } @@ -395,6 +430,7 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro existingDestProjectId, includeGit, mode, + planResult, sendDestinationPath, setAsDefaultFolder, source, @@ -434,7 +470,7 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro deleteConfirmed, }); - const busy = runState === "running"; + const busy = runState.status === "running"; return ( + {currentStep === "unavailable" ? ( +
+ + + This project's environment is not connected, or its server is too old to support + project sync. Reconnect or update it, then try again. + +
+ ) : null} + {currentStep === "origin" ? ( ) : null} - {currentStep === "progress" ? ( - - ) : null} + {currentStep === "progress" ? : null}
@@ -557,7 +596,9 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro ) : ( <> - {runState === "error" && runError?.canRetry ? ( + {runState.status === "failed" && + !runState.cancelled && + runState.error.canRetry ? ( @@ -590,7 +631,7 @@ export function SyncProjectDialog({ open, onOpenChange, initialSource }: SyncPro > Start sync - ) : ( + ) : currentStep === "unavailable" ? null : (