diff --git a/scripts/lib/pylon-consumer-lock.mjs b/scripts/lib/pylon-consumer-lock.mjs index bc3c6620e7..384b671ad3 100644 --- a/scripts/lib/pylon-consumer-lock.mjs +++ b/scripts/lib/pylon-consumer-lock.mjs @@ -1185,6 +1185,42 @@ async function readApplied(context, claim, terminal, options) { ); } +async function authenticateAppliedCommit(context, claim, terminal, options) { + for (const transaction of terminal.transactions) { + await revalidateAuthority(context, "read-applied-transition", options); + const existing = await readExactMetadata( + transitionPath(context, transaction.baseDigest), + options.metadataMaxBytes, + (value) => validateTransaction(value, transaction.baseDigest, options.stateMaxBytes).value, + "Consumer high-water applied transaction", + options, + ); + if (existing === null || !metadataBytes(existing).equals(metadataBytes(transaction))) { + throw new Error("Consumer high-water applied marker does not authenticate its exact immutable transaction chain."); + } + } + const tip = await walkTransactions(context, options); + const terminalDigest = terminal.transactions.at(-1).candidateDigest; + if (tip.tipDigest !== terminalDigest) { + await revalidateAuthority(context, "read-applied-continuation", options); + const continuation = await readExactMetadata( + transitionPath(context, terminalDigest), + options.metadataMaxBytes, + (value) => validateTransaction(value, terminalDigest, options.stateMaxBytes).value, + "Consumer high-water applied transaction continuation", + options, + ); + if (continuation === null) { + throw new Error("Consumer high-water applied marker does not authenticate its exact terminal digest."); + } + } + const repairedTip = await repairProjection(context, tip, options, claim); + const projection = await readProjection(context, "read-applied-projection", options); + if (projection.malformed || projection.sha256 !== repairedTip.tipDigest) { + throw new Error("Consumer high-water applied marker does not authenticate the current immutable tip and projection."); + } +} + async function finishCommitAtAuthenticatedDescendant(context, options) { const scan = await scanJournalRoot(context.statePath, context.journalDirectory, options); const head = scan.head; @@ -1206,7 +1242,11 @@ async function finishCommitAtAuthenticatedDescendant(context, options) { async function finishCommit(context, claim, terminal, options) { for (let attempt = 0; attempt < PROJECTION_RETRY_LIMIT; attempt += 1) { try { - await readApplied(context, claim, terminal, options); + const applied = await readApplied(context, claim, terminal, options); + if (applied !== null) { + await authenticateAppliedCommit(context, claim, terminal, options); + return; + } for (const transaction of terminal.transactions) await publishTransition(context, transaction, claim, options); const tip = await walkTransactions(context, options); await repairProjection(context, tip, options, claim); @@ -2785,7 +2825,16 @@ async function runRotation(statePath, rawOptions) { const initialScan = await scanEpoch(context, options); const completed = await recoverCompletedCurrentRotation(context, initialScan, options); if (completed) return completed; - const frontier = await resolveOperationFrontier(context, options); + const expectedIntent = rotationIntentFor(context, await effectiveTip(context, options)); + let frontier; + try { + frontier = await resolveOperationFrontier(context, options); + } catch (error) { + if (error?.message !== "Consumer high-water journal epoch changed and fenced a paused writer.") throw error; + const completedResult = await completedRotationResult(context, expectedIntent, options); + if (completedResult) return completedResult; + throw error; + } if (frontier.rotated) continue; if (frontier.active) { throw new Error("Consumer high-water state is actively locked; rotation will retry after the claim quiesces."); diff --git a/scripts/pylon-publication.test.mjs b/scripts/pylon-publication.test.mjs index 9c85a87307..37e359b446 100644 --- a/scripts/pylon-publication.test.mjs +++ b/scripts/pylon-publication.test.mjs @@ -2348,6 +2348,24 @@ test("consumer state locking, recovery, transaction fencing, durability, and pat ); } + const appliedReceiptPath = join(fixture, "already-applied-rotation.json"); + await withConsumerStateLock(appliedReceiptPath, async (_path, transaction) => { + await transaction.commitState(bytes("already-applied-anchor")); + }, manualRuntime({ value: 1 })); + const redundantAppliedWrites = []; + const appliedReceiptRotation = await rotateConsumerStateJournal(appliedReceiptPath, manualRuntime({ value: 2 }, { + afterFileSync: async ({ kind }) => { + if (!["transition", "applied"].includes(kind)) return; + redundantAppliedWrites.push(kind); + throw new Error(`already-applied claim attempted a redundant ${kind} publication`); + }, + })); + assert.deepEqual(redundantAppliedWrites, []); + assert.deepEqual(appliedReceiptRotation, { + epoch: 2, + tipSha256: sha256Bytes(bytes("already-applied-anchor")), + }); + const legacyRotationPath = join(fixture, "legacy-rotation.json"); writeFileSync(legacyRotationPath, bytes("legacy-anchor"), { mode: 0o600 }); assert.equal((await rotateConsumerStateJournal(legacyRotationPath, manualRuntime({ value: 1 }))).epoch, 2); @@ -2599,6 +2617,41 @@ test("consumer state locking, recovery, transaction fencing, durability, and pat rmSync(join(liveTemporaryDirectory, liveTemporaryName)); assert.equal((await rotateConsumerStateJournal(liveTemporaryPath, manualRuntime({ value: 4 }))).epoch, 2); + const frontierHandoffPath = join(fixture, "frontier-completion-handoff.json"); + await withConsumerStateLock(frontierHandoffPath, async (_path, transaction) => { + await transaction.commitState(bytes("frontier-handoff-anchor")); + }, manualRuntime({ value: 1 })); + const firstSecondScanReached = deferred(); + const secondSecondScanReached = deferred(); + const releaseFirstSecondScan = deferred(); + const releaseSecondSecondScan = deferred(); + const rotationAtSecondScanBarrier = (reached, release) => { + let claimScans = 0; + return rotateConsumerStateJournal(frontierHandoffPath, manualRuntime({ value: 2 }, { + beforePathOperation: async ({ operation }) => { + if (operation !== "scan-claims") return; + claimScans += 1; + if (claimScans !== 2) return; + reached.resolve(); + await release.promise; + }, + })); + }; + const frontierWinner = rotationAtSecondScanBarrier(firstSecondScanReached, releaseFirstSecondScan); + const frontierResumed = rotationAtSecondScanBarrier(secondSecondScanReached, releaseSecondSecondScan); + await Promise.all([firstSecondScanReached.promise, secondSecondScanReached.promise]); + releaseFirstSecondScan.resolve(); + const frontierWinnerResult = await frontierWinner; + releaseSecondSecondScan.resolve(); + const frontierResumedResult = await frontierResumed; + assert.equal(Buffer.from(JSON.stringify(frontierResumedResult)).equals( + Buffer.from(JSON.stringify(frontierWinnerResult)), + ), true, "the stale rotator returns the winner's byte-identical completion receipt"); + assert.deepEqual(frontierWinnerResult, { + epoch: 2, + tipSha256: sha256Bytes(bytes("frontier-handoff-anchor")), + }); + const concurrentRotationPath = join(fixture, "concurrent-rotation.json"); await withConsumerStateLock(concurrentRotationPath, async (_path, transaction) => { await transaction.commitState(bytes("concurrent-anchor"));