From 83615aa7cb5a131280c1c7d52aaa7098aa69ecc2 Mon Sep 17 00:00:00 2001 From: piotr Date: Wed, 16 Sep 2026 17:33:05 +0000 Subject: [PATCH 1/7] Post step: bounded incremental mirror maintenance; maintenance failure no longer vetoes the commit Replace the post-step `git gc --auto` with a geometric repack that only folds small packs and loose objects, marking packs >= 256 MiB with .keep so they are never rewritten. A maintenance timeout or failure is reported but no longer sets shouldCommit=false: the mirror was already synced, and refusing to persist it made every following job repeat the same repack. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- __test__/mirror-cleanup.test.ts | 172 +++++++++++++ __test__/mirror-maintenance-git.test.ts | 324 ++++++++++++++++++++++++ dist/index.js | 237 +++++++++++++---- src/blacksmith-cache.ts | 234 +++++++++++++---- src/main.ts | 6 +- 5 files changed, 873 insertions(+), 100 deletions(-) create mode 100644 __test__/mirror-cleanup.test.ts create mode 100644 __test__/mirror-maintenance-git.test.ts diff --git a/__test__/mirror-cleanup.test.ts b/__test__/mirror-cleanup.test.ts new file mode 100644 index 0000000..1f17fbb --- /dev/null +++ b/__test__/mirror-cleanup.test.ts @@ -0,0 +1,172 @@ +/** + * Post-step cleanup commit decision: mirror sync failure vetoes the commit, + * maintenance failure or timeout does not, and maintenance only runs when + * the result will be persisted. + */ +const mockCommitStickyDisk = jest.fn() + +jest.mock('@connectrpc/connect', () => ({ + createClient: jest.fn(() => ({commitStickyDisk: mockCommitStickyDisk})), + ConnectError: class ConnectError extends Error {}, + Code: {Aborted: 'ABORTED'} +})) + +jest.mock('@connectrpc/connect-node', () => ({ + createGrpcTransport: jest.fn() +})) + +jest.mock( + '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect', + () => ({ + StickyDiskService: {} + }) +) + +jest.mock('../src/container-detector', () => ({ + isRunningInContainer: jest.fn(() => false) +})) + +jest.mock('@actions/exec') + +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import * as exec from '@actions/exec' +import * as blacksmithCache from '../src/blacksmith-cache' + +const mockExec = exec.exec as jest.MockedFunction +const mockGetExecOutput = exec.getExecOutput as jest.MockedFunction< + typeof exec.getExecOutput +> + +function isRepack(args: string[] | undefined): boolean { + return (args || []).includes('repack') +} + +function commands(): string[][] { + return mockGetExecOutput.mock.calls.map(([tool, args]) => [ + tool, + ...(args || []) + ]) +} + +describe('cleanup commit decision', () => { + let mirrorPath: string + let repackExitCode: number + + beforeEach(() => { + jest.clearAllMocks() + process.env.BLACKSMITH_AGENT_ADDR = '127.0.0.1' + process.env.BLACKSMITH_STICKY_DISK_GRPC_PORT = '1' + mirrorPath = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-cleanup-')) + fs.mkdirSync(path.join(mirrorPath, 'objects', 'pack'), {recursive: true}) + repackExitCode = 0 + mockExec.mockResolvedValue(0) + mockGetExecOutput.mockImplementation(async (_tool, args) => ({ + exitCode: isRepack(args) ? repackExitCode : 0, + stdout: '', + stderr: '' + })) + mockCommitStickyDisk.mockResolvedValue({}) + }) + + afterEach(() => { + fs.rmSync(mirrorPath, {recursive: true, force: true}) + }) + + const base = { + exposeId: 'expose-1', + stickyDiskKey: 'key-1', + repoName: 'owner/repo', + shouldCommit: true, + vmHydratedGitMirror: true + } + + it('runs bounded maintenance instead of gc --auto and commits', async () => { + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult).toEqual({success: true, timedOut: false}) + const repack = commands().find(c => c.includes('repack')) + expect(repack).toBeDefined() + expect(repack).toEqual( + expect.arrayContaining([ + '-d', + '-l', + '-n', + '--geometric=2', + '--write-midx', + 'repack.writeBitmaps=false' + ]) + ) + expect(commands().some(c => c.includes('gc'))).toBe(false) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + + it('still commits when maintenance times out', async () => { + repackExitCode = 124 + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult.success).toBe(false) + expect(result.maintenanceResult.timedOut).toBe(true) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + + it('still commits when maintenance fails', async () => { + repackExitCode = 128 + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult).toEqual({ + success: false, + timedOut: false, + error: expect.stringContaining('128') + }) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + + it('does not commit or run maintenance when the mirror sync failed', async () => { + const result = await blacksmithCache.cleanup({ + ...base, + mirrorPath, + mirrorSyncFailed: true + }) + + expect(result.maintenanceResult).toEqual({success: true, timedOut: false}) + expect(commands().some(c => c.includes('repack'))).toBe(false) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: false, vmHydratedGitMirror: false}) + ) + }) + + it('does not commit or run maintenance when the mirror sync timed out', async () => { + await blacksmithCache.cleanup({ + ...base, + mirrorPath, + mirrorSyncTimedOut: true + }) + + expect(commands().some(c => c.includes('repack'))).toBe(false) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: false, vmHydratedGitMirror: false}) + ) + }) + + it('skips maintenance when the disk is released without commit', async () => { + await blacksmithCache.cleanup({ + ...base, + mirrorPath, + shouldCommit: false, + vmHydratedGitMirror: false + }) + + expect(commands().some(c => c.includes('repack'))).toBe(false) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: false}) + ) + }) +}) diff --git a/__test__/mirror-maintenance-git.test.ts b/__test__/mirror-maintenance-git.test.ts new file mode 100644 index 0000000..d6ce3f1 --- /dev/null +++ b/__test__/mirror-maintenance-git.test.ts @@ -0,0 +1,324 @@ +/** + * Real-git coverage for post-step mirror maintenance: large packs are + * marked kept and never rewritten, small packs and loose objects are + * rolled up, the mirror stays intact, repeated runs are no-ops, and a + * killed or failing maintenance leaves no temporary or lock files behind. + */ +jest.mock('@connectrpc/connect', () => ({ + createClient: jest.fn(), + ConnectError: class ConnectError extends Error {}, + Code: {Aborted: 'ABORTED'} +})) + +jest.mock('@connectrpc/connect-node', () => ({ + createGrpcTransport: jest.fn() +})) + +jest.mock( + '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect', + () => ({ + StickyDiskService: {} + }) +) + +jest.mock('../src/container-detector', () => ({ + isRunningInContainer: jest.fn(() => false) +})) + +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {execFileSync} from 'child_process' +import * as blacksmithCache from '../src/blacksmith-cache' + +const KEEP_BYTES = 1024 * 1024 + +function git(cwd: string, ...args: string[]): string { + return execFileSync('git', ['-C', cwd, ...args], {encoding: 'utf8'}).trim() +} + +function commitBlob(repo: string, name: string, bytes: number): void { + const buf = Buffer.alloc(bytes) + for (let i = 0; i < bytes; i += 4) { + buf.writeUInt32LE((Math.random() * 0xffffffff) >>> 0, i) + } + fs.writeFileSync(path.join(repo, name), buf) + git(repo, 'add', name) + git( + repo, + '-c', + 'user.email=test@example.com', + '-c', + 'user.name=Test', + 'commit', + '-q', + '-m', + name + ) +} + +function packs(mirror: string): string[] { + return fs + .readdirSync(path.join(mirror, 'objects', 'pack')) + .filter(f => f.endsWith('.pack')) + .sort() +} + +function looseObjects(mirror: string): number { + const objects = path.join(mirror, 'objects') + let n = 0 + for (const dir of fs.readdirSync(objects)) { + if (/^[0-9a-f]{2}$/.test(dir)) { + n += fs.readdirSync(path.join(objects, dir)).length + } + } + return n +} + +function refsResolve(mirror: string): void { + const out = git(mirror, 'for-each-ref', '--format=%(objectname)') + for (const sha of out.split('\n').filter(Boolean)) { + git(mirror, 'cat-file', '-e', `${sha}^{commit}`) + } +} + +function fsck(mirror: string): void { + execFileSync('git', ['-C', mirror, 'fsck', '--strict'], {stdio: 'pipe'}) +} + +/** + * Bare mirror with one base pack larger than KEEP_BYTES, several small + * packs (pushed with receive.unpackLimit=1) and a few loose objects. + */ +function buildMirror(root: string): {mirror: string; basePack: string} { + const src = path.join(root, 'src') + const mirror = path.join(root, 'mirror.git') + fs.mkdirSync(src) + git(root, 'init', '-q', '--bare', 'mirror.git') + git(root, 'init', '-q', '-b', 'main', 'src') + for (let i = 0; i < 6; i++) { + commitBlob(src, `base-${i}`, 300 * 1024) + git(src, 'push', '-q', mirror, `HEAD:refs/heads/base-${i}`) + } + git(mirror, '-c', 'repack.writeBitmaps=false', 'repack', '-a', '-d', '-q') + const basePacks = packs(mirror) + expect(basePacks).toHaveLength(1) + expect( + fs.statSync(path.join(mirror, 'objects', 'pack', basePacks[0])).size + ).toBeGreaterThan(KEEP_BYTES) + + for (let i = 0; i < 6; i++) { + commitBlob(src, `small-${i}`, 100 * 1024) + git( + src, + 'push', + '-q', + '--receive-pack=git -c receive.unpackLimit=1 receive-pack', + mirror, + `HEAD:refs/heads/small-${i}` + ) + } + commitBlob(src, 'loose', 512) + git(src, 'push', '-q', mirror, 'HEAD:refs/heads/loose') + + expect(packs(mirror).length).toBe(7) + expect(looseObjects(mirror)).toBeGreaterThan(0) + return {mirror, basePack: basePacks[0]} +} + +describe('runMirrorMaintenance (real git)', () => { + let root: string + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-maint-')) + }) + + afterEach(() => { + fs.rmSync(root, {recursive: true, force: true}) + }) + + it('keeps the base pack, rolls up small packs and loose objects', async () => { + const {mirror, basePack} = buildMirror(root) + const before = fs.statSync(path.join(mirror, 'objects', 'pack', basePack)) + + const result = await blacksmithCache.runMirrorMaintenance( + mirror, + 60, + KEEP_BYTES + ) + expect(result).toEqual({success: true, timedOut: false}) + + const packDir = path.join(mirror, 'objects', 'pack') + expect( + fs.existsSync(path.join(packDir, basePack.replace(/\.pack$/, '.keep'))) + ).toBe(true) + const after = fs.statSync(path.join(packDir, basePack)) + expect(after.ino).toBe(before.ino) + expect(after.size).toBe(before.size) + + const remaining = packs(mirror) + expect(remaining).toContain(basePack) + expect(remaining.length).toBeLessThanOrEqual(3) + expect(looseObjects(mirror)).toBe(0) + expect(fs.existsSync(path.join(packDir, 'multi-pack-index'))).toBe(true) + expect(fs.existsSync(path.join(mirror, 'packed-refs'))).toBe(true) + fsck(mirror) + refsResolve(mirror) + }) + + it('is a no-op on an already maintained mirror', async () => { + const {mirror} = buildMirror(root) + await blacksmithCache.runMirrorMaintenance(mirror, 60, KEEP_BYTES) + const first = packs(mirror) + + const result = await blacksmithCache.runMirrorMaintenance( + mirror, + 60, + KEEP_BYTES + ) + expect(result).toEqual({success: true, timedOut: false}) + expect(packs(mirror)).toEqual(first) + fsck(mirror) + }) + + it('excludes kept packs from the roll-up regardless of size', async () => { + const {mirror, basePack} = buildMirror(root) + // Kept by an earlier run; a larger pack arriving later must not pull + // it into the roll-up. + fs.writeFileSync( + path.join( + mirror, + 'objects', + 'pack', + basePack.replace(/\.pack$/, '.keep') + ), + '' + ) + const src = path.join(root, 'src') + commitBlob(src, 'huge', 4 * 1024 * 1024) + git( + src, + 'push', + '-q', + '--receive-pack=git -c receive.unpackLimit=1 receive-pack', + mirror, + 'HEAD:refs/heads/huge' + ) + + const result = await blacksmithCache.runMirrorMaintenance( + mirror, + 60, + Number.MAX_SAFE_INTEGER + ) + expect(result.success).toBe(true) + expect(packs(mirror)).toContain(basePack) + fsck(mirror) + refsResolve(mirror) + }) + + it('marks only packs at or above the threshold as kept', async () => { + const {mirror, basePack} = buildMirror(root) + const kept = await blacksmithCache.markKeepPacks(mirror, KEEP_BYTES) + expect(kept).toEqual([basePack.replace(/\.pack$/, '')]) + const keepFiles = fs + .readdirSync(path.join(mirror, 'objects', 'pack')) + .filter(f => f.endsWith('.keep')) + expect(keepFiles).toEqual([basePack.replace(/\.pack$/, '.keep')]) + + // Idempotent: already-kept packs are reported, no duplicates written. + expect(await blacksmithCache.markKeepPacks(mirror, KEEP_BYTES)).toEqual( + kept + ) + }) + + describe('with a misbehaving git', () => { + let binDir: string + let originalPath: string | undefined + + beforeEach(() => { + binDir = path.join(root, 'bin') + fs.mkdirSync(binDir) + originalPath = process.env.PATH + }) + + afterEach(() => { + process.env.PATH = originalPath + }) + + // Jest's sandboxed process.env is not what child processes inherit, so + // the shim is installed as a `timeout` found through the sandbox PATH + // that puts binDir first on the PATH of the real one. + function installFakeGit(repackScript: string): void { + const realGit = execFileSync('which', ['git'], {encoding: 'utf8'}).trim() + const realTimeout = execFileSync('which', ['timeout'], { + encoding: 'utf8' + }).trim() + fs.writeFileSync( + path.join(binDir, 'git'), + `#!/bin/sh +for arg in "$@"; do + if [ "$arg" = "repack" ]; then +${repackScript} + fi +done +exec ${realGit} "$@" +`, + {mode: 0o755} + ) + fs.writeFileSync( + path.join(binDir, 'timeout'), + `#!/bin/sh +PATH="${binDir}:$PATH" exec ${realTimeout} "$@" +`, + {mode: 0o755} + ) + process.env.PATH = `${binDir}:${originalPath}` + } + + it('reports a timeout and removes leftovers without touching the mirror', async () => { + const {mirror, basePack} = buildMirror(root) + const before = packs(mirror) + const packDir = path.join(mirror, 'objects', 'pack') + installFakeGit(` touch "${packDir}/tmp_pack_abc" "${packDir}/.tmp-1-pack-abc.pack" "${packDir}/multi-pack-index.lock" "${mirror}/packed-refs.lock" + sleep 30`) + + const result = await blacksmithCache.runMirrorMaintenance( + mirror, + 1, + KEEP_BYTES + ) + expect(result.success).toBe(false) + expect(result.timedOut).toBe(true) + + expect(packs(mirror)).toEqual(before) + expect(fs.existsSync(path.join(packDir, 'tmp_pack_abc'))).toBe(false) + expect(fs.existsSync(path.join(packDir, '.tmp-1-pack-abc.pack'))).toBe( + false + ) + expect(fs.existsSync(path.join(packDir, 'multi-pack-index.lock'))).toBe( + false + ) + expect(fs.existsSync(path.join(mirror, 'packed-refs.lock'))).toBe(false) + expect( + fs.existsSync(path.join(packDir, basePack.replace(/\.pack$/, '.keep'))) + ).toBe(true) + fsck(mirror) + refsResolve(mirror) + }) + + it('reports a failure without a timeout flag', async () => { + const {mirror} = buildMirror(root) + installFakeGit(' exit 128') + + const result = await blacksmithCache.runMirrorMaintenance( + mirror, + 60, + KEEP_BYTES + ) + expect(result.success).toBe(false) + expect(result.timedOut).toBe(false) + expect(result.error).toContain('128') + fsck(mirror) + }) + }) +}) diff --git a/dist/index.js b/dist/index.js index 32382c7..99f5443 100644 --- a/dist/index.js +++ b/dist/index.js @@ -65,6 +65,8 @@ exports.fetchRefsFromMirror = fetchRefsFromMirror; exports.writeAlternates = writeAlternates; exports.dissociate = dissociate; exports.hasCommitGraph = hasCommitGraph; +exports.markKeepPacks = markKeepPacks; +exports.runMirrorMaintenance = runMirrorMaintenance; exports.cleanup = cleanup; const core = __importStar(__nccwpck_require__(2186)); const exec = __importStar(__nccwpck_require__(1514)); @@ -73,7 +75,7 @@ const os = __importStar(__nccwpck_require__(2037)); const path = __importStar(__nccwpck_require__(1017)); const connect_1 = __nccwpck_require__(632); const connect_node_1 = __nccwpck_require__(1125); -const stickydisk_connect_1 = __nccwpck_require__(2880); +const stickydisk_connect_1 = __nccwpck_require__(783); const retryHelper = __importStar(__nccwpck_require__(2155)); const container_detector_1 = __nccwpck_require__(6424); // Without a deadline, a black-holed dial stalls the checkout until the OS @@ -89,7 +91,13 @@ const MIRROR_VERSION = 'v1'; // burns checkout time on a mirror disk that is having a bad day. The // window is kept generous so slow-but-healthy syncs still complete. const REFRESH_TIMEOUT_SECS = 120; // 2 minutes, single attempt -const GC_TIMEOUT_SECS = 120; // 2 minutes +// Post-step mirror maintenance is bounded by construction (it never rewrites +// packs at or above MAINTENANCE_KEEP_PACK_BYTES), so this deadline is a +// safety net rather than an expected cost. Running out of it does not +// affect the commit: the mirror was already synced, maintenance is a pure +// optimization. +const MAINTENANCE_TIMEOUT_SECS = 120; // 2 minutes +const MAINTENANCE_KEEP_PACK_BYTES = 256 * 1024 * 1024; const FLUSH_TIMEOUT_SECS = 10; // 10 seconds for durability flush const UMOUNT_TIMEOUT_SECS = 10; // 10 seconds for unmount const UMOUNT_MAX_RETRIES = 3; // Number of unmount retry attempts @@ -1338,13 +1346,12 @@ function dissociate(workspacePath, env) { /** * Write the mirror's commit-graph so subsequent commit parsing (e.g. the * per-ref walks inside fetch) is a lookup in one compact file instead of - * scattered pack reads. Reading commit-graphs is enabled by default in git; - * writing only happens during a real gc, so a freshly cloned mirror has - * none until the first threshold-tripping `gc --auto`. Failure is - * non-fatal - the graph is a pure cache. + * scattered pack reads. Reading commit-graphs is enabled by default in git, + * but a freshly cloned mirror has none until something writes it. Failure + * is non-fatal - the graph is a pure cache. */ function writeCommitGraph(mirrorPath_1) { - return __awaiter(this, arguments, void 0, function* (mirrorPath, timeoutSecs = GC_TIMEOUT_SECS) { + return __awaiter(this, arguments, void 0, function* (mirrorPath, timeoutSecs = MAINTENANCE_TIMEOUT_SECS) { try { const start = Date.now(); const result = yield exec.getExecOutput('timeout', [ @@ -1380,52 +1387,176 @@ function hasCommitGraph(mirrorPath) { fs.existsSync(path.join(mirrorPath, 'objects', 'info', 'commit-graphs', 'commit-graph-chain'))); } /** - * Run lightweight garbage collection on the mirror. - * Uses --auto to only run GC when git determines it's needed (based on loose object count). - * This avoids expensive full repacks on every run while still keeping the repo tidy over time. + * Mark every pack at or above MAINTENANCE_KEEP_PACK_BYTES with a `.keep` + * file so that no repack - this maintenance, or `gc` run by an older action + * version on the same mirror - ever rewrites or deletes it. Returns the + * names (without extension) of all kept packs. + * + * `.keep` files are used rather than `repack --keep-pack`: git 2.34's + * geometric repack does not exclude `--keep-pack` packs from the roll-up, + * so with `-d` it deletes the kept pack and loses its objects. Kept packs + * are ignored by `--honor-pack-keep` only in the repository they live in, + * so a workspace repack over the mirror alternate (dissociate) still copies + * their objects. */ -function runMirrorGC(mirrorPath_1) { - return __awaiter(this, arguments, void 0, function* (mirrorPath, timeoutSecs = GC_TIMEOUT_SECS) { - core.info(`[git-mirror] Running auto garbage collection (timeout: ${timeoutSecs}s)`); +function markKeepPacks(mirrorPath_1) { + return __awaiter(this, arguments, void 0, function* (mirrorPath, keepBytes = MAINTENANCE_KEEP_PACK_BYTES) { + const packDir = path.join(mirrorPath, 'objects', 'pack'); + const kept = []; + let entries; + try { + entries = yield fs.promises.readdir(packDir); + } + catch (_a) { + return kept; + } + const names = new Set(entries); + for (const entry of entries) { + if (!entry.startsWith('pack-') || !entry.endsWith('.pack')) { + continue; + } + const base = entry.slice(0, -'.pack'.length); + if (names.has(`${base}.keep`)) { + kept.push(base); + continue; + } + try { + const stat = yield fs.promises.stat(path.join(packDir, entry)); + if (stat.size < keepBytes) { + continue; + } + yield fs.promises.writeFile(path.join(packDir, `${base}.keep`), ''); + core.info(`[git-mirror] Marked ${base} (${Math.round(stat.size / (1024 * 1024))} MiB) as kept`); + kept.push(base); + } + catch (error) { + core.warning(`[git-mirror] Failed to mark ${base} as kept: ${error}`); + } + } + return kept.sort(); + }); +} +/** + * Remove temporary files and lock files a killed maintenance process may + * have left in the mirror. Only the post step touches the mirror, and + * `timeout` has already signalled the whole process group by the time this + * runs, so any such file is stale and would make the next job's sync fail + * on a lock that nobody holds. Best effort: a still-exiting process removing + * the same file is harmless. + */ +function removeMaintenanceLeftovers(mirrorPath) { + return __awaiter(this, void 0, void 0, function* () { + const sweeps = [ + { + dir: path.join(mirrorPath, 'objects', 'pack'), + match: name => name.startsWith('tmp_') || + name.startsWith('.tmp-') || + name.endsWith('.lock') + }, + { + dir: path.join(mirrorPath, 'objects', 'info'), + match: name => name.endsWith('.lock') + }, + { + dir: path.join(mirrorPath, 'objects', 'info', 'commit-graphs'), + match: name => name.endsWith('.lock') + }, + { + dir: mirrorPath, + match: name => name === 'packed-refs.lock' + } + ]; + for (const { dir, match } of sweeps) { + let entries; + try { + entries = yield fs.promises.readdir(dir); + } + catch (_a) { + continue; + } + for (const entry of entries) { + if (!match(entry)) { + continue; + } + try { + yield fs.promises.rm(path.join(dir, entry), { force: true }); + core.info(`[git-mirror] Removed stale maintenance file ${entry}`); + } + catch (error) { + core.warning(`[git-mirror] Failed to remove ${entry}: ${error}`); + } + } + } + }); +} +/** + * Bounded mirror maintenance, run in the post step once the mirror has been + * synced. Each sync leaves a new small pack (or loose objects) behind; + * left alone they accumulate until object lookup slows down. Instead of + * `gc --auto` - whose "auto packing" step rewrites every pack including the + * repository's multi-gigabyte base pack, taking minutes on large mirrors - + * this folds only the small packs and loose objects together with a + * geometric repack, explicitly keeping any pack at or above + * MAINTENANCE_KEEP_PACK_BYTES out of the rewrite. A rolled-up pack that + * grows past that size simply becomes another kept pack, so the cost of a + * single run is bounded by the keep threshold, never by the size of the + * repository. The multi-pack-index keeps lookups fast across the kept packs. + * + * Maintenance is optional: its failure or timeout is reported but must not + * prevent the synced mirror from being committed - otherwise a mirror that + * needs more maintenance than the deadline allows is never persisted, and + * every subsequent job repeats the same doomed work. + */ +function runMirrorMaintenance(mirrorPath_1) { + return __awaiter(this, arguments, void 0, function* (mirrorPath, timeoutSecs = MAINTENANCE_TIMEOUT_SECS, keepBytes = MAINTENANCE_KEEP_PACK_BYTES) { + const start = Date.now(); + const keptPacks = yield markKeepPacks(mirrorPath, keepBytes); + core.info(`[git-mirror] Running incremental maintenance (timeout: ${timeoutSecs}s, ${keptPacks.length} kept pack(s))`); try { - // --auto: only run if thresholds exceeded (default: 6700 loose objects or 50 packs) - // This is much faster than a full gc when not needed - // gc.autoDetach=false: prevent git from forking a background daemon for GC. - // Without this, the parent `git gc --auto` returns immediately while the - // daemonized child keeps running with cwd and mmap'd pack files on the - // mirror mount, causing the subsequent `umount` to fail with EBUSY. const result = yield exec.getExecOutput('timeout', [ String(timeoutSecs), 'git', '-c', - 'gc.autoDetach=false', + 'repack.writeBitmaps=false', '-C', mirrorPath, - 'gc', - '--auto' + 'repack', + '-d', + '-l', + '-n', + '--geometric=2', + '--write-midx' ], { ignoreReturnCode: true }); if (result.exitCode === TIMEOUT_EXIT_CODE) { - core.warning(`[git-mirror] GC timed out after ${timeoutSecs}s`); + core.warning(`[git-mirror] Maintenance timed out after ${timeoutSecs}s; committing the synced mirror without it`); + yield removeMaintenanceLeftovers(mirrorPath); return { success: false, timedOut: true, - error: `git gc timed out after ${timeoutSecs}s` + error: `git repack timed out after ${timeoutSecs}s` }; } if (result.exitCode !== 0) { - core.warning(`[git-mirror] GC failed with exit code ${result.exitCode}`); + core.warning(`[git-mirror] Maintenance failed with exit code ${result.exitCode}; committing the synced mirror without it`); + yield removeMaintenanceLeftovers(mirrorPath); return { success: false, timedOut: false, - error: `git gc failed with exit code ${result.exitCode}` + error: `git repack failed with exit code ${result.exitCode}` }; } - core.debug('[git-mirror] Completed git gc --auto'); + const packRefs = yield exec.getExecOutput('timeout', [String(timeoutSecs), 'git', '-C', mirrorPath, 'pack-refs', '--all'], { silent: true, ignoreReturnCode: true }); + if (packRefs.exitCode !== 0) { + core.warning(`[git-mirror] pack-refs failed with exit code ${packRefs.exitCode}`); + yield removeMaintenanceLeftovers(mirrorPath); + } + core.info(`[git-mirror] Incremental maintenance finished in ${Date.now() - start}ms`); return { success: true, timedOut: false }; } catch (error) { const msg = error.message || String(error); - core.warning(`[git-mirror] GC failed: ${msg}`); + core.warning(`[git-mirror] Maintenance failed: ${msg}`); + yield removeMaintenanceLeftovers(mirrorPath); return { success: false, timedOut: false, error: msg }; } }); @@ -1522,10 +1653,11 @@ function flushBlockDevice(devicePath) { }); } /** - * Cleanup: run GC, sync, unmount, and commit the sticky disk. + * Cleanup: run mirror maintenance, sync, unmount, and commit the sticky disk. * - * Execution order: GC → sync → unmount (with retry) → flush → commit - * If the mirror sync or GC failed or timed out, shouldCommit is set to false. + * Execution order: maintenance → sync → unmount (with retry) → flush → commit + * If the mirror sync failed or timed out, shouldCommit is set to false. + * Maintenance failure or timeout does not affect the commit. */ function cleanup(options) { return __awaiter(this, void 0, void 0, function* () { @@ -1533,12 +1665,12 @@ function cleanup(options) { const { exposeId, stickyDiskKey, repoName, mountPoint, mirrorPath, mirrorSyncFailed, mirrorSyncTimedOut } = options; let { shouldCommit } = options; // vmHydratedGitMirror must track shouldCommit: if we decide not to commit - // (due to GC/refresh failure), we must not tell the backend that - // hydration completed, otherwise it marks the entry as ready despite no - // valid disk being persisted. + // (due to sync failure), we must not tell the backend that hydration + // completed, otherwise it marks the entry as ready despite no valid disk + // being persisted. let vmHydratedGitMirror = options.vmHydratedGitMirror; const result = { - gcResult: { success: true, timedOut: false } + maintenanceResult: { success: true, timedOut: false } }; core.info(`[git-mirror] Starting cleanup: exposeId=${exposeId}, stickyDiskKey=${stickyDiskKey}, shouldCommit=${shouldCommit}, vmHydratedGitMirror=${vmHydratedGitMirror}`); // If the mirror sync failed or timed out, don't commit @@ -1548,19 +1680,14 @@ function cleanup(options) { shouldCommit = false; vmHydratedGitMirror = false; } - if (mirrorPath) { - // Run GC on the mirror - result.gcResult = yield runMirrorGC(mirrorPath); - if (!result.gcResult.success) { - core.warning('[git-mirror] GC failed or timed out, will not commit sticky disk'); - shouldCommit = false; - vmHydratedGitMirror = false; - } + // Maintenance only pays off if the result is persisted. + if (mirrorPath && shouldCommit) { + result.maintenanceResult = yield runMirrorMaintenance(mirrorPath); // Catch-up for mirrors that predate commit-graph writing: build the // initial graph here in the post step, off the checkout critical path. - // Once it exists, the sync fetch keeps it current incrementally. Only - // worth doing when the disk is being committed; failure is non-fatal. - if (shouldCommit && !hasCommitGraph(mirrorPath)) { + // Once it exists, the sync fetch keeps it current incrementally. + // Failure is non-fatal. + if (!hasCommitGraph(mirrorPath)) { yield writeCommitGraph(mirrorPath); } } @@ -4272,9 +4399,11 @@ function cleanup() { }); } if (cleanupResult) { - if (!cleanupResult.gcResult.success) { + if (!cleanupResult.maintenanceResult.success) { yield (0, internal_metrics_1.reportInternalMetric)('git_mirror_gc_failure', 1, { - reason: cleanupResult.gcResult.timedOut ? 'timeout' : 'failure' + reason: cleanupResult.maintenanceResult.timedOut + ? 'timeout' + : 'failure' }); } } @@ -61420,7 +61549,7 @@ module.exports = parseParams /***/ }), -/***/ 2880: +/***/ 783: /***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __nccwpck_require__) => { "use strict"; @@ -65151,7 +65280,7 @@ const proto3 = makeProtoRuntime("proto3", (fields) => { } }); -;// CONCATENATED MODULE: ./node_modules/@buf/blacksmith_vm-agent.connectrpc_es/node_modules/@buf/blacksmith_vm-agent.bufbuild_es/stickydisk/v1/stickydisk_pb.js +;// CONCATENATED MODULE: ./node_modules/@buf/blacksmith_vm-agent.bufbuild_es/stickydisk/v1/stickydisk_pb.js // @generated by protoc-gen-es v1.10.0 // @generated from file stickydisk/v1/stickydisk.proto (package stickydisk.v1, syntax proto3) /* eslint-disable */ @@ -65227,6 +65356,8 @@ const CommitSkipReason = /*@__PURE__*/ proto3.makeEnum( {no: 4, name: "COMMIT_SKIP_REASON_CLEANUP_ERROR", localName: "CLEANUP_ERROR"}, {no: 5, name: "COMMIT_SKIP_REASON_AMBIGUOUS", localName: "AMBIGUOUS"}, {no: 6, name: "COMMIT_SKIP_REASON_NO_EXPOSE", localName: "NO_EXPOSE"}, + {no: 7, name: "COMMIT_SKIP_REASON_SERVER_DECLINED", localName: "SERVER_DECLINED"}, + {no: 8, name: "COMMIT_SKIP_REASON_HOOK_SKIPPED", localName: "HOOK_SKIPPED"}, ], ); @@ -65453,6 +65584,12 @@ const DockerJobLifecycle = /*@__PURE__*/ proto3.makeMessageType( { no: 19, name: "buildkitd_sigkill_used", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, { no: 20, name: "history_export_timed_out", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, { no: 21, name: "history_prune_failed", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 22, name: "timeline_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 23, name: "timeline_truncated", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, + { no: 24, name: "history_export_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, + { no: 25, name: "traces_dropped_oversize", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, + { no: 26, name: "traces_dropped_payload_cap", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, + { no: 27, name: "records_dropped_payload_cap", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, ], ); diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index fa27cd6..c700ae0 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -23,7 +23,13 @@ const MIRROR_VERSION = 'v1' // burns checkout time on a mirror disk that is having a bad day. The // window is kept generous so slow-but-healthy syncs still complete. const REFRESH_TIMEOUT_SECS = 120 // 2 minutes, single attempt -const GC_TIMEOUT_SECS = 120 // 2 minutes +// Post-step mirror maintenance is bounded by construction (it never rewrites +// packs at or above MAINTENANCE_KEEP_PACK_BYTES), so this deadline is a +// safety net rather than an expected cost. Running out of it does not +// affect the commit: the mirror was already synced, maintenance is a pure +// optimization. +const MAINTENANCE_TIMEOUT_SECS = 120 // 2 minutes +const MAINTENANCE_KEEP_PACK_BYTES = 256 * 1024 * 1024 const FLUSH_TIMEOUT_SECS = 10 // 10 seconds for durability flush const UMOUNT_TIMEOUT_SECS = 10 // 10 seconds for unmount const UMOUNT_MAX_RETRIES = 3 // Number of unmount retry attempts @@ -57,7 +63,7 @@ export interface OperationResult { * Result of the cleanup phase, used for metric reporting. */ export interface CleanupResult { - gcResult: OperationResult + maintenanceResult: OperationResult } /** @@ -1598,14 +1604,13 @@ export async function dissociate( /** * Write the mirror's commit-graph so subsequent commit parsing (e.g. the * per-ref walks inside fetch) is a lookup in one compact file instead of - * scattered pack reads. Reading commit-graphs is enabled by default in git; - * writing only happens during a real gc, so a freshly cloned mirror has - * none until the first threshold-tripping `gc --auto`. Failure is - * non-fatal - the graph is a pure cache. + * scattered pack reads. Reading commit-graphs is enabled by default in git, + * but a freshly cloned mirror has none until something writes it. Failure + * is non-fatal - the graph is a pure cache. */ async function writeCommitGraph( mirrorPath: string, - timeoutSecs: number = GC_TIMEOUT_SECS + timeoutSecs: number = MAINTENANCE_TIMEOUT_SECS ): Promise { try { const start = Date.now() @@ -1660,60 +1665,199 @@ export function hasCommitGraph(mirrorPath: string): boolean { } /** - * Run lightweight garbage collection on the mirror. - * Uses --auto to only run GC when git determines it's needed (based on loose object count). - * This avoids expensive full repacks on every run while still keeping the repo tidy over time. + * Mark every pack at or above MAINTENANCE_KEEP_PACK_BYTES with a `.keep` + * file so that no repack - this maintenance, or `gc` run by an older action + * version on the same mirror - ever rewrites or deletes it. Returns the + * names (without extension) of all kept packs. + * + * `.keep` files are used rather than `repack --keep-pack`: git 2.34's + * geometric repack does not exclude `--keep-pack` packs from the roll-up, + * so with `-d` it deletes the kept pack and loses its objects. Kept packs + * are ignored by `--honor-pack-keep` only in the repository they live in, + * so a workspace repack over the mirror alternate (dissociate) still copies + * their objects. + */ +export async function markKeepPacks( + mirrorPath: string, + keepBytes: number = MAINTENANCE_KEEP_PACK_BYTES +): Promise { + const packDir = path.join(mirrorPath, 'objects', 'pack') + const kept: string[] = [] + let entries: string[] + try { + entries = await fs.promises.readdir(packDir) + } catch { + return kept + } + const names = new Set(entries) + for (const entry of entries) { + if (!entry.startsWith('pack-') || !entry.endsWith('.pack')) { + continue + } + const base = entry.slice(0, -'.pack'.length) + if (names.has(`${base}.keep`)) { + kept.push(base) + continue + } + try { + const stat = await fs.promises.stat(path.join(packDir, entry)) + if (stat.size < keepBytes) { + continue + } + await fs.promises.writeFile(path.join(packDir, `${base}.keep`), '') + core.info( + `[git-mirror] Marked ${base} (${Math.round(stat.size / (1024 * 1024))} MiB) as kept` + ) + kept.push(base) + } catch (error) { + core.warning(`[git-mirror] Failed to mark ${base} as kept: ${error}`) + } + } + return kept.sort() +} + +/** + * Remove temporary files and lock files a killed maintenance process may + * have left in the mirror. Only the post step touches the mirror, and + * `timeout` has already signalled the whole process group by the time this + * runs, so any such file is stale and would make the next job's sync fail + * on a lock that nobody holds. Best effort: a still-exiting process removing + * the same file is harmless. + */ +async function removeMaintenanceLeftovers(mirrorPath: string): Promise { + const sweeps: Array<{dir: string; match: (name: string) => boolean}> = [ + { + dir: path.join(mirrorPath, 'objects', 'pack'), + match: name => + name.startsWith('tmp_') || + name.startsWith('.tmp-') || + name.endsWith('.lock') + }, + { + dir: path.join(mirrorPath, 'objects', 'info'), + match: name => name.endsWith('.lock') + }, + { + dir: path.join(mirrorPath, 'objects', 'info', 'commit-graphs'), + match: name => name.endsWith('.lock') + }, + { + dir: mirrorPath, + match: name => name === 'packed-refs.lock' + } + ] + for (const {dir, match} of sweeps) { + let entries: string[] + try { + entries = await fs.promises.readdir(dir) + } catch { + continue + } + for (const entry of entries) { + if (!match(entry)) { + continue + } + try { + await fs.promises.rm(path.join(dir, entry), {force: true}) + core.info(`[git-mirror] Removed stale maintenance file ${entry}`) + } catch (error) { + core.warning(`[git-mirror] Failed to remove ${entry}: ${error}`) + } + } + } +} + +/** + * Bounded mirror maintenance, run in the post step once the mirror has been + * synced. Each sync leaves a new small pack (or loose objects) behind; + * left alone they accumulate until object lookup slows down. Instead of + * `gc --auto` - whose "auto packing" step rewrites every pack including the + * repository's multi-gigabyte base pack, taking minutes on large mirrors - + * this folds only the small packs and loose objects together with a + * geometric repack, explicitly keeping any pack at or above + * MAINTENANCE_KEEP_PACK_BYTES out of the rewrite. A rolled-up pack that + * grows past that size simply becomes another kept pack, so the cost of a + * single run is bounded by the keep threshold, never by the size of the + * repository. The multi-pack-index keeps lookups fast across the kept packs. + * + * Maintenance is optional: its failure or timeout is reported but must not + * prevent the synced mirror from being committed - otherwise a mirror that + * needs more maintenance than the deadline allows is never persisted, and + * every subsequent job repeats the same doomed work. */ -async function runMirrorGC( +export async function runMirrorMaintenance( mirrorPath: string, - timeoutSecs: number = GC_TIMEOUT_SECS + timeoutSecs: number = MAINTENANCE_TIMEOUT_SECS, + keepBytes: number = MAINTENANCE_KEEP_PACK_BYTES ): Promise { + const start = Date.now() + const keptPacks = await markKeepPacks(mirrorPath, keepBytes) core.info( - `[git-mirror] Running auto garbage collection (timeout: ${timeoutSecs}s)` + `[git-mirror] Running incremental maintenance (timeout: ${timeoutSecs}s, ${keptPacks.length} kept pack(s))` ) try { - // --auto: only run if thresholds exceeded (default: 6700 loose objects or 50 packs) - // This is much faster than a full gc when not needed - // gc.autoDetach=false: prevent git from forking a background daemon for GC. - // Without this, the parent `git gc --auto` returns immediately while the - // daemonized child keeps running with cwd and mmap'd pack files on the - // mirror mount, causing the subsequent `umount` to fail with EBUSY. const result = await exec.getExecOutput( 'timeout', [ String(timeoutSecs), 'git', '-c', - 'gc.autoDetach=false', + 'repack.writeBitmaps=false', '-C', mirrorPath, - 'gc', - '--auto' + 'repack', + '-d', + '-l', + '-n', + '--geometric=2', + '--write-midx' ], {ignoreReturnCode: true} ) if (result.exitCode === TIMEOUT_EXIT_CODE) { - core.warning(`[git-mirror] GC timed out after ${timeoutSecs}s`) + core.warning( + `[git-mirror] Maintenance timed out after ${timeoutSecs}s; committing the synced mirror without it` + ) + await removeMaintenanceLeftovers(mirrorPath) return { success: false, timedOut: true, - error: `git gc timed out after ${timeoutSecs}s` + error: `git repack timed out after ${timeoutSecs}s` } } if (result.exitCode !== 0) { - core.warning(`[git-mirror] GC failed with exit code ${result.exitCode}`) + core.warning( + `[git-mirror] Maintenance failed with exit code ${result.exitCode}; committing the synced mirror without it` + ) + await removeMaintenanceLeftovers(mirrorPath) return { success: false, timedOut: false, - error: `git gc failed with exit code ${result.exitCode}` + error: `git repack failed with exit code ${result.exitCode}` } } - core.debug('[git-mirror] Completed git gc --auto') + + const packRefs = await exec.getExecOutput( + 'timeout', + [String(timeoutSecs), 'git', '-C', mirrorPath, 'pack-refs', '--all'], + {silent: true, ignoreReturnCode: true} + ) + if (packRefs.exitCode !== 0) { + core.warning( + `[git-mirror] pack-refs failed with exit code ${packRefs.exitCode}` + ) + await removeMaintenanceLeftovers(mirrorPath) + } + + core.info( + `[git-mirror] Incremental maintenance finished in ${Date.now() - start}ms` + ) return {success: true, timedOut: false} } catch (error) { const msg = (error as Error).message || String(error) - core.warning(`[git-mirror] GC failed: ${msg}`) + core.warning(`[git-mirror] Maintenance failed: ${msg}`) + await removeMaintenanceLeftovers(mirrorPath) return {success: false, timedOut: false, error: msg} } } @@ -1852,10 +1996,11 @@ export interface CleanupOptions { } /** - * Cleanup: run GC, sync, unmount, and commit the sticky disk. + * Cleanup: run mirror maintenance, sync, unmount, and commit the sticky disk. * - * Execution order: GC → sync → unmount (with retry) → flush → commit - * If the mirror sync or GC failed or timed out, shouldCommit is set to false. + * Execution order: maintenance → sync → unmount (with retry) → flush → commit + * If the mirror sync failed or timed out, shouldCommit is set to false. + * Maintenance failure or timeout does not affect the commit. */ export async function cleanup(options: CleanupOptions): Promise { const { @@ -1869,13 +2014,13 @@ export async function cleanup(options: CleanupOptions): Promise { } = options let {shouldCommit} = options // vmHydratedGitMirror must track shouldCommit: if we decide not to commit - // (due to GC/refresh failure), we must not tell the backend that - // hydration completed, otherwise it marks the entry as ready despite no - // valid disk being persisted. + // (due to sync failure), we must not tell the backend that hydration + // completed, otherwise it marks the entry as ready despite no valid disk + // being persisted. let vmHydratedGitMirror = options.vmHydratedGitMirror const result: CleanupResult = { - gcResult: {success: true, timedOut: false} + maintenanceResult: {success: true, timedOut: false} } core.info( @@ -1892,21 +2037,14 @@ export async function cleanup(options: CleanupOptions): Promise { vmHydratedGitMirror = false } - if (mirrorPath) { - // Run GC on the mirror - result.gcResult = await runMirrorGC(mirrorPath) - if (!result.gcResult.success) { - core.warning( - '[git-mirror] GC failed or timed out, will not commit sticky disk' - ) - shouldCommit = false - vmHydratedGitMirror = false - } + // Maintenance only pays off if the result is persisted. + if (mirrorPath && shouldCommit) { + result.maintenanceResult = await runMirrorMaintenance(mirrorPath) // Catch-up for mirrors that predate commit-graph writing: build the // initial graph here in the post step, off the checkout critical path. - // Once it exists, the sync fetch keeps it current incrementally. Only - // worth doing when the disk is being committed; failure is non-fatal. - if (shouldCommit && !hasCommitGraph(mirrorPath)) { + // Once it exists, the sync fetch keeps it current incrementally. + // Failure is non-fatal. + if (!hasCommitGraph(mirrorPath)) { await writeCommitGraph(mirrorPath) } } diff --git a/src/main.ts b/src/main.ts index db8f1f1..596e440 100644 --- a/src/main.ts +++ b/src/main.ts @@ -164,9 +164,11 @@ async function cleanup(): Promise { }) } if (cleanupResult) { - if (!cleanupResult.gcResult.success) { + if (!cleanupResult.maintenanceResult.success) { await reportInternalMetric('git_mirror_gc_failure', 1, { - reason: cleanupResult.gcResult.timedOut ? 'timeout' : 'failure' + reason: cleanupResult.maintenanceResult.timedOut + ? 'timeout' + : 'failure' }) } } From 44e5d652dffbf366b0d0afed9c85c9946c35a1a7 Mon Sep 17 00:00:00 2001 From: piotr Date: Wed, 16 Sep 2026 17:35:29 +0000 Subject: [PATCH 2/7] Rebuild dist from npm ci dependencies Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- dist/index.js | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/dist/index.js b/dist/index.js index 99f5443..ddf05f0 100644 --- a/dist/index.js +++ b/dist/index.js @@ -75,7 +75,7 @@ const os = __importStar(__nccwpck_require__(2037)); const path = __importStar(__nccwpck_require__(1017)); const connect_1 = __nccwpck_require__(632); const connect_node_1 = __nccwpck_require__(1125); -const stickydisk_connect_1 = __nccwpck_require__(783); +const stickydisk_connect_1 = __nccwpck_require__(2880); const retryHelper = __importStar(__nccwpck_require__(2155)); const container_detector_1 = __nccwpck_require__(6424); // Without a deadline, a black-holed dial stalls the checkout until the OS @@ -61549,7 +61549,7 @@ module.exports = parseParams /***/ }), -/***/ 783: +/***/ 2880: /***/ ((__unused_webpack___webpack_module__, __webpack_exports__, __nccwpck_require__) => { "use strict"; @@ -65280,7 +65280,7 @@ const proto3 = makeProtoRuntime("proto3", (fields) => { } }); -;// CONCATENATED MODULE: ./node_modules/@buf/blacksmith_vm-agent.bufbuild_es/stickydisk/v1/stickydisk_pb.js +;// CONCATENATED MODULE: ./node_modules/@buf/blacksmith_vm-agent.connectrpc_es/node_modules/@buf/blacksmith_vm-agent.bufbuild_es/stickydisk/v1/stickydisk_pb.js // @generated by protoc-gen-es v1.10.0 // @generated from file stickydisk/v1/stickydisk.proto (package stickydisk.v1, syntax proto3) /* eslint-disable */ @@ -65356,8 +65356,6 @@ const CommitSkipReason = /*@__PURE__*/ proto3.makeEnum( {no: 4, name: "COMMIT_SKIP_REASON_CLEANUP_ERROR", localName: "CLEANUP_ERROR"}, {no: 5, name: "COMMIT_SKIP_REASON_AMBIGUOUS", localName: "AMBIGUOUS"}, {no: 6, name: "COMMIT_SKIP_REASON_NO_EXPOSE", localName: "NO_EXPOSE"}, - {no: 7, name: "COMMIT_SKIP_REASON_SERVER_DECLINED", localName: "SERVER_DECLINED"}, - {no: 8, name: "COMMIT_SKIP_REASON_HOOK_SKIPPED", localName: "HOOK_SKIPPED"}, ], ); @@ -65584,12 +65582,6 @@ const DockerJobLifecycle = /*@__PURE__*/ proto3.makeMessageType( { no: 19, name: "buildkitd_sigkill_used", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, { no: 20, name: "history_export_timed_out", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, { no: 21, name: "history_prune_failed", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, - { no: 22, name: "timeline_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, - { no: 23, name: "timeline_truncated", kind: "scalar", T: 8 /* ScalarType.BOOL */ }, - { no: 24, name: "history_export_bytes", kind: "scalar", T: 3 /* ScalarType.INT64 */ }, - { no: 25, name: "traces_dropped_oversize", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, - { no: 26, name: "traces_dropped_payload_cap", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, - { no: 27, name: "records_dropped_payload_cap", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, ], ); From fee5d7b314ead3c83a8898f7d26521b2e54dd23e Mon Sep 17 00:00:00 2001 From: piotr Date: Wed, 16 Sep 2026 17:49:50 +0000 Subject: [PATCH 3/7] Test fixture: disable receive-side auto maintenance so pack layout is version-independent Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- __test__/mirror-maintenance-git.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/__test__/mirror-maintenance-git.test.ts b/__test__/mirror-maintenance-git.test.ts index d6ce3f1..7a368a4 100644 --- a/__test__/mirror-maintenance-git.test.ts +++ b/__test__/mirror-maintenance-git.test.ts @@ -88,13 +88,18 @@ function fsck(mirror: string): void { /** * Bare mirror with one base pack larger than KEEP_BYTES, several small - * packs (pushed with receive.unpackLimit=1) and a few loose objects. + * packs (pushed with receive.unpackLimit=1) and a few loose objects. Auto + * maintenance on the receiving side is disabled so the pack layout is + * exactly what the pushes produced, whatever the git version. */ function buildMirror(root: string): {mirror: string; basePack: string} { const src = path.join(root, 'src') const mirror = path.join(root, 'mirror.git') fs.mkdirSync(src) git(root, 'init', '-q', '--bare', 'mirror.git') + git(mirror, 'config', 'gc.auto', '0') + git(mirror, 'config', 'receive.autogc', 'false') + git(mirror, 'config', 'maintenance.auto', 'false') git(root, 'init', '-q', '-b', 'main', 'src') for (let i = 0; i < 6; i++) { commitBlob(src, `base-${i}`, 300 * 1024) From 3a91c2f854b93c1149425867951c90340daac454 Mon Sep 17 00:00:00 2001 From: piotr Date: Wed, 16 Sep 2026 17:58:05 +0000 Subject: [PATCH 4/7] CI: container mirror jobs verify the direct-checkout fallback when branch protection withholds the mirror on untrusted triggers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-blacksmith.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/test-blacksmith.yml b/.github/workflows/test-blacksmith.yml index bb00214..d2c24d3 100644 --- a/.github/workflows/test-blacksmith.yml +++ b/.github/workflows/test-blacksmith.yml @@ -128,6 +128,11 @@ jobs: # checkout step mounts its own copy of the last commit, so ownership cannot be # flipped between two steps of one job; the takeover is only observable # across jobs, and in the VM direction only after a container job committed.) + # + # With sticky disk branch protection on, only trusted triggers (push to main) + # may hydrate the mirror. When no hydrated mirror exists yet, PR runs are + # sent down the direct-checkout fallback; they verify that fallback and + # report the mirror assertions as skipped instead of failing. test-git-mirror-container: runs-on: blacksmith container: @@ -168,6 +173,14 @@ jobs: test "$(id -u)" = 0 mirror="/blacksmith-git-mirror/${GITHUB_REPOSITORY}/v1/${GITHUB_REPOSITORY_OWNER}-${GITHUB_REPOSITORY#*/}.git" if ! mountpoint -q "$(dirname "$(dirname "$mirror")")"; then + if [ "$GITHUB_EVENT_NAME" != push ]; then + echo "::notice::Sticky disk not mounted (mirror not hydrated for a ${GITHUB_EVENT_NAME} run); verifying the direct-checkout fallback only" + cd host-owned + git config --global --add safe.directory "$PWD" + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + git fsck --no-dangling + exit 0 + fi echo "Sticky disk is not mounted inside the container; the mirror path was not exercised" exit 1 fi @@ -205,6 +218,16 @@ jobs: run: | set -euo pipefail mirror="/blacksmith-git-mirror/${GITHUB_REPOSITORY}/v1/${GITHUB_REPOSITORY_OWNER}-${GITHUB_REPOSITORY#*/}.git" + if ! mountpoint -q "$(dirname "$(dirname "$mirror")")"; then + if [ "$GITHUB_EVENT_NAME" != push ]; then + echo "::notice::Sticky disk not mounted (mirror not hydrated for a ${GITHUB_EVENT_NAME} run); verifying the direct-checkout fallback only" + test "$(git -C after-container rev-parse HEAD)" = "$GITHUB_SHA" + git -C after-container fsck --no-dangling + exit 0 + fi + echo "Sticky disk is not mounted; the mirror path was not exercised" + exit 1 + fi test -d "$mirror" test "$(stat -c %u "$mirror")" = "$(id -u)" grep -qF "$mirror/objects" after-container/.git/objects/info/alternates From 53d90179179bfe2f7414f6779a7d94f8fb1a66bc Mon Sep 17 00:00:00 2001 From: piotr Date: Wed, 16 Sep 2026 19:42:04 +0000 Subject: [PATCH 5/7] Maintenance: bound bytes rewritten per run, fail on pack-refs errors, reclaim kept packs periodically markKeepPacks now returns a KeepSelection: packs at or above the keep threshold stay permanently kept, and of the rest the largest are given a per-run .keep until the remaining candidates add up to less than the threshold. Geometric repack selects by object count, so without this several medium packs could combine into a rewrite far larger than any one of them. Deferred keeps are removed after the run; stale ones from a killed run are recognised by their marker content and cleared. pack-refs failures are now reported: timeout -> success:false/timedOut:true, other exits -> success:false/timedOut:false. cleanup() still commits the synced mirror either way. Once per MAINTENANCE_RECLAIM_INTERVAL_MS (14 days) the run lifts every .keep and does repack -a -d --write-midx + prune --expire=2.weeks.ago under MAINTENANCE_RECLAIM_TIMEOUT_SECS (600 s), so objects that became unreachable inside kept packs are eventually dropped. The stamp is written before the attempt so a reclaim that times out is not retried by every following job; a mirror without a stamp starts its interval rather than reclaiming at once. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- __test__/mirror-cleanup.test.ts | 40 ++- __test__/mirror-maintenance-git.test.ts | 382 ++++++++++++++++++++++-- dist/index.js | 246 +++++++++++---- src/blacksmith-cache.ts | 306 +++++++++++++++---- 4 files changed, 825 insertions(+), 149 deletions(-) diff --git a/__test__/mirror-cleanup.test.ts b/__test__/mirror-cleanup.test.ts index 1f17fbb..4243f58 100644 --- a/__test__/mirror-cleanup.test.ts +++ b/__test__/mirror-cleanup.test.ts @@ -43,6 +43,10 @@ function isRepack(args: string[] | undefined): boolean { return (args || []).includes('repack') } +function isPackRefs(args: string[] | undefined): boolean { + return (args || []).includes('pack-refs') +} + function commands(): string[][] { return mockGetExecOutput.mock.calls.map(([tool, args]) => [ tool, @@ -53,6 +57,7 @@ function commands(): string[][] { describe('cleanup commit decision', () => { let mirrorPath: string let repackExitCode: number + let packRefsExitCode: number beforeEach(() => { jest.clearAllMocks() @@ -61,9 +66,14 @@ describe('cleanup commit decision', () => { mirrorPath = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-cleanup-')) fs.mkdirSync(path.join(mirrorPath, 'objects', 'pack'), {recursive: true}) repackExitCode = 0 + packRefsExitCode = 0 mockExec.mockResolvedValue(0) mockGetExecOutput.mockImplementation(async (_tool, args) => ({ - exitCode: isRepack(args) ? repackExitCode : 0, + exitCode: isRepack(args) + ? repackExitCode + : isPackRefs(args) + ? packRefsExitCode + : 0, stdout: '', stderr: '' })) @@ -129,6 +139,34 @@ describe('cleanup commit decision', () => { ) }) + it('reports a pack-refs failure and still commits', async () => { + packRefsExitCode = 1 + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult).toEqual({ + success: false, + timedOut: false, + error: expect.stringContaining('pack-refs failed with exit code 1') + }) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + + it('reports a pack-refs timeout and still commits', async () => { + packRefsExitCode = 124 + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult).toEqual({ + success: false, + timedOut: true, + error: expect.stringContaining('pack-refs timed out') + }) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + it('does not commit or run maintenance when the mirror sync failed', async () => { const result = await blacksmithCache.cleanup({ ...base, diff --git a/__test__/mirror-maintenance-git.test.ts b/__test__/mirror-maintenance-git.test.ts index 7a368a4..ba8b2bf 100644 --- a/__test__/mirror-maintenance-git.test.ts +++ b/__test__/mirror-maintenance-git.test.ts @@ -86,6 +86,46 @@ function fsck(mirror: string): void { execFileSync('git', ['-C', mirror, 'fsck', '--strict'], {stdio: 'pipe'}) } +/** Pushes HEAD of `src` as one pack of its own, whatever its object count. */ +function pushPack(src: string, mirror: string, branch: string): void { + git( + src, + 'push', + '-q', + '--receive-pack=git -c receive.unpackLimit=1 receive-pack', + mirror, + `HEAD:refs/heads/${branch}` + ) +} + +function packSizes(mirror: string): Map { + const packDir = path.join(mirror, 'objects', 'pack') + return new Map( + packs(mirror).map(p => [p, fs.statSync(path.join(packDir, p)).size]) + ) +} + +/** Bytes of packs present after a run that were not present before it. */ +function bytesWritten( + before: Map, + after: Map +): number { + let n = 0 + for (const [name, size] of after) { + if (!before.has(name)) { + n += size + } + } + return n +} + +function stampReclaim(mirror: string, at: number): void { + fs.writeFileSync( + path.join(mirror, blacksmithCache.MAINTENANCE_RECLAIM_STAMP), + `${at}\n` + ) +} + /** * Bare mirror with one base pack larger than KEEP_BYTES, several small * packs (pushed with receive.unpackLimit=1) and a few loose objects. Auto @@ -146,11 +186,10 @@ describe('runMirrorMaintenance (real git)', () => { const {mirror, basePack} = buildMirror(root) const before = fs.statSync(path.join(mirror, 'objects', 'pack', basePack)) - const result = await blacksmithCache.runMirrorMaintenance( - mirror, - 60, - KEEP_BYTES - ) + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) expect(result).toEqual({success: true, timedOut: false}) const packDir = path.join(mirror, 'objects', 'pack') @@ -173,14 +212,16 @@ describe('runMirrorMaintenance (real git)', () => { it('is a no-op on an already maintained mirror', async () => { const {mirror} = buildMirror(root) - await blacksmithCache.runMirrorMaintenance(mirror, 60, KEEP_BYTES) + await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) const first = packs(mirror) - const result = await blacksmithCache.runMirrorMaintenance( - mirror, - 60, - KEEP_BYTES - ) + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) expect(result).toEqual({success: true, timedOut: false}) expect(packs(mirror)).toEqual(first) fsck(mirror) @@ -210,11 +251,10 @@ describe('runMirrorMaintenance (real git)', () => { 'HEAD:refs/heads/huge' ) - const result = await blacksmithCache.runMirrorMaintenance( - mirror, - 60, - Number.MAX_SAFE_INTEGER - ) + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: Number.MAX_SAFE_INTEGER + }) expect(result.success).toBe(true) expect(packs(mirror)).toContain(basePack) fsck(mirror) @@ -224,7 +264,10 @@ describe('runMirrorMaintenance (real git)', () => { it('marks only packs at or above the threshold as kept', async () => { const {mirror, basePack} = buildMirror(root) const kept = await blacksmithCache.markKeepPacks(mirror, KEEP_BYTES) - expect(kept).toEqual([basePack.replace(/\.pack$/, '')]) + expect(kept).toEqual({ + kept: [basePack.replace(/\.pack$/, '')], + deferred: [] + }) const keepFiles = fs .readdirSync(path.join(mirror, 'objects', 'pack')) .filter(f => f.endsWith('.keep')) @@ -236,6 +279,210 @@ describe('runMirrorMaintenance (real git)', () => { ) }) + it('never rewrites more than the keep threshold per run, whatever the object counts', async () => { + const {mirror, basePack} = buildMirror(root) + const src = path.join(root, 'src') + // Four medium packs of one large object each, and one pack of many tiny + // objects: all below the threshold on their own, well above it together. + // Geometric repack would pick them by object count; the byte bound must + // hold regardless. + for (let i = 0; i < 4; i++) { + commitBlob(src, `medium-${i}`, 400 * 1024) + pushPack(src, mirror, `medium-${i}`) + } + for (let i = 0; i < 200; i++) { + fs.writeFileSync(path.join(src, `tiny-${i}`), `tiny ${i}\n`) + } + git(src, 'add', '.') + git( + src, + '-c', + 'user.email=test@example.com', + '-c', + 'user.name=Test', + 'commit', + '-q', + '-m', + 'tiny' + ) + pushPack(src, mirror, 'tiny') + + const packDir = path.join(mirror, 'objects', 'pack') + let before = packSizes(mirror) + let unkept = 0 + for (const [name, size] of before) { + if (name !== basePack) { + unkept += size + } + } + expect(unkept).toBeGreaterThan(2 * KEEP_BYTES) + + for (let run = 0; run < 3; run++) { + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) + expect(result).toEqual({success: true, timedOut: false}) + const after = packSizes(mirror) + expect(bytesWritten(before, after)).toBeLessThan(KEEP_BYTES) + expect(after.has(basePack)).toBe(true) + // Only the base pack is kept for good; deferrals do not outlive the run. + const keepFiles = fs.readdirSync(packDir).filter(f => f.endsWith('.keep')) + expect(keepFiles).toEqual([basePack.replace(/\.pack$/, '.keep')]) + fsck(mirror) + refsResolve(mirror) + before = after + } + expect(looseObjects(mirror)).toBe(0) + }) + + it('defers the largest packs until the rest fit under the threshold', async () => { + const {mirror, basePack} = buildMirror(root) + const src = path.join(root, 'src') + for (let i = 0; i < 3; i++) { + commitBlob(src, `medium-${i}`, 400 * 1024) + pushPack(src, mirror, `medium-${i}`) + } + const sizes = packSizes(mirror) + const medium = [...sizes.entries()] + .filter(([, size]) => size > 300 * 1024 && size < KEEP_BYTES) + .map(([name]) => name.replace(/\.pack$/, '')) + .sort() + expect(medium).toHaveLength(3) + + const selection = await blacksmithCache.markKeepPacks(mirror, KEEP_BYTES) + expect(selection.kept).toEqual([basePack.replace(/\.pack$/, '')]) + // 3 x 400 KiB + 6 x 100 KiB + loose: dropping two of the medium packs + // brings the rest under 1 MiB. + expect(selection.deferred).toHaveLength(2) + for (const base of selection.deferred) { + expect(medium).toContain(base) + expect( + fs.readFileSync( + path.join(mirror, 'objects', 'pack', `${base}.keep`), + 'utf8' + ) + ).not.toBe('') + } + + // A deferral left behind by a killed run is lifted and re-evaluated. + const again = await blacksmithCache.markKeepPacks(mirror, KEEP_BYTES) + expect(again).toEqual(selection) + const withoutLimit = await blacksmithCache.markKeepPacks( + mirror, + Number.MAX_SAFE_INTEGER + ) + expect(withoutLimit.deferred).toEqual([]) + }) + + describe('reclaim', () => { + function addGarbage( + root: string, + mirror: string + ): {pack: string; blob: string} { + const src = path.join(root, 'src') + const before = new Set(packs(mirror)) + commitBlob(src, 'garbage', 600 * 1024) + const blob = git(src, 'rev-parse', 'HEAD:garbage') + pushPack(src, mirror, 'garbage') + const pack = packs(mirror).find(p => !before.has(p)) as string + fs.writeFileSync( + path.join(mirror, 'objects', 'pack', pack.replace(/\.pack$/, '.keep')), + '' + ) + git(mirror, 'update-ref', '-d', 'refs/heads/garbage') + return {pack, blob} + } + + function hasObject(mirror: string, oid: string): boolean { + try { + execFileSync('git', ['-C', mirror, 'cat-file', '-e', oid], { + stdio: 'pipe' + }) + return true + } catch { + return false + } + } + + it('starts the interval on first sight instead of reclaiming at once', async () => { + const {mirror} = buildMirror(root) + const {pack, blob} = addGarbage(root, mirror) + const stamp = path.join(mirror, blacksmithCache.MAINTENANCE_RECLAIM_STAMP) + expect(fs.existsSync(stamp)).toBe(false) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + now: 1000 + }) + expect(result).toEqual({success: true, timedOut: false}) + expect(fs.readFileSync(stamp, 'utf8').trim()).toBe('1000') + expect(packs(mirror)).toContain(pack) + expect(hasObject(mirror, blob)).toBe(true) + + // Within the interval: still incremental. + await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 5000 + }) + expect(fs.readFileSync(stamp, 'utf8').trim()).toBe('1000') + expect(packs(mirror)).toContain(pack) + fsck(mirror) + }) + + it('rewrites the whole mirror and drops unreachable objects when due', async () => { + const {mirror, basePack} = buildMirror(root) + const {pack, blob} = addGarbage(root, mirror) + stampReclaim(mirror, 0) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 20_000 + }) + expect(result).toEqual({success: true, timedOut: false}) + + const packDir = path.join(mirror, 'objects', 'pack') + const remaining = packs(mirror) + expect(remaining).toHaveLength(1) + expect(remaining).not.toContain(pack) + expect(remaining).not.toContain(basePack) + expect(hasObject(mirror, blob)).toBe(false) + expect(fs.readdirSync(packDir).filter(f => f.endsWith('.keep'))).toEqual( + [] + ) + expect(looseObjects(mirror)).toBe(0) + expect( + fs + .readFileSync( + path.join(mirror, blacksmithCache.MAINTENANCE_RECLAIM_STAMP), + 'utf8' + ) + .trim() + ).toBe('20000') + expect(fs.existsSync(path.join(packDir, 'multi-pack-index'))).toBe(true) + fsck(mirror) + refsResolve(mirror) + + // The next run keeps the new single pack and is incremental again. + const next = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 25_000 + }) + expect(next).toEqual({success: true, timedOut: false}) + expect(packs(mirror)).toEqual(remaining) + expect(fs.readdirSync(packDir).filter(f => f.endsWith('.keep'))).toEqual([ + remaining[0].replace(/\.pack$/, '.keep') + ]) + }) + }) + describe('with a misbehaving git', () => { let binDir: string let originalPath: string | undefined @@ -253,7 +500,7 @@ describe('runMirrorMaintenance (real git)', () => { // Jest's sandboxed process.env is not what child processes inherit, so // the shim is installed as a `timeout` found through the sandbox PATH // that puts binDir first on the PATH of the real one. - function installFakeGit(repackScript: string): void { + function installFakeGit(script: string, command = 'repack'): void { const realGit = execFileSync('which', ['git'], {encoding: 'utf8'}).trim() const realTimeout = execFileSync('which', ['timeout'], { encoding: 'utf8' @@ -262,8 +509,8 @@ describe('runMirrorMaintenance (real git)', () => { path.join(binDir, 'git'), `#!/bin/sh for arg in "$@"; do - if [ "$arg" = "repack" ]; then -${repackScript} + if [ "$arg" = "${command}" ]; then +${script} fi done exec ${realGit} "$@" @@ -287,11 +534,10 @@ PATH="${binDir}:$PATH" exec ${realTimeout} "$@" installFakeGit(` touch "${packDir}/tmp_pack_abc" "${packDir}/.tmp-1-pack-abc.pack" "${packDir}/multi-pack-index.lock" "${mirror}/packed-refs.lock" sleep 30`) - const result = await blacksmithCache.runMirrorMaintenance( - mirror, - 1, - KEEP_BYTES - ) + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 1, + keepBytes: KEEP_BYTES + }) expect(result.success).toBe(false) expect(result.timedOut).toBe(true) @@ -315,15 +561,91 @@ PATH="${binDir}:$PATH" exec ${realTimeout} "$@" const {mirror} = buildMirror(root) installFakeGit(' exit 128') - const result = await blacksmithCache.runMirrorMaintenance( - mirror, - 60, - KEEP_BYTES - ) + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) expect(result.success).toBe(false) expect(result.timedOut).toBe(false) expect(result.error).toContain('128') fsck(mirror) }) + it('reports a pack-refs failure without a timeout flag', async () => { + const {mirror} = buildMirror(root) + installFakeGit(' exit 3', 'pack-refs') + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) + expect(result).toEqual({ + success: false, + timedOut: false, + error: expect.stringContaining('pack-refs failed with exit code 3') + }) + fsck(mirror) + refsResolve(mirror) + }) + + it('reports a pack-refs timeout and removes its lock file', async () => { + const {mirror} = buildMirror(root) + installFakeGit( + ` touch "${mirror}/packed-refs.lock" + sleep 30`, + 'pack-refs' + ) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 1, + keepBytes: KEEP_BYTES + }) + expect(result).toEqual({ + success: false, + timedOut: true, + error: expect.stringContaining('pack-refs timed out') + }) + expect(fs.existsSync(path.join(mirror, 'packed-refs.lock'))).toBe(false) + fsck(mirror) + refsResolve(mirror) + }) + + it('a reclaim that times out leaves the mirror intact and is not retried', async () => { + const {mirror, basePack} = buildMirror(root) + const packDir = path.join(mirror, 'objects', 'pack') + const keepFile = path.join(packDir, basePack.replace(/\.pack$/, '.keep')) + fs.writeFileSync(keepFile, '') + stampReclaim(mirror, 0) + const before = packs(mirror) + installFakeGit(` touch "${packDir}/tmp_pack_reclaim" + sleep 30`) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + reclaimTimeoutSecs: 1, + now: 20_000 + }) + expect(result.success).toBe(false) + expect(result.timedOut).toBe(true) + expect(packs(mirror)).toEqual(before) + expect(fs.existsSync(path.join(packDir, 'tmp_pack_reclaim'))).toBe(false) + fsck(mirror) + refsResolve(mirror) + + // The stamp was advanced before the attempt, so the following run is + // incremental and re-marks the base pack instead of trying again. + process.env.PATH = originalPath + const next = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 25_000 + }) + expect(next).toEqual({success: true, timedOut: false}) + expect(fs.existsSync(keepFile)).toBe(true) + expect(packs(mirror)).toContain(basePack) + fsck(mirror) + }) }) }) diff --git a/dist/index.js b/dist/index.js index ddf05f0..dc2abbc 100644 --- a/dist/index.js +++ b/dist/index.js @@ -39,6 +39,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.MAINTENANCE_RECLAIM_STAMP = void 0; exports.getMountPoint = getMountPoint; exports.isBlacksmithEnvironment = isBlacksmithEnvironment; exports.getAgentAddr = getAgentAddr; @@ -91,13 +92,27 @@ const MIRROR_VERSION = 'v1'; // burns checkout time on a mirror disk that is having a bad day. The // window is kept generous so slow-but-healthy syncs still complete. const REFRESH_TIMEOUT_SECS = 120; // 2 minutes, single attempt -// Post-step mirror maintenance is bounded by construction (it never rewrites -// packs at or above MAINTENANCE_KEEP_PACK_BYTES), so this deadline is a -// safety net rather than an expected cost. Running out of it does not -// affect the commit: the mirror was already synced, maintenance is a pure -// optimization. +// Post-step mirror maintenance is bounded by construction (a run never +// rewrites more than MAINTENANCE_KEEP_PACK_BYTES of existing packs), so this +// deadline is a safety net rather than an expected cost. Running out of it +// does not affect the commit: the mirror was already synced, maintenance is +// a pure optimization. const MAINTENANCE_TIMEOUT_SECS = 120; // 2 minutes +// Packs at or above this size are kept permanently; the packs below it that +// one run rewrites are chosen so their combined size stays under it too. const MAINTENANCE_KEEP_PACK_BYTES = 256 * 1024 * 1024; +// Kept packs hold on to objects that later become unreachable (deleted or +// rewritten branches). A full rewrite with the keeps lifted reclaims them; +// it is unbounded in size, so it runs rarely and under its own deadline. +const MAINTENANCE_RECLAIM_INTERVAL_MS = 14 * 24 * 60 * 60 * 1000; +const MAINTENANCE_RECLAIM_TIMEOUT_SECS = 600; +// Loose objects younger than this survive a reclaim, the usual guard against +// deleting objects a concurrent writer has stored but not yet referenced. +const MAINTENANCE_RECLAIM_PRUNE_EXPIRE = '2.weeks.ago'; +// Marker in the mirror root recording the last reclaim attempt. +exports.MAINTENANCE_RECLAIM_STAMP = 'blacksmith-maintenance-reclaim'; +// Content of a `.keep` written only for the duration of one run. +const DEFERRED_KEEP_MARKER = 'blacksmith-checkout: deferred to a later run\n'; const FLUSH_TIMEOUT_SECS = 10; // 10 seconds for durability flush const UMOUNT_TIMEOUT_SECS = 10; // 10 seconds for unmount const UMOUNT_MAX_RETRIES = 3; // Number of unmount retry attempts @@ -1387,10 +1402,16 @@ function hasCommitGraph(mirrorPath) { fs.existsSync(path.join(mirrorPath, 'objects', 'info', 'commit-graphs', 'commit-graph-chain'))); } /** - * Mark every pack at or above MAINTENANCE_KEEP_PACK_BYTES with a `.keep` - * file so that no repack - this maintenance, or `gc` run by an older action - * version on the same mirror - ever rewrites or deletes it. Returns the - * names (without extension) of all kept packs. + * Choose which packs the geometric repack may rewrite. Every pack at or + * above `keepBytes` gets a permanent `.keep` file so that no repack - this + * maintenance, or `gc` run by an older action version on the same mirror - + * ever rewrites or deletes it. Of the remaining packs, the largest are given + * a `.keep` for this run only until the rest add up to less than `keepBytes`: + * git picks the packs to roll up by object count, so without this several + * medium packs could combine into a rewrite far larger than any one of them. + * A deferred pack is reconsidered on the next run, once the small packs + * around it have been folded. Returns the names (without extension) of both + * groups. * * `.keep` files are used rather than `repack --keep-pack`: git 2.34's * geometric repack does not exclude `--keep-pack` packs from the roll-up, @@ -1403,37 +1424,75 @@ function markKeepPacks(mirrorPath_1) { return __awaiter(this, arguments, void 0, function* (mirrorPath, keepBytes = MAINTENANCE_KEEP_PACK_BYTES) { const packDir = path.join(mirrorPath, 'objects', 'pack'); const kept = []; + const deferred = []; let entries; try { entries = yield fs.promises.readdir(packDir); } catch (_a) { - return kept; + return { kept, deferred }; } const names = new Set(entries); + const candidates = []; for (const entry of entries) { if (!entry.startsWith('pack-') || !entry.endsWith('.pack')) { continue; } const base = entry.slice(0, -'.pack'.length); - if (names.has(`${base}.keep`)) { - kept.push(base); - continue; - } + const keepFile = path.join(packDir, `${base}.keep`); try { + if (names.has(`${base}.keep`)) { + const content = yield fs.promises.readFile(keepFile, 'utf8'); + if (content !== DEFERRED_KEEP_MARKER) { + kept.push(base); + continue; + } + // Left behind by a run that was killed before it could clean up. + yield fs.promises.rm(keepFile, { force: true }); + } const stat = yield fs.promises.stat(path.join(packDir, entry)); - if (stat.size < keepBytes) { + if (stat.size >= keepBytes) { + yield fs.promises.writeFile(keepFile, ''); + core.info(`[git-mirror] Marked ${base} (${Math.round(stat.size / (1024 * 1024))} MiB) as kept`); + kept.push(base); continue; } - yield fs.promises.writeFile(path.join(packDir, `${base}.keep`), ''); - core.info(`[git-mirror] Marked ${base} (${Math.round(stat.size / (1024 * 1024))} MiB) as kept`); - kept.push(base); + candidates.push({ base, size: stat.size }); } catch (error) { core.warning(`[git-mirror] Failed to mark ${base} as kept: ${error}`); } } - return kept.sort(); + candidates.sort((a, b) => b.size - a.size); + let total = candidates.reduce((sum, c) => sum + c.size, 0); + for (const { base, size } of candidates) { + if (total < keepBytes) { + break; + } + try { + yield fs.promises.writeFile(path.join(packDir, `${base}.keep`), DEFERRED_KEEP_MARKER); + core.info(`[git-mirror] Deferred ${base} (${(size / (1024 * 1024)).toFixed(1)} MiB) to a later run`); + deferred.push(base); + total -= size; + } + catch (error) { + core.warning(`[git-mirror] Failed to defer ${base}: ${error}`); + } + } + return { kept: kept.sort(), deferred: deferred.sort() }; + }); +} +function removeKeepFiles(mirrorPath, packs) { + return __awaiter(this, void 0, void 0, function* () { + for (const base of packs) { + const keepFile = path.join(mirrorPath, 'objects', 'pack', `${base}.keep`); + try { + yield fs.promises.rm(keepFile, { force: true }); + } + catch (error) { + core.warning(`[git-mirror] Failed to remove ${base}.keep: ${error}`); + } + } }); } /** @@ -1489,18 +1548,62 @@ function removeMaintenanceLeftovers(mirrorPath) { } }); } +/** + * Whether a reclaim is due, and if so, record this attempt. The stamp is + * written before the reclaim runs so that a reclaim that times out is not + * retried by every following job. A mirror without a stamp (freshly + * hydrated, or maintained by an older action version) starts its interval + * now rather than reclaiming immediately, so a rollout does not make every + * job in the fleet rewrite its mirror at once. + */ +function reclaimDue(mirrorPath, now, intervalMs) { + return __awaiter(this, void 0, void 0, function* () { + const stamp = path.join(mirrorPath, exports.MAINTENANCE_RECLAIM_STAMP); + let last; + try { + const parsed = Number((yield fs.promises.readFile(stamp, 'utf8')).trim()); + if (Number.isFinite(parsed)) { + last = parsed; + } + } + catch (_a) { + // no stamp yet + } + if (last !== undefined && now - last < intervalMs) { + return false; + } + try { + yield fs.promises.writeFile(stamp, `${now}\n`); + } + catch (error) { + core.warning(`[git-mirror] Failed to write the reclaim stamp: ${error}`); + return false; + } + return last !== undefined; + }); +} /** * Bounded mirror maintenance, run in the post step once the mirror has been * synced. Each sync leaves a new small pack (or loose objects) behind; * left alone they accumulate until object lookup slows down. Instead of * `gc --auto` - whose "auto packing" step rewrites every pack including the * repository's multi-gigabyte base pack, taking minutes on large mirrors - - * this folds only the small packs and loose objects together with a - * geometric repack, explicitly keeping any pack at or above - * MAINTENANCE_KEEP_PACK_BYTES out of the rewrite. A rolled-up pack that - * grows past that size simply becomes another kept pack, so the cost of a - * single run is bounded by the keep threshold, never by the size of the - * repository. The multi-pack-index keeps lookups fast across the kept packs. + * this folds only small packs and loose objects together with a geometric + * repack. Packs at or above MAINTENANCE_KEEP_PACK_BYTES are kept out of the + * rewrite for good, and the packs below it that one run does rewrite are + * chosen so their combined size stays under that bound as well (see + * markKeepPacks). A rolled-up pack that grows past the threshold simply + * becomes another kept pack, so the cost of a single run is bounded by the + * threshold, never by the size of the repository. The multi-pack-index + * keeps lookups fast across the kept packs. + * + * Kept packs never lose objects, so history that becomes unreachable stays + * on disk. Once per MAINTENANCE_RECLAIM_INTERVAL_MS the run instead lifts + * every `.keep` and rewrites the whole mirror into a single pack, dropping + * unreachable objects, under the longer MAINTENANCE_RECLAIM_TIMEOUT_SECS. + * That rewrite is the one unbounded step; the stamp written beforehand + * keeps a mirror too large for the deadline from retrying it every job, + * and the incremental runs in between are unaffected either way. * * Maintenance is optional: its failure or timeout is reported but must not * prevent the synced mirror from being committed - otherwise a mirror that @@ -1508,56 +1611,89 @@ function removeMaintenanceLeftovers(mirrorPath) { * every subsequent job repeats the same doomed work. */ function runMirrorMaintenance(mirrorPath_1) { - return __awaiter(this, arguments, void 0, function* (mirrorPath, timeoutSecs = MAINTENANCE_TIMEOUT_SECS, keepBytes = MAINTENANCE_KEEP_PACK_BYTES) { + return __awaiter(this, arguments, void 0, function* (mirrorPath, options = {}) { + var _a, _b, _c, _d, _e; + const timeoutSecs = (_a = options.timeoutSecs) !== null && _a !== void 0 ? _a : MAINTENANCE_TIMEOUT_SECS; + const keepBytes = (_b = options.keepBytes) !== null && _b !== void 0 ? _b : MAINTENANCE_KEEP_PACK_BYTES; + const reclaimIntervalMs = (_c = options.reclaimIntervalMs) !== null && _c !== void 0 ? _c : MAINTENANCE_RECLAIM_INTERVAL_MS; + const reclaimTimeoutSecs = (_d = options.reclaimTimeoutSecs) !== null && _d !== void 0 ? _d : MAINTENANCE_RECLAIM_TIMEOUT_SECS; + const now = (_e = options.now) !== null && _e !== void 0 ? _e : Date.now(); const start = Date.now(); - const keptPacks = yield markKeepPacks(mirrorPath, keepBytes); - core.info(`[git-mirror] Running incremental maintenance (timeout: ${timeoutSecs}s, ${keptPacks.length} kept pack(s))`); + const reclaim = yield reclaimDue(mirrorPath, now, reclaimIntervalMs); + const budgetSecs = reclaim ? reclaimTimeoutSecs : timeoutSecs; + const deadline = start + budgetSecs * 1000; + const remainingSecs = () => Math.max(1, Math.ceil((deadline - Date.now()) / 1000)); + let deferred = []; + let label; + let repackArgs; + if (reclaim) { + const { kept } = yield markKeepPacks(mirrorPath, Number.MAX_SAFE_INTEGER); + yield removeKeepFiles(mirrorPath, kept); + label = 'Reclaim'; + repackArgs = ['-a', '-d', '-l', '-n', '--write-midx']; + core.info(`[git-mirror] Running reclaim maintenance (timeout: ${budgetSecs}s, ${kept.length} kept pack(s) released)`); + } + else { + const selection = yield markKeepPacks(mirrorPath, keepBytes); + deferred = selection.deferred; + label = 'Incremental'; + repackArgs = ['-d', '-l', '-n', '--geometric=2', '--write-midx']; + core.info(`[git-mirror] Running incremental maintenance (timeout: ${budgetSecs}s, ${selection.kept.length} kept pack(s), ${deferred.length} deferred)`); + } + const fail = (timedOut, error) => __awaiter(this, void 0, void 0, function* () { + core.warning(`[git-mirror] ${timedOut ? `Maintenance timed out after ${budgetSecs}s` : error}; committing the synced mirror without it`); + yield removeMaintenanceLeftovers(mirrorPath); + return { success: false, timedOut, error }; + }); try { const result = yield exec.getExecOutput('timeout', [ - String(timeoutSecs), + String(remainingSecs()), 'git', '-c', 'repack.writeBitmaps=false', '-C', mirrorPath, 'repack', - '-d', - '-l', - '-n', - '--geometric=2', - '--write-midx' + ...repackArgs ], { ignoreReturnCode: true }); if (result.exitCode === TIMEOUT_EXIT_CODE) { - core.warning(`[git-mirror] Maintenance timed out after ${timeoutSecs}s; committing the synced mirror without it`); - yield removeMaintenanceLeftovers(mirrorPath); - return { - success: false, - timedOut: true, - error: `git repack timed out after ${timeoutSecs}s` - }; + return yield fail(true, `git repack timed out after ${budgetSecs}s`); } if (result.exitCode !== 0) { - core.warning(`[git-mirror] Maintenance failed with exit code ${result.exitCode}; committing the synced mirror without it`); - yield removeMaintenanceLeftovers(mirrorPath); - return { - success: false, - timedOut: false, - error: `git repack failed with exit code ${result.exitCode}` - }; + return yield fail(false, `git repack failed with exit code ${result.exitCode}`); + } + if (reclaim) { + const prune = yield exec.getExecOutput('timeout', [ + String(remainingSecs()), + 'git', + '-C', + mirrorPath, + 'prune', + `--expire=${MAINTENANCE_RECLAIM_PRUNE_EXPIRE}` + ], { silent: true, ignoreReturnCode: true }); + if (prune.exitCode === TIMEOUT_EXIT_CODE) { + return yield fail(true, `git prune timed out after ${budgetSecs}s`); + } + if (prune.exitCode !== 0) { + return yield fail(false, `git prune failed with exit code ${prune.exitCode}`); + } + } + const packRefs = yield exec.getExecOutput('timeout', [String(remainingSecs()), 'git', '-C', mirrorPath, 'pack-refs', '--all'], { silent: true, ignoreReturnCode: true }); + if (packRefs.exitCode === TIMEOUT_EXIT_CODE) { + return yield fail(true, `git pack-refs timed out after ${budgetSecs}s`); } - const packRefs = yield exec.getExecOutput('timeout', [String(timeoutSecs), 'git', '-C', mirrorPath, 'pack-refs', '--all'], { silent: true, ignoreReturnCode: true }); if (packRefs.exitCode !== 0) { - core.warning(`[git-mirror] pack-refs failed with exit code ${packRefs.exitCode}`); - yield removeMaintenanceLeftovers(mirrorPath); + return yield fail(false, `git pack-refs failed with exit code ${packRefs.exitCode}`); } - core.info(`[git-mirror] Incremental maintenance finished in ${Date.now() - start}ms`); + core.info(`[git-mirror] ${label} maintenance finished in ${Date.now() - start}ms`); return { success: true, timedOut: false }; } catch (error) { const msg = error.message || String(error); - core.warning(`[git-mirror] Maintenance failed: ${msg}`); - yield removeMaintenanceLeftovers(mirrorPath); - return { success: false, timedOut: false, error: msg }; + return yield fail(false, msg); + } + finally { + yield removeKeepFiles(mirrorPath, deferred); } }); } diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index c700ae0..0a7a852 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -23,13 +23,27 @@ const MIRROR_VERSION = 'v1' // burns checkout time on a mirror disk that is having a bad day. The // window is kept generous so slow-but-healthy syncs still complete. const REFRESH_TIMEOUT_SECS = 120 // 2 minutes, single attempt -// Post-step mirror maintenance is bounded by construction (it never rewrites -// packs at or above MAINTENANCE_KEEP_PACK_BYTES), so this deadline is a -// safety net rather than an expected cost. Running out of it does not -// affect the commit: the mirror was already synced, maintenance is a pure -// optimization. +// Post-step mirror maintenance is bounded by construction (a run never +// rewrites more than MAINTENANCE_KEEP_PACK_BYTES of existing packs), so this +// deadline is a safety net rather than an expected cost. Running out of it +// does not affect the commit: the mirror was already synced, maintenance is +// a pure optimization. const MAINTENANCE_TIMEOUT_SECS = 120 // 2 minutes +// Packs at or above this size are kept permanently; the packs below it that +// one run rewrites are chosen so their combined size stays under it too. const MAINTENANCE_KEEP_PACK_BYTES = 256 * 1024 * 1024 +// Kept packs hold on to objects that later become unreachable (deleted or +// rewritten branches). A full rewrite with the keeps lifted reclaims them; +// it is unbounded in size, so it runs rarely and under its own deadline. +const MAINTENANCE_RECLAIM_INTERVAL_MS = 14 * 24 * 60 * 60 * 1000 +const MAINTENANCE_RECLAIM_TIMEOUT_SECS = 600 +// Loose objects younger than this survive a reclaim, the usual guard against +// deleting objects a concurrent writer has stored but not yet referenced. +const MAINTENANCE_RECLAIM_PRUNE_EXPIRE = '2.weeks.ago' +// Marker in the mirror root recording the last reclaim attempt. +export const MAINTENANCE_RECLAIM_STAMP = 'blacksmith-maintenance-reclaim' +// Content of a `.keep` written only for the duration of one run. +const DEFERRED_KEEP_MARKER = 'blacksmith-checkout: deferred to a later run\n' const FLUSH_TIMEOUT_SECS = 10 // 10 seconds for durability flush const UMOUNT_TIMEOUT_SECS = 10 // 10 seconds for unmount const UMOUNT_MAX_RETRIES = 3 // Number of unmount retry attempts @@ -1664,11 +1678,28 @@ export function hasCommitGraph(mirrorPath: string): boolean { ) } +export interface KeepSelection { + /** Packs with a permanent `.keep`: at or above the keep threshold. */ + kept: string[] + /** + * Packs under the threshold that this run leaves alone so that the packs + * it does rewrite stay under the threshold combined. Their `.keep` is + * removed once the run is over. + */ + deferred: string[] +} + /** - * Mark every pack at or above MAINTENANCE_KEEP_PACK_BYTES with a `.keep` - * file so that no repack - this maintenance, or `gc` run by an older action - * version on the same mirror - ever rewrites or deletes it. Returns the - * names (without extension) of all kept packs. + * Choose which packs the geometric repack may rewrite. Every pack at or + * above `keepBytes` gets a permanent `.keep` file so that no repack - this + * maintenance, or `gc` run by an older action version on the same mirror - + * ever rewrites or deletes it. Of the remaining packs, the largest are given + * a `.keep` for this run only until the rest add up to less than `keepBytes`: + * git picks the packs to roll up by object count, so without this several + * medium packs could combine into a rewrite far larger than any one of them. + * A deferred pack is reconsidered on the next run, once the small packs + * around it have been folded. Returns the names (without extension) of both + * groups. * * `.keep` files are used rather than `repack --keep-pack`: git 2.34's * geometric repack does not exclude `--keep-pack` packs from the roll-up, @@ -1680,40 +1711,84 @@ export function hasCommitGraph(mirrorPath: string): boolean { export async function markKeepPacks( mirrorPath: string, keepBytes: number = MAINTENANCE_KEEP_PACK_BYTES -): Promise { +): Promise { const packDir = path.join(mirrorPath, 'objects', 'pack') const kept: string[] = [] + const deferred: string[] = [] let entries: string[] try { entries = await fs.promises.readdir(packDir) } catch { - return kept + return {kept, deferred} } const names = new Set(entries) + const candidates: Array<{base: string; size: number}> = [] for (const entry of entries) { if (!entry.startsWith('pack-') || !entry.endsWith('.pack')) { continue } const base = entry.slice(0, -'.pack'.length) - if (names.has(`${base}.keep`)) { - kept.push(base) - continue - } + const keepFile = path.join(packDir, `${base}.keep`) try { + if (names.has(`${base}.keep`)) { + const content = await fs.promises.readFile(keepFile, 'utf8') + if (content !== DEFERRED_KEEP_MARKER) { + kept.push(base) + continue + } + // Left behind by a run that was killed before it could clean up. + await fs.promises.rm(keepFile, {force: true}) + } const stat = await fs.promises.stat(path.join(packDir, entry)) - if (stat.size < keepBytes) { + if (stat.size >= keepBytes) { + await fs.promises.writeFile(keepFile, '') + core.info( + `[git-mirror] Marked ${base} (${Math.round(stat.size / (1024 * 1024))} MiB) as kept` + ) + kept.push(base) continue } - await fs.promises.writeFile(path.join(packDir, `${base}.keep`), '') + candidates.push({base, size: stat.size}) + } catch (error) { + core.warning(`[git-mirror] Failed to mark ${base} as kept: ${error}`) + } + } + + candidates.sort((a, b) => b.size - a.size) + let total = candidates.reduce((sum, c) => sum + c.size, 0) + for (const {base, size} of candidates) { + if (total < keepBytes) { + break + } + try { + await fs.promises.writeFile( + path.join(packDir, `${base}.keep`), + DEFERRED_KEEP_MARKER + ) core.info( - `[git-mirror] Marked ${base} (${Math.round(stat.size / (1024 * 1024))} MiB) as kept` + `[git-mirror] Deferred ${base} (${(size / (1024 * 1024)).toFixed(1)} MiB) to a later run` ) - kept.push(base) + deferred.push(base) + total -= size } catch (error) { - core.warning(`[git-mirror] Failed to mark ${base} as kept: ${error}`) + core.warning(`[git-mirror] Failed to defer ${base}: ${error}`) + } + } + return {kept: kept.sort(), deferred: deferred.sort()} +} + +async function removeKeepFiles( + mirrorPath: string, + packs: string[] +): Promise { + for (const base of packs) { + const keepFile = path.join(mirrorPath, 'objects', 'pack', `${base}.keep`) + try { + await fs.promises.rm(keepFile, {force: true}) + } catch (error) { + core.warning(`[git-mirror] Failed to remove ${base}.keep: ${error}`) } } - return kept.sort() } /** @@ -1767,18 +1842,71 @@ async function removeMaintenanceLeftovers(mirrorPath: string): Promise { } } +/** + * Whether a reclaim is due, and if so, record this attempt. The stamp is + * written before the reclaim runs so that a reclaim that times out is not + * retried by every following job. A mirror without a stamp (freshly + * hydrated, or maintained by an older action version) starts its interval + * now rather than reclaiming immediately, so a rollout does not make every + * job in the fleet rewrite its mirror at once. + */ +async function reclaimDue( + mirrorPath: string, + now: number, + intervalMs: number +): Promise { + const stamp = path.join(mirrorPath, MAINTENANCE_RECLAIM_STAMP) + let last: number | undefined + try { + const parsed = Number((await fs.promises.readFile(stamp, 'utf8')).trim()) + if (Number.isFinite(parsed)) { + last = parsed + } + } catch { + // no stamp yet + } + if (last !== undefined && now - last < intervalMs) { + return false + } + try { + await fs.promises.writeFile(stamp, `${now}\n`) + } catch (error) { + core.warning(`[git-mirror] Failed to write the reclaim stamp: ${error}`) + return false + } + return last !== undefined +} + +export interface MaintenanceOptions { + timeoutSecs?: number + keepBytes?: number + reclaimIntervalMs?: number + reclaimTimeoutSecs?: number + now?: number +} + /** * Bounded mirror maintenance, run in the post step once the mirror has been * synced. Each sync leaves a new small pack (or loose objects) behind; * left alone they accumulate until object lookup slows down. Instead of * `gc --auto` - whose "auto packing" step rewrites every pack including the * repository's multi-gigabyte base pack, taking minutes on large mirrors - - * this folds only the small packs and loose objects together with a - * geometric repack, explicitly keeping any pack at or above - * MAINTENANCE_KEEP_PACK_BYTES out of the rewrite. A rolled-up pack that - * grows past that size simply becomes another kept pack, so the cost of a - * single run is bounded by the keep threshold, never by the size of the - * repository. The multi-pack-index keeps lookups fast across the kept packs. + * this folds only small packs and loose objects together with a geometric + * repack. Packs at or above MAINTENANCE_KEEP_PACK_BYTES are kept out of the + * rewrite for good, and the packs below it that one run does rewrite are + * chosen so their combined size stays under that bound as well (see + * markKeepPacks). A rolled-up pack that grows past the threshold simply + * becomes another kept pack, so the cost of a single run is bounded by the + * threshold, never by the size of the repository. The multi-pack-index + * keeps lookups fast across the kept packs. + * + * Kept packs never lose objects, so history that becomes unreachable stays + * on disk. Once per MAINTENANCE_RECLAIM_INTERVAL_MS the run instead lifts + * every `.keep` and rewrites the whole mirror into a single pack, dropping + * unreachable objects, under the longer MAINTENANCE_RECLAIM_TIMEOUT_SECS. + * That rewrite is the one unbounded step; the stamp written beforehand + * keeps a mirror too large for the deadline from retrying it every job, + * and the incremental runs in between are unaffected either way. * * Maintenance is optional: its failure or timeout is reported but must not * prevent the synced mirror from being committed - otherwise a mirror that @@ -1787,78 +1915,130 @@ async function removeMaintenanceLeftovers(mirrorPath: string): Promise { */ export async function runMirrorMaintenance( mirrorPath: string, - timeoutSecs: number = MAINTENANCE_TIMEOUT_SECS, - keepBytes: number = MAINTENANCE_KEEP_PACK_BYTES + options: MaintenanceOptions = {} ): Promise { + const timeoutSecs = options.timeoutSecs ?? MAINTENANCE_TIMEOUT_SECS + const keepBytes = options.keepBytes ?? MAINTENANCE_KEEP_PACK_BYTES + const reclaimIntervalMs = + options.reclaimIntervalMs ?? MAINTENANCE_RECLAIM_INTERVAL_MS + const reclaimTimeoutSecs = + options.reclaimTimeoutSecs ?? MAINTENANCE_RECLAIM_TIMEOUT_SECS + const now = options.now ?? Date.now() const start = Date.now() - const keptPacks = await markKeepPacks(mirrorPath, keepBytes) - core.info( - `[git-mirror] Running incremental maintenance (timeout: ${timeoutSecs}s, ${keptPacks.length} kept pack(s))` - ) + + const reclaim = await reclaimDue(mirrorPath, now, reclaimIntervalMs) + const budgetSecs = reclaim ? reclaimTimeoutSecs : timeoutSecs + const deadline = start + budgetSecs * 1000 + const remainingSecs = (): number => + Math.max(1, Math.ceil((deadline - Date.now()) / 1000)) + + let deferred: string[] = [] + let label: string + let repackArgs: string[] + if (reclaim) { + const {kept} = await markKeepPacks(mirrorPath, Number.MAX_SAFE_INTEGER) + await removeKeepFiles(mirrorPath, kept) + label = 'Reclaim' + repackArgs = ['-a', '-d', '-l', '-n', '--write-midx'] + core.info( + `[git-mirror] Running reclaim maintenance (timeout: ${budgetSecs}s, ${kept.length} kept pack(s) released)` + ) + } else { + const selection = await markKeepPacks(mirrorPath, keepBytes) + deferred = selection.deferred + label = 'Incremental' + repackArgs = ['-d', '-l', '-n', '--geometric=2', '--write-midx'] + core.info( + `[git-mirror] Running incremental maintenance (timeout: ${budgetSecs}s, ${selection.kept.length} kept pack(s), ${deferred.length} deferred)` + ) + } + + const fail = async ( + timedOut: boolean, + error: string + ): Promise => { + core.warning( + `[git-mirror] ${ + timedOut ? `Maintenance timed out after ${budgetSecs}s` : error + }; committing the synced mirror without it` + ) + await removeMaintenanceLeftovers(mirrorPath) + return {success: false, timedOut, error} + } try { const result = await exec.getExecOutput( 'timeout', [ - String(timeoutSecs), + String(remainingSecs()), 'git', '-c', 'repack.writeBitmaps=false', '-C', mirrorPath, 'repack', - '-d', - '-l', - '-n', - '--geometric=2', - '--write-midx' + ...repackArgs ], {ignoreReturnCode: true} ) if (result.exitCode === TIMEOUT_EXIT_CODE) { - core.warning( - `[git-mirror] Maintenance timed out after ${timeoutSecs}s; committing the synced mirror without it` - ) - await removeMaintenanceLeftovers(mirrorPath) - return { - success: false, - timedOut: true, - error: `git repack timed out after ${timeoutSecs}s` - } + return await fail(true, `git repack timed out after ${budgetSecs}s`) } if (result.exitCode !== 0) { - core.warning( - `[git-mirror] Maintenance failed with exit code ${result.exitCode}; committing the synced mirror without it` + return await fail( + false, + `git repack failed with exit code ${result.exitCode}` ) - await removeMaintenanceLeftovers(mirrorPath) - return { - success: false, - timedOut: false, - error: `git repack failed with exit code ${result.exitCode}` + } + + if (reclaim) { + const prune = await exec.getExecOutput( + 'timeout', + [ + String(remainingSecs()), + 'git', + '-C', + mirrorPath, + 'prune', + `--expire=${MAINTENANCE_RECLAIM_PRUNE_EXPIRE}` + ], + {silent: true, ignoreReturnCode: true} + ) + if (prune.exitCode === TIMEOUT_EXIT_CODE) { + return await fail(true, `git prune timed out after ${budgetSecs}s`) + } + if (prune.exitCode !== 0) { + return await fail( + false, + `git prune failed with exit code ${prune.exitCode}` + ) } } const packRefs = await exec.getExecOutput( 'timeout', - [String(timeoutSecs), 'git', '-C', mirrorPath, 'pack-refs', '--all'], + [String(remainingSecs()), 'git', '-C', mirrorPath, 'pack-refs', '--all'], {silent: true, ignoreReturnCode: true} ) + if (packRefs.exitCode === TIMEOUT_EXIT_CODE) { + return await fail(true, `git pack-refs timed out after ${budgetSecs}s`) + } if (packRefs.exitCode !== 0) { - core.warning( - `[git-mirror] pack-refs failed with exit code ${packRefs.exitCode}` + return await fail( + false, + `git pack-refs failed with exit code ${packRefs.exitCode}` ) - await removeMaintenanceLeftovers(mirrorPath) } core.info( - `[git-mirror] Incremental maintenance finished in ${Date.now() - start}ms` + `[git-mirror] ${label} maintenance finished in ${Date.now() - start}ms` ) return {success: true, timedOut: false} } catch (error) { const msg = (error as Error).message || String(error) - core.warning(`[git-mirror] Maintenance failed: ${msg}`) - await removeMaintenanceLeftovers(mirrorPath) - return {success: false, timedOut: false, error: msg} + return await fail(false, msg) + } finally { + await removeKeepFiles(mirrorPath, deferred) } } From 2f902a82ba6a47d34228408616af602dffb76896 Mon Sep 17 00:00:00 2001 From: piotr Date: Wed, 16 Sep 2026 19:51:55 +0000 Subject: [PATCH 6/7] Maintenance: rebuild the commit-graph after a reclaim repack -a -d + prune drop unreachable commits the existing commit-graph still lists; fsck and incremental graph writes then fail on those entries. Remove the graph (single file or chain) after a successful reclaim and write it again from the reachable commits, within the reclaim's remaining budget. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- __test__/mirror-maintenance-git.test.ts | 24 ++++++++++++++++++++---- dist/index.js | 21 +++++++++++++++++++++ src/blacksmith-cache.ts | 19 +++++++++++++++++++ 3 files changed, 60 insertions(+), 4 deletions(-) diff --git a/__test__/mirror-maintenance-git.test.ts b/__test__/mirror-maintenance-git.test.ts index ba8b2bf..ad21e9d 100644 --- a/__test__/mirror-maintenance-git.test.ts +++ b/__test__/mirror-maintenance-git.test.ts @@ -376,22 +376,29 @@ describe('runMirrorMaintenance (real git)', () => { }) describe('reclaim', () => { + // Pushes a branch into its own kept pack, optionally records it in the + // commit-graph, then deletes the branch so its objects are unreachable. function addGarbage( root: string, - mirror: string - ): {pack: string; blob: string} { + mirror: string, + options: {commitGraph?: boolean} = {} + ): {pack: string; blob: string; commit: string} { const src = path.join(root, 'src') const before = new Set(packs(mirror)) commitBlob(src, 'garbage', 600 * 1024) const blob = git(src, 'rev-parse', 'HEAD:garbage') + const commit = git(src, 'rev-parse', 'HEAD') pushPack(src, mirror, 'garbage') const pack = packs(mirror).find(p => !before.has(p)) as string fs.writeFileSync( path.join(mirror, 'objects', 'pack', pack.replace(/\.pack$/, '.keep')), '' ) + if (options.commitGraph) { + git(mirror, 'commit-graph', 'write', '--reachable', '--split') + } git(mirror, 'update-ref', '-d', 'refs/heads/garbage') - return {pack, blob} + return {pack, blob, commit} } function hasObject(mirror: string, oid: string): boolean { @@ -435,7 +442,12 @@ describe('runMirrorMaintenance (real git)', () => { it('rewrites the whole mirror and drops unreachable objects when due', async () => { const {mirror, basePack} = buildMirror(root) - const {pack, blob} = addGarbage(root, mirror) + const { + pack, + blob, + commit: garbageCommit + } = addGarbage(root, mirror, {commitGraph: true}) + expect(blacksmithCache.hasCommitGraph(mirror)).toBe(true) stampReclaim(mirror, 0) const result = await blacksmithCache.runMirrorMaintenance(mirror, { @@ -465,6 +477,10 @@ describe('runMirrorMaintenance (real git)', () => { .trim() ).toBe('20000') expect(fs.existsSync(path.join(packDir, 'multi-pack-index'))).toBe(true) + // The graph was rebuilt without the pruned commit. + expect(blacksmithCache.hasCommitGraph(mirror)).toBe(true) + git(mirror, 'commit-graph', 'verify') + expect(hasObject(mirror, garbageCommit)).toBe(false) fsck(mirror) refsResolve(mirror) diff --git a/dist/index.js b/dist/index.js index dc2abbc..196daad 100644 --- a/dist/index.js +++ b/dist/index.js @@ -1393,6 +1393,22 @@ function writeCommitGraph(mirrorPath_1) { } }); } +function removeCommitGraph(mirrorPath) { + return __awaiter(this, void 0, void 0, function* () { + const info = path.join(mirrorPath, 'objects', 'info'); + for (const target of [ + path.join(info, 'commit-graph'), + path.join(info, 'commit-graphs') + ]) { + try { + yield fs.promises.rm(target, { recursive: true, force: true }); + } + catch (error) { + core.warning(`[git-mirror] Failed to remove ${target}: ${error}`); + } + } + }); +} /** * Whether the mirror has a commit-graph (single file or split chain). * Determines if the sync fetch can write incrementally. @@ -1677,6 +1693,11 @@ function runMirrorMaintenance(mirrorPath_1) { if (prune.exitCode !== 0) { return yield fail(false, `git prune failed with exit code ${prune.exitCode}`); } + // The commit-graph still lists the commits just pruned; fsck and + // incremental graph writes fail on such entries. Rebuild it from + // what is reachable now. + yield removeCommitGraph(mirrorPath); + yield writeCommitGraph(mirrorPath, remainingSecs()); } const packRefs = yield exec.getExecOutput('timeout', [String(remainingSecs()), 'git', '-C', mirrorPath, 'pack-refs', '--all'], { silent: true, ignoreReturnCode: true }); if (packRefs.exitCode === TIMEOUT_EXIT_CODE) { diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index 0a7a852..9b9d9cf 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -1659,6 +1659,20 @@ async function writeCommitGraph( } } +async function removeCommitGraph(mirrorPath: string): Promise { + const info = path.join(mirrorPath, 'objects', 'info') + for (const target of [ + path.join(info, 'commit-graph'), + path.join(info, 'commit-graphs') + ]) { + try { + await fs.promises.rm(target, {recursive: true, force: true}) + } catch (error) { + core.warning(`[git-mirror] Failed to remove ${target}: ${error}`) + } + } +} + /** * Whether the mirror has a commit-graph (single file or split chain). * Determines if the sync fetch can write incrementally. @@ -2013,6 +2027,11 @@ export async function runMirrorMaintenance( `git prune failed with exit code ${prune.exitCode}` ) } + // The commit-graph still lists the commits just pruned; fsck and + // incremental graph writes fail on such entries. Rebuild it from + // what is reachable now. + await removeCommitGraph(mirrorPath) + await writeCommitGraph(mirrorPath, remainingSecs()) } const packRefs = await exec.getExecOutput( From 6572b50b6e3ec6283df3394e13d54f4a2f383480 Mon Sep 17 00:00:00 2001 From: piotr Date: Wed, 16 Sep 2026 20:01:54 +0000 Subject: [PATCH 7/7] Maintenance: drop the commit-graph before a reclaim; store synced objects packed The reclaim repack deletes unreachable commits; a graph that still lists them fails fsck in every later job if prune then fails, so the graph is removed before the repack and reclaim is skipped when that removal fails. Sync fetches use fetch.unpackLimit=1 so objects always land in a pack whose size the per-run byte budget can see. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- __test__/mirror-maintenance-git.test.ts | 93 +++++++++++++------- __test__/mirror-sync-negotiation-git.test.ts | 27 ++++-- dist/index.js | 21 +++-- src/blacksmith-cache.ts | 24 +++-- 4 files changed, 116 insertions(+), 49 deletions(-) diff --git a/__test__/mirror-maintenance-git.test.ts b/__test__/mirror-maintenance-git.test.ts index ad21e9d..4475006 100644 --- a/__test__/mirror-maintenance-git.test.ts +++ b/__test__/mirror-maintenance-git.test.ts @@ -375,43 +375,43 @@ describe('runMirrorMaintenance (real git)', () => { expect(withoutLimit.deferred).toEqual([]) }) - describe('reclaim', () => { - // Pushes a branch into its own kept pack, optionally records it in the - // commit-graph, then deletes the branch so its objects are unreachable. - function addGarbage( - root: string, - mirror: string, - options: {commitGraph?: boolean} = {} - ): {pack: string; blob: string; commit: string} { - const src = path.join(root, 'src') - const before = new Set(packs(mirror)) - commitBlob(src, 'garbage', 600 * 1024) - const blob = git(src, 'rev-parse', 'HEAD:garbage') - const commit = git(src, 'rev-parse', 'HEAD') - pushPack(src, mirror, 'garbage') - const pack = packs(mirror).find(p => !before.has(p)) as string - fs.writeFileSync( - path.join(mirror, 'objects', 'pack', pack.replace(/\.pack$/, '.keep')), - '' - ) - if (options.commitGraph) { - git(mirror, 'commit-graph', 'write', '--reachable', '--split') - } - git(mirror, 'update-ref', '-d', 'refs/heads/garbage') - return {pack, blob, commit} + // Pushes a branch into its own kept pack, optionally records it in the + // commit-graph, then deletes the branch so its objects are unreachable. + function addGarbage( + root: string, + mirror: string, + options: {commitGraph?: boolean} = {} + ): {pack: string; blob: string; commit: string} { + const src = path.join(root, 'src') + const before = new Set(packs(mirror)) + commitBlob(src, 'garbage', 600 * 1024) + const blob = git(src, 'rev-parse', 'HEAD:garbage') + const commit = git(src, 'rev-parse', 'HEAD') + pushPack(src, mirror, 'garbage') + const pack = packs(mirror).find(p => !before.has(p)) as string + fs.writeFileSync( + path.join(mirror, 'objects', 'pack', pack.replace(/\.pack$/, '.keep')), + '' + ) + if (options.commitGraph) { + git(mirror, 'commit-graph', 'write', '--reachable', '--split') } + git(mirror, 'update-ref', '-d', 'refs/heads/garbage') + return {pack, blob, commit} + } - function hasObject(mirror: string, oid: string): boolean { - try { - execFileSync('git', ['-C', mirror, 'cat-file', '-e', oid], { - stdio: 'pipe' - }) - return true - } catch { - return false - } + function hasObject(mirror: string, oid: string): boolean { + try { + execFileSync('git', ['-C', mirror, 'cat-file', '-e', oid], { + stdio: 'pipe' + }) + return true + } catch { + return false } + } + describe('reclaim', () => { it('starts the interval on first sight instead of reclaiming at once', async () => { const {mirror} = buildMirror(root) const {pack, blob} = addGarbage(root, mirror) @@ -663,5 +663,32 @@ PATH="${binDir}:$PATH" exec ${realTimeout} "$@" expect(packs(mirror)).toContain(basePack) fsck(mirror) }) + + it('a prune failure after the reclaim repack leaves no stale commit-graph', async () => { + const {mirror} = buildMirror(root) + const {commit} = addGarbage(root, mirror, {commitGraph: true}) + expect(blacksmithCache.hasCommitGraph(mirror)).toBe(true) + stampReclaim(mirror, 0) + installFakeGit(' exit 7', 'prune') + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 20_000 + }) + expect(result).toEqual({ + success: false, + timedOut: false, + error: expect.stringContaining('prune failed with exit code 7') + }) + + // The repack already dropped the unreachable commit; a graph still + // listing it would fail fsck in every later job. + expect(hasObject(mirror, commit)).toBe(false) + expect(blacksmithCache.hasCommitGraph(mirror)).toBe(false) + fsck(mirror) + refsResolve(mirror) + }) }) }) diff --git a/__test__/mirror-sync-negotiation-git.test.ts b/__test__/mirror-sync-negotiation-git.test.ts index 4bc942d..ac34cb7 100644 --- a/__test__/mirror-sync-negotiation-git.test.ts +++ b/__test__/mirror-sync-negotiation-git.test.ts @@ -54,16 +54,21 @@ function commit(repo: string, msg: string): string { return git(repo, 'rev-parse', 'HEAD') } -/** Loose + packed object count, i.e. everything a fetch wrote so far. */ -function objectCount(repo: string): number { - let total = 0 +function countObjects(repo: string): {loose: number; packed: number} { + const counts = {loose: 0, packed: 0} for (const line of git(repo, 'count-objects', '-v').split('\n')) { const match = /^(count|in-pack): (\d+)$/.exec(line.trim()) if (match) { - total += parseInt(match[2], 10) + counts[match[1] === 'count' ? 'loose' : 'packed'] = parseInt(match[2], 10) } } - return total + return counts +} + +/** Loose + packed object count, i.e. everything a fetch wrote so far. */ +function objectCount(repo: string): number { + const {loose, packed} = countObjects(repo) + return loose + packed } function packetLines(packetTrace: string, re: RegExp): Set { @@ -163,6 +168,10 @@ describe('mirror sync negotiation with real git', () => { const packetTrace = path.join(tmpDir, 'sync-packets') process.env['GIT_TRACE_PACKET'] = packetTrace const before = objectCount(mirrorPath) + const looseBefore = countObjects(mirrorPath).loose + const packsBefore = fs + .readdirSync(path.join(mirrorPath, 'objects', 'pack')) + .filter(f => f.endsWith('.pack')) const result = await blacksmithCache.syncMirrorFromRemote( mirrorPath, @@ -192,5 +201,13 @@ describe('mirror sync negotiation with real git', () => { const received = objectCount(mirrorPath) - before expect(received).toBeGreaterThan(0) expect(received).toBeLessThan(TRUNK_COMMITS_AFTER_FORK) + + // Small as it is, the fetch is stored as a pack rather than loose + // objects, so maintenance sees its size. + expect(countObjects(mirrorPath).loose).toBe(looseBefore) + const packsAfter = fs + .readdirSync(path.join(mirrorPath, 'objects', 'pack')) + .filter(f => f.endsWith('.pack')) + expect(packsAfter.length).toBe(packsBefore.length + 1) }) }) diff --git a/dist/index.js b/dist/index.js index 196daad..18e420e 100644 --- a/dist/index.js +++ b/dist/index.js @@ -869,6 +869,12 @@ function syncMirrorFromRemote(mirrorPath_1, repoUrl_1, authToken_1) { 'gc.auto=0', '-c', 'fetch.negotiationAlgorithm=skipping', + // Always store what a sync receives as a pack, never as loose + // objects: maintenance bounds the bytes it rewrites per run by + // pack size (see markKeepPacks), and a large blob that arrived + // loose would not be counted. + '-c', + 'fetch.unpackLimit=1', // Keep the commit-graph current so ref-tip commit parsing // (mark_complete_local_refs and negotiation walks) reads one // compact mmap'd file instead of scattered pack entries. Only @@ -1405,8 +1411,10 @@ function removeCommitGraph(mirrorPath) { } catch (error) { core.warning(`[git-mirror] Failed to remove ${target}: ${error}`); + return false; } } + return true; }); } /** @@ -1635,7 +1643,14 @@ function runMirrorMaintenance(mirrorPath_1) { const reclaimTimeoutSecs = (_d = options.reclaimTimeoutSecs) !== null && _d !== void 0 ? _d : MAINTENANCE_RECLAIM_TIMEOUT_SECS; const now = (_e = options.now) !== null && _e !== void 0 ? _e : Date.now(); const start = Date.now(); - const reclaim = yield reclaimDue(mirrorPath, now, reclaimIntervalMs); + // A commit-graph that outlived the full rewrite would still list the + // commits it drops; should prune or a later step then fail, the mirror + // is committed with that graph and fsck breaks in every following job. + // So the graph is removed first, and reclaim is skipped when that fails. + // Without a graph git only walks commits the slow way until a new one is + // written after a successful prune. + const reclaim = (yield reclaimDue(mirrorPath, now, reclaimIntervalMs)) && + (yield removeCommitGraph(mirrorPath)); const budgetSecs = reclaim ? reclaimTimeoutSecs : timeoutSecs; const deadline = start + budgetSecs * 1000; const remainingSecs = () => Math.max(1, Math.ceil((deadline - Date.now()) / 1000)); @@ -1693,10 +1708,6 @@ function runMirrorMaintenance(mirrorPath_1) { if (prune.exitCode !== 0) { return yield fail(false, `git prune failed with exit code ${prune.exitCode}`); } - // The commit-graph still lists the commits just pruned; fsck and - // incremental graph writes fail on such entries. Rebuild it from - // what is reachable now. - yield removeCommitGraph(mirrorPath); yield writeCommitGraph(mirrorPath, remainingSecs()); } const packRefs = yield exec.getExecOutput('timeout', [String(remainingSecs()), 'git', '-C', mirrorPath, 'pack-refs', '--all'], { silent: true, ignoreReturnCode: true }); diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index 9b9d9cf..baf402f 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -1010,6 +1010,12 @@ export async function syncMirrorFromRemote( 'gc.auto=0', '-c', 'fetch.negotiationAlgorithm=skipping', + // Always store what a sync receives as a pack, never as loose + // objects: maintenance bounds the bytes it rewrites per run by + // pack size (see markKeepPacks), and a large blob that arrived + // loose would not be counted. + '-c', + 'fetch.unpackLimit=1', // Keep the commit-graph current so ref-tip commit parsing // (mark_complete_local_refs and negotiation walks) reads one // compact mmap'd file instead of scattered pack entries. Only @@ -1659,7 +1665,7 @@ async function writeCommitGraph( } } -async function removeCommitGraph(mirrorPath: string): Promise { +async function removeCommitGraph(mirrorPath: string): Promise { const info = path.join(mirrorPath, 'objects', 'info') for (const target of [ path.join(info, 'commit-graph'), @@ -1669,8 +1675,10 @@ async function removeCommitGraph(mirrorPath: string): Promise { await fs.promises.rm(target, {recursive: true, force: true}) } catch (error) { core.warning(`[git-mirror] Failed to remove ${target}: ${error}`) + return false } } + return true } /** @@ -1940,7 +1948,15 @@ export async function runMirrorMaintenance( const now = options.now ?? Date.now() const start = Date.now() - const reclaim = await reclaimDue(mirrorPath, now, reclaimIntervalMs) + // A commit-graph that outlived the full rewrite would still list the + // commits it drops; should prune or a later step then fail, the mirror + // is committed with that graph and fsck breaks in every following job. + // So the graph is removed first, and reclaim is skipped when that fails. + // Without a graph git only walks commits the slow way until a new one is + // written after a successful prune. + const reclaim = + (await reclaimDue(mirrorPath, now, reclaimIntervalMs)) && + (await removeCommitGraph(mirrorPath)) const budgetSecs = reclaim ? reclaimTimeoutSecs : timeoutSecs const deadline = start + budgetSecs * 1000 const remainingSecs = (): number => @@ -2027,10 +2043,6 @@ export async function runMirrorMaintenance( `git prune failed with exit code ${prune.exitCode}` ) } - // The commit-graph still lists the commits just pruned; fsck and - // incremental graph writes fail on such entries. Rebuild it from - // what is reachable now. - await removeCommitGraph(mirrorPath) await writeCommitGraph(mirrorPath, remainingSecs()) }