diff --git a/.github/workflows/test-blacksmith.yml b/.github/workflows/test-blacksmith.yml index bb00214..d2c24d3 100644 --- a/.github/workflows/test-blacksmith.yml +++ b/.github/workflows/test-blacksmith.yml @@ -128,6 +128,11 @@ jobs: # checkout step mounts its own copy of the last commit, so ownership cannot be # flipped between two steps of one job; the takeover is only observable # across jobs, and in the VM direction only after a container job committed.) + # + # With sticky disk branch protection on, only trusted triggers (push to main) + # may hydrate the mirror. When no hydrated mirror exists yet, PR runs are + # sent down the direct-checkout fallback; they verify that fallback and + # report the mirror assertions as skipped instead of failing. test-git-mirror-container: runs-on: blacksmith container: @@ -168,6 +173,14 @@ jobs: test "$(id -u)" = 0 mirror="/blacksmith-git-mirror/${GITHUB_REPOSITORY}/v1/${GITHUB_REPOSITORY_OWNER}-${GITHUB_REPOSITORY#*/}.git" if ! mountpoint -q "$(dirname "$(dirname "$mirror")")"; then + if [ "$GITHUB_EVENT_NAME" != push ]; then + echo "::notice::Sticky disk not mounted (mirror not hydrated for a ${GITHUB_EVENT_NAME} run); verifying the direct-checkout fallback only" + cd host-owned + git config --global --add safe.directory "$PWD" + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + git fsck --no-dangling + exit 0 + fi echo "Sticky disk is not mounted inside the container; the mirror path was not exercised" exit 1 fi @@ -205,6 +218,16 @@ jobs: run: | set -euo pipefail mirror="/blacksmith-git-mirror/${GITHUB_REPOSITORY}/v1/${GITHUB_REPOSITORY_OWNER}-${GITHUB_REPOSITORY#*/}.git" + if ! mountpoint -q "$(dirname "$(dirname "$mirror")")"; then + if [ "$GITHUB_EVENT_NAME" != push ]; then + echo "::notice::Sticky disk not mounted (mirror not hydrated for a ${GITHUB_EVENT_NAME} run); verifying the direct-checkout fallback only" + test "$(git -C after-container rev-parse HEAD)" = "$GITHUB_SHA" + git -C after-container fsck --no-dangling + exit 0 + fi + echo "Sticky disk is not mounted; the mirror path was not exercised" + exit 1 + fi test -d "$mirror" test "$(stat -c %u "$mirror")" = "$(id -u)" grep -qF "$mirror/objects" after-container/.git/objects/info/alternates diff --git a/__test__/mirror-cleanup.test.ts b/__test__/mirror-cleanup.test.ts new file mode 100644 index 0000000..4243f58 --- /dev/null +++ b/__test__/mirror-cleanup.test.ts @@ -0,0 +1,210 @@ +/** + * Post-step cleanup commit decision: mirror sync failure vetoes the commit, + * maintenance failure or timeout does not, and maintenance only runs when + * the result will be persisted. + */ +const mockCommitStickyDisk = jest.fn() + +jest.mock('@connectrpc/connect', () => ({ + createClient: jest.fn(() => ({commitStickyDisk: mockCommitStickyDisk})), + ConnectError: class ConnectError extends Error {}, + Code: {Aborted: 'ABORTED'} +})) + +jest.mock('@connectrpc/connect-node', () => ({ + createGrpcTransport: jest.fn() +})) + +jest.mock( + '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect', + () => ({ + StickyDiskService: {} + }) +) + +jest.mock('../src/container-detector', () => ({ + isRunningInContainer: jest.fn(() => false) +})) + +jest.mock('@actions/exec') + +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import * as exec from '@actions/exec' +import * as blacksmithCache from '../src/blacksmith-cache' + +const mockExec = exec.exec as jest.MockedFunction +const mockGetExecOutput = exec.getExecOutput as jest.MockedFunction< + typeof exec.getExecOutput +> + +function isRepack(args: string[] | undefined): boolean { + return (args || []).includes('repack') +} + +function isPackRefs(args: string[] | undefined): boolean { + return (args || []).includes('pack-refs') +} + +function commands(): string[][] { + return mockGetExecOutput.mock.calls.map(([tool, args]) => [ + tool, + ...(args || []) + ]) +} + +describe('cleanup commit decision', () => { + let mirrorPath: string + let repackExitCode: number + let packRefsExitCode: number + + beforeEach(() => { + jest.clearAllMocks() + process.env.BLACKSMITH_AGENT_ADDR = '127.0.0.1' + process.env.BLACKSMITH_STICKY_DISK_GRPC_PORT = '1' + mirrorPath = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-cleanup-')) + fs.mkdirSync(path.join(mirrorPath, 'objects', 'pack'), {recursive: true}) + repackExitCode = 0 + packRefsExitCode = 0 + mockExec.mockResolvedValue(0) + mockGetExecOutput.mockImplementation(async (_tool, args) => ({ + exitCode: isRepack(args) + ? repackExitCode + : isPackRefs(args) + ? packRefsExitCode + : 0, + stdout: '', + stderr: '' + })) + mockCommitStickyDisk.mockResolvedValue({}) + }) + + afterEach(() => { + fs.rmSync(mirrorPath, {recursive: true, force: true}) + }) + + const base = { + exposeId: 'expose-1', + stickyDiskKey: 'key-1', + repoName: 'owner/repo', + shouldCommit: true, + vmHydratedGitMirror: true + } + + it('runs bounded maintenance instead of gc --auto and commits', async () => { + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult).toEqual({success: true, timedOut: false}) + const repack = commands().find(c => c.includes('repack')) + expect(repack).toBeDefined() + expect(repack).toEqual( + expect.arrayContaining([ + '-d', + '-l', + '-n', + '--geometric=2', + '--write-midx', + 'repack.writeBitmaps=false' + ]) + ) + expect(commands().some(c => c.includes('gc'))).toBe(false) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + + it('still commits when maintenance times out', async () => { + repackExitCode = 124 + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult.success).toBe(false) + expect(result.maintenanceResult.timedOut).toBe(true) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + + it('still commits when maintenance fails', async () => { + repackExitCode = 128 + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult).toEqual({ + success: false, + timedOut: false, + error: expect.stringContaining('128') + }) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + + it('reports a pack-refs failure and still commits', async () => { + packRefsExitCode = 1 + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult).toEqual({ + success: false, + timedOut: false, + error: expect.stringContaining('pack-refs failed with exit code 1') + }) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + + it('reports a pack-refs timeout and still commits', async () => { + packRefsExitCode = 124 + const result = await blacksmithCache.cleanup({...base, mirrorPath}) + + expect(result.maintenanceResult).toEqual({ + success: false, + timedOut: true, + error: expect.stringContaining('pack-refs timed out') + }) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: true, vmHydratedGitMirror: true}) + ) + }) + + it('does not commit or run maintenance when the mirror sync failed', async () => { + const result = await blacksmithCache.cleanup({ + ...base, + mirrorPath, + mirrorSyncFailed: true + }) + + expect(result.maintenanceResult).toEqual({success: true, timedOut: false}) + expect(commands().some(c => c.includes('repack'))).toBe(false) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: false, vmHydratedGitMirror: false}) + ) + }) + + it('does not commit or run maintenance when the mirror sync timed out', async () => { + await blacksmithCache.cleanup({ + ...base, + mirrorPath, + mirrorSyncTimedOut: true + }) + + expect(commands().some(c => c.includes('repack'))).toBe(false) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: false, vmHydratedGitMirror: false}) + ) + }) + + it('skips maintenance when the disk is released without commit', async () => { + await blacksmithCache.cleanup({ + ...base, + mirrorPath, + shouldCommit: false, + vmHydratedGitMirror: false + }) + + expect(commands().some(c => c.includes('repack'))).toBe(false) + expect(mockCommitStickyDisk).toHaveBeenCalledWith( + expect.objectContaining({shouldCommit: false}) + ) + }) +}) diff --git a/__test__/mirror-maintenance-git.test.ts b/__test__/mirror-maintenance-git.test.ts new file mode 100644 index 0000000..4475006 --- /dev/null +++ b/__test__/mirror-maintenance-git.test.ts @@ -0,0 +1,694 @@ +/** + * Real-git coverage for post-step mirror maintenance: large packs are + * marked kept and never rewritten, small packs and loose objects are + * rolled up, the mirror stays intact, repeated runs are no-ops, and a + * killed or failing maintenance leaves no temporary or lock files behind. + */ +jest.mock('@connectrpc/connect', () => ({ + createClient: jest.fn(), + ConnectError: class ConnectError extends Error {}, + Code: {Aborted: 'ABORTED'} +})) + +jest.mock('@connectrpc/connect-node', () => ({ + createGrpcTransport: jest.fn() +})) + +jest.mock( + '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect', + () => ({ + StickyDiskService: {} + }) +) + +jest.mock('../src/container-detector', () => ({ + isRunningInContainer: jest.fn(() => false) +})) + +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {execFileSync} from 'child_process' +import * as blacksmithCache from '../src/blacksmith-cache' + +const KEEP_BYTES = 1024 * 1024 + +function git(cwd: string, ...args: string[]): string { + return execFileSync('git', ['-C', cwd, ...args], {encoding: 'utf8'}).trim() +} + +function commitBlob(repo: string, name: string, bytes: number): void { + const buf = Buffer.alloc(bytes) + for (let i = 0; i < bytes; i += 4) { + buf.writeUInt32LE((Math.random() * 0xffffffff) >>> 0, i) + } + fs.writeFileSync(path.join(repo, name), buf) + git(repo, 'add', name) + git( + repo, + '-c', + 'user.email=test@example.com', + '-c', + 'user.name=Test', + 'commit', + '-q', + '-m', + name + ) +} + +function packs(mirror: string): string[] { + return fs + .readdirSync(path.join(mirror, 'objects', 'pack')) + .filter(f => f.endsWith('.pack')) + .sort() +} + +function looseObjects(mirror: string): number { + const objects = path.join(mirror, 'objects') + let n = 0 + for (const dir of fs.readdirSync(objects)) { + if (/^[0-9a-f]{2}$/.test(dir)) { + n += fs.readdirSync(path.join(objects, dir)).length + } + } + return n +} + +function refsResolve(mirror: string): void { + const out = git(mirror, 'for-each-ref', '--format=%(objectname)') + for (const sha of out.split('\n').filter(Boolean)) { + git(mirror, 'cat-file', '-e', `${sha}^{commit}`) + } +} + +function fsck(mirror: string): void { + execFileSync('git', ['-C', mirror, 'fsck', '--strict'], {stdio: 'pipe'}) +} + +/** Pushes HEAD of `src` as one pack of its own, whatever its object count. */ +function pushPack(src: string, mirror: string, branch: string): void { + git( + src, + 'push', + '-q', + '--receive-pack=git -c receive.unpackLimit=1 receive-pack', + mirror, + `HEAD:refs/heads/${branch}` + ) +} + +function packSizes(mirror: string): Map { + const packDir = path.join(mirror, 'objects', 'pack') + return new Map( + packs(mirror).map(p => [p, fs.statSync(path.join(packDir, p)).size]) + ) +} + +/** Bytes of packs present after a run that were not present before it. */ +function bytesWritten( + before: Map, + after: Map +): number { + let n = 0 + for (const [name, size] of after) { + if (!before.has(name)) { + n += size + } + } + return n +} + +function stampReclaim(mirror: string, at: number): void { + fs.writeFileSync( + path.join(mirror, blacksmithCache.MAINTENANCE_RECLAIM_STAMP), + `${at}\n` + ) +} + +/** + * Bare mirror with one base pack larger than KEEP_BYTES, several small + * packs (pushed with receive.unpackLimit=1) and a few loose objects. Auto + * maintenance on the receiving side is disabled so the pack layout is + * exactly what the pushes produced, whatever the git version. + */ +function buildMirror(root: string): {mirror: string; basePack: string} { + const src = path.join(root, 'src') + const mirror = path.join(root, 'mirror.git') + fs.mkdirSync(src) + git(root, 'init', '-q', '--bare', 'mirror.git') + git(mirror, 'config', 'gc.auto', '0') + git(mirror, 'config', 'receive.autogc', 'false') + git(mirror, 'config', 'maintenance.auto', 'false') + git(root, 'init', '-q', '-b', 'main', 'src') + for (let i = 0; i < 6; i++) { + commitBlob(src, `base-${i}`, 300 * 1024) + git(src, 'push', '-q', mirror, `HEAD:refs/heads/base-${i}`) + } + git(mirror, '-c', 'repack.writeBitmaps=false', 'repack', '-a', '-d', '-q') + const basePacks = packs(mirror) + expect(basePacks).toHaveLength(1) + expect( + fs.statSync(path.join(mirror, 'objects', 'pack', basePacks[0])).size + ).toBeGreaterThan(KEEP_BYTES) + + for (let i = 0; i < 6; i++) { + commitBlob(src, `small-${i}`, 100 * 1024) + git( + src, + 'push', + '-q', + '--receive-pack=git -c receive.unpackLimit=1 receive-pack', + mirror, + `HEAD:refs/heads/small-${i}` + ) + } + commitBlob(src, 'loose', 512) + git(src, 'push', '-q', mirror, 'HEAD:refs/heads/loose') + + expect(packs(mirror).length).toBe(7) + expect(looseObjects(mirror)).toBeGreaterThan(0) + return {mirror, basePack: basePacks[0]} +} + +describe('runMirrorMaintenance (real git)', () => { + let root: string + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-maint-')) + }) + + afterEach(() => { + fs.rmSync(root, {recursive: true, force: true}) + }) + + it('keeps the base pack, rolls up small packs and loose objects', async () => { + const {mirror, basePack} = buildMirror(root) + const before = fs.statSync(path.join(mirror, 'objects', 'pack', basePack)) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) + expect(result).toEqual({success: true, timedOut: false}) + + const packDir = path.join(mirror, 'objects', 'pack') + expect( + fs.existsSync(path.join(packDir, basePack.replace(/\.pack$/, '.keep'))) + ).toBe(true) + const after = fs.statSync(path.join(packDir, basePack)) + expect(after.ino).toBe(before.ino) + expect(after.size).toBe(before.size) + + const remaining = packs(mirror) + expect(remaining).toContain(basePack) + expect(remaining.length).toBeLessThanOrEqual(3) + expect(looseObjects(mirror)).toBe(0) + expect(fs.existsSync(path.join(packDir, 'multi-pack-index'))).toBe(true) + expect(fs.existsSync(path.join(mirror, 'packed-refs'))).toBe(true) + fsck(mirror) + refsResolve(mirror) + }) + + it('is a no-op on an already maintained mirror', async () => { + const {mirror} = buildMirror(root) + await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) + const first = packs(mirror) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) + expect(result).toEqual({success: true, timedOut: false}) + expect(packs(mirror)).toEqual(first) + fsck(mirror) + }) + + it('excludes kept packs from the roll-up regardless of size', async () => { + const {mirror, basePack} = buildMirror(root) + // Kept by an earlier run; a larger pack arriving later must not pull + // it into the roll-up. + fs.writeFileSync( + path.join( + mirror, + 'objects', + 'pack', + basePack.replace(/\.pack$/, '.keep') + ), + '' + ) + const src = path.join(root, 'src') + commitBlob(src, 'huge', 4 * 1024 * 1024) + git( + src, + 'push', + '-q', + '--receive-pack=git -c receive.unpackLimit=1 receive-pack', + mirror, + 'HEAD:refs/heads/huge' + ) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: Number.MAX_SAFE_INTEGER + }) + expect(result.success).toBe(true) + expect(packs(mirror)).toContain(basePack) + fsck(mirror) + refsResolve(mirror) + }) + + it('marks only packs at or above the threshold as kept', async () => { + const {mirror, basePack} = buildMirror(root) + const kept = await blacksmithCache.markKeepPacks(mirror, KEEP_BYTES) + expect(kept).toEqual({ + kept: [basePack.replace(/\.pack$/, '')], + deferred: [] + }) + const keepFiles = fs + .readdirSync(path.join(mirror, 'objects', 'pack')) + .filter(f => f.endsWith('.keep')) + expect(keepFiles).toEqual([basePack.replace(/\.pack$/, '.keep')]) + + // Idempotent: already-kept packs are reported, no duplicates written. + expect(await blacksmithCache.markKeepPacks(mirror, KEEP_BYTES)).toEqual( + kept + ) + }) + + it('never rewrites more than the keep threshold per run, whatever the object counts', async () => { + const {mirror, basePack} = buildMirror(root) + const src = path.join(root, 'src') + // Four medium packs of one large object each, and one pack of many tiny + // objects: all below the threshold on their own, well above it together. + // Geometric repack would pick them by object count; the byte bound must + // hold regardless. + for (let i = 0; i < 4; i++) { + commitBlob(src, `medium-${i}`, 400 * 1024) + pushPack(src, mirror, `medium-${i}`) + } + for (let i = 0; i < 200; i++) { + fs.writeFileSync(path.join(src, `tiny-${i}`), `tiny ${i}\n`) + } + git(src, 'add', '.') + git( + src, + '-c', + 'user.email=test@example.com', + '-c', + 'user.name=Test', + 'commit', + '-q', + '-m', + 'tiny' + ) + pushPack(src, mirror, 'tiny') + + const packDir = path.join(mirror, 'objects', 'pack') + let before = packSizes(mirror) + let unkept = 0 + for (const [name, size] of before) { + if (name !== basePack) { + unkept += size + } + } + expect(unkept).toBeGreaterThan(2 * KEEP_BYTES) + + for (let run = 0; run < 3; run++) { + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) + expect(result).toEqual({success: true, timedOut: false}) + const after = packSizes(mirror) + expect(bytesWritten(before, after)).toBeLessThan(KEEP_BYTES) + expect(after.has(basePack)).toBe(true) + // Only the base pack is kept for good; deferrals do not outlive the run. + const keepFiles = fs.readdirSync(packDir).filter(f => f.endsWith('.keep')) + expect(keepFiles).toEqual([basePack.replace(/\.pack$/, '.keep')]) + fsck(mirror) + refsResolve(mirror) + before = after + } + expect(looseObjects(mirror)).toBe(0) + }) + + it('defers the largest packs until the rest fit under the threshold', async () => { + const {mirror, basePack} = buildMirror(root) + const src = path.join(root, 'src') + for (let i = 0; i < 3; i++) { + commitBlob(src, `medium-${i}`, 400 * 1024) + pushPack(src, mirror, `medium-${i}`) + } + const sizes = packSizes(mirror) + const medium = [...sizes.entries()] + .filter(([, size]) => size > 300 * 1024 && size < KEEP_BYTES) + .map(([name]) => name.replace(/\.pack$/, '')) + .sort() + expect(medium).toHaveLength(3) + + const selection = await blacksmithCache.markKeepPacks(mirror, KEEP_BYTES) + expect(selection.kept).toEqual([basePack.replace(/\.pack$/, '')]) + // 3 x 400 KiB + 6 x 100 KiB + loose: dropping two of the medium packs + // brings the rest under 1 MiB. + expect(selection.deferred).toHaveLength(2) + for (const base of selection.deferred) { + expect(medium).toContain(base) + expect( + fs.readFileSync( + path.join(mirror, 'objects', 'pack', `${base}.keep`), + 'utf8' + ) + ).not.toBe('') + } + + // A deferral left behind by a killed run is lifted and re-evaluated. + const again = await blacksmithCache.markKeepPacks(mirror, KEEP_BYTES) + expect(again).toEqual(selection) + const withoutLimit = await blacksmithCache.markKeepPacks( + mirror, + Number.MAX_SAFE_INTEGER + ) + expect(withoutLimit.deferred).toEqual([]) + }) + + // Pushes a branch into its own kept pack, optionally records it in the + // commit-graph, then deletes the branch so its objects are unreachable. + function addGarbage( + root: string, + mirror: string, + options: {commitGraph?: boolean} = {} + ): {pack: string; blob: string; commit: string} { + const src = path.join(root, 'src') + const before = new Set(packs(mirror)) + commitBlob(src, 'garbage', 600 * 1024) + const blob = git(src, 'rev-parse', 'HEAD:garbage') + const commit = git(src, 'rev-parse', 'HEAD') + pushPack(src, mirror, 'garbage') + const pack = packs(mirror).find(p => !before.has(p)) as string + fs.writeFileSync( + path.join(mirror, 'objects', 'pack', pack.replace(/\.pack$/, '.keep')), + '' + ) + if (options.commitGraph) { + git(mirror, 'commit-graph', 'write', '--reachable', '--split') + } + git(mirror, 'update-ref', '-d', 'refs/heads/garbage') + return {pack, blob, commit} + } + + function hasObject(mirror: string, oid: string): boolean { + try { + execFileSync('git', ['-C', mirror, 'cat-file', '-e', oid], { + stdio: 'pipe' + }) + return true + } catch { + return false + } + } + + describe('reclaim', () => { + it('starts the interval on first sight instead of reclaiming at once', async () => { + const {mirror} = buildMirror(root) + const {pack, blob} = addGarbage(root, mirror) + const stamp = path.join(mirror, blacksmithCache.MAINTENANCE_RECLAIM_STAMP) + expect(fs.existsSync(stamp)).toBe(false) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + now: 1000 + }) + expect(result).toEqual({success: true, timedOut: false}) + expect(fs.readFileSync(stamp, 'utf8').trim()).toBe('1000') + expect(packs(mirror)).toContain(pack) + expect(hasObject(mirror, blob)).toBe(true) + + // Within the interval: still incremental. + await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 5000 + }) + expect(fs.readFileSync(stamp, 'utf8').trim()).toBe('1000') + expect(packs(mirror)).toContain(pack) + fsck(mirror) + }) + + it('rewrites the whole mirror and drops unreachable objects when due', async () => { + const {mirror, basePack} = buildMirror(root) + const { + pack, + blob, + commit: garbageCommit + } = addGarbage(root, mirror, {commitGraph: true}) + expect(blacksmithCache.hasCommitGraph(mirror)).toBe(true) + stampReclaim(mirror, 0) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 20_000 + }) + expect(result).toEqual({success: true, timedOut: false}) + + const packDir = path.join(mirror, 'objects', 'pack') + const remaining = packs(mirror) + expect(remaining).toHaveLength(1) + expect(remaining).not.toContain(pack) + expect(remaining).not.toContain(basePack) + expect(hasObject(mirror, blob)).toBe(false) + expect(fs.readdirSync(packDir).filter(f => f.endsWith('.keep'))).toEqual( + [] + ) + expect(looseObjects(mirror)).toBe(0) + expect( + fs + .readFileSync( + path.join(mirror, blacksmithCache.MAINTENANCE_RECLAIM_STAMP), + 'utf8' + ) + .trim() + ).toBe('20000') + expect(fs.existsSync(path.join(packDir, 'multi-pack-index'))).toBe(true) + // The graph was rebuilt without the pruned commit. + expect(blacksmithCache.hasCommitGraph(mirror)).toBe(true) + git(mirror, 'commit-graph', 'verify') + expect(hasObject(mirror, garbageCommit)).toBe(false) + fsck(mirror) + refsResolve(mirror) + + // The next run keeps the new single pack and is incremental again. + const next = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 25_000 + }) + expect(next).toEqual({success: true, timedOut: false}) + expect(packs(mirror)).toEqual(remaining) + expect(fs.readdirSync(packDir).filter(f => f.endsWith('.keep'))).toEqual([ + remaining[0].replace(/\.pack$/, '.keep') + ]) + }) + }) + + describe('with a misbehaving git', () => { + let binDir: string + let originalPath: string | undefined + + beforeEach(() => { + binDir = path.join(root, 'bin') + fs.mkdirSync(binDir) + originalPath = process.env.PATH + }) + + afterEach(() => { + process.env.PATH = originalPath + }) + + // Jest's sandboxed process.env is not what child processes inherit, so + // the shim is installed as a `timeout` found through the sandbox PATH + // that puts binDir first on the PATH of the real one. + function installFakeGit(script: string, command = 'repack'): void { + const realGit = execFileSync('which', ['git'], {encoding: 'utf8'}).trim() + const realTimeout = execFileSync('which', ['timeout'], { + encoding: 'utf8' + }).trim() + fs.writeFileSync( + path.join(binDir, 'git'), + `#!/bin/sh +for arg in "$@"; do + if [ "$arg" = "${command}" ]; then +${script} + fi +done +exec ${realGit} "$@" +`, + {mode: 0o755} + ) + fs.writeFileSync( + path.join(binDir, 'timeout'), + `#!/bin/sh +PATH="${binDir}:$PATH" exec ${realTimeout} "$@" +`, + {mode: 0o755} + ) + process.env.PATH = `${binDir}:${originalPath}` + } + + it('reports a timeout and removes leftovers without touching the mirror', async () => { + const {mirror, basePack} = buildMirror(root) + const before = packs(mirror) + const packDir = path.join(mirror, 'objects', 'pack') + installFakeGit(` touch "${packDir}/tmp_pack_abc" "${packDir}/.tmp-1-pack-abc.pack" "${packDir}/multi-pack-index.lock" "${mirror}/packed-refs.lock" + sleep 30`) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 1, + keepBytes: KEEP_BYTES + }) + expect(result.success).toBe(false) + expect(result.timedOut).toBe(true) + + expect(packs(mirror)).toEqual(before) + expect(fs.existsSync(path.join(packDir, 'tmp_pack_abc'))).toBe(false) + expect(fs.existsSync(path.join(packDir, '.tmp-1-pack-abc.pack'))).toBe( + false + ) + expect(fs.existsSync(path.join(packDir, 'multi-pack-index.lock'))).toBe( + false + ) + expect(fs.existsSync(path.join(mirror, 'packed-refs.lock'))).toBe(false) + expect( + fs.existsSync(path.join(packDir, basePack.replace(/\.pack$/, '.keep'))) + ).toBe(true) + fsck(mirror) + refsResolve(mirror) + }) + + it('reports a failure without a timeout flag', async () => { + const {mirror} = buildMirror(root) + installFakeGit(' exit 128') + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) + expect(result.success).toBe(false) + expect(result.timedOut).toBe(false) + expect(result.error).toContain('128') + fsck(mirror) + }) + it('reports a pack-refs failure without a timeout flag', async () => { + const {mirror} = buildMirror(root) + installFakeGit(' exit 3', 'pack-refs') + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES + }) + expect(result).toEqual({ + success: false, + timedOut: false, + error: expect.stringContaining('pack-refs failed with exit code 3') + }) + fsck(mirror) + refsResolve(mirror) + }) + + it('reports a pack-refs timeout and removes its lock file', async () => { + const {mirror} = buildMirror(root) + installFakeGit( + ` touch "${mirror}/packed-refs.lock" + sleep 30`, + 'pack-refs' + ) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 1, + keepBytes: KEEP_BYTES + }) + expect(result).toEqual({ + success: false, + timedOut: true, + error: expect.stringContaining('pack-refs timed out') + }) + expect(fs.existsSync(path.join(mirror, 'packed-refs.lock'))).toBe(false) + fsck(mirror) + refsResolve(mirror) + }) + + it('a reclaim that times out leaves the mirror intact and is not retried', async () => { + const {mirror, basePack} = buildMirror(root) + const packDir = path.join(mirror, 'objects', 'pack') + const keepFile = path.join(packDir, basePack.replace(/\.pack$/, '.keep')) + fs.writeFileSync(keepFile, '') + stampReclaim(mirror, 0) + const before = packs(mirror) + installFakeGit(` touch "${packDir}/tmp_pack_reclaim" + sleep 30`) + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + reclaimTimeoutSecs: 1, + now: 20_000 + }) + expect(result.success).toBe(false) + expect(result.timedOut).toBe(true) + expect(packs(mirror)).toEqual(before) + expect(fs.existsSync(path.join(packDir, 'tmp_pack_reclaim'))).toBe(false) + fsck(mirror) + refsResolve(mirror) + + // The stamp was advanced before the attempt, so the following run is + // incremental and re-marks the base pack instead of trying again. + process.env.PATH = originalPath + const next = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 25_000 + }) + expect(next).toEqual({success: true, timedOut: false}) + expect(fs.existsSync(keepFile)).toBe(true) + expect(packs(mirror)).toContain(basePack) + fsck(mirror) + }) + + it('a prune failure after the reclaim repack leaves no stale commit-graph', async () => { + const {mirror} = buildMirror(root) + const {commit} = addGarbage(root, mirror, {commitGraph: true}) + expect(blacksmithCache.hasCommitGraph(mirror)).toBe(true) + stampReclaim(mirror, 0) + installFakeGit(' exit 7', 'prune') + + const result = await blacksmithCache.runMirrorMaintenance(mirror, { + timeoutSecs: 60, + keepBytes: KEEP_BYTES, + reclaimIntervalMs: 10_000, + now: 20_000 + }) + expect(result).toEqual({ + success: false, + timedOut: false, + error: expect.stringContaining('prune failed with exit code 7') + }) + + // The repack already dropped the unreachable commit; a graph still + // listing it would fail fsck in every later job. + expect(hasObject(mirror, commit)).toBe(false) + expect(blacksmithCache.hasCommitGraph(mirror)).toBe(false) + fsck(mirror) + refsResolve(mirror) + }) + }) +}) diff --git a/__test__/mirror-sync-negotiation-git.test.ts b/__test__/mirror-sync-negotiation-git.test.ts index 4bc942d..ac34cb7 100644 --- a/__test__/mirror-sync-negotiation-git.test.ts +++ b/__test__/mirror-sync-negotiation-git.test.ts @@ -54,16 +54,21 @@ function commit(repo: string, msg: string): string { return git(repo, 'rev-parse', 'HEAD') } -/** Loose + packed object count, i.e. everything a fetch wrote so far. */ -function objectCount(repo: string): number { - let total = 0 +function countObjects(repo: string): {loose: number; packed: number} { + const counts = {loose: 0, packed: 0} for (const line of git(repo, 'count-objects', '-v').split('\n')) { const match = /^(count|in-pack): (\d+)$/.exec(line.trim()) if (match) { - total += parseInt(match[2], 10) + counts[match[1] === 'count' ? 'loose' : 'packed'] = parseInt(match[2], 10) } } - return total + return counts +} + +/** Loose + packed object count, i.e. everything a fetch wrote so far. */ +function objectCount(repo: string): number { + const {loose, packed} = countObjects(repo) + return loose + packed } function packetLines(packetTrace: string, re: RegExp): Set { @@ -163,6 +168,10 @@ describe('mirror sync negotiation with real git', () => { const packetTrace = path.join(tmpDir, 'sync-packets') process.env['GIT_TRACE_PACKET'] = packetTrace const before = objectCount(mirrorPath) + const looseBefore = countObjects(mirrorPath).loose + const packsBefore = fs + .readdirSync(path.join(mirrorPath, 'objects', 'pack')) + .filter(f => f.endsWith('.pack')) const result = await blacksmithCache.syncMirrorFromRemote( mirrorPath, @@ -192,5 +201,13 @@ describe('mirror sync negotiation with real git', () => { const received = objectCount(mirrorPath) - before expect(received).toBeGreaterThan(0) expect(received).toBeLessThan(TRUNK_COMMITS_AFTER_FORK) + + // Small as it is, the fetch is stored as a pack rather than loose + // objects, so maintenance sees its size. + expect(countObjects(mirrorPath).loose).toBe(looseBefore) + const packsAfter = fs + .readdirSync(path.join(mirrorPath, 'objects', 'pack')) + .filter(f => f.endsWith('.pack')) + expect(packsAfter.length).toBe(packsBefore.length + 1) }) }) diff --git a/dist/index.js b/dist/index.js index 32382c7..18e420e 100644 --- a/dist/index.js +++ b/dist/index.js @@ -39,6 +39,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.MAINTENANCE_RECLAIM_STAMP = void 0; exports.getMountPoint = getMountPoint; exports.isBlacksmithEnvironment = isBlacksmithEnvironment; exports.getAgentAddr = getAgentAddr; @@ -65,6 +66,8 @@ exports.fetchRefsFromMirror = fetchRefsFromMirror; exports.writeAlternates = writeAlternates; exports.dissociate = dissociate; exports.hasCommitGraph = hasCommitGraph; +exports.markKeepPacks = markKeepPacks; +exports.runMirrorMaintenance = runMirrorMaintenance; exports.cleanup = cleanup; const core = __importStar(__nccwpck_require__(2186)); const exec = __importStar(__nccwpck_require__(1514)); @@ -89,7 +92,27 @@ const MIRROR_VERSION = 'v1'; // burns checkout time on a mirror disk that is having a bad day. The // window is kept generous so slow-but-healthy syncs still complete. const REFRESH_TIMEOUT_SECS = 120; // 2 minutes, single attempt -const GC_TIMEOUT_SECS = 120; // 2 minutes +// Post-step mirror maintenance is bounded by construction (a run never +// rewrites more than MAINTENANCE_KEEP_PACK_BYTES of existing packs), so this +// deadline is a safety net rather than an expected cost. Running out of it +// does not affect the commit: the mirror was already synced, maintenance is +// a pure optimization. +const MAINTENANCE_TIMEOUT_SECS = 120; // 2 minutes +// Packs at or above this size are kept permanently; the packs below it that +// one run rewrites are chosen so their combined size stays under it too. +const MAINTENANCE_KEEP_PACK_BYTES = 256 * 1024 * 1024; +// Kept packs hold on to objects that later become unreachable (deleted or +// rewritten branches). A full rewrite with the keeps lifted reclaims them; +// it is unbounded in size, so it runs rarely and under its own deadline. +const MAINTENANCE_RECLAIM_INTERVAL_MS = 14 * 24 * 60 * 60 * 1000; +const MAINTENANCE_RECLAIM_TIMEOUT_SECS = 600; +// Loose objects younger than this survive a reclaim, the usual guard against +// deleting objects a concurrent writer has stored but not yet referenced. +const MAINTENANCE_RECLAIM_PRUNE_EXPIRE = '2.weeks.ago'; +// Marker in the mirror root recording the last reclaim attempt. +exports.MAINTENANCE_RECLAIM_STAMP = 'blacksmith-maintenance-reclaim'; +// Content of a `.keep` written only for the duration of one run. +const DEFERRED_KEEP_MARKER = 'blacksmith-checkout: deferred to a later run\n'; const FLUSH_TIMEOUT_SECS = 10; // 10 seconds for durability flush const UMOUNT_TIMEOUT_SECS = 10; // 10 seconds for unmount const UMOUNT_MAX_RETRIES = 3; // Number of unmount retry attempts @@ -846,6 +869,12 @@ function syncMirrorFromRemote(mirrorPath_1, repoUrl_1, authToken_1) { 'gc.auto=0', '-c', 'fetch.negotiationAlgorithm=skipping', + // Always store what a sync receives as a pack, never as loose + // objects: maintenance bounds the bytes it rewrites per run by + // pack size (see markKeepPacks), and a large blob that arrived + // loose would not be counted. + '-c', + 'fetch.unpackLimit=1', // Keep the commit-graph current so ref-tip commit parsing // (mark_complete_local_refs and negotiation walks) reads one // compact mmap'd file instead of scattered pack entries. Only @@ -1338,13 +1367,12 @@ function dissociate(workspacePath, env) { /** * Write the mirror's commit-graph so subsequent commit parsing (e.g. the * per-ref walks inside fetch) is a lookup in one compact file instead of - * scattered pack reads. Reading commit-graphs is enabled by default in git; - * writing only happens during a real gc, so a freshly cloned mirror has - * none until the first threshold-tripping `gc --auto`. Failure is - * non-fatal - the graph is a pure cache. + * scattered pack reads. Reading commit-graphs is enabled by default in git, + * but a freshly cloned mirror has none until something writes it. Failure + * is non-fatal - the graph is a pure cache. */ function writeCommitGraph(mirrorPath_1) { - return __awaiter(this, arguments, void 0, function* (mirrorPath, timeoutSecs = GC_TIMEOUT_SECS) { + return __awaiter(this, arguments, void 0, function* (mirrorPath, timeoutSecs = MAINTENANCE_TIMEOUT_SECS) { try { const start = Date.now(); const result = yield exec.getExecOutput('timeout', [ @@ -1371,6 +1399,24 @@ function writeCommitGraph(mirrorPath_1) { } }); } +function removeCommitGraph(mirrorPath) { + return __awaiter(this, void 0, void 0, function* () { + const info = path.join(mirrorPath, 'objects', 'info'); + for (const target of [ + path.join(info, 'commit-graph'), + path.join(info, 'commit-graphs') + ]) { + try { + yield fs.promises.rm(target, { recursive: true, force: true }); + } + catch (error) { + core.warning(`[git-mirror] Failed to remove ${target}: ${error}`); + return false; + } + } + return true; + }); +} /** * Whether the mirror has a commit-graph (single file or split chain). * Determines if the sync fetch can write incrementally. @@ -1380,53 +1426,306 @@ function hasCommitGraph(mirrorPath) { fs.existsSync(path.join(mirrorPath, 'objects', 'info', 'commit-graphs', 'commit-graph-chain'))); } /** - * Run lightweight garbage collection on the mirror. - * Uses --auto to only run GC when git determines it's needed (based on loose object count). - * This avoids expensive full repacks on every run while still keeping the repo tidy over time. + * Choose which packs the geometric repack may rewrite. Every pack at or + * above `keepBytes` gets a permanent `.keep` file so that no repack - this + * maintenance, or `gc` run by an older action version on the same mirror - + * ever rewrites or deletes it. Of the remaining packs, the largest are given + * a `.keep` for this run only until the rest add up to less than `keepBytes`: + * git picks the packs to roll up by object count, so without this several + * medium packs could combine into a rewrite far larger than any one of them. + * A deferred pack is reconsidered on the next run, once the small packs + * around it have been folded. Returns the names (without extension) of both + * groups. + * + * `.keep` files are used rather than `repack --keep-pack`: git 2.34's + * geometric repack does not exclude `--keep-pack` packs from the roll-up, + * so with `-d` it deletes the kept pack and loses its objects. Kept packs + * are ignored by `--honor-pack-keep` only in the repository they live in, + * so a workspace repack over the mirror alternate (dissociate) still copies + * their objects. + */ +function markKeepPacks(mirrorPath_1) { + return __awaiter(this, arguments, void 0, function* (mirrorPath, keepBytes = MAINTENANCE_KEEP_PACK_BYTES) { + const packDir = path.join(mirrorPath, 'objects', 'pack'); + const kept = []; + const deferred = []; + let entries; + try { + entries = yield fs.promises.readdir(packDir); + } + catch (_a) { + return { kept, deferred }; + } + const names = new Set(entries); + const candidates = []; + for (const entry of entries) { + if (!entry.startsWith('pack-') || !entry.endsWith('.pack')) { + continue; + } + const base = entry.slice(0, -'.pack'.length); + const keepFile = path.join(packDir, `${base}.keep`); + try { + if (names.has(`${base}.keep`)) { + const content = yield fs.promises.readFile(keepFile, 'utf8'); + if (content !== DEFERRED_KEEP_MARKER) { + kept.push(base); + continue; + } + // Left behind by a run that was killed before it could clean up. + yield fs.promises.rm(keepFile, { force: true }); + } + const stat = yield fs.promises.stat(path.join(packDir, entry)); + if (stat.size >= keepBytes) { + yield fs.promises.writeFile(keepFile, ''); + core.info(`[git-mirror] Marked ${base} (${Math.round(stat.size / (1024 * 1024))} MiB) as kept`); + kept.push(base); + continue; + } + candidates.push({ base, size: stat.size }); + } + catch (error) { + core.warning(`[git-mirror] Failed to mark ${base} as kept: ${error}`); + } + } + candidates.sort((a, b) => b.size - a.size); + let total = candidates.reduce((sum, c) => sum + c.size, 0); + for (const { base, size } of candidates) { + if (total < keepBytes) { + break; + } + try { + yield fs.promises.writeFile(path.join(packDir, `${base}.keep`), DEFERRED_KEEP_MARKER); + core.info(`[git-mirror] Deferred ${base} (${(size / (1024 * 1024)).toFixed(1)} MiB) to a later run`); + deferred.push(base); + total -= size; + } + catch (error) { + core.warning(`[git-mirror] Failed to defer ${base}: ${error}`); + } + } + return { kept: kept.sort(), deferred: deferred.sort() }; + }); +} +function removeKeepFiles(mirrorPath, packs) { + return __awaiter(this, void 0, void 0, function* () { + for (const base of packs) { + const keepFile = path.join(mirrorPath, 'objects', 'pack', `${base}.keep`); + try { + yield fs.promises.rm(keepFile, { force: true }); + } + catch (error) { + core.warning(`[git-mirror] Failed to remove ${base}.keep: ${error}`); + } + } + }); +} +/** + * Remove temporary files and lock files a killed maintenance process may + * have left in the mirror. Only the post step touches the mirror, and + * `timeout` has already signalled the whole process group by the time this + * runs, so any such file is stale and would make the next job's sync fail + * on a lock that nobody holds. Best effort: a still-exiting process removing + * the same file is harmless. */ -function runMirrorGC(mirrorPath_1) { - return __awaiter(this, arguments, void 0, function* (mirrorPath, timeoutSecs = GC_TIMEOUT_SECS) { - core.info(`[git-mirror] Running auto garbage collection (timeout: ${timeoutSecs}s)`); +function removeMaintenanceLeftovers(mirrorPath) { + return __awaiter(this, void 0, void 0, function* () { + const sweeps = [ + { + dir: path.join(mirrorPath, 'objects', 'pack'), + match: name => name.startsWith('tmp_') || + name.startsWith('.tmp-') || + name.endsWith('.lock') + }, + { + dir: path.join(mirrorPath, 'objects', 'info'), + match: name => name.endsWith('.lock') + }, + { + dir: path.join(mirrorPath, 'objects', 'info', 'commit-graphs'), + match: name => name.endsWith('.lock') + }, + { + dir: mirrorPath, + match: name => name === 'packed-refs.lock' + } + ]; + for (const { dir, match } of sweeps) { + let entries; + try { + entries = yield fs.promises.readdir(dir); + } + catch (_a) { + continue; + } + for (const entry of entries) { + if (!match(entry)) { + continue; + } + try { + yield fs.promises.rm(path.join(dir, entry), { force: true }); + core.info(`[git-mirror] Removed stale maintenance file ${entry}`); + } + catch (error) { + core.warning(`[git-mirror] Failed to remove ${entry}: ${error}`); + } + } + } + }); +} +/** + * Whether a reclaim is due, and if so, record this attempt. The stamp is + * written before the reclaim runs so that a reclaim that times out is not + * retried by every following job. A mirror without a stamp (freshly + * hydrated, or maintained by an older action version) starts its interval + * now rather than reclaiming immediately, so a rollout does not make every + * job in the fleet rewrite its mirror at once. + */ +function reclaimDue(mirrorPath, now, intervalMs) { + return __awaiter(this, void 0, void 0, function* () { + const stamp = path.join(mirrorPath, exports.MAINTENANCE_RECLAIM_STAMP); + let last; + try { + const parsed = Number((yield fs.promises.readFile(stamp, 'utf8')).trim()); + if (Number.isFinite(parsed)) { + last = parsed; + } + } + catch (_a) { + // no stamp yet + } + if (last !== undefined && now - last < intervalMs) { + return false; + } + try { + yield fs.promises.writeFile(stamp, `${now}\n`); + } + catch (error) { + core.warning(`[git-mirror] Failed to write the reclaim stamp: ${error}`); + return false; + } + return last !== undefined; + }); +} +/** + * Bounded mirror maintenance, run in the post step once the mirror has been + * synced. Each sync leaves a new small pack (or loose objects) behind; + * left alone they accumulate until object lookup slows down. Instead of + * `gc --auto` - whose "auto packing" step rewrites every pack including the + * repository's multi-gigabyte base pack, taking minutes on large mirrors - + * this folds only small packs and loose objects together with a geometric + * repack. Packs at or above MAINTENANCE_KEEP_PACK_BYTES are kept out of the + * rewrite for good, and the packs below it that one run does rewrite are + * chosen so their combined size stays under that bound as well (see + * markKeepPacks). A rolled-up pack that grows past the threshold simply + * becomes another kept pack, so the cost of a single run is bounded by the + * threshold, never by the size of the repository. The multi-pack-index + * keeps lookups fast across the kept packs. + * + * Kept packs never lose objects, so history that becomes unreachable stays + * on disk. Once per MAINTENANCE_RECLAIM_INTERVAL_MS the run instead lifts + * every `.keep` and rewrites the whole mirror into a single pack, dropping + * unreachable objects, under the longer MAINTENANCE_RECLAIM_TIMEOUT_SECS. + * That rewrite is the one unbounded step; the stamp written beforehand + * keeps a mirror too large for the deadline from retrying it every job, + * and the incremental runs in between are unaffected either way. + * + * Maintenance is optional: its failure or timeout is reported but must not + * prevent the synced mirror from being committed - otherwise a mirror that + * needs more maintenance than the deadline allows is never persisted, and + * every subsequent job repeats the same doomed work. + */ +function runMirrorMaintenance(mirrorPath_1) { + return __awaiter(this, arguments, void 0, function* (mirrorPath, options = {}) { + var _a, _b, _c, _d, _e; + const timeoutSecs = (_a = options.timeoutSecs) !== null && _a !== void 0 ? _a : MAINTENANCE_TIMEOUT_SECS; + const keepBytes = (_b = options.keepBytes) !== null && _b !== void 0 ? _b : MAINTENANCE_KEEP_PACK_BYTES; + const reclaimIntervalMs = (_c = options.reclaimIntervalMs) !== null && _c !== void 0 ? _c : MAINTENANCE_RECLAIM_INTERVAL_MS; + const reclaimTimeoutSecs = (_d = options.reclaimTimeoutSecs) !== null && _d !== void 0 ? _d : MAINTENANCE_RECLAIM_TIMEOUT_SECS; + const now = (_e = options.now) !== null && _e !== void 0 ? _e : Date.now(); + const start = Date.now(); + // A commit-graph that outlived the full rewrite would still list the + // commits it drops; should prune or a later step then fail, the mirror + // is committed with that graph and fsck breaks in every following job. + // So the graph is removed first, and reclaim is skipped when that fails. + // Without a graph git only walks commits the slow way until a new one is + // written after a successful prune. + const reclaim = (yield reclaimDue(mirrorPath, now, reclaimIntervalMs)) && + (yield removeCommitGraph(mirrorPath)); + const budgetSecs = reclaim ? reclaimTimeoutSecs : timeoutSecs; + const deadline = start + budgetSecs * 1000; + const remainingSecs = () => Math.max(1, Math.ceil((deadline - Date.now()) / 1000)); + let deferred = []; + let label; + let repackArgs; + if (reclaim) { + const { kept } = yield markKeepPacks(mirrorPath, Number.MAX_SAFE_INTEGER); + yield removeKeepFiles(mirrorPath, kept); + label = 'Reclaim'; + repackArgs = ['-a', '-d', '-l', '-n', '--write-midx']; + core.info(`[git-mirror] Running reclaim maintenance (timeout: ${budgetSecs}s, ${kept.length} kept pack(s) released)`); + } + else { + const selection = yield markKeepPacks(mirrorPath, keepBytes); + deferred = selection.deferred; + label = 'Incremental'; + repackArgs = ['-d', '-l', '-n', '--geometric=2', '--write-midx']; + core.info(`[git-mirror] Running incremental maintenance (timeout: ${budgetSecs}s, ${selection.kept.length} kept pack(s), ${deferred.length} deferred)`); + } + const fail = (timedOut, error) => __awaiter(this, void 0, void 0, function* () { + core.warning(`[git-mirror] ${timedOut ? `Maintenance timed out after ${budgetSecs}s` : error}; committing the synced mirror without it`); + yield removeMaintenanceLeftovers(mirrorPath); + return { success: false, timedOut, error }; + }); try { - // --auto: only run if thresholds exceeded (default: 6700 loose objects or 50 packs) - // This is much faster than a full gc when not needed - // gc.autoDetach=false: prevent git from forking a background daemon for GC. - // Without this, the parent `git gc --auto` returns immediately while the - // daemonized child keeps running with cwd and mmap'd pack files on the - // mirror mount, causing the subsequent `umount` to fail with EBUSY. const result = yield exec.getExecOutput('timeout', [ - String(timeoutSecs), + String(remainingSecs()), 'git', '-c', - 'gc.autoDetach=false', + 'repack.writeBitmaps=false', '-C', mirrorPath, - 'gc', - '--auto' + 'repack', + ...repackArgs ], { ignoreReturnCode: true }); if (result.exitCode === TIMEOUT_EXIT_CODE) { - core.warning(`[git-mirror] GC timed out after ${timeoutSecs}s`); - return { - success: false, - timedOut: true, - error: `git gc timed out after ${timeoutSecs}s` - }; + return yield fail(true, `git repack timed out after ${budgetSecs}s`); } if (result.exitCode !== 0) { - core.warning(`[git-mirror] GC failed with exit code ${result.exitCode}`); - return { - success: false, - timedOut: false, - error: `git gc failed with exit code ${result.exitCode}` - }; + return yield fail(false, `git repack failed with exit code ${result.exitCode}`); + } + if (reclaim) { + const prune = yield exec.getExecOutput('timeout', [ + String(remainingSecs()), + 'git', + '-C', + mirrorPath, + 'prune', + `--expire=${MAINTENANCE_RECLAIM_PRUNE_EXPIRE}` + ], { silent: true, ignoreReturnCode: true }); + if (prune.exitCode === TIMEOUT_EXIT_CODE) { + return yield fail(true, `git prune timed out after ${budgetSecs}s`); + } + if (prune.exitCode !== 0) { + return yield fail(false, `git prune failed with exit code ${prune.exitCode}`); + } + yield writeCommitGraph(mirrorPath, remainingSecs()); + } + const packRefs = yield exec.getExecOutput('timeout', [String(remainingSecs()), 'git', '-C', mirrorPath, 'pack-refs', '--all'], { silent: true, ignoreReturnCode: true }); + if (packRefs.exitCode === TIMEOUT_EXIT_CODE) { + return yield fail(true, `git pack-refs timed out after ${budgetSecs}s`); } - core.debug('[git-mirror] Completed git gc --auto'); + if (packRefs.exitCode !== 0) { + return yield fail(false, `git pack-refs failed with exit code ${packRefs.exitCode}`); + } + core.info(`[git-mirror] ${label} maintenance finished in ${Date.now() - start}ms`); return { success: true, timedOut: false }; } catch (error) { const msg = error.message || String(error); - core.warning(`[git-mirror] GC failed: ${msg}`); - return { success: false, timedOut: false, error: msg }; + return yield fail(false, msg); + } + finally { + yield removeKeepFiles(mirrorPath, deferred); } }); } @@ -1522,10 +1821,11 @@ function flushBlockDevice(devicePath) { }); } /** - * Cleanup: run GC, sync, unmount, and commit the sticky disk. + * Cleanup: run mirror maintenance, sync, unmount, and commit the sticky disk. * - * Execution order: GC → sync → unmount (with retry) → flush → commit - * If the mirror sync or GC failed or timed out, shouldCommit is set to false. + * Execution order: maintenance → sync → unmount (with retry) → flush → commit + * If the mirror sync failed or timed out, shouldCommit is set to false. + * Maintenance failure or timeout does not affect the commit. */ function cleanup(options) { return __awaiter(this, void 0, void 0, function* () { @@ -1533,12 +1833,12 @@ function cleanup(options) { const { exposeId, stickyDiskKey, repoName, mountPoint, mirrorPath, mirrorSyncFailed, mirrorSyncTimedOut } = options; let { shouldCommit } = options; // vmHydratedGitMirror must track shouldCommit: if we decide not to commit - // (due to GC/refresh failure), we must not tell the backend that - // hydration completed, otherwise it marks the entry as ready despite no - // valid disk being persisted. + // (due to sync failure), we must not tell the backend that hydration + // completed, otherwise it marks the entry as ready despite no valid disk + // being persisted. let vmHydratedGitMirror = options.vmHydratedGitMirror; const result = { - gcResult: { success: true, timedOut: false } + maintenanceResult: { success: true, timedOut: false } }; core.info(`[git-mirror] Starting cleanup: exposeId=${exposeId}, stickyDiskKey=${stickyDiskKey}, shouldCommit=${shouldCommit}, vmHydratedGitMirror=${vmHydratedGitMirror}`); // If the mirror sync failed or timed out, don't commit @@ -1548,19 +1848,14 @@ function cleanup(options) { shouldCommit = false; vmHydratedGitMirror = false; } - if (mirrorPath) { - // Run GC on the mirror - result.gcResult = yield runMirrorGC(mirrorPath); - if (!result.gcResult.success) { - core.warning('[git-mirror] GC failed or timed out, will not commit sticky disk'); - shouldCommit = false; - vmHydratedGitMirror = false; - } + // Maintenance only pays off if the result is persisted. + if (mirrorPath && shouldCommit) { + result.maintenanceResult = yield runMirrorMaintenance(mirrorPath); // Catch-up for mirrors that predate commit-graph writing: build the // initial graph here in the post step, off the checkout critical path. - // Once it exists, the sync fetch keeps it current incrementally. Only - // worth doing when the disk is being committed; failure is non-fatal. - if (shouldCommit && !hasCommitGraph(mirrorPath)) { + // Once it exists, the sync fetch keeps it current incrementally. + // Failure is non-fatal. + if (!hasCommitGraph(mirrorPath)) { yield writeCommitGraph(mirrorPath); } } @@ -4272,9 +4567,11 @@ function cleanup() { }); } if (cleanupResult) { - if (!cleanupResult.gcResult.success) { + if (!cleanupResult.maintenanceResult.success) { yield (0, internal_metrics_1.reportInternalMetric)('git_mirror_gc_failure', 1, { - reason: cleanupResult.gcResult.timedOut ? 'timeout' : 'failure' + reason: cleanupResult.maintenanceResult.timedOut + ? 'timeout' + : 'failure' }); } } diff --git a/src/blacksmith-cache.ts b/src/blacksmith-cache.ts index fa27cd6..baf402f 100644 --- a/src/blacksmith-cache.ts +++ b/src/blacksmith-cache.ts @@ -23,7 +23,27 @@ const MIRROR_VERSION = 'v1' // burns checkout time on a mirror disk that is having a bad day. The // window is kept generous so slow-but-healthy syncs still complete. const REFRESH_TIMEOUT_SECS = 120 // 2 minutes, single attempt -const GC_TIMEOUT_SECS = 120 // 2 minutes +// Post-step mirror maintenance is bounded by construction (a run never +// rewrites more than MAINTENANCE_KEEP_PACK_BYTES of existing packs), so this +// deadline is a safety net rather than an expected cost. Running out of it +// does not affect the commit: the mirror was already synced, maintenance is +// a pure optimization. +const MAINTENANCE_TIMEOUT_SECS = 120 // 2 minutes +// Packs at or above this size are kept permanently; the packs below it that +// one run rewrites are chosen so their combined size stays under it too. +const MAINTENANCE_KEEP_PACK_BYTES = 256 * 1024 * 1024 +// Kept packs hold on to objects that later become unreachable (deleted or +// rewritten branches). A full rewrite with the keeps lifted reclaims them; +// it is unbounded in size, so it runs rarely and under its own deadline. +const MAINTENANCE_RECLAIM_INTERVAL_MS = 14 * 24 * 60 * 60 * 1000 +const MAINTENANCE_RECLAIM_TIMEOUT_SECS = 600 +// Loose objects younger than this survive a reclaim, the usual guard against +// deleting objects a concurrent writer has stored but not yet referenced. +const MAINTENANCE_RECLAIM_PRUNE_EXPIRE = '2.weeks.ago' +// Marker in the mirror root recording the last reclaim attempt. +export const MAINTENANCE_RECLAIM_STAMP = 'blacksmith-maintenance-reclaim' +// Content of a `.keep` written only for the duration of one run. +const DEFERRED_KEEP_MARKER = 'blacksmith-checkout: deferred to a later run\n' const FLUSH_TIMEOUT_SECS = 10 // 10 seconds for durability flush const UMOUNT_TIMEOUT_SECS = 10 // 10 seconds for unmount const UMOUNT_MAX_RETRIES = 3 // Number of unmount retry attempts @@ -57,7 +77,7 @@ export interface OperationResult { * Result of the cleanup phase, used for metric reporting. */ export interface CleanupResult { - gcResult: OperationResult + maintenanceResult: OperationResult } /** @@ -990,6 +1010,12 @@ export async function syncMirrorFromRemote( 'gc.auto=0', '-c', 'fetch.negotiationAlgorithm=skipping', + // Always store what a sync receives as a pack, never as loose + // objects: maintenance bounds the bytes it rewrites per run by + // pack size (see markKeepPacks), and a large blob that arrived + // loose would not be counted. + '-c', + 'fetch.unpackLimit=1', // Keep the commit-graph current so ref-tip commit parsing // (mark_complete_local_refs and negotiation walks) reads one // compact mmap'd file instead of scattered pack entries. Only @@ -1598,14 +1624,13 @@ export async function dissociate( /** * Write the mirror's commit-graph so subsequent commit parsing (e.g. the * per-ref walks inside fetch) is a lookup in one compact file instead of - * scattered pack reads. Reading commit-graphs is enabled by default in git; - * writing only happens during a real gc, so a freshly cloned mirror has - * none until the first threshold-tripping `gc --auto`. Failure is - * non-fatal - the graph is a pure cache. + * scattered pack reads. Reading commit-graphs is enabled by default in git, + * but a freshly cloned mirror has none until something writes it. Failure + * is non-fatal - the graph is a pure cache. */ async function writeCommitGraph( mirrorPath: string, - timeoutSecs: number = GC_TIMEOUT_SECS + timeoutSecs: number = MAINTENANCE_TIMEOUT_SECS ): Promise { try { const start = Date.now() @@ -1640,6 +1665,22 @@ async function writeCommitGraph( } } +async function removeCommitGraph(mirrorPath: string): Promise { + const info = path.join(mirrorPath, 'objects', 'info') + for (const target of [ + path.join(info, 'commit-graph'), + path.join(info, 'commit-graphs') + ]) { + try { + await fs.promises.rm(target, {recursive: true, force: true}) + } catch (error) { + core.warning(`[git-mirror] Failed to remove ${target}: ${error}`) + return false + } + } + return true +} + /** * Whether the mirror has a commit-graph (single file or split chain). * Determines if the sync fetch can write incrementally. @@ -1659,62 +1700,376 @@ export function hasCommitGraph(mirrorPath: string): boolean { ) } +export interface KeepSelection { + /** Packs with a permanent `.keep`: at or above the keep threshold. */ + kept: string[] + /** + * Packs under the threshold that this run leaves alone so that the packs + * it does rewrite stay under the threshold combined. Their `.keep` is + * removed once the run is over. + */ + deferred: string[] +} + +/** + * Choose which packs the geometric repack may rewrite. Every pack at or + * above `keepBytes` gets a permanent `.keep` file so that no repack - this + * maintenance, or `gc` run by an older action version on the same mirror - + * ever rewrites or deletes it. Of the remaining packs, the largest are given + * a `.keep` for this run only until the rest add up to less than `keepBytes`: + * git picks the packs to roll up by object count, so without this several + * medium packs could combine into a rewrite far larger than any one of them. + * A deferred pack is reconsidered on the next run, once the small packs + * around it have been folded. Returns the names (without extension) of both + * groups. + * + * `.keep` files are used rather than `repack --keep-pack`: git 2.34's + * geometric repack does not exclude `--keep-pack` packs from the roll-up, + * so with `-d` it deletes the kept pack and loses its objects. Kept packs + * are ignored by `--honor-pack-keep` only in the repository they live in, + * so a workspace repack over the mirror alternate (dissociate) still copies + * their objects. + */ +export async function markKeepPacks( + mirrorPath: string, + keepBytes: number = MAINTENANCE_KEEP_PACK_BYTES +): Promise { + const packDir = path.join(mirrorPath, 'objects', 'pack') + const kept: string[] = [] + const deferred: string[] = [] + let entries: string[] + try { + entries = await fs.promises.readdir(packDir) + } catch { + return {kept, deferred} + } + const names = new Set(entries) + const candidates: Array<{base: string; size: number}> = [] + for (const entry of entries) { + if (!entry.startsWith('pack-') || !entry.endsWith('.pack')) { + continue + } + const base = entry.slice(0, -'.pack'.length) + const keepFile = path.join(packDir, `${base}.keep`) + try { + if (names.has(`${base}.keep`)) { + const content = await fs.promises.readFile(keepFile, 'utf8') + if (content !== DEFERRED_KEEP_MARKER) { + kept.push(base) + continue + } + // Left behind by a run that was killed before it could clean up. + await fs.promises.rm(keepFile, {force: true}) + } + const stat = await fs.promises.stat(path.join(packDir, entry)) + if (stat.size >= keepBytes) { + await fs.promises.writeFile(keepFile, '') + core.info( + `[git-mirror] Marked ${base} (${Math.round(stat.size / (1024 * 1024))} MiB) as kept` + ) + kept.push(base) + continue + } + candidates.push({base, size: stat.size}) + } catch (error) { + core.warning(`[git-mirror] Failed to mark ${base} as kept: ${error}`) + } + } + + candidates.sort((a, b) => b.size - a.size) + let total = candidates.reduce((sum, c) => sum + c.size, 0) + for (const {base, size} of candidates) { + if (total < keepBytes) { + break + } + try { + await fs.promises.writeFile( + path.join(packDir, `${base}.keep`), + DEFERRED_KEEP_MARKER + ) + core.info( + `[git-mirror] Deferred ${base} (${(size / (1024 * 1024)).toFixed(1)} MiB) to a later run` + ) + deferred.push(base) + total -= size + } catch (error) { + core.warning(`[git-mirror] Failed to defer ${base}: ${error}`) + } + } + return {kept: kept.sort(), deferred: deferred.sort()} +} + +async function removeKeepFiles( + mirrorPath: string, + packs: string[] +): Promise { + for (const base of packs) { + const keepFile = path.join(mirrorPath, 'objects', 'pack', `${base}.keep`) + try { + await fs.promises.rm(keepFile, {force: true}) + } catch (error) { + core.warning(`[git-mirror] Failed to remove ${base}.keep: ${error}`) + } + } +} + /** - * Run lightweight garbage collection on the mirror. - * Uses --auto to only run GC when git determines it's needed (based on loose object count). - * This avoids expensive full repacks on every run while still keeping the repo tidy over time. + * Remove temporary files and lock files a killed maintenance process may + * have left in the mirror. Only the post step touches the mirror, and + * `timeout` has already signalled the whole process group by the time this + * runs, so any such file is stale and would make the next job's sync fail + * on a lock that nobody holds. Best effort: a still-exiting process removing + * the same file is harmless. */ -async function runMirrorGC( +async function removeMaintenanceLeftovers(mirrorPath: string): Promise { + const sweeps: Array<{dir: string; match: (name: string) => boolean}> = [ + { + dir: path.join(mirrorPath, 'objects', 'pack'), + match: name => + name.startsWith('tmp_') || + name.startsWith('.tmp-') || + name.endsWith('.lock') + }, + { + dir: path.join(mirrorPath, 'objects', 'info'), + match: name => name.endsWith('.lock') + }, + { + dir: path.join(mirrorPath, 'objects', 'info', 'commit-graphs'), + match: name => name.endsWith('.lock') + }, + { + dir: mirrorPath, + match: name => name === 'packed-refs.lock' + } + ] + for (const {dir, match} of sweeps) { + let entries: string[] + try { + entries = await fs.promises.readdir(dir) + } catch { + continue + } + for (const entry of entries) { + if (!match(entry)) { + continue + } + try { + await fs.promises.rm(path.join(dir, entry), {force: true}) + core.info(`[git-mirror] Removed stale maintenance file ${entry}`) + } catch (error) { + core.warning(`[git-mirror] Failed to remove ${entry}: ${error}`) + } + } + } +} + +/** + * Whether a reclaim is due, and if so, record this attempt. The stamp is + * written before the reclaim runs so that a reclaim that times out is not + * retried by every following job. A mirror without a stamp (freshly + * hydrated, or maintained by an older action version) starts its interval + * now rather than reclaiming immediately, so a rollout does not make every + * job in the fleet rewrite its mirror at once. + */ +async function reclaimDue( mirrorPath: string, - timeoutSecs: number = GC_TIMEOUT_SECS + now: number, + intervalMs: number +): Promise { + const stamp = path.join(mirrorPath, MAINTENANCE_RECLAIM_STAMP) + let last: number | undefined + try { + const parsed = Number((await fs.promises.readFile(stamp, 'utf8')).trim()) + if (Number.isFinite(parsed)) { + last = parsed + } + } catch { + // no stamp yet + } + if (last !== undefined && now - last < intervalMs) { + return false + } + try { + await fs.promises.writeFile(stamp, `${now}\n`) + } catch (error) { + core.warning(`[git-mirror] Failed to write the reclaim stamp: ${error}`) + return false + } + return last !== undefined +} + +export interface MaintenanceOptions { + timeoutSecs?: number + keepBytes?: number + reclaimIntervalMs?: number + reclaimTimeoutSecs?: number + now?: number +} + +/** + * Bounded mirror maintenance, run in the post step once the mirror has been + * synced. Each sync leaves a new small pack (or loose objects) behind; + * left alone they accumulate until object lookup slows down. Instead of + * `gc --auto` - whose "auto packing" step rewrites every pack including the + * repository's multi-gigabyte base pack, taking minutes on large mirrors - + * this folds only small packs and loose objects together with a geometric + * repack. Packs at or above MAINTENANCE_KEEP_PACK_BYTES are kept out of the + * rewrite for good, and the packs below it that one run does rewrite are + * chosen so their combined size stays under that bound as well (see + * markKeepPacks). A rolled-up pack that grows past the threshold simply + * becomes another kept pack, so the cost of a single run is bounded by the + * threshold, never by the size of the repository. The multi-pack-index + * keeps lookups fast across the kept packs. + * + * Kept packs never lose objects, so history that becomes unreachable stays + * on disk. Once per MAINTENANCE_RECLAIM_INTERVAL_MS the run instead lifts + * every `.keep` and rewrites the whole mirror into a single pack, dropping + * unreachable objects, under the longer MAINTENANCE_RECLAIM_TIMEOUT_SECS. + * That rewrite is the one unbounded step; the stamp written beforehand + * keeps a mirror too large for the deadline from retrying it every job, + * and the incremental runs in between are unaffected either way. + * + * Maintenance is optional: its failure or timeout is reported but must not + * prevent the synced mirror from being committed - otherwise a mirror that + * needs more maintenance than the deadline allows is never persisted, and + * every subsequent job repeats the same doomed work. + */ +export async function runMirrorMaintenance( + mirrorPath: string, + options: MaintenanceOptions = {} ): Promise { - core.info( - `[git-mirror] Running auto garbage collection (timeout: ${timeoutSecs}s)` - ) + const timeoutSecs = options.timeoutSecs ?? MAINTENANCE_TIMEOUT_SECS + const keepBytes = options.keepBytes ?? MAINTENANCE_KEEP_PACK_BYTES + const reclaimIntervalMs = + options.reclaimIntervalMs ?? MAINTENANCE_RECLAIM_INTERVAL_MS + const reclaimTimeoutSecs = + options.reclaimTimeoutSecs ?? MAINTENANCE_RECLAIM_TIMEOUT_SECS + const now = options.now ?? Date.now() + const start = Date.now() + + // A commit-graph that outlived the full rewrite would still list the + // commits it drops; should prune or a later step then fail, the mirror + // is committed with that graph and fsck breaks in every following job. + // So the graph is removed first, and reclaim is skipped when that fails. + // Without a graph git only walks commits the slow way until a new one is + // written after a successful prune. + const reclaim = + (await reclaimDue(mirrorPath, now, reclaimIntervalMs)) && + (await removeCommitGraph(mirrorPath)) + const budgetSecs = reclaim ? reclaimTimeoutSecs : timeoutSecs + const deadline = start + budgetSecs * 1000 + const remainingSecs = (): number => + Math.max(1, Math.ceil((deadline - Date.now()) / 1000)) + + let deferred: string[] = [] + let label: string + let repackArgs: string[] + if (reclaim) { + const {kept} = await markKeepPacks(mirrorPath, Number.MAX_SAFE_INTEGER) + await removeKeepFiles(mirrorPath, kept) + label = 'Reclaim' + repackArgs = ['-a', '-d', '-l', '-n', '--write-midx'] + core.info( + `[git-mirror] Running reclaim maintenance (timeout: ${budgetSecs}s, ${kept.length} kept pack(s) released)` + ) + } else { + const selection = await markKeepPacks(mirrorPath, keepBytes) + deferred = selection.deferred + label = 'Incremental' + repackArgs = ['-d', '-l', '-n', '--geometric=2', '--write-midx'] + core.info( + `[git-mirror] Running incremental maintenance (timeout: ${budgetSecs}s, ${selection.kept.length} kept pack(s), ${deferred.length} deferred)` + ) + } + + const fail = async ( + timedOut: boolean, + error: string + ): Promise => { + core.warning( + `[git-mirror] ${ + timedOut ? `Maintenance timed out after ${budgetSecs}s` : error + }; committing the synced mirror without it` + ) + await removeMaintenanceLeftovers(mirrorPath) + return {success: false, timedOut, error} + } try { - // --auto: only run if thresholds exceeded (default: 6700 loose objects or 50 packs) - // This is much faster than a full gc when not needed - // gc.autoDetach=false: prevent git from forking a background daemon for GC. - // Without this, the parent `git gc --auto` returns immediately while the - // daemonized child keeps running with cwd and mmap'd pack files on the - // mirror mount, causing the subsequent `umount` to fail with EBUSY. const result = await exec.getExecOutput( 'timeout', [ - String(timeoutSecs), + String(remainingSecs()), 'git', '-c', - 'gc.autoDetach=false', + 'repack.writeBitmaps=false', '-C', mirrorPath, - 'gc', - '--auto' + 'repack', + ...repackArgs ], {ignoreReturnCode: true} ) if (result.exitCode === TIMEOUT_EXIT_CODE) { - core.warning(`[git-mirror] GC timed out after ${timeoutSecs}s`) - return { - success: false, - timedOut: true, - error: `git gc timed out after ${timeoutSecs}s` - } + return await fail(true, `git repack timed out after ${budgetSecs}s`) } if (result.exitCode !== 0) { - core.warning(`[git-mirror] GC failed with exit code ${result.exitCode}`) - return { - success: false, - timedOut: false, - error: `git gc failed with exit code ${result.exitCode}` + return await fail( + false, + `git repack failed with exit code ${result.exitCode}` + ) + } + + if (reclaim) { + const prune = await exec.getExecOutput( + 'timeout', + [ + String(remainingSecs()), + 'git', + '-C', + mirrorPath, + 'prune', + `--expire=${MAINTENANCE_RECLAIM_PRUNE_EXPIRE}` + ], + {silent: true, ignoreReturnCode: true} + ) + if (prune.exitCode === TIMEOUT_EXIT_CODE) { + return await fail(true, `git prune timed out after ${budgetSecs}s`) } + if (prune.exitCode !== 0) { + return await fail( + false, + `git prune failed with exit code ${prune.exitCode}` + ) + } + await writeCommitGraph(mirrorPath, remainingSecs()) + } + + const packRefs = await exec.getExecOutput( + 'timeout', + [String(remainingSecs()), 'git', '-C', mirrorPath, 'pack-refs', '--all'], + {silent: true, ignoreReturnCode: true} + ) + if (packRefs.exitCode === TIMEOUT_EXIT_CODE) { + return await fail(true, `git pack-refs timed out after ${budgetSecs}s`) } - core.debug('[git-mirror] Completed git gc --auto') + if (packRefs.exitCode !== 0) { + return await fail( + false, + `git pack-refs failed with exit code ${packRefs.exitCode}` + ) + } + + core.info( + `[git-mirror] ${label} maintenance finished in ${Date.now() - start}ms` + ) return {success: true, timedOut: false} } catch (error) { const msg = (error as Error).message || String(error) - core.warning(`[git-mirror] GC failed: ${msg}`) - return {success: false, timedOut: false, error: msg} + return await fail(false, msg) + } finally { + await removeKeepFiles(mirrorPath, deferred) } } @@ -1852,10 +2207,11 @@ export interface CleanupOptions { } /** - * Cleanup: run GC, sync, unmount, and commit the sticky disk. + * Cleanup: run mirror maintenance, sync, unmount, and commit the sticky disk. * - * Execution order: GC → sync → unmount (with retry) → flush → commit - * If the mirror sync or GC failed or timed out, shouldCommit is set to false. + * Execution order: maintenance → sync → unmount (with retry) → flush → commit + * If the mirror sync failed or timed out, shouldCommit is set to false. + * Maintenance failure or timeout does not affect the commit. */ export async function cleanup(options: CleanupOptions): Promise { const { @@ -1869,13 +2225,13 @@ export async function cleanup(options: CleanupOptions): Promise { } = options let {shouldCommit} = options // vmHydratedGitMirror must track shouldCommit: if we decide not to commit - // (due to GC/refresh failure), we must not tell the backend that - // hydration completed, otherwise it marks the entry as ready despite no - // valid disk being persisted. + // (due to sync failure), we must not tell the backend that hydration + // completed, otherwise it marks the entry as ready despite no valid disk + // being persisted. let vmHydratedGitMirror = options.vmHydratedGitMirror const result: CleanupResult = { - gcResult: {success: true, timedOut: false} + maintenanceResult: {success: true, timedOut: false} } core.info( @@ -1892,21 +2248,14 @@ export async function cleanup(options: CleanupOptions): Promise { vmHydratedGitMirror = false } - if (mirrorPath) { - // Run GC on the mirror - result.gcResult = await runMirrorGC(mirrorPath) - if (!result.gcResult.success) { - core.warning( - '[git-mirror] GC failed or timed out, will not commit sticky disk' - ) - shouldCommit = false - vmHydratedGitMirror = false - } + // Maintenance only pays off if the result is persisted. + if (mirrorPath && shouldCommit) { + result.maintenanceResult = await runMirrorMaintenance(mirrorPath) // Catch-up for mirrors that predate commit-graph writing: build the // initial graph here in the post step, off the checkout critical path. - // Once it exists, the sync fetch keeps it current incrementally. Only - // worth doing when the disk is being committed; failure is non-fatal. - if (shouldCommit && !hasCommitGraph(mirrorPath)) { + // Once it exists, the sync fetch keeps it current incrementally. + // Failure is non-fatal. + if (!hasCommitGraph(mirrorPath)) { await writeCommitGraph(mirrorPath) } } diff --git a/src/main.ts b/src/main.ts index db8f1f1..596e440 100644 --- a/src/main.ts +++ b/src/main.ts @@ -164,9 +164,11 @@ async function cleanup(): Promise { }) } if (cleanupResult) { - if (!cleanupResult.gcResult.success) { + if (!cleanupResult.maintenanceResult.success) { await reportInternalMetric('git_mirror_gc_failure', 1, { - reason: cleanupResult.gcResult.timedOut ? 'timeout' : 'failure' + reason: cleanupResult.maintenanceResult.timedOut + ? 'timeout' + : 'failure' }) } }