telemetry: structured git mirror checkout, hydration, and maintenance reports

__test__/mirror-cleanup.test.ts

@@ -95,7 +95,10 @@ describe('cleanup commit decision', () => {
   it('runs bounded maintenance instead of gc --auto and commits', async () => {
     const result = await blacksmithCache.cleanup({...base, mirrorPath})
 
-    expect(result.maintenanceResult).toEqual({success: true, timedOut: false})
+    expect(result.maintenanceResult).toMatchObject({
+      success: true,
+      timedOut: false
+    })
     const repack = commands().find(c => c.includes('repack'))
     expect(repack).toBeDefined()
     expect(repack).toEqual(
@@ -129,7 +132,7 @@ describe('cleanup commit decision', () => {
     repackExitCode = 128
     const result = await blacksmithCache.cleanup({...base, mirrorPath})
 
-    expect(result.maintenanceResult).toEqual({
+    expect(result.maintenanceResult).toMatchObject({
       success: false,
       timedOut: false,
       error: expect.stringContaining('128')
@@ -143,7 +146,7 @@ describe('cleanup commit decision', () => {
     packRefsExitCode = 1
     const result = await blacksmithCache.cleanup({...base, mirrorPath})
 
-    expect(result.maintenanceResult).toEqual({
+    expect(result.maintenanceResult).toMatchObject({
       success: false,
       timedOut: false,
       error: expect.stringContaining('pack-refs failed with exit code 1')
@@ -157,7 +160,7 @@ describe('cleanup commit decision', () => {
     packRefsExitCode = 124
     const result = await blacksmithCache.cleanup({...base, mirrorPath})
 
-    expect(result.maintenanceResult).toEqual({
+    expect(result.maintenanceResult).toMatchObject({
       success: false,
       timedOut: true,
       error: expect.stringContaining('pack-refs timed out')
@@ -174,7 +177,11 @@ describe('cleanup commit decision', () => {
       mirrorSyncFailed: true
     })
 
-    expect(result.maintenanceResult).toEqual({success: true, timedOut: false})
+    expect(result.maintenanceResult).toEqual({
+      success: true,
+      timedOut: false,
+      skipped: true
+    })
     expect(commands().some(c => c.includes('repack'))).toBe(false)
     expect(mockCommitStickyDisk).toHaveBeenCalledWith(
       expect.objectContaining({shouldCommit: false, vmHydratedGitMirror: false})

__test__/mirror-gc-git.test.ts

@@ -0,0 +1,213 @@
+/**
+ * Real-git coverage for post-step maintenance telemetry: a geometric repack
+ * that finds nothing to roll up is reported as skipped (no maintenance
+ * row), while one that consolidates packs is a real maintenance result.
+ */
+jest.mock('@connectrpc/connect', () => ({
+  createClient: jest.fn(),
+  ConnectError: class ConnectError extends Error {},
+  Code: {Aborted: 'ABORTED'}
+}))
+
+jest.mock('@connectrpc/connect-node', () => ({
+  createGrpcTransport: jest.fn()
+}))
+
+jest.mock(
+  '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect',
+  () => ({
+    StickyDiskService: {}
+  })
+)
+
+jest.mock('../src/container-detector', () => ({
+  isRunningInContainer: jest.fn(() => false)
+}))
+
+import * as fs from 'fs'
+import * as os from 'os'
+import * as path from 'path'
+import {execFileSync} from 'child_process'
+import * as blacksmithCache from '../src/blacksmith-cache'
+import * as mirrorTelemetry from '../src/mirror-telemetry'
+
+const gitEnv = {
+  ...process.env,
+  GIT_AUTHOR_NAME: 'test',
+  GIT_AUTHOR_EMAIL: 'test@example.com',
+  GIT_COMMITTER_NAME: 'test',
+  GIT_COMMITTER_EMAIL: 'test@example.com'
+}
+
+function git(cwd: string, ...args: string[]): string {
+  return execFileSync('git', ['-C', cwd, ...args], {
+    encoding: 'utf8',
+    env: gitEnv
+  }).trim()
+}
+
+function packCount(repo: string): number {
+  const packDir = path.join(repo, 'objects', 'pack')
+  return fs.readdirSync(packDir).filter(f => f.endsWith('.pack')).length
+}
+
+// Writes one pack holding exactly the given objects; returns its basename.
+function writePack(repo: string, objects: string): string {
+  const hash = execFileSync(
+    'git',
+    [
+      '-C',
+      repo,
+      'pack-objects',
+      '-q',
+      path.join(repo, 'objects', 'pack', 'pack')
+    ],
+    {input: objects, encoding: 'utf8', env: gitEnv}
+  ).trim()
+  return `pack-${hash}`
+}
+
+// A bare repo whose objects live in exactly two similarly sized packs (one
+// per commit) and no loose objects: a geometric repack rolls them into one.
+function twoPackMirror(root: string): string {
+  const work = path.join(root, 'work')
+  fs.mkdirSync(work)
+  git(work, 'init', '-q')
+  fs.writeFileSync(path.join(work, 'a.txt'), 'one\n')
+  git(work, 'add', 'a.txt')
+  git(work, 'commit', '-q', '-m', 'one')
+  fs.writeFileSync(path.join(work, 'b.txt'), 'two\n')
+  git(work, 'add', 'b.txt')
+  git(work, 'commit', '-q', '-m', 'two')
+
+  const mirror = path.join(root, 'mirror.git')
+  git(root, 'clone', '-q', '--mirror', work, mirror)
+  const keep = new Set([
+    writePack(mirror, git(mirror, 'rev-list', '--objects', 'HEAD~1')),
+    writePack(mirror, git(mirror, 'rev-list', '--objects', 'HEAD~1..HEAD'))
+  ])
+  const packDir = path.join(mirror, 'objects', 'pack')
+  for (const f of fs.readdirSync(packDir)) {
+    if (!keep.has(f.replace(/\.(pack|idx|rev)$/, ''))) {
+      fs.rmSync(path.join(packDir, f))
+    }
+  }
+  git(mirror, 'prune-packed', '-q')
+  expect(packCount(mirror)).toBe(2)
+  return mirror
+}
+
+// A bare repo already consolidated into a single pack with no loose
+// objects, so a geometric repack has nothing to do.
+function onePackMirror(root: string): string {
+  const mirror = twoPackMirror(root)
+  git(mirror, '-c', 'repack.writeBitmaps=false', 'repack', '-a', '-d', '-q')
+  git(mirror, 'prune-packed', '-q')
+  expect(packCount(mirror)).toBe(1)
+  return mirror
+}
+
+describe('mirror maintenance telemetry', () => {
+  let root: string
+
+  beforeEach(() => {
+    root = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-gc-'))
+    jest.spyOn(console, 'log').mockImplementation(() => {})
+  })
+
+  afterEach(() => {
+    fs.rmSync(root, {recursive: true, force: true})
+    jest.restoreAllMocks()
+  })
+
+  it('reports a repack that rolled nothing up as skipped', async () => {
+    const mirror = onePackMirror(root)
+    const packsBefore = await mirrorTelemetry.packNamesOrNull(mirror)
+
+    const result = await blacksmithCache.cleanup({
+      exposeId: 'expose-1',
+      stickyDiskKey: 'owner-repo',
+      mirrorPath: mirror,
+      shouldCommit: true,
+      vmHydratedGitMirror: false
+    })
+
+    expect(await mirrorTelemetry.packNamesOrNull(mirror)).toEqual(packsBefore)
+    expect(result.maintenanceResult.success).toBe(true)
+    expect(result.maintenanceResult.skipped).toBe(true)
+  })
+
+  it('reports a repack that consolidated packs as a real run', async () => {
+    const mirror = twoPackMirror(root)
+
+    const result = await blacksmithCache.cleanup({
+      exposeId: 'expose-1',
+      stickyDiskKey: 'owner-repo',
+      mirrorPath: mirror,
+      shouldCommit: true,
+      vmHydratedGitMirror: false
+    })
+
+    expect(packCount(mirror)).toBe(1)
+    expect(result.maintenanceResult.success).toBe(true)
+    expect(result.maintenanceResult.skipped).toBeUndefined()
+    expect(result.maintenanceResult.durationMs).toBeGreaterThanOrEqual(0)
+    expect(result.maintenanceResult.mirrorSizeBytes).toBeGreaterThan(0)
+  })
+
+  it('reports no maintenance when the disk is not committed', async () => {
+    const mirror = twoPackMirror(root)
+
+    const result = await blacksmithCache.cleanup({
+      exposeId: 'expose-1',
+      stickyDiskKey: 'owner-repo',
+      mirrorPath: mirror,
+      shouldCommit: false,
+      vmHydratedGitMirror: false
+    })
+
+    expect(packCount(mirror)).toBe(2)
+    expect(result.maintenanceResult.skipped).toBe(true)
+  })
+})
+
+describe('pack measurement', () => {
+  let root: string
+
+  beforeEach(() => {
+    root = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-pack-'))
+  })
+
+  afterEach(() => {
+    fs.rmSync(root, {recursive: true, force: true})
+  })
+
+  it('sums only the files under objects/pack', async () => {
+    const mirror = twoPackMirror(root)
+    // Loose objects and refs must not count.
+    fs.mkdirSync(path.join(mirror, 'objects', 'ab'))
+    fs.writeFileSync(
+      path.join(mirror, 'objects', 'ab', 'cdef'),
+      'x'.repeat(4096)
+    )
+    const packDir = path.join(mirror, 'objects', 'pack')
+    const expected = fs
+      .readdirSync(packDir)
+      .reduce((sum, f) => sum + fs.statSync(path.join(packDir, f)).size, 0)
+
+    expect(await mirrorTelemetry.packSizeBytesOrNull(mirror)).toBe(expected)
+    expect(await mirrorTelemetry.packNamesOrNull(mirror)).toHaveLength(2)
+  })
+
+  it('measures a repository without packs as empty, not unmeasurable', async () => {
+    const bare = path.join(root, 'empty.git')
+    git(root, 'init', '-q', '--bare', 'empty.git')
+    fs.rmSync(path.join(bare, 'objects', 'pack'), {recursive: true})
+
+    expect(await mirrorTelemetry.packSizeBytesOrNull(bare)).toBe(0)
+    expect(await mirrorTelemetry.packNamesOrNull(bare)).toEqual([])
+    expect(
+      await mirrorTelemetry.packSizeBytesOrNull(path.join(root, 'missing'))
+    ).toBe(0)
+  })
+})

__test__/mirror-maintenance-git.test.ts

@@ -190,7 +190,9 @@ describe('runMirrorMaintenance (real git)', () => {
       timeoutSecs: 60,
       keepBytes: KEEP_BYTES
     })
-    expect(result).toEqual({success: true, timedOut: false})
+    expect(result).toMatchObject({success: true, timedOut: false})
+    expect(result.skipped).toBeUndefined()
+    expect(result.mirrorSizeBytes).toBeGreaterThan(before.size)
 
     const packDir = path.join(mirror, 'objects', 'pack')
     expect(
@@ -222,7 +224,11 @@ describe('runMirrorMaintenance (real git)', () => {
       timeoutSecs: 60,
       keepBytes: KEEP_BYTES
     })
-    expect(result).toEqual({success: true, timedOut: false})
+    expect(result).toMatchObject({
+      success: true,
+      timedOut: false,
+      skipped: true
+    })
     expect(packs(mirror)).toEqual(first)
     fsck(mirror)
   })
@@ -322,7 +328,7 @@ describe('runMirrorMaintenance (real git)', () => {
         timeoutSecs: 60,
         keepBytes: KEEP_BYTES
       })
-      expect(result).toEqual({success: true, timedOut: false})
+      expect(result).toMatchObject({success: true, timedOut: false})
       const after = packSizes(mirror)
       expect(bytesWritten(before, after)).toBeLessThan(KEEP_BYTES)
       expect(after.has(basePack)).toBe(true)
@@ -423,7 +429,7 @@ describe('runMirrorMaintenance (real git)', () => {
         keepBytes: KEEP_BYTES,
         now: 1000
       })
-      expect(result).toEqual({success: true, timedOut: false})
+      expect(result).toMatchObject({success: true, timedOut: false})
       expect(fs.readFileSync(stamp, 'utf8').trim()).toBe('1000')
       expect(packs(mirror)).toContain(pack)
       expect(hasObject(mirror, blob)).toBe(true)
@@ -456,7 +462,7 @@ describe('runMirrorMaintenance (real git)', () => {
         reclaimIntervalMs: 10_000,
         now: 20_000
       })
-      expect(result).toEqual({success: true, timedOut: false})
+      expect(result).toMatchObject({success: true, timedOut: false})
 
       const packDir = path.join(mirror, 'objects', 'pack')
       const remaining = packs(mirror)
@@ -491,7 +497,7 @@ describe('runMirrorMaintenance (real git)', () => {
         reclaimIntervalMs: 10_000,
         now: 25_000
       })
-      expect(next).toEqual({success: true, timedOut: false})
+      expect(next).toMatchObject({success: true, timedOut: false})
       expect(packs(mirror)).toEqual(remaining)
       expect(fs.readdirSync(packDir).filter(f => f.endsWith('.keep'))).toEqual([
         remaining[0].replace(/\.pack$/, '.keep')
@@ -594,7 +600,7 @@ PATH="${binDir}:$PATH" exec ${realTimeout} "$@"
         timeoutSecs: 60,
         keepBytes: KEEP_BYTES
       })
-      expect(result).toEqual({
+      expect(result).toMatchObject({
         success: false,
         timedOut: false,
         error: expect.stringContaining('pack-refs failed with exit code 3')
@@ -615,7 +621,7 @@ PATH="${binDir}:$PATH" exec ${realTimeout} "$@"
         timeoutSecs: 1,
         keepBytes: KEEP_BYTES
       })
-      expect(result).toEqual({
+      expect(result).toMatchObject({
         success: false,
         timedOut: true,
         error: expect.stringContaining('pack-refs timed out')
@@ -658,7 +664,7 @@ PATH="${binDir}:$PATH" exec ${realTimeout} "$@"
         reclaimIntervalMs: 10_000,
         now: 25_000
       })
-      expect(next).toEqual({success: true, timedOut: false})
+      expect(next).toMatchObject({success: true, timedOut: false})
       expect(fs.existsSync(keepFile)).toBe(true)
       expect(packs(mirror)).toContain(basePack)
       fsck(mirror)
@@ -677,7 +683,7 @@ PATH="${binDir}:$PATH" exec ${realTimeout} "$@"
         reclaimIntervalMs: 10_000,
         now: 20_000
       })
-      expect(result).toEqual({
+      expect(result).toMatchObject({
         success: false,
         timedOut: false,
         error: expect.stringContaining('prune failed with exit code 7')

__test__/mirror-sync-negotiation-git.test.ts

@@ -173,17 +173,20 @@ describe('mirror sync negotiation with real git', () => {
       .readdirSync(path.join(mirrorPath, 'objects', 'pack'))
       .filter(f => f.endsWith('.pack'))
 
+    let changedCallbacks = 0
     const result = await blacksmithCache.syncMirrorFromRemote(
       mirrorPath,
       'https://example.invalid/owner/repo',
       'token',
       false,
-      60
+      60,
+      () => changedCallbacks++
     )
 
     expect(result.success).toBe(true)
     expect(result.timedOut).toBe(false)
     expect(result.changed).toBe(true)
+    expect(changedCallbacks).toBe(1)
     expect(git(mirrorPath, 'rev-parse', 'refs/heads/feature')).toBe(
       newFeatureTip
     )