Fetch without an object filter when the workspace shares the mirror

.github/workflows/test-blacksmith.yml

@@ -114,3 +114,35 @@ jobs:
 
           git fsck --no-dangling
           echo "Reused deep checkout verified"
+
+      - name: Test fetch-depth 0 checkout with filter (mirror provides the objects)
+        uses: ./
+        with:
+          path: filter-checkout
+          fetch-depth: 0
+          filter: blob:none
+
+      # Verified before the next step mounts its own copy of the sticky disk
+      # over the same path: objects that only the current mount's sync added
+      # are not visible through a later mount.
+      - name: Verify filtered deep checkout is a full clone backed by the mirror
+        run: |
+          __test__/verify-mirror-no-filter.sh filter-checkout
+          test -f filter-checkout/action.yml
+
+      - name: Test sparse checkout (mirror provides the objects)
+        uses: ./
+        with:
+          path: sparse-checkout
+          sparse-checkout: |
+            __test__
+            .github
+
+      - name: Verify sparse checkout is a full clone backed by the mirror
+        run: |
+          __test__/verify-mirror-no-filter.sh sparse-checkout
+          test -f sparse-checkout/.github/workflows/test-blacksmith.yml
+          if [ -d sparse-checkout/src ]; then
+            echo "sparse-checkout: cone pattern was not applied"
+            exit 1
+          fi

README.md

@@ -52,6 +52,8 @@ For shallow checkouts (`fetch-depth` greater than 0), the fetch tells GitHub whi
 
 The mirror always contains full history, but the workspace still respects inputs such as `fetch-depth`, `fetch-tags`, sparse checkout, LFS, and submodules. For example, `fetch-depth: 1` still produces a shallow workspace checkout.
 
+The one input that is not passed through is `filter` (and the `blob:none` filter that sparse checkout implies). Every object a filter would omit is already available through the mirror, so the workspace is fetched without a filter and is a regular, not partial, clone. Honouring the filter would make the fetch far slower, not faster: on Git 2.48 and later, indexing a filtered pack repacks every mirror object reachable from the fetched commits into the workspace. When the action falls back to a standard checkout, `filter` behaves as in `actions/checkout`.
+
 ### 3. Post-job refresh
 
 The action refreshes an existing mirror with `git fetch --prune` during post-job cleanup, outside the critical checkout path. This prepares the mirror for subsequent workflow runs without delaying the checkout step itself.

__test__/mirror-filter-fetch-git.test.ts

@@ -0,0 +1,181 @@
+/**
+ * Real-git coverage for fetching into a workspace that shares the mirror's
+ * objects through alternates when the caller asked for an object filter: the
+ * unfiltered fetch the action issues instead only transfers the delta and
+ * leaves the workspace a plain (non-partial) clone, whereas honouring the
+ * filter would repack mirror objects into a local promisor pack.
+ */
+// Mock the gRPC dependencies before importing blacksmith-cache
+jest.mock('@connectrpc/connect', () => ({
+  createClient: jest.fn(),
+  ConnectError: class ConnectError extends Error {},
+  Code: {Aborted: 'ABORTED'}
+}))
+
+jest.mock('@connectrpc/connect-node', () => ({
+  createGrpcTransport: jest.fn()
+}))
+
+jest.mock(
+  '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect',
+  () => ({
+    StickyDiskService: {}
+  })
+)
+
+jest.mock('../src/container-detector', () => ({
+  isRunningInContainer: jest.fn(() => false)
+}))
+
+import * as fs from 'fs'
+import * as os from 'os'
+import * as path from 'path'
+import {execFileSync} from 'child_process'
+import * as blacksmithCache from '../src/blacksmith-cache'
+import * as gitCommandManager from '../src/git-command-manager'
+import {GitVersion} from '../src/git-version'
+
+function git(cwd: string, ...args: string[]): string {
+  return execFileSync('git', ['-C', cwd, ...args], {encoding: 'utf8'}).trim()
+}
+
+function commit(repo: string, msg: string): string {
+  fs.writeFileSync(path.join(repo, `${msg}.txt`), msg)
+  git(repo, 'add', `${msg}.txt`)
+  git(
+    repo,
+    '-c',
+    'user.email=test@example.com',
+    '-c',
+    'user.name=Test',
+    'commit',
+    '-q',
+    '-m',
+    msg
+  )
+  return git(repo, 'rev-parse', 'HEAD')
+}
+
+function objectCount(workspace: string): number {
+  const counts = git(workspace, 'count-objects', '-v')
+  return (
+    Number(/^count: (\d+)/m.exec(counts)?.[1]) +
+    Number(/^in-pack: (\d+)/m.exec(counts)?.[1])
+  )
+}
+
+function promisorPacks(workspace: string): string[] {
+  const packDir = path.join(workspace, '.git', 'objects', 'pack')
+  if (!fs.existsSync(packDir)) {
+    return []
+  }
+  return fs.readdirSync(packDir).filter(name => name.endsWith('.promisor'))
+}
+
+/** git 2.48 started repacking locally available objects linked from a promisor pack. */
+const RepacksLocalLinksVersion = new GitVersion('2.48')
+
+describe('object filters with mirror alternates and real git', () => {
+  let tmpDir: string
+  let sourceRepo: string
+  let mirrorPath: string
+  let mirrorCommitCount: number
+  let sourceMain: string
+
+  const initWorkspace = (): string => {
+    const workspace = fs.mkdtempSync(path.join(tmpDir, 'ws-'))
+    git(workspace, 'init', '-q', '.')
+    git(workspace, 'remote', 'add', 'origin', sourceRepo)
+    const infoDir = path.join(workspace, '.git', 'objects', 'info')
+    fs.mkdirSync(infoDir, {recursive: true})
+    fs.writeFileSync(
+      path.join(infoDir, 'alternates'),
+      `${mirrorPath}/objects\n`
+    )
+    return workspace
+  }
+
+  const fetchMain = async (
+    workspace: string,
+    options: gitCommandManager.FetchOptions
+  ): Promise<GitVersion> => {
+    const git = await gitCommandManager.createCommandManager(
+      workspace,
+      false,
+      false
+    )
+    await git.fetch(['+refs/heads/main:refs/remotes/origin/main'], options)
+    return git.version()
+  }
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-filter-test-'))
+    sourceRepo = path.join(tmpDir, 'source')
+    mirrorPath = path.join(tmpDir, 'mirror')
+
+    fs.mkdirSync(sourceRepo)
+    git(sourceRepo, 'init', '-q', '-b', 'main', '.')
+    git(sourceRepo, 'config', 'uploadpack.allowFilter', 'true')
+    for (let i = 0; i < 20; i++) {
+      commit(sourceRepo, `history-${i}`)
+    }
+    execFileSync('git', ['clone', '-q', '--mirror', sourceRepo, mirrorPath])
+    mirrorCommitCount = Number(git(mirrorPath, 'rev-list', '--count', 'HEAD'))
+
+    // The mirror now lags the source by two commits on main.
+    commit(sourceRepo, 'new-1')
+    sourceMain = commit(sourceRepo, 'new-2')
+  })
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, {recursive: true, force: true})
+  })
+
+  it('shares the mirror object store', async () => {
+    await expect(
+      blacksmithCache.sharesMirrorObjects(initWorkspace(), mirrorPath)
+    ).resolves.toBe(true)
+  })
+
+  it.each([{fetchDepth: undefined}, {fetchDepth: 2}])(
+    'an unfiltered fetch (%o) only stores the delta and stays a full clone',
+    async ({fetchDepth}) => {
+      const workspace = initWorkspace()
+      await fetchMain(workspace, fetchDepth ? {fetchDepth} : {})
+
+      expect(git(workspace, 'rev-parse', 'refs/remotes/origin/main')).toBe(
+        sourceMain
+      )
+      // 2 commits + 2 trees + 2 blobs
+      expect(objectCount(workspace)).toBeLessThanOrEqual(6)
+      expect(promisorPacks(workspace)).toEqual([])
+      expect(
+        fs.readFileSync(path.join(workspace, '.git', 'config'), 'utf8')
+      ).not.toMatch(/promisor|partialclonefilter/i)
+
+      // Old and new blobs alike are readable and the tree checks out whole.
+      expect(
+        git(workspace, 'cat-file', '-t', `${sourceMain}:history-0.txt`)
+      ).toBe('blob')
+      git(workspace, 'checkout', '-q', '--detach', 'refs/remotes/origin/main')
+      expect(
+        fs.readdirSync(workspace).filter(f => f.endsWith('.txt'))
+      ).toHaveLength(22)
+      git(workspace, 'fsck', '--connectivity-only', '--no-dangling')
+    }
+  )
+
+  it('a blob:none fetch would repack mirror objects into a promisor pack', async () => {
+    const workspace = initWorkspace()
+    const version = await fetchMain(workspace, {filter: 'blob:none'})
+
+    expect(git(workspace, 'rev-parse', 'refs/remotes/origin/main')).toBe(
+      sourceMain
+    )
+    expect(promisorPacks(workspace)).not.toEqual([])
+    if (version.checkMinimum(RepacksLocalLinksVersion)) {
+      // The mirror's whole history behind the fetched commits was copied.
+      expect(objectCount(workspace)).toBeGreaterThan(mirrorCommitCount)
+    }
+  })
+})

__test__/verify-mirror-no-filter.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# Verify that a checkout requesting an object filter, but backed by the
+# git mirror, is a regular clone that only stores the objects the mirror
+# lacked. Usage: verify-mirror-no-filter.sh <workspace>
+set -euo pipefail
+
+ws="$1"
+cd "$ws"
+
+# No partial-clone state: no promisor packs, no promisor remote
+if ls .git/objects/pack/*.promisor >/dev/null 2>&1; then
+  echo "$ws: unexpected promisor pack"
+  exit 1
+fi
+if git config --local --get remote.origin.promisor >/dev/null \
+   || git config --local --get remote.origin.partialclonefilter >/dev/null; then
+  echo "$ws: unexpected partial clone configuration"
+  exit 1
+fi
+
+# Only the objects the mirror lacked may live in the workspace
+in_pack=$(git count-objects -v | awk '/^in-pack:/ {print $2}')
+loose=$(git count-objects -v | awk '/^count:/ {print $2}')
+echo "$ws: $in_pack packed + $loose loose local objects"
+if [ "$((in_pack + loose))" -gt 1000 ]; then
+  echo "$ws: mirror history was copied into the workspace"
+  exit 1
+fi
+
+git fsck --connectivity-only --no-dangling
+echo "$ws: verified"

action.yml

@@ -61,6 +61,8 @@ inputs:
     description: >
       Partially clone against a given filter.
       Overrides sparse-checkout if set.
+      Ignored when the workspace shares a Blacksmith git mirror, which already
+      holds every object the filter would omit.
     default: null
   sparse-checkout:
     description: >

dist/index.js

@@ -3304,11 +3304,22 @@ function getSource(settings) {
             // Fetch
             core.startGroup('Fetching the repository');
             const fetchOptions = {};
-            if (settings.filter) {
-                fetchOptions.filter = settings.filter;
+            const sharesMirrorObjects = !!cacheInfo &&
+                (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath));
+            const filter = settings.filter
+                ? settings.filter
+                : settings.sparseCheckout
+                    ? 'blob:none'
+                    : undefined;
+            if (filter && sharesMirrorObjects) {
+                // The mirror alternate already holds every object the filter would omit,
+                // and a filtered fetch makes the workspace a partial clone whose
+                // index-pack (git >= 2.48) repacks all mirror objects reachable from the
+                // fetched commits into a local promisor pack.
+                core.info(`[git-mirror] Fetching without --filter=${filter}: the mirror already provides every object`);
             }
-            else if (settings.sparseCheckout) {
-                fetchOptions.filter = 'blob:none';
+            else if (filter) {
+                fetchOptions.filter = filter;
             }
             if (settings.fetchDepth <= 0) {
                 // When the Blacksmith mirror is available and synced with the remote,
@@ -3367,8 +3378,8 @@ function getSource(settings) {
                 // tips explicitly instead, so the server still sends only the delta
                 // from the mirror (see resolveShallowNegotiationTips).
                 if (cacheInfo &&
-                    (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion) &&
-                    (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath))) {
+                    sharesMirrorObjects &&
+                    (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion)) {
                     fetchOptions.ignoreAlternateRefs = true;
                     fetchOptions.negotiationTips =
                         yield blacksmithCache.resolveShallowNegotiationTips(cacheInfo.mirrorPath, settings.ref, process.env['GITHUB_BASE_REF'] || '');

src/git-source-provider.ts

@@ -252,10 +252,28 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
     core.startGroup('Fetching the repository')
     const fetchOptions: FetchOptions = {}
 
-    if (settings.filter) {
-      fetchOptions.filter = settings.filter
-    } else if (settings.sparseCheckout) {
-      fetchOptions.filter = 'blob:none'
+    const sharesMirrorObjects =
+      !!cacheInfo &&
+      (await blacksmithCache.sharesMirrorObjects(
+        settings.repositoryPath,
+        cacheInfo.mirrorPath
+      ))
+
+    const filter = settings.filter
+      ? settings.filter
+      : settings.sparseCheckout
+        ? 'blob:none'
+        : undefined
+    if (filter && sharesMirrorObjects) {
+      // The mirror alternate already holds every object the filter would omit,
+      // and a filtered fetch makes the workspace a partial clone whose
+      // index-pack (git >= 2.48) repacks all mirror objects reachable from the
+      // fetched commits into a local promisor pack.
+      core.info(
+        `[git-mirror] Fetching without --filter=${filter}: the mirror already provides every object`
+      )
+    } else if (filter) {
+      fetchOptions.filter = filter
     }
 
     if (settings.fetchDepth <= 0) {
@@ -327,13 +345,10 @@ export async function getSource(settings: IGitSourceSettings): Promise<void> {
       // from the mirror (see resolveShallowNegotiationTips).
       if (
         cacheInfo &&
+        sharesMirrorObjects &&
         (await git.version()).checkMinimum(
           MinimumGitAlternateRefsCommandVersion
-        ) &&
-        (await blacksmithCache.sharesMirrorObjects(
-          settings.repositoryPath,
-          cacheInfo.mirrorPath
-        ))
+        )
       ) {
         fetchOptions.ignoreAlternateRefs = true
         fetchOptions.negotiationTips =