From 79a968cadaef24d57977d40405af77ec723e1a81 Mon Sep 17 00:00:00 2001 From: piotr Date: Sun, 6 Sep 2026 13:23:16 +0000 Subject: [PATCH 1/3] Fetch without an object filter when the workspace shares the mirror With the mirror attached as an alternate every object a filter would omit is already local, so the filter saves nothing. It does make the workspace a partial clone, and since git 2.48 index-pack --promisor repacks every locally available object linked from the received pack into a promisor pack: with a full mirror behind the alternates that copies the mirror's reachable history into the workspace on every filtered fetch that receives objects, costing time proportional to the mirror rather than to the delta. Drop the filter (explicit or the blob:none that sparse-checkout implies) whenever the workspace verifiably shares the mirror object store. As a side effect fetch-depth 0 with a filter now takes the direct ref-copy path instead of a full network fetch of all heads and tags. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-blacksmith.yml | 53 +++++++ README.md | 2 + __test__/mirror-filter-fetch-git.test.ts | 181 +++++++++++++++++++++++ action.yml | 2 + dist/index.js | 23 ++- src/git-source-provider.ts | 33 +++-- 6 files changed, 279 insertions(+), 15 deletions(-) create mode 100644 __test__/mirror-filter-fetch-git.test.ts diff --git a/.github/workflows/test-blacksmith.yml b/.github/workflows/test-blacksmith.yml index d6390e8..0c11db7 100644 --- a/.github/workflows/test-blacksmith.yml +++ b/.github/workflows/test-blacksmith.yml @@ -114,3 +114,56 @@ jobs: git fsck --no-dangling echo "Reused deep checkout verified" + + - name: Test fetch-depth 0 checkout with filter (mirror provides the objects) + uses: ./ + with: + path: filter-checkout + fetch-depth: 0 + filter: blob:none + + - name: Test sparse checkout (mirror provides the objects) + uses: ./ + with: + path: sparse-checkout + sparse-checkout: | + __test__ + .github + + - name: Verify filtered checkouts are full clones backed by the mirror + run: | + set -euo pipefail + for ws in filter-checkout sparse-checkout; do + cd "$ws" + + # No partial-clone state: no promisor packs, no promisor remote + if ls .git/objects/pack/*.promisor >/dev/null 2>&1; then + echo "$ws: unexpected promisor pack" + exit 1 + fi + if git config --local --get remote.origin.promisor >/dev/null \ + || git config --local --get remote.origin.partialclonefilter >/dev/null; then + echo "$ws: unexpected partial clone configuration" + exit 1 + fi + + # Only the objects the mirror lacked may live in the workspace + in_pack=$(git count-objects -v | awk '/^in-pack:/ {print $2}') + loose=$(git count-objects -v | awk '/^count:/ {print $2}') + echo "$ws: $in_pack packed + $loose loose local objects" + if [ "$((in_pack + loose))" -gt 1000 ]; then + echo "$ws: mirror history was copied into the workspace" + exit 1 + fi + + git fsck --connectivity-only --no-dangling + cd .. + done + + test -f filter-checkout/action.yml + test -f sparse-checkout/.github/workflows/test-blacksmith.yml + if [ -f sparse-checkout/action.yml ]; then + echo "sparse-checkout: cone pattern was not applied" + exit 1 + fi + echo "Filtered checkouts verified" diff --git a/README.md b/README.md index 9e993ad..ee5f01b 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ For shallow checkouts (`fetch-depth` greater than 0), the fetch tells GitHub whi The mirror always contains full history, but the workspace still respects inputs such as `fetch-depth`, `fetch-tags`, sparse checkout, LFS, and submodules. For example, `fetch-depth: 1` still produces a shallow workspace checkout. +The one input that is not passed through is `filter` (and the `blob:none` filter that sparse checkout implies). Every object a filter would omit is already available through the mirror, so the workspace is fetched without a filter and is a regular, not partial, clone. Honouring the filter would make the fetch far slower, not faster: on Git 2.48 and later, indexing a filtered pack repacks every mirror object reachable from the fetched commits into the workspace. When the action falls back to a standard checkout, `filter` behaves as in `actions/checkout`. + ### 3. Post-job refresh The action refreshes an existing mirror with `git fetch --prune` during post-job cleanup, outside the critical checkout path. This prepares the mirror for subsequent workflow runs without delaying the checkout step itself. diff --git a/__test__/mirror-filter-fetch-git.test.ts b/__test__/mirror-filter-fetch-git.test.ts new file mode 100644 index 0000000..01fd177 --- /dev/null +++ b/__test__/mirror-filter-fetch-git.test.ts @@ -0,0 +1,181 @@ +/** + * Real-git coverage for fetching into a workspace that shares the mirror's + * objects through alternates when the caller asked for an object filter: the + * unfiltered fetch the action issues instead only transfers the delta and + * leaves the workspace a plain (non-partial) clone, whereas honouring the + * filter would repack mirror objects into a local promisor pack. + */ +// Mock the gRPC dependencies before importing blacksmith-cache +jest.mock('@connectrpc/connect', () => ({ + createClient: jest.fn(), + ConnectError: class ConnectError extends Error {}, + Code: {Aborted: 'ABORTED'} +})) + +jest.mock('@connectrpc/connect-node', () => ({ + createGrpcTransport: jest.fn() +})) + +jest.mock( + '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect', + () => ({ + StickyDiskService: {} + }) +) + +jest.mock('../src/container-detector', () => ({ + isRunningInContainer: jest.fn(() => false) +})) + +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {execFileSync} from 'child_process' +import * as blacksmithCache from '../src/blacksmith-cache' +import * as gitCommandManager from '../src/git-command-manager' +import {GitVersion} from '../src/git-version' + +function git(cwd: string, ...args: string[]): string { + return execFileSync('git', ['-C', cwd, ...args], {encoding: 'utf8'}).trim() +} + +function commit(repo: string, msg: string): string { + fs.writeFileSync(path.join(repo, `${msg}.txt`), msg) + git(repo, 'add', `${msg}.txt`) + git( + repo, + '-c', + 'user.email=test@example.com', + '-c', + 'user.name=Test', + 'commit', + '-q', + '-m', + msg + ) + return git(repo, 'rev-parse', 'HEAD') +} + +function objectCount(workspace: string): number { + const counts = git(workspace, 'count-objects', '-v') + return ( + Number(/^count: (\d+)/m.exec(counts)?.[1]) + + Number(/^in-pack: (\d+)/m.exec(counts)?.[1]) + ) +} + +function promisorPacks(workspace: string): string[] { + const packDir = path.join(workspace, '.git', 'objects', 'pack') + if (!fs.existsSync(packDir)) { + return [] + } + return fs.readdirSync(packDir).filter(name => name.endsWith('.promisor')) +} + +/** git 2.48 started repacking locally available objects linked from a promisor pack. */ +const RepacksLocalLinksVersion = new GitVersion('2.48') + +describe('object filters with mirror alternates and real git', () => { + let tmpDir: string + let sourceRepo: string + let mirrorPath: string + let mirrorCommitCount: number + let sourceMain: string + + const initWorkspace = (): string => { + const workspace = fs.mkdtempSync(path.join(tmpDir, 'ws-')) + git(workspace, 'init', '-q', '.') + git(workspace, 'remote', 'add', 'origin', sourceRepo) + const infoDir = path.join(workspace, '.git', 'objects', 'info') + fs.mkdirSync(infoDir, {recursive: true}) + fs.writeFileSync( + path.join(infoDir, 'alternates'), + `${mirrorPath}/objects\n` + ) + return workspace + } + + const fetchMain = async ( + workspace: string, + options: gitCommandManager.FetchOptions + ): Promise => { + const git = await gitCommandManager.createCommandManager( + workspace, + false, + false + ) + await git.fetch(['+refs/heads/main:refs/remotes/origin/main'], options) + return git.version() + } + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-filter-test-')) + sourceRepo = path.join(tmpDir, 'source') + mirrorPath = path.join(tmpDir, 'mirror') + + fs.mkdirSync(sourceRepo) + git(sourceRepo, 'init', '-q', '-b', 'main', '.') + git(sourceRepo, 'config', 'uploadpack.allowFilter', 'true') + for (let i = 0; i < 20; i++) { + commit(sourceRepo, `history-${i}`) + } + execFileSync('git', ['clone', '-q', '--mirror', sourceRepo, mirrorPath]) + mirrorCommitCount = Number(git(mirrorPath, 'rev-list', '--count', 'HEAD')) + + // The mirror now lags the source by two commits on main. + commit(sourceRepo, 'new-1') + sourceMain = commit(sourceRepo, 'new-2') + }) + + afterEach(() => { + fs.rmSync(tmpDir, {recursive: true, force: true}) + }) + + it('shares the mirror object store', async () => { + await expect( + blacksmithCache.sharesMirrorObjects(initWorkspace(), mirrorPath) + ).resolves.toBe(true) + }) + + it.each([{fetchDepth: undefined}, {fetchDepth: 2}])( + 'an unfiltered fetch (%o) only stores the delta and stays a full clone', + async ({fetchDepth}) => { + const workspace = initWorkspace() + await fetchMain(workspace, fetchDepth ? {fetchDepth} : {}) + + expect(git(workspace, 'rev-parse', 'refs/remotes/origin/main')).toBe( + sourceMain + ) + // 2 commits + 2 trees + 2 blobs + expect(objectCount(workspace)).toBeLessThanOrEqual(6) + expect(promisorPacks(workspace)).toEqual([]) + expect( + fs.readFileSync(path.join(workspace, '.git', 'config'), 'utf8') + ).not.toMatch(/promisor|partialclonefilter/i) + + // Old and new blobs alike are readable and the tree checks out whole. + expect( + git(workspace, 'cat-file', '-t', `${sourceMain}:history-0.txt`) + ).toBe('blob') + git(workspace, 'checkout', '-q', '--detach', 'refs/remotes/origin/main') + expect( + fs.readdirSync(workspace).filter(f => f.endsWith('.txt')) + ).toHaveLength(22) + git(workspace, 'fsck', '--connectivity-only', '--no-dangling') + } + ) + + it('a blob:none fetch would repack mirror objects into a promisor pack', async () => { + const workspace = initWorkspace() + const version = await fetchMain(workspace, {filter: 'blob:none'}) + + expect(git(workspace, 'rev-parse', 'refs/remotes/origin/main')).toBe( + sourceMain + ) + expect(promisorPacks(workspace)).not.toEqual([]) + if (version.checkMinimum(RepacksLocalLinksVersion)) { + // The mirror's whole history behind the fetched commits was copied. + expect(objectCount(workspace)).toBeGreaterThan(mirrorCommitCount) + } + }) +}) diff --git a/action.yml b/action.yml index 4c4863e..d29527f 100644 --- a/action.yml +++ b/action.yml @@ -61,6 +61,8 @@ inputs: description: > Partially clone against a given filter. Overrides sparse-checkout if set. + Ignored when the workspace shares a Blacksmith git mirror, which already + holds every object the filter would omit. default: null sparse-checkout: description: > diff --git a/dist/index.js b/dist/index.js index 2b950bf..7693bc6 100644 --- a/dist/index.js +++ b/dist/index.js @@ -3304,11 +3304,22 @@ function getSource(settings) { // Fetch core.startGroup('Fetching the repository'); const fetchOptions = {}; - if (settings.filter) { - fetchOptions.filter = settings.filter; + const sharesMirrorObjects = !!cacheInfo && + (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath)); + const filter = settings.filter + ? settings.filter + : settings.sparseCheckout + ? 'blob:none' + : undefined; + if (filter && sharesMirrorObjects) { + // The mirror alternate already holds every object the filter would omit, + // and a filtered fetch makes the workspace a partial clone whose + // index-pack (git >= 2.48) repacks all mirror objects reachable from the + // fetched commits into a local promisor pack. + core.info(`[git-mirror] Fetching without --filter=${filter}: the mirror already provides every object`); } - else if (settings.sparseCheckout) { - fetchOptions.filter = 'blob:none'; + else if (filter) { + fetchOptions.filter = filter; } if (settings.fetchDepth <= 0) { // When the Blacksmith mirror is available and synced with the remote, @@ -3367,8 +3378,8 @@ function getSource(settings) { // tips explicitly instead, so the server still sends only the delta // from the mirror (see resolveShallowNegotiationTips). if (cacheInfo && - (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion) && - (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath))) { + sharesMirrorObjects && + (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion)) { fetchOptions.ignoreAlternateRefs = true; fetchOptions.negotiationTips = yield blacksmithCache.resolveShallowNegotiationTips(cacheInfo.mirrorPath, settings.ref, process.env['GITHUB_BASE_REF'] || ''); diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index 9e756b4..63f5074 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -252,10 +252,28 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.startGroup('Fetching the repository') const fetchOptions: FetchOptions = {} - if (settings.filter) { - fetchOptions.filter = settings.filter - } else if (settings.sparseCheckout) { - fetchOptions.filter = 'blob:none' + const sharesMirrorObjects = + !!cacheInfo && + (await blacksmithCache.sharesMirrorObjects( + settings.repositoryPath, + cacheInfo.mirrorPath + )) + + const filter = settings.filter + ? settings.filter + : settings.sparseCheckout + ? 'blob:none' + : undefined + if (filter && sharesMirrorObjects) { + // The mirror alternate already holds every object the filter would omit, + // and a filtered fetch makes the workspace a partial clone whose + // index-pack (git >= 2.48) repacks all mirror objects reachable from the + // fetched commits into a local promisor pack. + core.info( + `[git-mirror] Fetching without --filter=${filter}: the mirror already provides every object` + ) + } else if (filter) { + fetchOptions.filter = filter } if (settings.fetchDepth <= 0) { @@ -327,13 +345,10 @@ export async function getSource(settings: IGitSourceSettings): Promise { // from the mirror (see resolveShallowNegotiationTips). if ( cacheInfo && + sharesMirrorObjects && (await git.version()).checkMinimum( MinimumGitAlternateRefsCommandVersion - ) && - (await blacksmithCache.sharesMirrorObjects( - settings.repositoryPath, - cacheInfo.mirrorPath - )) + ) ) { fetchOptions.ignoreAlternateRefs = true fetchOptions.negotiationTips = From 63709f430c976cd71f0896c5f66b43e8549af2a5 Mon Sep 17 00:00:00 2001 From: piotr Date: Sun, 6 Sep 2026 13:27:11 +0000 Subject: [PATCH 2/3] test-blacksmith: verify each filtered checkout before the next mirror mount A later checkout step in the same job mounts its own copy of the sticky disk over the same path, so objects that only the earlier mount's sync added are no longer visible through the earlier workspace's alternates. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-blacksmith.yml | 41 +++++++-------------------- __test__/verify-mirror-no-filter.sh | 31 ++++++++++++++++++++ 2 files changed, 41 insertions(+), 31 deletions(-) create mode 100755 __test__/verify-mirror-no-filter.sh diff --git a/.github/workflows/test-blacksmith.yml b/.github/workflows/test-blacksmith.yml index 0c11db7..00ef67e 100644 --- a/.github/workflows/test-blacksmith.yml +++ b/.github/workflows/test-blacksmith.yml @@ -122,6 +122,14 @@ jobs: fetch-depth: 0 filter: blob:none + # Verified before the next step mounts its own copy of the sticky disk + # over the same path: objects that only the current mount's sync added + # are not visible through a later mount. + - name: Verify filtered deep checkout is a full clone backed by the mirror + run: | + __test__/verify-mirror-no-filter.sh filter-checkout + test -f filter-checkout/action.yml + - name: Test sparse checkout (mirror provides the objects) uses: ./ with: @@ -130,40 +138,11 @@ jobs: __test__ .github - - name: Verify filtered checkouts are full clones backed by the mirror + - name: Verify sparse checkout is a full clone backed by the mirror run: | - set -euo pipefail - for ws in filter-checkout sparse-checkout; do - cd "$ws" - - # No partial-clone state: no promisor packs, no promisor remote - if ls .git/objects/pack/*.promisor >/dev/null 2>&1; then - echo "$ws: unexpected promisor pack" - exit 1 - fi - if git config --local --get remote.origin.promisor >/dev/null \ - || git config --local --get remote.origin.partialclonefilter >/dev/null; then - echo "$ws: unexpected partial clone configuration" - exit 1 - fi - - # Only the objects the mirror lacked may live in the workspace - in_pack=$(git count-objects -v | awk '/^in-pack:/ {print $2}') - loose=$(git count-objects -v | awk '/^count:/ {print $2}') - echo "$ws: $in_pack packed + $loose loose local objects" - if [ "$((in_pack + loose))" -gt 1000 ]; then - echo "$ws: mirror history was copied into the workspace" - exit 1 - fi - - git fsck --connectivity-only --no-dangling - cd .. - done - - test -f filter-checkout/action.yml + __test__/verify-mirror-no-filter.sh sparse-checkout test -f sparse-checkout/.github/workflows/test-blacksmith.yml if [ -f sparse-checkout/action.yml ]; then echo "sparse-checkout: cone pattern was not applied" exit 1 fi - echo "Filtered checkouts verified" diff --git a/__test__/verify-mirror-no-filter.sh b/__test__/verify-mirror-no-filter.sh new file mode 100755 index 0000000..4413673 --- /dev/null +++ b/__test__/verify-mirror-no-filter.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Verify that a checkout requesting an object filter, but backed by the +# git mirror, is a regular clone that only stores the objects the mirror +# lacked. Usage: verify-mirror-no-filter.sh +set -euo pipefail + +ws="$1" +cd "$ws" + +# No partial-clone state: no promisor packs, no promisor remote +if ls .git/objects/pack/*.promisor >/dev/null 2>&1; then + echo "$ws: unexpected promisor pack" + exit 1 +fi +if git config --local --get remote.origin.promisor >/dev/null \ + || git config --local --get remote.origin.partialclonefilter >/dev/null; then + echo "$ws: unexpected partial clone configuration" + exit 1 +fi + +# Only the objects the mirror lacked may live in the workspace +in_pack=$(git count-objects -v | awk '/^in-pack:/ {print $2}') +loose=$(git count-objects -v | awk '/^count:/ {print $2}') +echo "$ws: $in_pack packed + $loose loose local objects" +if [ "$((in_pack + loose))" -gt 1000 ]; then + echo "$ws: mirror history was copied into the workspace" + exit 1 +fi + +git fsck --connectivity-only --no-dangling +echo "$ws: verified" From 70484dfbe4d66a6bf5d477c6d0cdcd004db4cd9e Mon Sep 17 00:00:00 2001 From: piotr Date: Sun, 6 Sep 2026 13:30:18 +0000 Subject: [PATCH 3/3] test-blacksmith: cone mode keeps root files; assert on an unselected directory Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-blacksmith.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-blacksmith.yml b/.github/workflows/test-blacksmith.yml index 00ef67e..c4addc4 100644 --- a/.github/workflows/test-blacksmith.yml +++ b/.github/workflows/test-blacksmith.yml @@ -142,7 +142,7 @@ jobs: run: | __test__/verify-mirror-no-filter.sh sparse-checkout test -f sparse-checkout/.github/workflows/test-blacksmith.yml - if [ -f sparse-checkout/action.yml ]; then + if [ -d sparse-checkout/src ]; then echo "sparse-checkout: cone pattern was not applied" exit 1 fi