diff --git a/.github/workflows/test-blacksmith.yml b/.github/workflows/test-blacksmith.yml index d6390e8..c4addc4 100644 --- a/.github/workflows/test-blacksmith.yml +++ b/.github/workflows/test-blacksmith.yml @@ -114,3 +114,35 @@ jobs: git fsck --no-dangling echo "Reused deep checkout verified" + + - name: Test fetch-depth 0 checkout with filter (mirror provides the objects) + uses: ./ + with: + path: filter-checkout + fetch-depth: 0 + filter: blob:none + + # Verified before the next step mounts its own copy of the sticky disk + # over the same path: objects that only the current mount's sync added + # are not visible through a later mount. + - name: Verify filtered deep checkout is a full clone backed by the mirror + run: | + __test__/verify-mirror-no-filter.sh filter-checkout + test -f filter-checkout/action.yml + + - name: Test sparse checkout (mirror provides the objects) + uses: ./ + with: + path: sparse-checkout + sparse-checkout: | + __test__ + .github + + - name: Verify sparse checkout is a full clone backed by the mirror + run: | + __test__/verify-mirror-no-filter.sh sparse-checkout + test -f sparse-checkout/.github/workflows/test-blacksmith.yml + if [ -d sparse-checkout/src ]; then + echo "sparse-checkout: cone pattern was not applied" + exit 1 + fi diff --git a/README.md b/README.md index 9e993ad..ee5f01b 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,8 @@ For shallow checkouts (`fetch-depth` greater than 0), the fetch tells GitHub whi The mirror always contains full history, but the workspace still respects inputs such as `fetch-depth`, `fetch-tags`, sparse checkout, LFS, and submodules. For example, `fetch-depth: 1` still produces a shallow workspace checkout. +The one input that is not passed through is `filter` (and the `blob:none` filter that sparse checkout implies). Every object a filter would omit is already available through the mirror, so the workspace is fetched without a filter and is a regular, not partial, clone. Honouring the filter would make the fetch far slower, not faster: on Git 2.48 and later, indexing a filtered pack repacks every mirror object reachable from the fetched commits into the workspace. When the action falls back to a standard checkout, `filter` behaves as in `actions/checkout`. + ### 3. Post-job refresh The action refreshes an existing mirror with `git fetch --prune` during post-job cleanup, outside the critical checkout path. This prepares the mirror for subsequent workflow runs without delaying the checkout step itself. diff --git a/__test__/mirror-filter-fetch-git.test.ts b/__test__/mirror-filter-fetch-git.test.ts new file mode 100644 index 0000000..01fd177 --- /dev/null +++ b/__test__/mirror-filter-fetch-git.test.ts @@ -0,0 +1,181 @@ +/** + * Real-git coverage for fetching into a workspace that shares the mirror's + * objects through alternates when the caller asked for an object filter: the + * unfiltered fetch the action issues instead only transfers the delta and + * leaves the workspace a plain (non-partial) clone, whereas honouring the + * filter would repack mirror objects into a local promisor pack. + */ +// Mock the gRPC dependencies before importing blacksmith-cache +jest.mock('@connectrpc/connect', () => ({ + createClient: jest.fn(), + ConnectError: class ConnectError extends Error {}, + Code: {Aborted: 'ABORTED'} +})) + +jest.mock('@connectrpc/connect-node', () => ({ + createGrpcTransport: jest.fn() +})) + +jest.mock( + '@buf/blacksmith_vm-agent.connectrpc_es/stickydisk/v1/stickydisk_connect', + () => ({ + StickyDiskService: {} + }) +) + +jest.mock('../src/container-detector', () => ({ + isRunningInContainer: jest.fn(() => false) +})) + +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import {execFileSync} from 'child_process' +import * as blacksmithCache from '../src/blacksmith-cache' +import * as gitCommandManager from '../src/git-command-manager' +import {GitVersion} from '../src/git-version' + +function git(cwd: string, ...args: string[]): string { + return execFileSync('git', ['-C', cwd, ...args], {encoding: 'utf8'}).trim() +} + +function commit(repo: string, msg: string): string { + fs.writeFileSync(path.join(repo, `${msg}.txt`), msg) + git(repo, 'add', `${msg}.txt`) + git( + repo, + '-c', + 'user.email=test@example.com', + '-c', + 'user.name=Test', + 'commit', + '-q', + '-m', + msg + ) + return git(repo, 'rev-parse', 'HEAD') +} + +function objectCount(workspace: string): number { + const counts = git(workspace, 'count-objects', '-v') + return ( + Number(/^count: (\d+)/m.exec(counts)?.[1]) + + Number(/^in-pack: (\d+)/m.exec(counts)?.[1]) + ) +} + +function promisorPacks(workspace: string): string[] { + const packDir = path.join(workspace, '.git', 'objects', 'pack') + if (!fs.existsSync(packDir)) { + return [] + } + return fs.readdirSync(packDir).filter(name => name.endsWith('.promisor')) +} + +/** git 2.48 started repacking locally available objects linked from a promisor pack. */ +const RepacksLocalLinksVersion = new GitVersion('2.48') + +describe('object filters with mirror alternates and real git', () => { + let tmpDir: string + let sourceRepo: string + let mirrorPath: string + let mirrorCommitCount: number + let sourceMain: string + + const initWorkspace = (): string => { + const workspace = fs.mkdtempSync(path.join(tmpDir, 'ws-')) + git(workspace, 'init', '-q', '.') + git(workspace, 'remote', 'add', 'origin', sourceRepo) + const infoDir = path.join(workspace, '.git', 'objects', 'info') + fs.mkdirSync(infoDir, {recursive: true}) + fs.writeFileSync( + path.join(infoDir, 'alternates'), + `${mirrorPath}/objects\n` + ) + return workspace + } + + const fetchMain = async ( + workspace: string, + options: gitCommandManager.FetchOptions + ): Promise => { + const git = await gitCommandManager.createCommandManager( + workspace, + false, + false + ) + await git.fetch(['+refs/heads/main:refs/remotes/origin/main'], options) + return git.version() + } + + beforeEach(() => { + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'mirror-filter-test-')) + sourceRepo = path.join(tmpDir, 'source') + mirrorPath = path.join(tmpDir, 'mirror') + + fs.mkdirSync(sourceRepo) + git(sourceRepo, 'init', '-q', '-b', 'main', '.') + git(sourceRepo, 'config', 'uploadpack.allowFilter', 'true') + for (let i = 0; i < 20; i++) { + commit(sourceRepo, `history-${i}`) + } + execFileSync('git', ['clone', '-q', '--mirror', sourceRepo, mirrorPath]) + mirrorCommitCount = Number(git(mirrorPath, 'rev-list', '--count', 'HEAD')) + + // The mirror now lags the source by two commits on main. + commit(sourceRepo, 'new-1') + sourceMain = commit(sourceRepo, 'new-2') + }) + + afterEach(() => { + fs.rmSync(tmpDir, {recursive: true, force: true}) + }) + + it('shares the mirror object store', async () => { + await expect( + blacksmithCache.sharesMirrorObjects(initWorkspace(), mirrorPath) + ).resolves.toBe(true) + }) + + it.each([{fetchDepth: undefined}, {fetchDepth: 2}])( + 'an unfiltered fetch (%o) only stores the delta and stays a full clone', + async ({fetchDepth}) => { + const workspace = initWorkspace() + await fetchMain(workspace, fetchDepth ? {fetchDepth} : {}) + + expect(git(workspace, 'rev-parse', 'refs/remotes/origin/main')).toBe( + sourceMain + ) + // 2 commits + 2 trees + 2 blobs + expect(objectCount(workspace)).toBeLessThanOrEqual(6) + expect(promisorPacks(workspace)).toEqual([]) + expect( + fs.readFileSync(path.join(workspace, '.git', 'config'), 'utf8') + ).not.toMatch(/promisor|partialclonefilter/i) + + // Old and new blobs alike are readable and the tree checks out whole. + expect( + git(workspace, 'cat-file', '-t', `${sourceMain}:history-0.txt`) + ).toBe('blob') + git(workspace, 'checkout', '-q', '--detach', 'refs/remotes/origin/main') + expect( + fs.readdirSync(workspace).filter(f => f.endsWith('.txt')) + ).toHaveLength(22) + git(workspace, 'fsck', '--connectivity-only', '--no-dangling') + } + ) + + it('a blob:none fetch would repack mirror objects into a promisor pack', async () => { + const workspace = initWorkspace() + const version = await fetchMain(workspace, {filter: 'blob:none'}) + + expect(git(workspace, 'rev-parse', 'refs/remotes/origin/main')).toBe( + sourceMain + ) + expect(promisorPacks(workspace)).not.toEqual([]) + if (version.checkMinimum(RepacksLocalLinksVersion)) { + // The mirror's whole history behind the fetched commits was copied. + expect(objectCount(workspace)).toBeGreaterThan(mirrorCommitCount) + } + }) +}) diff --git a/__test__/verify-mirror-no-filter.sh b/__test__/verify-mirror-no-filter.sh new file mode 100755 index 0000000..4413673 --- /dev/null +++ b/__test__/verify-mirror-no-filter.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Verify that a checkout requesting an object filter, but backed by the +# git mirror, is a regular clone that only stores the objects the mirror +# lacked. Usage: verify-mirror-no-filter.sh +set -euo pipefail + +ws="$1" +cd "$ws" + +# No partial-clone state: no promisor packs, no promisor remote +if ls .git/objects/pack/*.promisor >/dev/null 2>&1; then + echo "$ws: unexpected promisor pack" + exit 1 +fi +if git config --local --get remote.origin.promisor >/dev/null \ + || git config --local --get remote.origin.partialclonefilter >/dev/null; then + echo "$ws: unexpected partial clone configuration" + exit 1 +fi + +# Only the objects the mirror lacked may live in the workspace +in_pack=$(git count-objects -v | awk '/^in-pack:/ {print $2}') +loose=$(git count-objects -v | awk '/^count:/ {print $2}') +echo "$ws: $in_pack packed + $loose loose local objects" +if [ "$((in_pack + loose))" -gt 1000 ]; then + echo "$ws: mirror history was copied into the workspace" + exit 1 +fi + +git fsck --connectivity-only --no-dangling +echo "$ws: verified" diff --git a/action.yml b/action.yml index 4c4863e..d29527f 100644 --- a/action.yml +++ b/action.yml @@ -61,6 +61,8 @@ inputs: description: > Partially clone against a given filter. Overrides sparse-checkout if set. + Ignored when the workspace shares a Blacksmith git mirror, which already + holds every object the filter would omit. default: null sparse-checkout: description: > diff --git a/dist/index.js b/dist/index.js index 2b950bf..7693bc6 100644 --- a/dist/index.js +++ b/dist/index.js @@ -3304,11 +3304,22 @@ function getSource(settings) { // Fetch core.startGroup('Fetching the repository'); const fetchOptions = {}; - if (settings.filter) { - fetchOptions.filter = settings.filter; + const sharesMirrorObjects = !!cacheInfo && + (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath)); + const filter = settings.filter + ? settings.filter + : settings.sparseCheckout + ? 'blob:none' + : undefined; + if (filter && sharesMirrorObjects) { + // The mirror alternate already holds every object the filter would omit, + // and a filtered fetch makes the workspace a partial clone whose + // index-pack (git >= 2.48) repacks all mirror objects reachable from the + // fetched commits into a local promisor pack. + core.info(`[git-mirror] Fetching without --filter=${filter}: the mirror already provides every object`); } - else if (settings.sparseCheckout) { - fetchOptions.filter = 'blob:none'; + else if (filter) { + fetchOptions.filter = filter; } if (settings.fetchDepth <= 0) { // When the Blacksmith mirror is available and synced with the remote, @@ -3367,8 +3378,8 @@ function getSource(settings) { // tips explicitly instead, so the server still sends only the delta // from the mirror (see resolveShallowNegotiationTips). if (cacheInfo && - (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion) && - (yield blacksmithCache.sharesMirrorObjects(settings.repositoryPath, cacheInfo.mirrorPath))) { + sharesMirrorObjects && + (yield git.version()).checkMinimum(git_command_manager_1.MinimumGitAlternateRefsCommandVersion)) { fetchOptions.ignoreAlternateRefs = true; fetchOptions.negotiationTips = yield blacksmithCache.resolveShallowNegotiationTips(cacheInfo.mirrorPath, settings.ref, process.env['GITHUB_BASE_REF'] || ''); diff --git a/src/git-source-provider.ts b/src/git-source-provider.ts index 9e756b4..63f5074 100644 --- a/src/git-source-provider.ts +++ b/src/git-source-provider.ts @@ -252,10 +252,28 @@ export async function getSource(settings: IGitSourceSettings): Promise { core.startGroup('Fetching the repository') const fetchOptions: FetchOptions = {} - if (settings.filter) { - fetchOptions.filter = settings.filter - } else if (settings.sparseCheckout) { - fetchOptions.filter = 'blob:none' + const sharesMirrorObjects = + !!cacheInfo && + (await blacksmithCache.sharesMirrorObjects( + settings.repositoryPath, + cacheInfo.mirrorPath + )) + + const filter = settings.filter + ? settings.filter + : settings.sparseCheckout + ? 'blob:none' + : undefined + if (filter && sharesMirrorObjects) { + // The mirror alternate already holds every object the filter would omit, + // and a filtered fetch makes the workspace a partial clone whose + // index-pack (git >= 2.48) repacks all mirror objects reachable from the + // fetched commits into a local promisor pack. + core.info( + `[git-mirror] Fetching without --filter=${filter}: the mirror already provides every object` + ) + } else if (filter) { + fetchOptions.filter = filter } if (settings.fetchDepth <= 0) { @@ -327,13 +345,10 @@ export async function getSource(settings: IGitSourceSettings): Promise { // from the mirror (see resolveShallowNegotiationTips). if ( cacheInfo && + sharesMirrorObjects && (await git.version()).checkMinimum( MinimumGitAlternateRefsCommandVersion - ) && - (await blacksmithCache.sharesMirrorObjects( - settings.repositoryPath, - cacheInfo.mirrorPath - )) + ) ) { fetchOptions.ignoreAlternateRefs = true fetchOptions.negotiationTips =