[build-tools] Load local composite function catalog at build runtime

CHANGELOG.md

@@ -9,6 +9,7 @@ This is the log of notable changes to EAS CLI and related packages.
 ### 🎉 New features
 
 - [build-tools] Add `ref` input to the `eas/checkout` step to check out a different git ref (branch, tag, or commit SHA) than the one that triggered the job. ([#4035](https://github.com/expo/eas-cli/pull/4035) by [@sswrk](https://github.com/sswrk))
+- [build-tools] Load local composite function catalogs at build time so custom builds and generic jobs can resolve local composite functions referenced via `uses:`. ([#3930](https://github.com/expo/eas-cli/pull/3930) by [@sswrk](https://github.com/sswrk))
 
 ### 🐛 Bug fixes
 

packages/build-tools/src/__tests__/generic.test.ts

@@ -52,6 +52,7 @@ describe(runGenericJobAsync, () => {
         child: jest.fn().mockReturnThis(),
       },
       runBuildPhase: jest.fn(async (_phase: BuildPhase, fn: () => Promise<any>) => fn()),
+      getReactNativeProjectDirectory: jest.fn(() => '/tmp/src'),
     };
 
     mockUploadJobOutputsToWwwAsync.mockResolvedValue(undefined);

packages/build-tools/src/builders/__tests__/custom.test.ts

@@ -163,4 +163,71 @@ describe(runCustomBuildAsync, () => {
     drainSpy.mockRestore();
     executeSpy.mockRestore();
   });
+
+  describe('with inline job steps (StepsConfigParser path)', () => {
+    let executeSpy: jest.SpyInstance;
+
+    function createStepsCtx(steps: unknown[]): BuildContext<BuildJob> {
+      const job = createTestIosJob();
+      return new BuildContext(
+        {
+          ...job,
+          steps,
+        } as unknown as BuildJob,
+        {
+          workingdir: '/workingdir',
+          logBuffer: { getLogs: () => [], getPhaseLogs: () => [] },
+          logger: createMockLogger(),
+          env: {
+            __API_SERVER_URL: 'http://api.expo.test',
+          },
+          uploadArtifact: jest.fn(),
+        }
+      );
+    }
+
+    beforeEach(() => {
+      executeSpy = jest.spyOn(BuildWorkflow.prototype, 'executeAsync').mockResolvedValue(undefined);
+    });
+
+    afterEach(() => {
+      executeSpy.mockRestore();
+    });
+
+    it('builds the composite function catalog and resolves a local composite function referenced by a step', async () => {
+      jest.mocked(prepareProjectSourcesAsync).mockImplementation(async () => {
+        vol.mkdirSync('/workingdir/env', { recursive: true });
+        vol.fromJSON(
+          {
+            '.eas/functions/hello/function.yml': `
+            name: Hello
+            runs:
+              steps:
+                - run: echo "hello from composite function"
+            `,
+          },
+          '/workingdir/temporary-custom-build'
+        );
+        return { handled: true };
+      });
+
+      const stepsCtx = createStepsCtx([{ uses: './.eas/functions/hello', id: 'hello' }]);
+
+      await expect(runCustomBuildAsync(stepsCtx)).resolves.toBeDefined();
+      expect(executeSpy).toHaveBeenCalledTimes(1);
+    });
+
+    it('fails to parse when a referenced local composite function does not exist', async () => {
+      jest.mocked(prepareProjectSourcesAsync).mockImplementation(async () => {
+        vol.mkdirSync('/workingdir/env', { recursive: true });
+        vol.mkdirSync('/workingdir/temporary-custom-build', { recursive: true });
+        return { handled: true };
+      });
+
+      const stepsCtx = createStepsCtx([{ uses: './.eas/functions/missing', id: 'missing' }]);
+
+      await expect(runCustomBuildAsync(stepsCtx)).rejects.toThrow();
+      expect(executeSpy).not.toHaveBeenCalled();
+    });
+  });
 });

packages/build-tools/src/builders/custom.ts

@@ -17,6 +17,7 @@ import { Artifacts, BuildContext } from '../context';
 import { CustomBuildContext } from '../customBuildContext';
 import { Datadog } from '../datadog';
 import { findAndUploadXcodeBuildLogsAsync } from '../ios/xcodeBuildLogs';
+import { buildCompositeFunctionCatalogAsync } from '../steps/compositeFunctions';
 import { getEasFunctionGroups } from '../steps/easFunctionGroups';
 import { getEasFunctions } from '../steps/easFunctions';
 import { retryAsync } from '../utils/retry';
@@ -58,26 +59,30 @@ export async function runCustomBuildAsync(ctx: BuildContext<BuildJob>): Promise<
   const globalContext = new BuildStepGlobalContext(customBuildCtx, false);
   const easFunctions = getEasFunctions(customBuildCtx);
   const easFunctionGroups = getEasFunctionGroups(customBuildCtx);
-  const parser = ctx.job.steps
-    ? new StepsConfigParser(globalContext, {
-        externalFunctions: easFunctions,
-        externalFunctionGroups: easFunctionGroups,
-        steps: ctx.job.steps,
-        hooks: ctx.job.hooks,
-      })
-    : new BuildConfigParser(globalContext, {
-        externalFunctions: easFunctions,
-        externalFunctionGroups: easFunctionGroups,
-        configPath: path.join(
-          ctx.getReactNativeProjectDirectory(customBuildCtx.projectSourceDirectory),
-          nullthrows(
-            ctx.job.customBuildConfig?.path,
-            'Steps or custom build config path are required in custom jobs'
-          )
-        ),
-      });
   const workflow = await ctx.runBuildPhase(BuildPhase.PARSE_CUSTOM_WORKFLOW_CONFIG, async () => {
     try {
+      const parser = ctx.job.steps
+        ? new StepsConfigParser(globalContext, {
+            externalFunctions: easFunctions,
+            externalFunctionGroups: easFunctionGroups,
+            steps: ctx.job.steps,
+            hooks: ctx.job.hooks,
+            compositeFunctionCatalog: await buildCompositeFunctionCatalogAsync(
+              ctx.getReactNativeProjectDirectory(customBuildCtx.projectSourceDirectory),
+              { steps: ctx.job.steps, logger: ctx.logger }
+            ),
+          })
+        : new BuildConfigParser(globalContext, {
+            externalFunctions: easFunctions,
+            externalFunctionGroups: easFunctionGroups,
+            configPath: path.join(
+              ctx.getReactNativeProjectDirectory(customBuildCtx.projectSourceDirectory),
+              nullthrows(
+                ctx.job.customBuildConfig?.path,
+                'Steps or custom build config path are required in custom jobs'
+              )
+            ),
+          });
       return await parser.parseAsync();
     } catch (parseError: any) {
       ctx.logger.error('Failed to parse the custom build config file.');

packages/build-tools/src/generic.ts

@@ -7,6 +7,7 @@ import nullthrows from 'nullthrows';
 import { prepareProjectSourcesAsync } from './common/projectSources';
 import { BuildContext } from './context';
 import { CustomBuildContext } from './customBuildContext';
+import { buildCompositeFunctionCatalogAsync } from './steps/compositeFunctions';
 import { getEasFunctionGroups } from './steps/easFunctionGroups';
 import { getEasFunctions } from './steps/easFunctions';
 import { uploadJobOutputsToWwwAsync } from './utils/outputs';
@@ -43,15 +44,21 @@ export async function runGenericJobAsync(
 
   const globalContext = new BuildStepGlobalContext(customBuildCtx, false);
 
-  const parser = new StepsConfigParser(globalContext, {
-    externalFunctions: getEasFunctions(customBuildCtx),
-    externalFunctionGroups: getEasFunctionGroups(customBuildCtx),
-    steps: ctx.job.steps,
-    hooks: ctx.job.hooks,
-  });
-
   const workflow = await ctx.runBuildPhase(BuildPhase.PARSE_CUSTOM_WORKFLOW_CONFIG, async () => {
     try {
+      const compositeFunctionCatalog = await buildCompositeFunctionCatalogAsync(
+        ctx.getReactNativeProjectDirectory(customBuildCtx.projectSourceDirectory),
+        { steps: ctx.job.steps, logger: ctx.logger }
+      );
+
+      const parser = new StepsConfigParser(globalContext, {
+        externalFunctions: getEasFunctions(customBuildCtx),
+        externalFunctionGroups: getEasFunctionGroups(customBuildCtx),
+        steps: ctx.job.steps,
+        hooks: ctx.job.hooks,
+        compositeFunctionCatalog,
+      });
+
       return await parser.parseAsync();
     } catch (parseError: any) {
       ctx.logger.error('Failed to parse the job definition file.');

packages/build-tools/src/steps/__tests__/compositeFunctions.test.ts

@@ -0,0 +1,173 @@
+import { promises as fs } from 'fs';
+import os from 'os';
+import path from 'path';
+
+import { buildCompositeFunctionCatalogAsync } from '../compositeFunctions';
+
+async function makeProjectWithCompositeFunctionAsync(
+  functionName: string,
+  contents: string
+): Promise<string> {
+  const projectRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-functions-test-'));
+  const functionDir = path.join(projectRoot, '.eas', 'functions', functionName);
+  await fs.mkdir(functionDir, { recursive: true });
+  await fs.writeFile(path.join(functionDir, 'function.yml'), contents, 'utf-8');
+  return projectRoot;
+}
+
+const setupCompositeFunctionContents = [
+  'name: Setup',
+  'inputs:',
+  '  - name: greeting',
+  '    type: string',
+  '    default_value: hello',
+  'outputs:',
+  '  version:',
+  '    value: ${{ steps.read.outputs.version }}',
+  'runs:',
+  '  steps:',
+  '    - id: read',
+  '      run: set-output version "1.0.0"',
+].join('\n');
+
+describe(buildCompositeFunctionCatalogAsync, () => {
+  it('discovers, reads and validates referenced local composite functions keyed by normalized ref', async () => {
+    const projectRoot = await makeProjectWithCompositeFunctionAsync(
+      'setup',
+      setupCompositeFunctionContents
+    );
+
+    const catalog = await buildCompositeFunctionCatalogAsync(projectRoot, {
+      steps: [{ uses: './.eas/functions/setup', id: 'setup' }],
+    });
+
+    expect(Object.keys(catalog)).toEqual(['./.eas/functions/setup']);
+    const action = catalog['./.eas/functions/setup'];
+    expect(action.name).toBe('Setup');
+    expect(action.runs.steps).toHaveLength(1);
+    expect(action.outputs?.version.value).toBe('${{ steps.read.outputs.version }}');
+  });
+
+  it('loads nested composite functions transitively referenced by other composite functions', async () => {
+    const projectRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-functions-nested-'));
+    const innerDir = path.join(projectRoot, '.eas', 'functions', 'inner');
+    const outerDir = path.join(projectRoot, '.eas', 'functions', 'outer');
+    await fs.mkdir(innerDir, { recursive: true });
+    await fs.mkdir(outerDir, { recursive: true });
+    await fs.writeFile(
+      path.join(innerDir, 'function.yml'),
+      ['runs:', '  steps:', '    - run: echo inner'].join('\n'),
+      'utf-8'
+    );
+    await fs.writeFile(
+      path.join(outerDir, 'function.yml'),
+      ['runs:', '  steps:', '    - uses: ./.eas/functions/inner'].join('\n'),
+      'utf-8'
+    );
+
+    const catalog = await buildCompositeFunctionCatalogAsync(projectRoot, {
+      steps: [{ uses: './.eas/functions/outer', id: 'outer' }],
+    });
+
+    expect(Object.keys(catalog).sort()).toEqual([
+      './.eas/functions/inner',
+      './.eas/functions/outer',
+    ]);
+  });
+
+  it('returns an empty catalog when there are no referenced composite functions', async () => {
+    const projectRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-functions-empty-'));
+    const catalog = await buildCompositeFunctionCatalogAsync(projectRoot, {
+      steps: [{ run: 'echo hi' }],
+    });
+    expect(catalog).toEqual({});
+  });
+
+  it('ignores unreferenced malformed composite functions on disk', async () => {
+    const projectRoot = await makeProjectWithCompositeFunctionAsync(
+      'broken',
+      ['name: Broken', 'runs:', '  steps: []'].join('\n')
+    );
+
+    const catalog = await buildCompositeFunctionCatalogAsync(projectRoot, {
+      steps: [{ run: 'echo hi' }],
+    });
+    expect(catalog).toEqual({});
+  });
+
+  it('resolves composite functions referenced by an arbitrary path (arbitrary path style)', async () => {
+    const projectRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-functions-arbitrary-'));
+    const functionDir = path.join(projectRoot, 'internal-functions', 'deploy');
+    await fs.mkdir(functionDir, { recursive: true });
+    await fs.writeFile(
+      path.join(functionDir, 'function.yml'),
+      ['runs:', '  steps:', '    - run: echo deploy'].join('\n'),
+      'utf-8'
+    );
+
+    const catalog = await buildCompositeFunctionCatalogAsync(projectRoot, {
+      steps: [{ uses: './internal-functions/deploy', id: 'deploy' }],
+    });
+
+    expect(Object.keys(catalog)).toEqual(['./internal-functions/deploy']);
+  });
+
+  it('resolves composite functions defined with a function.yaml extension', async () => {
+    const projectRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-functions-yaml-'));
+    const functionDir = path.join(projectRoot, '.eas', 'functions', 'setup');
+    await fs.mkdir(functionDir, { recursive: true });
+    await fs.writeFile(
+      path.join(functionDir, 'function.yaml'),
+      ['runs:', '  steps:', '    - run: echo hi'].join('\n'),
+      'utf-8'
+    );
+
+    const catalog = await buildCompositeFunctionCatalogAsync(projectRoot, {
+      steps: [{ uses: './.eas/functions/setup', id: 'setup' }],
+    });
+
+    expect(Object.keys(catalog)).toEqual(['./.eas/functions/setup']);
+  });
+
+  it('loads a shared composite function above the EAS project root', async () => {
+    const sourceRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-functions-monorepo-'));
+    const projectRoot = path.join(sourceRoot, 'apps', 'my-app');
+    const functionDir = path.join(sourceRoot, 'shared', 'functions', 'setup');
+    await fs.mkdir(projectRoot, { recursive: true });
+    await fs.mkdir(functionDir, { recursive: true });
+    await fs.writeFile(
+      path.join(functionDir, 'function.yml'),
+      ['runs:', '  steps:', '    - run: echo shared'].join('\n'),
+      'utf-8'
+    );
+
+    const catalog = await buildCompositeFunctionCatalogAsync(projectRoot, {
+      steps: [{ uses: '../../shared/functions/setup', id: 'setup' }],
+    });
+
+    expect(Object.keys(catalog)).toEqual(['../../shared/functions/setup']);
+  });
+
+  it('throws a clear error for a referenced composite function that does not exist', async () => {
+    const projectRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'eas-functions-missing-'));
+    await expect(
+      buildCompositeFunctionCatalogAsync(projectRoot, {
+        steps: [{ uses: './.eas/functions/missing', id: 'missing' }],
+      })
+    ).rejects.toThrow(
+      /Local composite function "\.\/\.eas\/functions\/missing" was referenced by a step but no such composite function exists/
+    );
+  });
+
+  it('throws a clear error for a malformed referenced composite function config', async () => {
+    const projectRoot = await makeProjectWithCompositeFunctionAsync(
+      'broken',
+      ['name: Broken', 'runs:', '  steps: []'].join('\n')
+    );
+    await expect(
+      buildCompositeFunctionCatalogAsync(projectRoot, {
+        steps: [{ uses: './.eas/functions/broken', id: 'broken' }],
+      })
+    ).rejects.toThrow(/must declare at least one step under "runs.steps"/);
+  });
+});