Add live setup progress indicators

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cloudflare-github-actions-runner",
-  "version": "1.0.8",
+  "version": "1.0.9",
   "description": "Run GitHub Actions jobs as disposable Cloudflare Containers.",
   "keywords": [
     "ci",

scripts/setup.ts

@@ -52,6 +52,19 @@ function colorStatus(color, status) {
   return `${terminalColor[color]}${status}${terminalColor.reset}`;
 }
 
+export function formatSetupStepDuration(durationMs) {
+  const roundedDurationMs = Math.max(0, Math.round(durationMs));
+  if (roundedDurationMs < 1_000) {
+    return `${roundedDurationMs}ms`;
+  }
+  if (roundedDurationMs < 60_000) {
+    return `${(roundedDurationMs / 1_000).toFixed(1)}s`;
+  }
+  const minutes = Math.floor(roundedDurationMs / 60_000);
+  const seconds = Math.floor((roundedDurationMs % 60_000) / 1_000);
+  return `${minutes}m ${seconds}s`;
+}
+
 async function withSpinner(checkingStatus, completeStatus, operation) {
   let frame = 0;
   let timer;
@@ -81,10 +94,21 @@ async function withSpinner(checkingStatus, completeStatus, operation) {
       render();
     }
   };
+  const runStep = async (status, completedStatus, step) => {
+    updateStatus(status);
+    const startedAt = Date.now();
+    const result = await step();
+    pause();
+    process.stdout.write(
+      `${colorStatus("green", `✔ ${completedStatus} (${formatSetupStepDuration(Date.now() - startedAt)})`)}\n`,
+    );
+    resume();
+    return result;
+  };
 
   resume();
   try {
-    const result = await operation({ pause, resume, updateStatus });
+    const result = await operation({ pause, resume, runStep, updateStatus });
     pause();
     const status = completeStatus instanceof Function ? completeStatus(result) : { message: completeStatus };
     process.stdout.write(`${colorStatus(status.color ?? "green", `${status.marker ?? "✔"} ${status.message}`)}\n`);
@@ -330,6 +354,24 @@ export function hasValidRunnerSetupTokenStatus(status) {
   );
 }
 
+/** Non-sensitive credential check results shown before setup changes any stored credentials. */
+export function existingWorkerTokenStatusMessages(status) {
+  const cloudflareTokenValid =
+    status.cloudflareContainersToken && status.cloudflareRegistryPush && status.cloudflareResourceTagging;
+  const githubAppValid = status.githubApp && status.githubAppWebhookSecret;
+  return [
+    `${cloudflareTokenValid ? "✔" : "✘"} Cloudflare Containers Write + Tag Read/Write token: ${cloudflareTokenValid ? "valid (reusing)" : "needs attention"}`,
+    `${githubAppValid ? "✔" : "✘"} GitHub App credentials: ${githubAppValid ? "valid (reusing)" : "unavailable or rejected"}`,
+    `${status.resourceTraceSigningKey ? "✔" : "✘"} Runner resource-trace signing key: ${status.resourceTraceSigningKey ? "present (reusing)" : "missing"}`,
+    `${status.runnerCacheSigningKey ? "✔" : "✘"} Runner R2-cache signing key: ${status.runnerCacheSigningKey ? "present (reusing)" : "missing"}`,
+  ];
+}
+
+/** Keep discoverable App credentials until the user explicitly chooses a replacement. */
+export function shouldCreateInitialGitHubApp(existingTokens, existingGitHubAppConfiguration) {
+  return !(existingTokens.githubApp && existingTokens.githubAppWebhookSecret) && !existingGitHubAppConfiguration;
+}
+
 export function githubAppManifest(name, workerBaseUrl, redirectUrl) {
   return {
     name,
@@ -1394,6 +1436,8 @@ export function remoteRunnerImageBuildProgressMessage(status) {
   const phase = status?.progress?.phase;
   const phases = {
     queued: "Waiting for Cloudflare to schedule the image build",
+    "bootstrapping-builder": "Bootstrapping Cloudflare's private daemonless image builder",
+    "rolling-out-builder": "Rolling Cloudflare's private daemonless image builder to its private image",
     "downloading-source": "Downloading the runner-image source from GitHub",
     "starting-builder": "Starting Cloudflare's isolated daemonless image builder",
     "preparing-build-context": "Preparing the runner image build context",
@@ -1403,7 +1447,19 @@ export function remoteRunnerImageBuildProgressMessage(status) {
   };
   const parsedPhase = z.enum(Object.keys(phases)).safeParse(phase);
   if (parsedPhase.success) {
-    return phases[parsedPhase.data];
+    const message = phases[parsedPhase.data];
+    if (parsedPhase.data !== "rolling-out") {
+      return message;
+    }
+    const rollout = z
+      .object({
+        processedApplications: z.number().int().nonnegative(),
+        totalApplications: z.number().int().nonnegative(),
+      })
+      .safeParse(status?.progress?.rollout);
+    return rollout.success
+      ? `${message} (${rollout.data.processedApplications}/${rollout.data.totalApplications} profiles checked)`
+      : message;
   }
   if (status?.status === "queued") {
     return phases.queued;
@@ -1793,51 +1849,79 @@ export async function main() {
     });
 
   const setupValidationToken = generateWebhookSecret();
+  const existingGitHubAppConfiguration = cloudflareAccount.runnerPool.githubAppConfigured;
   const existingTokens = await retryWorkerValidationAuthorization(() =>
-    withSpinner("Checking existing Worker token configuration", "Worker token configuration: checked", async () => {
-      await putWorkerSecret("CLOUDFLARE_ACCOUNT_ID", cloudflareAccount.account.id, cloudflareEnvironment);
-      await putWorkerSecret("RUNNER_SETUP_VALIDATION_TOKEN", setupValidationToken, cloudflareEnvironment);
-      return retryWorkerTokenValidation(() => validateExistingWorkerTokens(workerBaseUrl, setupValidationToken));
-    }),
+    withSpinner(
+      "Checking existing Worker token configuration",
+      "Worker token configuration: checked",
+      async ({ runStep }) => {
+        await runStep(
+          "Saving the selected Cloudflare account for the Worker",
+          "Selected Cloudflare account saved for the Worker",
+          () => putWorkerSecret("CLOUDFLARE_ACCOUNT_ID", cloudflareAccount.account.id, cloudflareEnvironment),
+        );
+        await runStep(
+          "Authorizing this setup session with the Worker",
+          "Setup session authorized with the Worker",
+          () => putWorkerSecret("RUNNER_SETUP_VALIDATION_TOKEN", setupValidationToken, cloudflareEnvironment),
+        );
+        return runStep(
+          "Validating the Worker's existing Cloudflare and GitHub App credentials",
+          "Existing Worker credentials validated",
+          () =>
+            retryWorkerTokenValidation(() => validateExistingWorkerTokens(workerBaseUrl, setupValidationToken), {
+              // An existing App can take a short time to become visible after the
+              // deployment that introduced this setup session. Do not create a
+              // duplicate App merely because GitHub has not accepted its JWT yet.
+              isValid: existingGitHubAppConfiguration
+                ? (status) => status.githubApp && status.githubAppWebhookSecret
+                : () => true,
+            }),
+        );
+      },
+    ),
   );
   if (existingTokens === undefined) {
     await deleteWorkerSecret("RUNNER_SETUP_VALIDATION_TOKEN", cloudflareEnvironment);
     console.log("Setup stopped. The temporary setup credential was removed.");
     return;
   }
+  console.log("\nExisting Worker credential status:");
+  for (const message of existingWorkerTokenStatusMessages(existingTokens)) {
+    console.log(`  ${message}`);
+  }
 
   let discardUninstalledInitialGitHubAppCredentials = false;
   try {
     let cloudflareToken;
-    if (
+    const existingCloudflareTokenIsValid =
       existingTokens.cloudflareContainersToken &&
       existingTokens.cloudflareRegistryPush &&
-      existingTokens.cloudflareResourceTagging
-    ) {
-      console.log("  \u2714 Reusing the valid account-owned Cloudflare Containers Write + Tag Read/Write token");
-    } else {
+      existingTokens.cloudflareResourceTagging;
+    if (!existingCloudflareTokenIsValid) {
       cloudflareToken = await promptForValidatedCloudflareToken(cloudflareAccount.account, { showTokenForm: true });
       await putWorkerSecret("CLOUDFLARE_CONTAINERS_API_TOKEN", cloudflareToken, cloudflareEnvironment);
     }
 
     let createdGitHubApp;
     let replacementGitHubAppIsPending = false;
-    if (existingTokens.githubApp && existingTokens.githubAppWebhookSecret) {
-      console.log("  \u2714 Reusing the valid GitHub App credentials");
-    } else {
-      createdGitHubApp = await createGitHubApp();
-      await storeGitHubAppCredentials(createdGitHubApp, cloudflareEnvironment);
-      discardUninstalledInitialGitHubAppCredentials = true;
+    const existingGitHubAppIsValid = existingTokens.githubApp && existingTokens.githubAppWebhookSecret;
+    if (!existingGitHubAppIsValid) {
+      if (!shouldCreateInitialGitHubApp(existingTokens, existingGitHubAppConfiguration)) {
+        console.log(
+          "  ! Found an existing GitHub App configuration, but GitHub did not validate it. Keeping it until you choose recovery.",
+        );
+      } else {
+        createdGitHubApp = await createGitHubApp();
+        await storeGitHubAppCredentials(createdGitHubApp, cloudflareEnvironment);
+        discardUninstalledInitialGitHubAppCredentials = true;
+      }
     }
 
-    if (existingTokens.resourceTraceSigningKey) {
-      console.log("  ✔ Reusing the runner resource-trace signing key");
-    } else {
+    if (!existingTokens.resourceTraceSigningKey) {
       await putWorkerSecret("RESOURCE_TRACE_SIGNING_KEY", generateResourceTraceSigningKey(), cloudflareEnvironment);
     }
-    if (existingTokens.runnerCacheSigningKey) {
-      console.log("  ✔ Reusing the runner R2-cache signing key");
-    } else {
+    if (!existingTokens.runnerCacheSigningKey) {
       await putWorkerSecret("RUNNER_CACHE_SIGNING_KEY", generateResourceTraceSigningKey(), cloudflareEnvironment);
     }
 

src/cloudflare-containers.ts

@@ -92,6 +92,13 @@ export interface RolloutRunnerApplicationImagesOptions {
    * recent PATCH reached its own rollout (for example A→B→A).
    */
   reissueMatchingImageRollouts?: boolean;
+  /** Reports how many runner applications this rollout pass has inspected. */
+  onProgress?: (progress: RunnerApplicationImageRolloutProgress) => Promise<void>;
+}
+
+export interface RunnerApplicationImageRolloutProgress {
+  processedApplications: number;
+  totalApplications: number;
 }
 
 export interface RolloutRunnerImageBuilderOptions {
@@ -450,60 +457,78 @@ export async function rolloutRunnerApplicationImages(
   );
   const updatedApplications: string[] = [];
   const skippedApplications: string[] = [];
-
-  for (const listedApplication of applications) {
-    // The scheduler can resize an idle custom application while an image build
-    // is finishing. Re-read its configuration immediately before patching so
-    // an image rollout never writes the stale resources from the initial list.
-    // eslint-disable-next-line no-await-in-loop -- obtain the current configuration for this exact application.
-    const application = await findRunnerApplication(env, listedApplication.name, dependencies);
-    // eslint-disable-next-line no-await-in-loop -- a configuration rollout must be observed before the next application.
-    const rollouts = await listRollouts(env, application.id, dependencies);
-    if (rollouts.some((rollout) => rollout.status === "pending" || rollout.status === "progressing")) {
-      skippedApplications.push(application.name);
-      continue;
+  const reportProgress = async (progress: RunnerApplicationImageRolloutProgress): Promise<void> => {
+    try {
+      await options.onProgress?.(progress);
+    } catch (error) {
+      // Progress is telemetry for the setup UI, not part of the Container API
+      // transaction. A transient Durable Object error must not repeat a
+      // successful rollout or abandon the build lease.
+      console.error("Cloudflare runner image rollout progress reporting failed", {
+        error: error instanceof Error ? error.message : String(error),
+      });
     }
-    if (
-      !options.reissueMatchingImageRollouts &&
-      application.configuration.image === image &&
-      rollouts.some((rollout) => rollout.status === "completed" && rollout.target_configuration.image === image)
-    ) {
-      continue;
-    }
-    if (applicationHasLiveInstances(application)) {
-      skippedApplications.push(application.name);
-      continue;
-    }
-    if (application.configuration.image !== image) {
-      // eslint-disable-next-line no-await-in-loop -- preserve capacity by finishing one image patch before the next.
-      await patchApplicationImage(env, application, image, dependencies);
-    }
-    // A scheduler admission may have resized this custom application while
-    // the image-only PATCH was in flight. Read its current resources again so
-    // the full rollout target cannot restore that stale machine shape.
-    // eslint-disable-next-line no-await-in-loop -- this exact application owns its next rollout body.
-    const current = await findRunnerApplication(env, application.name, dependencies);
-    if (applicationHasLiveInstances(current)) {
-      skippedApplications.push(application.name);
-      continue;
+  };
+  await reportProgress({ processedApplications: 0, totalApplications: applications.length });
+
+  for (const [index, listedApplication] of applications.entries()) {
+    try {
+      // The scheduler can resize an idle custom application while an image build
+      // is finishing. Re-read its configuration immediately before patching so
+      // an image rollout never writes the stale resources from the initial list.
+      // eslint-disable-next-line no-await-in-loop -- obtain the current configuration for this exact application.
+      const application = await findRunnerApplication(env, listedApplication.name, dependencies);
+      // eslint-disable-next-line no-await-in-loop -- a configuration rollout must be observed before the next application.
+      const rollouts = await listRollouts(env, application.id, dependencies);
+      if (rollouts.some((rollout) => rollout.status === "pending" || rollout.status === "progressing")) {
+        skippedApplications.push(application.name);
+        continue;
+      }
+      if (
+        !options.reissueMatchingImageRollouts &&
+        application.configuration.image === image &&
+        rollouts.some((rollout) => rollout.status === "completed" && rollout.target_configuration.image === image)
+      ) {
+        continue;
+      }
+      if (applicationHasLiveInstances(application)) {
+        skippedApplications.push(application.name);
+        continue;
+      }
+      if (application.configuration.image !== image) {
+        // eslint-disable-next-line no-await-in-loop -- preserve capacity by finishing one image patch before the next.
+        await patchApplicationImage(env, application, image, dependencies);
+      }
+      // A scheduler admission may have resized this custom application while
+      // the image-only PATCH was in flight. Read its current resources again so
+      // the full rollout target cannot restore that stale machine shape.
+      // eslint-disable-next-line no-await-in-loop -- this exact application owns its next rollout body.
+      const current = await findRunnerApplication(env, application.name, dependencies);
+      if (applicationHasLiveInstances(current)) {
+        skippedApplications.push(application.name);
+        continue;
+      }
+      // PATCH acceptance does not make the requested image visible in the
+      // application's active configuration until a rollout applies it. Build
+      // the rollout target from this freshly read configuration so concurrent
+      // scheduler-owned resource changes survive while the new image remains
+      // explicit below.
+      // If a prior PATCH made it through but its rollout call did not, the
+      // application already has this image here. Still create the rollout: a
+      // desired configuration alone does not prove live instances moved.
+      // eslint-disable-next-line no-await-in-loop -- each explicit rollout belongs to this application.
+      await createApplicationImageRollout(
+        env,
+        current,
+        image,
+        "Cloudflare GitHub Actions runner image update",
+        dependencies,
+      );
+      updatedApplications.push(application.name);
+    } finally {
+      // eslint-disable-next-line no-await-in-loop -- progress must correspond to the application that just finished inspection.
+      await reportProgress({ processedApplications: index + 1, totalApplications: applications.length });
     }
-    // PATCH acceptance does not make the requested image visible in the
-    // application's active configuration until a rollout applies it. Build
-    // the rollout target from this freshly read configuration so concurrent
-    // scheduler-owned resource changes survive while the new image remains
-    // explicit below.
-    // If a prior PATCH made it through but its rollout call did not, the
-    // application already has this image here. Still create the rollout: a
-    // desired configuration alone does not prove live instances moved.
-    // eslint-disable-next-line no-await-in-loop -- each explicit rollout belongs to this application.
-    await createApplicationImageRollout(
-      env,
-      current,
-      image,
-      "Cloudflare GitHub Actions runner image update",
-      dependencies,
-    );
-    updatedApplications.push(application.name);
   }
   return { updatedApplications, skippedApplications };
 }

src/runner-image-build-workflow.ts

@@ -23,6 +23,7 @@ import {
 import { runnerImageBuilderExitError } from "./runner-image-builder-command";
 import {
   runnerImageBuilderProtocolVersion,
+  runnerImageBuildPhaseForOwner,
   type RunnerImageBuildResult,
   type RunnerImageBuildStatus,
 } from "./runner-image-builder";
@@ -306,10 +307,6 @@ export class RunnerImageBuildWorkflow extends WorkflowEntrypoint<
       }
       let completed: RunnerImageBuildResult | undefined;
       for (let buildRound = 1; buildRound <= maximumBuildQueueRounds; buildRound += 1) {
-        // eslint-disable-next-line no-await-in-loop -- each queued source is staged only after its predecessor completes.
-        await withFreshRunnerImageBuilder(builderNamespace, (builder) =>
-          builder.updateBuildProgress(event.payload.workflowId, "downloading-source"),
-        );
         // A joining Workflow must wait for its predecessor, then stage its own source in a fresh slot.
         // eslint-disable-next-line no-await-in-loop -- the durable step serializes this exact queue position.
         ownsBuild = await step.do(`start runner image build in Cloudflare ${buildRound}`, buildStep, async () => {
@@ -341,6 +338,10 @@ export class RunnerImageBuildWorkflow extends WorkflowEntrypoint<
                 if (checked.kind === "failed") {
                   return { kind: "failed", exitCode: checked.exitCode, diagnostic: checked.diagnostic };
                 }
+                const phase = runnerImageBuildPhaseForOwner(ownsBuild, checked);
+                if (phase !== undefined) {
+                  await builder.updateBuildProgress(event.payload.workflowId, phase);
+                }
                 return { kind: "running" };
               }),
           );
@@ -402,6 +403,8 @@ export class RunnerImageBuildWorkflow extends WorkflowEntrypoint<
               await builder.updateBuildProgress(event.payload.workflowId, "rolling-out");
               const result = await rolloutRunnerApplicationImages(this.env, completed.imageReference, undefined, {
                 reissueMatchingImageRollouts: lease.reissueMatchingImageRollouts,
+                onProgress: (progress) =>
+                  builder.updateBuildProgress(event.payload.workflowId, "rolling-out", progress),
               });
               await builder.completeRollOutAttempt(event.payload.workflowId, completed.imageReference);
               // Keep the lease while busy applications are pending. The next