ci: publish amd64 rootfs images for AWS, and document the bucket semantics

.github/workflows/build-images.yaml

@@ -27,6 +27,9 @@ jobs:
       - name: Checkout GitOps config
         uses: actions/checkout@v4
 
+      - name: Check image push backend selection
+        run: bash ./scripts/test-push-images.sh
+
       - name: Checkout Firework (pinned config contract)
         uses: actions/checkout@v4
         with:
@@ -162,17 +165,34 @@ jobs:
         run: |
           case "${{ matrix.target_arch }}" in
             arm64)
-              echo "S3_IMAGES_BUCKET=${S3_IMAGES_BUCKET_ARM64:-${S3_IMAGES_BUCKET:-}}" >> "$GITHUB_ENV"
-              echo "GCS_IMAGES_BUCKET=${GCS_IMAGES_BUCKET_ARM64:-}" >> "$GITHUB_ENV"
+              S3_TARGET="${S3_IMAGES_BUCKET_ARM64:-}"
+              GCS_TARGET="${GCS_IMAGES_BUCKET_ARM64:-}"
               ;;
             amd64)
-              echo "S3_IMAGES_BUCKET=${S3_IMAGES_BUCKET_AMD64:-}" >> "$GITHUB_ENV"
-              echo "GCS_IMAGES_BUCKET=${GCS_IMAGES_BUCKET_AMD64:-${GCS_IMAGES_BUCKET:-}}" >> "$GITHUB_ENV"
+              S3_TARGET="${S3_IMAGES_BUCKET_AMD64:-${S3_IMAGES_BUCKET:-}}"
+              GCS_TARGET="${GCS_IMAGES_BUCKET_AMD64:-${GCS_IMAGES_BUCKET:-}}"
               ;;
           esac
 
+          # Publishing an architecture nobody consumes is intentional and cheap;
+          # silently publishing nothing is how an architecture mismatch reaches
+          # a running deployment. Make the no-op loud.
+          if [ -z "$S3_TARGET" ] && [ -z "$GCS_TARGET" ]; then
+            echo "::warning title=No ${{ matrix.target_arch }} image bucket configured::The ${{ matrix.target_arch }} build will produce rootfs images and publish none of them. Set the ${{ matrix.target_arch }} bucket variables to publish it."
+          fi
+
+          echo "S3_IMAGES_BUCKET=$S3_TARGET" >> "$GITHUB_ENV"
+          echo "GCS_IMAGES_BUCKET=$GCS_TARGET" >> "$GITHUB_ENV"
+
       - name: Upload images to S3
-        if: env.SHOULD_UPLOAD == 'true'
+        # Gated the same way as the GCS credentials below, so the arm64 leg is
+        # not handed AWS credentials for an upload it will never perform.
+        if: >-
+          env.SHOULD_UPLOAD == 'true' &&
+          (
+            (matrix.target_arch == 'arm64' && vars.S3_IMAGES_BUCKET_ARM64 != '') ||
+            (matrix.target_arch == 'amd64' && (vars.S3_IMAGES_BUCKET_AMD64 != '' || vars.S3_IMAGES_BUCKET != ''))
+          )
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -198,7 +218,14 @@ jobs:
           project_id: ${{ secrets.GCP_PROJECT_ID }}
 
       - name: Upload images to GCS
-        if: env.SHOULD_UPLOAD == 'true'
+        # Matches the credentials step above; without this the arm64 leg runs an
+        # upload step that can only no-op, having never authenticated.
+        if: >-
+          env.SHOULD_UPLOAD == 'true' &&
+          (
+            (matrix.target_arch == 'arm64' && vars.GCS_IMAGES_BUCKET_ARM64 != '') ||
+            (matrix.target_arch == 'amd64' && (vars.GCS_IMAGES_BUCKET_AMD64 != '' || vars.GCS_IMAGES_BUCKET != ''))
+          )
         run: |
           if [ -z "${GCS_IMAGES_BUCKET:-}" ]; then
             echo "Skipping GCS upload for ${{ matrix.target_platform }}; no bucket configured"

AGENTS.md

@@ -2,7 +2,7 @@
 
 ## Project
 
-This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish both ARM64 and amd64 artifacts to S3 and GCS via architecture-specific buckets. Public routing is provider-neutral via `metadata.subdomain`; there is no provider-specific runtime config tree.
+This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish them to S3 and GCS via architecture-specific buckets. Both amd64 and ARM64 are built; amd64 is the published default because both data planes run x86_64 nodes, and ARM64 publishing is opt-in via the `*_ARM64` bucket variables. Public routing is provider-neutral via `metadata.subdomain`; there is no provider-specific runtime config tree.
 
 ## Layout
 
@@ -13,7 +13,8 @@ This is the example GitOps input repo for Firework. It defines tenant service YA
 - `configs/<service>/` and `configs/<tenant>-<service>/`: rootfs overlays; tenant-specific overlays take precedence.
 - `scripts/build-images.sh`: resolves `fc-init` and builds tenant rootfs images. Skips a tenant service when its inputs (own YAML, shared/tenant overlays) are unchanged since `COMPARE_BASE_SHA`, unless `FORCE_REBUILD=true` or a shared pipeline file changed (see workflow for how these are set in CI).
 - `scripts/docker-to-rootfs.sh`: converts Docker images into ext4 rootfs images.
-- `scripts/push-images.sh`: uploads generated rootfs images to the selected object store bucket.
+- `scripts/push-images.sh`: uploads generated rootfs images to the object store named by its `s3`/`gcs` backend argument.
+- `scripts/test-push-images.sh`: regression checks for that backend selection.
 - `scripts/fc-init/`: fallback bundled `fc-init` source for CI.
 
 ## Conventions
@@ -29,7 +30,8 @@ For YAML-only changes, inspect schema consistency against the main repo docs.
 For image pipeline changes, validate:
 
 - `shellcheck scripts/docker-to-rootfs.sh`
-- A targeted local rootfs build when Docker, `jq`, `mkfs.ext4`, and a linux/arm64 `fc-init` are available.
+- `bash scripts/test-push-images.sh` when touching `push-images.sh`, the `push-*` Makefile targets, or the workflow's bucket resolution. CI exports both bucket variables, so the backend must be selected explicitly rather than inferred.
+- A targeted local rootfs build when Docker, `jq`, `mkfs.ext4`, and a linux/amd64 `fc-init` are available.
 
 For CI-equivalent validation, run `make build`, but skip `make push`.
 

Makefile

@@ -1,4 +1,4 @@
-TARGET_PLATFORM ?= linux/arm64
+TARGET_PLATFORM ?= linux/amd64
 
 .PHONY: build build-amd64 build-arm64 push push-s3 push-gcs
 
@@ -15,7 +15,7 @@ push:
 	bash ./scripts/push-images.sh
 
 push-s3:
-	S3_IMAGES_BUCKET="$(S3_IMAGES_BUCKET)" bash ./scripts/push-images.sh
+	S3_IMAGES_BUCKET="$(S3_IMAGES_BUCKET)" bash ./scripts/push-images.sh s3
 
 push-gcs:
-	GCS_IMAGES_BUCKET="$(GCS_IMAGES_BUCKET)" bash ./scripts/push-images.sh
+	GCS_IMAGES_BUCKET="$(GCS_IMAGES_BUCKET)" bash ./scripts/push-images.sh gcs

README.md

@@ -2,7 +2,7 @@
 
 > This is an example deployment intended for demonstration and learning purposes only. It is not hardened, audited, etc.
 
-Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images and publishing both ARM64 and X86 images to S3 and GCS via architecture-specific buckets.
+Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images and publishing them to S3 and GCS via architecture-specific buckets. Both X86 and ARM64 images are built; X86 is published by default, because both the AWS and GCP data planes run x86_64 nodes, and ARM64 publishing is opt-in.
 
 ## Related Repositories
 

docs/ci-pipeline.md

@@ -37,10 +37,39 @@ before this feature existed.
 
 ## Bucket configuration
 
-Legacy variables keep their original meanings: `S3_IMAGES_BUCKET` is the arm64
-S3 bucket and `GCS_IMAGES_BUCKET` is the amd64 GCS bucket. Configure
-`S3_IMAGES_BUCKET_AMD64` and `GCS_IMAGES_BUCKET_ARM64` to enable the extra
-cross-backend uploads.
+Both legacy variables mean the amd64 bucket, because both providers default to
+x86_64 nodes: `S3_IMAGES_BUCKET` is the amd64 S3 bucket and `GCS_IMAGES_BUCKET`
+is the amd64 GCS bucket. The explicit `*_AMD64` names are still supported and
+take precedence when set. Publishing the arm64 build requires opting in with
+`S3_IMAGES_BUCKET_ARM64` or `GCS_IMAGES_BUCKET_ARM64`; without them the arm64
+build still runs but uploads nothing.
+
+Resolved upload targets per build:
+
+| Build | S3 bucket | GCS bucket |
+| --- | --- | --- |
+| amd64 | `S3_IMAGES_BUCKET_AMD64`, else `S3_IMAGES_BUCKET` | `GCS_IMAGES_BUCKET_AMD64`, else `GCS_IMAGES_BUCKET` |
+| arm64 | `S3_IMAGES_BUCKET_ARM64` | `GCS_IMAGES_BUCKET_ARM64` |
+
+Both variables are exported for the amd64 build, so `push-images.sh` takes an
+explicit `s3` or `gcs` backend argument rather than inferring one from whichever
+bucket is set. `make push-s3` and `make push-gcs` pass it. Inference is still
+accepted when exactly one bucket variable is set, and errors when both are, so
+a publish can never silently go to the wrong object store.
+`scripts/test-push-images.sh` covers this and runs in CI.
+
+Host and guest architecture must match, and a mismatch fails at microVM start
+rather than at deploy time. The AWS data plane in
+`firework-deployment-example` now defaults to x86_64 nodes using nested
+virtualization rather than bare-metal Graviton, so it consumes the amd64 rootfs
+images; the GCP data plane has always been x86_64.
+
+`S3_IMAGES_BUCKET` previously meant the arm64 S3 bucket, so an existing
+deployment that keeps its value will now receive amd64 images in that same
+bucket, replacing the arm64 objects under identical names. That is intended for
+the default x86_64 AWS node. A deployment that still runs Graviton nodes
+(`node_ami_architecture = "arm64"`) must set `S3_IMAGES_BUCKET_ARM64` and point
+`s3_images_bucket_id` at that bucket instead.
 
 ## CI config validation
 

scripts/build-images.sh

@@ -11,7 +11,7 @@ if [[ "$CACHE_BIN_DIR" != /* ]]; then
     CACHE_BIN_DIR="$REPO_ROOT/$CACHE_BIN_DIR"
 fi
 
-TARGET_PLATFORM="${TARGET_PLATFORM:-linux/arm64}"
+TARGET_PLATFORM="${TARGET_PLATFORM:-linux/amd64}"
 FC_INIT_BIN_INPUT="${FC_INIT_BIN:-}"
 if [[ -n "$FC_INIT_BIN_INPUT" && "$FC_INIT_BIN_INPUT" != /* ]]; then
     FC_INIT_BIN_INPUT="$REPO_ROOT/$FC_INIT_BIN_INPUT"

scripts/docker-to-rootfs.sh

@@ -17,7 +17,7 @@
 #                  4) build from ../firework/cmd/fc-init (requires Go)
 #                  5) fc-init from PATH
 #   target_platform Optional Docker platform to export. Defaults to
-#                   TARGET_PLATFORM env var, then linux/arm64.
+#                   TARGET_PLATFORM env var, then linux/amd64.
 #
 # Requires: docker with buildx, mkfs.ext4 (e2fsprogs), jq
 #
@@ -29,7 +29,7 @@ OUTPUT="${2:?Usage: $0 <docker-image> <output.ext4> [size_mb] [overlay_dir] [fc_
 SIZE_MB="${3:-512}"
 OVERLAY_DIR="${4:-}"
 FC_INIT_BIN_INPUT="${5:-${FC_INIT_BIN:-}}"
-TARGET_PLATFORM="${6:-${TARGET_PLATFORM:-linux/arm64}}"
+TARGET_PLATFORM="${6:-${TARGET_PLATFORM:-linux/amd64}}"
 TARGET_ARCH="${TARGET_PLATFORM##*/}"
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

scripts/push-images.sh

@@ -1,24 +1,72 @@
 #!/usr/bin/env bash
 
+# Usage: ./scripts/push-images.sh [backend]
+#
+#   backend  Optional object store to upload to: "s3" or "gcs". When given, the
+#            matching bucket variable is required and the other is ignored.
+#            When omitted the backend is inferred from whichever single bucket
+#            variable is set, and having both set is an error rather than a
+#            silent preference — CI exports both, so inference cannot
+#            distinguish "push to S3" from "push to GCS" on its own.
+#
+# Environment:
+#   S3_IMAGES_BUCKET   destination bucket for the s3 backend
+#   GCS_IMAGES_BUCKET  destination bucket for the gcs backend
+
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
 cd "$REPO_ROOT"
 
-if [ -n "${GCS_IMAGES_BUCKET:-}" ]; then
+BACKEND="${1:-}"
+
+push_s3() {
+    if [ -z "${S3_IMAGES_BUCKET:-}" ]; then
+        echo "ERROR: S3_IMAGES_BUCKET must be set for the s3 backend" >&2
+        exit 1
+    fi
     for ext4 in *-rootfs.ext4; do
         [ -f "$ext4" ] || continue
-        echo "Uploading $ext4 to gs://${GCS_IMAGES_BUCKET}/${ext4}"
-        gcloud storage cp "$ext4" "gs://${GCS_IMAGES_BUCKET}/${ext4}"
+        echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}"
+        aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}"
     done
-elif [ -n "${S3_IMAGES_BUCKET:-}" ]; then
+}
+
+push_gcs() {
+    if [ -z "${GCS_IMAGES_BUCKET:-}" ]; then
+        echo "ERROR: GCS_IMAGES_BUCKET must be set for the gcs backend" >&2
+        exit 1
+    fi
     for ext4 in *-rootfs.ext4; do
         [ -f "$ext4" ] || continue
-        echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}"
-        aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}"
+        echo "Uploading $ext4 to gs://${GCS_IMAGES_BUCKET}/${ext4}"
+        gcloud storage cp "$ext4" "gs://${GCS_IMAGES_BUCKET}/${ext4}"
     done
-else
-    echo "ERROR: S3_IMAGES_BUCKET or GCS_IMAGES_BUCKET must be set" >&2
-    exit 1
-fi
+}
+
+case "$BACKEND" in
+    s3)
+        push_s3
+        ;;
+    gcs)
+        push_gcs
+        ;;
+    "")
+        if [ -n "${S3_IMAGES_BUCKET:-}" ] && [ -n "${GCS_IMAGES_BUCKET:-}" ]; then
+            echo "ERROR: S3_IMAGES_BUCKET and GCS_IMAGES_BUCKET are both set; pass an explicit backend (s3 or gcs)" >&2
+            exit 1
+        elif [ -n "${GCS_IMAGES_BUCKET:-}" ]; then
+            push_gcs
+        elif [ -n "${S3_IMAGES_BUCKET:-}" ]; then
+            push_s3
+        else
+            echo "ERROR: S3_IMAGES_BUCKET or GCS_IMAGES_BUCKET must be set" >&2
+            exit 1
+        fi
+        ;;
+    *)
+        echo "ERROR: unknown backend: $BACKEND (expected s3 or gcs)" >&2
+        exit 1
+        ;;
+esac

scripts/test-push-images.sh

@@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+
+# Regression checks for scripts/push-images.sh backend selection.
+#
+# CI exports both S3_IMAGES_BUCKET and GCS_IMAGES_BUCKET for the amd64 build, so
+# `make push-s3` must upload to S3 even though a GCS bucket is also configured.
+# An earlier version inferred the backend and preferred GCS, which silently sent
+# the AWS images to GCS and left the S3 bucket untouched.
+#
+# Runs the real Makefile targets with `aws` and `gcloud` stubbed on PATH.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+
+mkdir -p "$WORKDIR/bin"
+printf '#!/bin/sh\necho "aws $*"\n' > "$WORKDIR/bin/aws"
+printf '#!/bin/sh\necho "gcloud $*"\n' > "$WORKDIR/bin/gcloud"
+chmod +x "$WORKDIR/bin/aws" "$WORKDIR/bin/gcloud"
+
+FAILURES=0
+
+print_indented() {
+    while IFS= read -r line; do
+        printf '    %s\n' "$line"
+    done <<< "$1"
+}
+
+# Run a make target from a scratch copy of the repo's pipeline entrypoints so a
+# stray *-rootfs.ext4 in the working tree cannot affect the result.
+run_case() {
+    local desc="$1" target="$2" expected="$3"
+    shift 3
+
+    local sandbox="$WORKDIR/case"
+    rm -rf "$sandbox"
+    mkdir -p "$sandbox/scripts"
+    cp "$REPO_ROOT/Makefile" "$sandbox/Makefile"
+    cp "$REPO_ROOT/scripts/push-images.sh" "$sandbox/scripts/push-images.sh"
+    touch "$sandbox/demo-rootfs.ext4"
+
+    local output status=0
+    output="$(cd "$sandbox" && env PATH="$WORKDIR/bin:$PATH" "$@" make "$target" 2>&1)" || status=$?
+
+    if [ "$expected" = "FAIL" ]; then
+        if [ "$status" -eq 0 ]; then
+            echo "FAIL: $desc — expected a non-zero exit, got 0"
+            print_indented "$output"
+            FAILURES=$((FAILURES + 1))
+        else
+            echo "ok: $desc"
+        fi
+        return
+    fi
+
+    if [ "$status" -ne 0 ]; then
+        echo "FAIL: $desc — command exited $status"
+        print_indented "$output"
+        FAILURES=$((FAILURES + 1))
+    elif printf '%s\n' "$output" | grep -q "$expected"; then
+        echo "ok: $desc"
+    else
+        echo "FAIL: $desc — expected output matching: $expected"
+        print_indented "$output"
+        FAILURES=$((FAILURES + 1))
+    fi
+}
+
+# The regression: both buckets set, as the amd64 CI leg exports them.
+run_case "push-s3 uploads to S3 when both buckets are set" \
+    push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/demo-rootfs.ext4" \
+    S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket
+
+run_case "push-gcs uploads to GCS when both buckets are set" \
+    push-gcs "gcloud storage cp demo-rootfs.ext4 gs://gcs-bucket/demo-rootfs.ext4" \
+    S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket
+
+run_case "push-s3 uploads to S3 when only the S3 bucket is set" \
+    push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/demo-rootfs.ext4" \
+    S3_IMAGES_BUCKET=s3-bucket
+
+run_case "push-gcs uploads to GCS when only the GCS bucket is set" \
+    push-gcs "gcloud storage cp demo-rootfs.ext4 gs://gcs-bucket/demo-rootfs.ext4" \
+    GCS_IMAGES_BUCKET=gcs-bucket
+
+run_case "push-s3 fails when the S3 bucket is unset" \
+    push-s3 FAIL GCS_IMAGES_BUCKET=gcs-bucket
+
+run_case "push without a backend fails when both buckets are set" \
+    push FAIL S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket
+
+if [ "$FAILURES" -ne 0 ]; then
+    echo "$FAILURES check(s) failed" >&2
+    exit 1
+fi
+
+echo "All push-images backend checks passed"