diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index 47fd5ae..b660fc9 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -27,6 +27,9 @@ jobs: - name: Checkout GitOps config uses: actions/checkout@v4 + - name: Check image push backend selection + run: bash ./scripts/test-push-images.sh + - name: Checkout Firework (pinned config contract) uses: actions/checkout@v4 with: @@ -162,17 +165,34 @@ jobs: run: | case "${{ matrix.target_arch }}" in arm64) - echo "S3_IMAGES_BUCKET=${S3_IMAGES_BUCKET_ARM64:-${S3_IMAGES_BUCKET:-}}" >> "$GITHUB_ENV" - echo "GCS_IMAGES_BUCKET=${GCS_IMAGES_BUCKET_ARM64:-}" >> "$GITHUB_ENV" + S3_TARGET="${S3_IMAGES_BUCKET_ARM64:-}" + GCS_TARGET="${GCS_IMAGES_BUCKET_ARM64:-}" ;; amd64) - echo "S3_IMAGES_BUCKET=${S3_IMAGES_BUCKET_AMD64:-}" >> "$GITHUB_ENV" - echo "GCS_IMAGES_BUCKET=${GCS_IMAGES_BUCKET_AMD64:-${GCS_IMAGES_BUCKET:-}}" >> "$GITHUB_ENV" + S3_TARGET="${S3_IMAGES_BUCKET_AMD64:-${S3_IMAGES_BUCKET:-}}" + GCS_TARGET="${GCS_IMAGES_BUCKET_AMD64:-${GCS_IMAGES_BUCKET:-}}" ;; esac + # Publishing an architecture nobody consumes is intentional and cheap; + # silently publishing nothing is how an architecture mismatch reaches + # a running deployment. Make the no-op loud. + if [ -z "$S3_TARGET" ] && [ -z "$GCS_TARGET" ]; then + echo "::warning title=No ${{ matrix.target_arch }} image bucket configured::The ${{ matrix.target_arch }} build will produce rootfs images and publish none of them. Set the ${{ matrix.target_arch }} bucket variables to publish it." + fi + + echo "S3_IMAGES_BUCKET=$S3_TARGET" >> "$GITHUB_ENV" + echo "GCS_IMAGES_BUCKET=$GCS_TARGET" >> "$GITHUB_ENV" + - name: Upload images to S3 - if: env.SHOULD_UPLOAD == 'true' + # Gated the same way as the GCS credentials below, so the arm64 leg is + # not handed AWS credentials for an upload it will never perform. + if: >- + env.SHOULD_UPLOAD == 'true' && + ( + (matrix.target_arch == 'arm64' && vars.S3_IMAGES_BUCKET_ARM64 != '') || + (matrix.target_arch == 'amd64' && (vars.S3_IMAGES_BUCKET_AMD64 != '' || vars.S3_IMAGES_BUCKET != '')) + ) env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -198,7 +218,14 @@ jobs: project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Upload images to GCS - if: env.SHOULD_UPLOAD == 'true' + # Matches the credentials step above; without this the arm64 leg runs an + # upload step that can only no-op, having never authenticated. + if: >- + env.SHOULD_UPLOAD == 'true' && + ( + (matrix.target_arch == 'arm64' && vars.GCS_IMAGES_BUCKET_ARM64 != '') || + (matrix.target_arch == 'amd64' && (vars.GCS_IMAGES_BUCKET_AMD64 != '' || vars.GCS_IMAGES_BUCKET != '')) + ) run: | if [ -z "${GCS_IMAGES_BUCKET:-}" ]; then echo "Skipping GCS upload for ${{ matrix.target_platform }}; no bucket configured" diff --git a/AGENTS.md b/AGENTS.md index f945c74..9f345d6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project -This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish both ARM64 and amd64 artifacts to S3 and GCS via architecture-specific buckets. Public routing is provider-neutral via `metadata.subdomain`; there is no provider-specific runtime config tree. +This is the example GitOps input repo for Firework. It defines tenant service YAML and config overlays used to build Firecracker-ready rootfs images and publish them to S3 and GCS via architecture-specific buckets. Both amd64 and ARM64 are built; amd64 is the published default because both data planes run x86_64 nodes, and ARM64 publishing is opt-in via the `*_ARM64` bucket variables. Public routing is provider-neutral via `metadata.subdomain`; there is no provider-specific runtime config tree. ## Layout @@ -13,7 +13,8 @@ This is the example GitOps input repo for Firework. It defines tenant service YA - `configs//` and `configs/-/`: rootfs overlays; tenant-specific overlays take precedence. - `scripts/build-images.sh`: resolves `fc-init` and builds tenant rootfs images. Skips a tenant service when its inputs (own YAML, shared/tenant overlays) are unchanged since `COMPARE_BASE_SHA`, unless `FORCE_REBUILD=true` or a shared pipeline file changed (see workflow for how these are set in CI). - `scripts/docker-to-rootfs.sh`: converts Docker images into ext4 rootfs images. -- `scripts/push-images.sh`: uploads generated rootfs images to the selected object store bucket. +- `scripts/push-images.sh`: uploads generated rootfs images to the object store named by its `s3`/`gcs` backend argument. +- `scripts/test-push-images.sh`: regression checks for that backend selection. - `scripts/fc-init/`: fallback bundled `fc-init` source for CI. ## Conventions @@ -29,7 +30,8 @@ For YAML-only changes, inspect schema consistency against the main repo docs. For image pipeline changes, validate: - `shellcheck scripts/docker-to-rootfs.sh` -- A targeted local rootfs build when Docker, `jq`, `mkfs.ext4`, and a linux/arm64 `fc-init` are available. +- `bash scripts/test-push-images.sh` when touching `push-images.sh`, the `push-*` Makefile targets, or the workflow's bucket resolution. CI exports both bucket variables, so the backend must be selected explicitly rather than inferred. +- A targeted local rootfs build when Docker, `jq`, `mkfs.ext4`, and a linux/amd64 `fc-init` are available. For CI-equivalent validation, run `make build`, but skip `make push`. diff --git a/Makefile b/Makefile index ab1b620..3f456c0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -TARGET_PLATFORM ?= linux/arm64 +TARGET_PLATFORM ?= linux/amd64 .PHONY: build build-amd64 build-arm64 push push-s3 push-gcs @@ -15,7 +15,7 @@ push: bash ./scripts/push-images.sh push-s3: - S3_IMAGES_BUCKET="$(S3_IMAGES_BUCKET)" bash ./scripts/push-images.sh + S3_IMAGES_BUCKET="$(S3_IMAGES_BUCKET)" bash ./scripts/push-images.sh s3 push-gcs: - GCS_IMAGES_BUCKET="$(GCS_IMAGES_BUCKET)" bash ./scripts/push-images.sh + GCS_IMAGES_BUCKET="$(GCS_IMAGES_BUCKET)" bash ./scripts/push-images.sh gcs diff --git a/README.md b/README.md index b9127a7..2911429 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ > This is an example deployment intended for demonstration and learning purposes only. It is not hardened, audited, etc. -Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images and publishing both ARM64 and X86 images to S3 and GCS via architecture-specific buckets. +Example GitOps repository for [Firework](https://github.com/artemnikitin/firework), focused on building Firecracker-ready rootfs images and publishing them to S3 and GCS via architecture-specific buckets. Both X86 and ARM64 images are built; X86 is published by default, because both the AWS and GCP data planes run x86_64 nodes, and ARM64 publishing is opt-in. ## Related Repositories diff --git a/docs/ci-pipeline.md b/docs/ci-pipeline.md index 315df23..7581fc6 100644 --- a/docs/ci-pipeline.md +++ b/docs/ci-pipeline.md @@ -37,10 +37,39 @@ before this feature existed. ## Bucket configuration -Legacy variables keep their original meanings: `S3_IMAGES_BUCKET` is the arm64 -S3 bucket and `GCS_IMAGES_BUCKET` is the amd64 GCS bucket. Configure -`S3_IMAGES_BUCKET_AMD64` and `GCS_IMAGES_BUCKET_ARM64` to enable the extra -cross-backend uploads. +Both legacy variables mean the amd64 bucket, because both providers default to +x86_64 nodes: `S3_IMAGES_BUCKET` is the amd64 S3 bucket and `GCS_IMAGES_BUCKET` +is the amd64 GCS bucket. The explicit `*_AMD64` names are still supported and +take precedence when set. Publishing the arm64 build requires opting in with +`S3_IMAGES_BUCKET_ARM64` or `GCS_IMAGES_BUCKET_ARM64`; without them the arm64 +build still runs but uploads nothing. + +Resolved upload targets per build: + +| Build | S3 bucket | GCS bucket | +| --- | --- | --- | +| amd64 | `S3_IMAGES_BUCKET_AMD64`, else `S3_IMAGES_BUCKET` | `GCS_IMAGES_BUCKET_AMD64`, else `GCS_IMAGES_BUCKET` | +| arm64 | `S3_IMAGES_BUCKET_ARM64` | `GCS_IMAGES_BUCKET_ARM64` | + +Both variables are exported for the amd64 build, so `push-images.sh` takes an +explicit `s3` or `gcs` backend argument rather than inferring one from whichever +bucket is set. `make push-s3` and `make push-gcs` pass it. Inference is still +accepted when exactly one bucket variable is set, and errors when both are, so +a publish can never silently go to the wrong object store. +`scripts/test-push-images.sh` covers this and runs in CI. + +Host and guest architecture must match, and a mismatch fails at microVM start +rather than at deploy time. The AWS data plane in +`firework-deployment-example` now defaults to x86_64 nodes using nested +virtualization rather than bare-metal Graviton, so it consumes the amd64 rootfs +images; the GCP data plane has always been x86_64. + +`S3_IMAGES_BUCKET` previously meant the arm64 S3 bucket, so an existing +deployment that keeps its value will now receive amd64 images in that same +bucket, replacing the arm64 objects under identical names. That is intended for +the default x86_64 AWS node. A deployment that still runs Graviton nodes +(`node_ami_architecture = "arm64"`) must set `S3_IMAGES_BUCKET_ARM64` and point +`s3_images_bucket_id` at that bucket instead. ## CI config validation diff --git a/scripts/build-images.sh b/scripts/build-images.sh index ac75a56..ae2f1fc 100644 --- a/scripts/build-images.sh +++ b/scripts/build-images.sh @@ -11,7 +11,7 @@ if [[ "$CACHE_BIN_DIR" != /* ]]; then CACHE_BIN_DIR="$REPO_ROOT/$CACHE_BIN_DIR" fi -TARGET_PLATFORM="${TARGET_PLATFORM:-linux/arm64}" +TARGET_PLATFORM="${TARGET_PLATFORM:-linux/amd64}" FC_INIT_BIN_INPUT="${FC_INIT_BIN:-}" if [[ -n "$FC_INIT_BIN_INPUT" && "$FC_INIT_BIN_INPUT" != /* ]]; then FC_INIT_BIN_INPUT="$REPO_ROOT/$FC_INIT_BIN_INPUT" diff --git a/scripts/docker-to-rootfs.sh b/scripts/docker-to-rootfs.sh index 366bc08..209a43f 100755 --- a/scripts/docker-to-rootfs.sh +++ b/scripts/docker-to-rootfs.sh @@ -17,7 +17,7 @@ # 4) build from ../firework/cmd/fc-init (requires Go) # 5) fc-init from PATH # target_platform Optional Docker platform to export. Defaults to -# TARGET_PLATFORM env var, then linux/arm64. +# TARGET_PLATFORM env var, then linux/amd64. # # Requires: docker with buildx, mkfs.ext4 (e2fsprogs), jq # @@ -29,7 +29,7 @@ OUTPUT="${2:?Usage: $0 [size_mb] [overlay_dir] [fc_ SIZE_MB="${3:-512}" OVERLAY_DIR="${4:-}" FC_INIT_BIN_INPUT="${5:-${FC_INIT_BIN:-}}" -TARGET_PLATFORM="${6:-${TARGET_PLATFORM:-linux/arm64}}" +TARGET_PLATFORM="${6:-${TARGET_PLATFORM:-linux/amd64}}" TARGET_ARCH="${TARGET_PLATFORM##*/}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/scripts/push-images.sh b/scripts/push-images.sh index 81c2766..e7c246d 100644 --- a/scripts/push-images.sh +++ b/scripts/push-images.sh @@ -1,24 +1,72 @@ #!/usr/bin/env bash +# Usage: ./scripts/push-images.sh [backend] +# +# backend Optional object store to upload to: "s3" or "gcs". When given, the +# matching bucket variable is required and the other is ignored. +# When omitted the backend is inferred from whichever single bucket +# variable is set, and having both set is an error rather than a +# silent preference — CI exports both, so inference cannot +# distinguish "push to S3" from "push to GCS" on its own. +# +# Environment: +# S3_IMAGES_BUCKET destination bucket for the s3 backend +# GCS_IMAGES_BUCKET destination bucket for the gcs backend + set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" cd "$REPO_ROOT" -if [ -n "${GCS_IMAGES_BUCKET:-}" ]; then +BACKEND="${1:-}" + +push_s3() { + if [ -z "${S3_IMAGES_BUCKET:-}" ]; then + echo "ERROR: S3_IMAGES_BUCKET must be set for the s3 backend" >&2 + exit 1 + fi for ext4 in *-rootfs.ext4; do [ -f "$ext4" ] || continue - echo "Uploading $ext4 to gs://${GCS_IMAGES_BUCKET}/${ext4}" - gcloud storage cp "$ext4" "gs://${GCS_IMAGES_BUCKET}/${ext4}" + echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}" + aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}" done -elif [ -n "${S3_IMAGES_BUCKET:-}" ]; then +} + +push_gcs() { + if [ -z "${GCS_IMAGES_BUCKET:-}" ]; then + echo "ERROR: GCS_IMAGES_BUCKET must be set for the gcs backend" >&2 + exit 1 + fi for ext4 in *-rootfs.ext4; do [ -f "$ext4" ] || continue - echo "Uploading $ext4 to s3://${S3_IMAGES_BUCKET}/${ext4}" - aws s3 cp "$ext4" "s3://${S3_IMAGES_BUCKET}/${ext4}" + echo "Uploading $ext4 to gs://${GCS_IMAGES_BUCKET}/${ext4}" + gcloud storage cp "$ext4" "gs://${GCS_IMAGES_BUCKET}/${ext4}" done -else - echo "ERROR: S3_IMAGES_BUCKET or GCS_IMAGES_BUCKET must be set" >&2 - exit 1 -fi +} + +case "$BACKEND" in + s3) + push_s3 + ;; + gcs) + push_gcs + ;; + "") + if [ -n "${S3_IMAGES_BUCKET:-}" ] && [ -n "${GCS_IMAGES_BUCKET:-}" ]; then + echo "ERROR: S3_IMAGES_BUCKET and GCS_IMAGES_BUCKET are both set; pass an explicit backend (s3 or gcs)" >&2 + exit 1 + elif [ -n "${GCS_IMAGES_BUCKET:-}" ]; then + push_gcs + elif [ -n "${S3_IMAGES_BUCKET:-}" ]; then + push_s3 + else + echo "ERROR: S3_IMAGES_BUCKET or GCS_IMAGES_BUCKET must be set" >&2 + exit 1 + fi + ;; + *) + echo "ERROR: unknown backend: $BACKEND (expected s3 or gcs)" >&2 + exit 1 + ;; +esac diff --git a/scripts/test-push-images.sh b/scripts/test-push-images.sh new file mode 100755 index 0000000..319b13b --- /dev/null +++ b/scripts/test-push-images.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +# Regression checks for scripts/push-images.sh backend selection. +# +# CI exports both S3_IMAGES_BUCKET and GCS_IMAGES_BUCKET for the amd64 build, so +# `make push-s3` must upload to S3 even though a GCS bucket is also configured. +# An earlier version inferred the backend and preferred GCS, which silently sent +# the AWS images to GCS and left the S3 bucket untouched. +# +# Runs the real Makefile targets with `aws` and `gcloud` stubbed on PATH. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +WORKDIR="$(mktemp -d)" +trap 'rm -rf "$WORKDIR"' EXIT + +mkdir -p "$WORKDIR/bin" +printf '#!/bin/sh\necho "aws $*"\n' > "$WORKDIR/bin/aws" +printf '#!/bin/sh\necho "gcloud $*"\n' > "$WORKDIR/bin/gcloud" +chmod +x "$WORKDIR/bin/aws" "$WORKDIR/bin/gcloud" + +FAILURES=0 + +print_indented() { + while IFS= read -r line; do + printf ' %s\n' "$line" + done <<< "$1" +} + +# Run a make target from a scratch copy of the repo's pipeline entrypoints so a +# stray *-rootfs.ext4 in the working tree cannot affect the result. +run_case() { + local desc="$1" target="$2" expected="$3" + shift 3 + + local sandbox="$WORKDIR/case" + rm -rf "$sandbox" + mkdir -p "$sandbox/scripts" + cp "$REPO_ROOT/Makefile" "$sandbox/Makefile" + cp "$REPO_ROOT/scripts/push-images.sh" "$sandbox/scripts/push-images.sh" + touch "$sandbox/demo-rootfs.ext4" + + local output status=0 + output="$(cd "$sandbox" && env PATH="$WORKDIR/bin:$PATH" "$@" make "$target" 2>&1)" || status=$? + + if [ "$expected" = "FAIL" ]; then + if [ "$status" -eq 0 ]; then + echo "FAIL: $desc — expected a non-zero exit, got 0" + print_indented "$output" + FAILURES=$((FAILURES + 1)) + else + echo "ok: $desc" + fi + return + fi + + if [ "$status" -ne 0 ]; then + echo "FAIL: $desc — command exited $status" + print_indented "$output" + FAILURES=$((FAILURES + 1)) + elif printf '%s\n' "$output" | grep -q "$expected"; then + echo "ok: $desc" + else + echo "FAIL: $desc — expected output matching: $expected" + print_indented "$output" + FAILURES=$((FAILURES + 1)) + fi +} + +# The regression: both buckets set, as the amd64 CI leg exports them. +run_case "push-s3 uploads to S3 when both buckets are set" \ + push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/demo-rootfs.ext4" \ + S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket + +run_case "push-gcs uploads to GCS when both buckets are set" \ + push-gcs "gcloud storage cp demo-rootfs.ext4 gs://gcs-bucket/demo-rootfs.ext4" \ + S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket + +run_case "push-s3 uploads to S3 when only the S3 bucket is set" \ + push-s3 "aws s3 cp demo-rootfs.ext4 s3://s3-bucket/demo-rootfs.ext4" \ + S3_IMAGES_BUCKET=s3-bucket + +run_case "push-gcs uploads to GCS when only the GCS bucket is set" \ + push-gcs "gcloud storage cp demo-rootfs.ext4 gs://gcs-bucket/demo-rootfs.ext4" \ + GCS_IMAGES_BUCKET=gcs-bucket + +run_case "push-s3 fails when the S3 bucket is unset" \ + push-s3 FAIL GCS_IMAGES_BUCKET=gcs-bucket + +run_case "push without a backend fails when both buckets are set" \ + push FAIL S3_IMAGES_BUCKET=s3-bucket GCS_IMAGES_BUCKET=gcs-bucket + +if [ "$FAILURES" -ne 0 ]; then + echo "$FAILURES check(s) failed" >&2 + exit 1 +fi + +echo "All push-images backend checks passed"