Support immutable hosts: install under /opt/unbounded and Ignition provisioning

.github/actions/agent-e2e-kind-control-plane/action.yaml

@@ -30,8 +30,10 @@ runs:
       shell: bash
       run: |
         sudo apt-get update
+        # ovmf is the UEFI firmware an Ignition host boots through; qemu-utils
+        # also provides the qemu-nbd used to patch its boot command line.
         sudo apt-get install -y --no-install-recommends \
-          qemu-system-x86 qemu-utils genisoimage \
+          qemu-system-x86 qemu-utils genisoimage ovmf \
           iptables
 
     - name: Create Kind cluster

.github/workflows/agent-e2e-kind.yaml

@@ -57,31 +57,78 @@ permissions:
   packages: read
 
 jobs:
+  # The Azure Container Linux image is not on a public mirror. It lives in a
+  # storage account that disables anonymous access and shared keys alike, so it
+  # is reachable only with a federated Azure login, and GitHub withholds secrets
+  # from workflows triggered by a fork. Every other host downloads from a public
+  # mirror and is unaffected.
+  #
+  # A job-level `if` would skip the whole matrix, and a static matrix cannot
+  # drop one entry conditionally, so the list is built here instead. On a fork
+  # the ACL entry is absent rather than failing, which is the difference between
+  # a contributor seeing their PR pass and seeing a red check they cannot fix.
+  select-hosts:
+    name: select host matrix
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    outputs:
+      matrix: ${{ steps.select.outputs.matrix }}
+    steps:
+      - name: Select hosts
+        id: select
+        env:
+          # Absent for non-pull_request events, where secrets are available.
+          HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+          # Presence only. The image needs a federated login, and the entry is
+          # left out entirely until one is configured, rather than added and
+          # failed. GitHub masks the value; nothing here reads it.
+          ACL_CREDENTIAL: ${{ secrets.ACL_IMAGE_CLIENT_ID }}
+        run: |
+          set -euo pipefail
+
+          # Fedora, AlmaLinux, and CentOS Stream cover RPM hosts with Azure
+          # Linux nspawn. Azure Linux 3 does not currently publish a QEMU-ready
+          # VHD that this e2e can boot directly as the host VM.
+          hosts='[
+            {"host-base-os":"ubuntu2404","nspawn-base-os":"ubuntu2404","timeout":60},
+            {"host-base-os":"fedora","nspawn-base-os":"azlinux3","timeout":60},
+            {"host-base-os":"almalinux9","nspawn-base-os":"azlinux3","timeout":60},
+            {"host-base-os":"almalinux10","nspawn-base-os":"azlinux3","timeout":60},
+            {"host-base-os":"centosstream9","nspawn-base-os":"azlinux3","timeout":60},
+            {"host-base-os":"centosstream10","nspawn-base-os":"azlinux3","timeout":60},
+            {"host-base-os":"ubuntu2604","nspawn-base-os":"ubuntu2604","timeout":60}
+          ]'
+
+          # Azure Container Linux gets longer: it downloads a 630 MiB image and
+          # boots a 31 GiB sparse overlay through OVMF.
+          acl='{"host-base-os":"acl","nspawn-base-os":"azlinux3","timeout":75}'
+
+          if [ -n "${HEAD_REPO}" ] && [ "${HEAD_REPO}" != "${GITHUB_REPOSITORY}" ]; then
+            echo "::notice::Azure Container Linux is skipped for forks: its image needs Azure credentials"
+          elif [ -z "${ACL_CREDENTIAL}" ]; then
+            echo "::notice::Azure Container Linux is skipped: ACL_IMAGE_CLIENT_ID is not configured"
+          else
+            hosts="$(printf '%s' "${hosts}" | jq -c ". + [${acl}]")"
+          fi
+
+          hosts="$(printf '%s' "${hosts}" | jq -c .)"
+
+          printf 'matrix=%s\n' "$(printf '%s' "${hosts}" | jq -c '{include: .}')" >> "${GITHUB_OUTPUT}"
+
   agent-e2e:
     name: agent e2e (host ${{ matrix.host-base-os }}, nspawn ${{ matrix.nspawn-base-os }})
+    needs: select-hosts
     runs-on: ubuntu-24.04
-    timeout-minutes: 60
+    timeout-minutes: ${{ matrix.timeout }}
+    permissions:
+      contents: read
+      packages: read
+      # Federated login for the Azure Container Linux image. Requested on this
+      # job alone rather than for the workflow, because no other step needs it.
+      id-token: write
     strategy:
       fail-fast: false
-      matrix:
-        include:
-          - host-base-os: ubuntu2404
-            nspawn-base-os: ubuntu2404
-          # Fedora, AlmaLinux, and CentOS Stream cover RPM hosts with Azure Linux nspawn.
-          # Azure Linux 3 does not currently publish a QEMU-ready VHD that this
-          # e2e can boot directly as the host VM.
-          - host-base-os: fedora
-            nspawn-base-os: azlinux3
-          - host-base-os: almalinux9
-            nspawn-base-os: azlinux3
-          - host-base-os: almalinux10
-            nspawn-base-os: azlinux3
-          - host-base-os: centosstream9
-            nspawn-base-os: azlinux3
-          - host-base-os: centosstream10
-            nspawn-base-os: azlinux3
-          - host-base-os: ubuntu2604
-            nspawn-base-os: ubuntu2604
+      matrix: ${{ fromJSON(needs.select-hosts.outputs.matrix) }}
     env:
       KIND_CLUSTER_NAME: agent-e2e-${{ matrix.host-base-os }}-${{ matrix.nspawn-base-os }}
       VM_NAME: agent-e2e-${{ matrix.host-base-os }}-${{ matrix.nspawn-base-os }}
@@ -93,12 +140,50 @@ jobs:
       - name: Checkout
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
+      - name: Azure login
+        if: matrix.host-base-os == 'acl'
+        uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0
+        with:
+          client-id: ${{ secrets.ACL_IMAGE_CLIENT_ID }}
+          tenant-id: ${{ secrets.ACL_IMAGE_TENANT_ID }}
+          subscription-id: ${{ secrets.ACL_IMAGE_SUBSCRIPTION_ID }}
+
+      # Resolved once for the job. The image URL, digest and build are
+      # exported to the environment, so later e2e.py processes use this build
+      # rather than reading the manifest again, which may have moved on.
+      - name: Resolve the Azure Container Linux build
+        if: matrix.host-base-os == 'acl'
+        id: acl-image
+        run: python3 ./hack/agent/e2e-kind/e2e.py resolve-host-image
+
+      # Keyed on the build, so a refreshed image misses and is downloaded once
+      # rather than every run re-fetching 630 MiB from the storage account.
+      # Restore and save are separate steps because Cleanup deletes .vm-e2e
+      # before a combined action's post step would save it.
+      - name: Restore the host image
+        if: matrix.host-base-os == 'acl'
+        id: acl-cache
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: .vm-e2e/acl-${{ steps.acl-image.outputs.build }}.qcow2
+          key: acl-image-${{ steps.acl-image.outputs.build }}
+
       - name: Set up test control plane
         uses: ./.github/actions/agent-e2e-kind-control-plane
         with:
           cluster-name: ${{ env.KIND_CLUSTER_NAME }}
           vm-subnet: ${{ env.VM_SUBNET }}
 
+      # create-vm, in the step above, has downloaded and verified the image.
+      # Saving here rather than at the end means a later test failure does not
+      # stop the next run reusing it.
+      - name: Save the host image
+        if: matrix.host-base-os == 'acl' && steps.acl-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: .vm-e2e/acl-${{ steps.acl-image.outputs.build }}.qcow2
+          key: acl-image-${{ steps.acl-image.outputs.build }}
+
       - name: Set up machina resources
         uses: ./.github/actions/agent-e2e-machina-setup
 
@@ -155,6 +240,42 @@ jobs:
         if: always()
         run: python3 ./hack/agent/e2e-kind/e2e.py --verbose cleanup
 
+  # A host installed by a release before the host root, moved to this build and
+  # back. The older release cannot be installed on an immutable host at all,
+  # and on a migrated host the units still run the same legacy paths, so one
+  # conventional host covers it.
+  agent-host-root-migration:
+    name: agent host root migration (Ubuntu)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    env:
+      KIND_CLUSTER_NAME: agent-host-root-migration
+      VM_NAME: agent-host-root-migration
+      VM_SUBNET: "192.168.100"
+      VM_IP: "192.168.100.10"
+      AGENT_MACHINE_NAME: agent-host-root-migration
+      HOST_BASE_OS: ubuntu2404
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Set up test control plane
+        uses: ./.github/actions/agent-e2e-kind-control-plane
+        with:
+          cluster-name: ${{ env.KIND_CLUSTER_NAME }}
+          vm-subnet: ${{ env.VM_SUBNET }}
+      - name: Set up machina resources
+        uses: ./.github/actions/agent-e2e-machina-setup
+      - name: Upgrade, reboot, downgrade and reset a migrated host
+        run: python3 ./hack/agent/e2e-kind/e2e.py --verbose run-suite --suite migration
+      - name: Collect logs
+        if: always()
+        uses: ./.github/actions/agent-e2e-kind-logs
+        with:
+          artifact-name: agent-host-root-migration-logs
+      - name: Cleanup
+        if: always()
+        run: python3 ./hack/agent/e2e-kind/e2e.py --verbose cleanup
+
   agent-config-e2e:
     name: agent config e2e
     runs-on: ubuntu-24.04

cmd/agent/internal/bootstrap/coordinator.go

@@ -10,12 +10,22 @@ package bootstrap
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"log/slog"
+	"time"
 
 	"github.com/Azure/unbounded/cmd/agent/internal/installstate"
 )
 
+// defaultLockWait bounds how long Run waits for another lifecycle operation to
+// release the installation lock. On a reboot the daemon holds it briefly while
+// it migrates the host on startup, and the first-boot unit runs start then.
+const (
+	defaultLockWait  = 30 * time.Second
+	lockPollInterval = 250 * time.Millisecond
+)
+
 type Identity struct{ MachineName, ConfigFingerprint string }
 
 type Stages interface {
@@ -51,16 +61,25 @@ type Coordinator struct {
 	store    *installstate.Store
 	stages   Stages
 	reporter Reporter
+	lockWait time.Duration
+	lockPoll time.Duration
 }
 
 func New(log *slog.Logger, store *installstate.Store, stages Stages, reporter Reporter) *Coordinator {
-	return &Coordinator{log: log, store: store, stages: stages, reporter: reporter}
+	return &Coordinator{
+		log:      log,
+		store:    store,
+		stages:   stages,
+		reporter: reporter,
+		lockWait: defaultLockWait,
+		lockPoll: lockPollInterval,
+	}
 }
 
 type Outcome struct{ AlreadyComplete bool }
 
 func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) {
-	lock, err := c.store.AcquireLock()
+	lock, err := c.acquireLock(ctx)
 	if err != nil {
 		return Outcome{}, err
 	}
@@ -91,18 +110,24 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) {
 	}
 
 	if disposition == installstate.AlreadyComplete {
-		if err := c.stages.VerifyInstalled(ctx); err != nil {
+		verifyErr := c.stages.VerifyInstalled(ctx)
+		if verifyErr != nil {
 			if err := c.stages.RepairDaemon(ctx); err != nil {
-				return Outcome{}, err
+				return Outcome{}, fmt.Errorf("repair daemon after %w: %w", verifyErr, err)
 			}
 
 			if err := c.stages.VerifyInstalled(ctx); err != nil {
 				return Outcome{}, err
 			}
-		}
 
-		if err := c.store.MarkComplete(r); err != nil {
-			return Outcome{}, err
+			// Only a repair can have changed anything, so only a repair needs
+			// to be committed. The record already says complete: rewriting it
+			// on a healthy host would be a durable write for no change, on
+			// every boot of every Ignition-provisioned node, since that unit
+			// has no completion condition and runs each time.
+			if err := c.store.MarkComplete(r); err != nil {
+				return Outcome{}, err
+			}
 		}
 
 		return Outcome{AlreadyComplete: true}, nil
@@ -149,3 +174,29 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) {
 
 	return Outcome{}, nil
 }
+
+// acquireLock waits up to lockWait for the installation lock, and returns
+// installstate.ErrLockHeld if it is still held after that.
+func (c *Coordinator) acquireLock(ctx context.Context) (*installstate.Lock, error) {
+	deadline := time.Now().Add(c.lockWait)
+	logged := false
+
+	for {
+		lock, err := c.store.AcquireLock()
+		if !errors.Is(err, installstate.ErrLockHeld) || !time.Now().Before(deadline) {
+			return lock, err
+		}
+
+		if !logged {
+			c.log.Info("waiting for another lifecycle operation to release the installation lock", "timeout", c.lockWait)
+
+			logged = true
+		}
+
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(c.lockPoll):
+		}
+	}
+}