diff --git a/.github/actions/agent-e2e-kind-control-plane/action.yaml b/.github/actions/agent-e2e-kind-control-plane/action.yaml index a10b398e5..53686cbc8 100644 --- a/.github/actions/agent-e2e-kind-control-plane/action.yaml +++ b/.github/actions/agent-e2e-kind-control-plane/action.yaml @@ -30,8 +30,10 @@ runs: shell: bash run: | sudo apt-get update + # ovmf is the UEFI firmware an Ignition host boots through; qemu-utils + # also provides the qemu-nbd used to patch its boot command line. sudo apt-get install -y --no-install-recommends \ - qemu-system-x86 qemu-utils genisoimage \ + qemu-system-x86 qemu-utils genisoimage ovmf \ iptables - name: Create Kind cluster diff --git a/.github/workflows/agent-e2e-kind.yaml b/.github/workflows/agent-e2e-kind.yaml index 19db28bd4..54f9ed817 100644 --- a/.github/workflows/agent-e2e-kind.yaml +++ b/.github/workflows/agent-e2e-kind.yaml @@ -57,31 +57,78 @@ permissions: packages: read jobs: + # The Azure Container Linux image is not on a public mirror. It lives in a + # storage account that disables anonymous access and shared keys alike, so it + # is reachable only with a federated Azure login, and GitHub withholds secrets + # from workflows triggered by a fork. Every other host downloads from a public + # mirror and is unaffected. + # + # A job-level `if` would skip the whole matrix, and a static matrix cannot + # drop one entry conditionally, so the list is built here instead. On a fork + # the ACL entry is absent rather than failing, which is the difference between + # a contributor seeing their PR pass and seeing a red check they cannot fix. + select-hosts: + name: select host matrix + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + matrix: ${{ steps.select.outputs.matrix }} + steps: + - name: Select hosts + id: select + env: + # Absent for non-pull_request events, where secrets are available. + HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }} + # Presence only. The image needs a federated login, and the entry is + # left out entirely until one is configured, rather than added and + # failed. GitHub masks the value; nothing here reads it. + ACL_CREDENTIAL: ${{ secrets.ACL_IMAGE_CLIENT_ID }} + run: | + set -euo pipefail + + # Fedora, AlmaLinux, and CentOS Stream cover RPM hosts with Azure + # Linux nspawn. Azure Linux 3 does not currently publish a QEMU-ready + # VHD that this e2e can boot directly as the host VM. + hosts='[ + {"host-base-os":"ubuntu2404","nspawn-base-os":"ubuntu2404","timeout":60}, + {"host-base-os":"fedora","nspawn-base-os":"azlinux3","timeout":60}, + {"host-base-os":"almalinux9","nspawn-base-os":"azlinux3","timeout":60}, + {"host-base-os":"almalinux10","nspawn-base-os":"azlinux3","timeout":60}, + {"host-base-os":"centosstream9","nspawn-base-os":"azlinux3","timeout":60}, + {"host-base-os":"centosstream10","nspawn-base-os":"azlinux3","timeout":60}, + {"host-base-os":"ubuntu2604","nspawn-base-os":"ubuntu2604","timeout":60} + ]' + + # Azure Container Linux gets longer: it downloads a 630 MiB image and + # boots a 31 GiB sparse overlay through OVMF. + acl='{"host-base-os":"acl","nspawn-base-os":"azlinux3","timeout":75}' + + if [ -n "${HEAD_REPO}" ] && [ "${HEAD_REPO}" != "${GITHUB_REPOSITORY}" ]; then + echo "::notice::Azure Container Linux is skipped for forks: its image needs Azure credentials" + elif [ -z "${ACL_CREDENTIAL}" ]; then + echo "::notice::Azure Container Linux is skipped: ACL_IMAGE_CLIENT_ID is not configured" + else + hosts="$(printf '%s' "${hosts}" | jq -c ". + [${acl}]")" + fi + + hosts="$(printf '%s' "${hosts}" | jq -c .)" + + printf 'matrix=%s\n' "$(printf '%s' "${hosts}" | jq -c '{include: .}')" >> "${GITHUB_OUTPUT}" + agent-e2e: name: agent e2e (host ${{ matrix.host-base-os }}, nspawn ${{ matrix.nspawn-base-os }}) + needs: select-hosts runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: ${{ matrix.timeout }} + permissions: + contents: read + packages: read + # Federated login for the Azure Container Linux image. Requested on this + # job alone rather than for the workflow, because no other step needs it. + id-token: write strategy: fail-fast: false - matrix: - include: - - host-base-os: ubuntu2404 - nspawn-base-os: ubuntu2404 - # Fedora, AlmaLinux, and CentOS Stream cover RPM hosts with Azure Linux nspawn. - # Azure Linux 3 does not currently publish a QEMU-ready VHD that this - # e2e can boot directly as the host VM. - - host-base-os: fedora - nspawn-base-os: azlinux3 - - host-base-os: almalinux9 - nspawn-base-os: azlinux3 - - host-base-os: almalinux10 - nspawn-base-os: azlinux3 - - host-base-os: centosstream9 - nspawn-base-os: azlinux3 - - host-base-os: centosstream10 - nspawn-base-os: azlinux3 - - host-base-os: ubuntu2604 - nspawn-base-os: ubuntu2604 + matrix: ${{ fromJSON(needs.select-hosts.outputs.matrix) }} env: KIND_CLUSTER_NAME: agent-e2e-${{ matrix.host-base-os }}-${{ matrix.nspawn-base-os }} VM_NAME: agent-e2e-${{ matrix.host-base-os }}-${{ matrix.nspawn-base-os }} @@ -93,12 +140,50 @@ jobs: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Azure login + if: matrix.host-base-os == 'acl' + uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0 + with: + client-id: ${{ secrets.ACL_IMAGE_CLIENT_ID }} + tenant-id: ${{ secrets.ACL_IMAGE_TENANT_ID }} + subscription-id: ${{ secrets.ACL_IMAGE_SUBSCRIPTION_ID }} + + # Resolved once for the job. The image URL, digest and build are + # exported to the environment, so later e2e.py processes use this build + # rather than reading the manifest again, which may have moved on. + - name: Resolve the Azure Container Linux build + if: matrix.host-base-os == 'acl' + id: acl-image + run: python3 ./hack/agent/e2e-kind/e2e.py resolve-host-image + + # Keyed on the build, so a refreshed image misses and is downloaded once + # rather than every run re-fetching 630 MiB from the storage account. + # Restore and save are separate steps because Cleanup deletes .vm-e2e + # before a combined action's post step would save it. + - name: Restore the host image + if: matrix.host-base-os == 'acl' + id: acl-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .vm-e2e/acl-${{ steps.acl-image.outputs.build }}.qcow2 + key: acl-image-${{ steps.acl-image.outputs.build }} + - name: Set up test control plane uses: ./.github/actions/agent-e2e-kind-control-plane with: cluster-name: ${{ env.KIND_CLUSTER_NAME }} vm-subnet: ${{ env.VM_SUBNET }} + # create-vm, in the step above, has downloaded and verified the image. + # Saving here rather than at the end means a later test failure does not + # stop the next run reusing it. + - name: Save the host image + if: matrix.host-base-os == 'acl' && steps.acl-cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .vm-e2e/acl-${{ steps.acl-image.outputs.build }}.qcow2 + key: acl-image-${{ steps.acl-image.outputs.build }} + - name: Set up machina resources uses: ./.github/actions/agent-e2e-machina-setup @@ -155,6 +240,42 @@ jobs: if: always() run: python3 ./hack/agent/e2e-kind/e2e.py --verbose cleanup + # A host installed by a release before the host root, moved to this build and + # back. The older release cannot be installed on an immutable host at all, + # and on a migrated host the units still run the same legacy paths, so one + # conventional host covers it. + agent-host-root-migration: + name: agent host root migration (Ubuntu) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + KIND_CLUSTER_NAME: agent-host-root-migration + VM_NAME: agent-host-root-migration + VM_SUBNET: "192.168.100" + VM_IP: "192.168.100.10" + AGENT_MACHINE_NAME: agent-host-root-migration + HOST_BASE_OS: ubuntu2404 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up test control plane + uses: ./.github/actions/agent-e2e-kind-control-plane + with: + cluster-name: ${{ env.KIND_CLUSTER_NAME }} + vm-subnet: ${{ env.VM_SUBNET }} + - name: Set up machina resources + uses: ./.github/actions/agent-e2e-machina-setup + - name: Upgrade, reboot, downgrade and reset a migrated host + run: python3 ./hack/agent/e2e-kind/e2e.py --verbose run-suite --suite migration + - name: Collect logs + if: always() + uses: ./.github/actions/agent-e2e-kind-logs + with: + artifact-name: agent-host-root-migration-logs + - name: Cleanup + if: always() + run: python3 ./hack/agent/e2e-kind/e2e.py --verbose cleanup + agent-config-e2e: name: agent config e2e runs-on: ubuntu-24.04 diff --git a/cmd/agent/internal/bootstrap/coordinator.go b/cmd/agent/internal/bootstrap/coordinator.go index d5339bdbf..e478e50b3 100644 --- a/cmd/agent/internal/bootstrap/coordinator.go +++ b/cmd/agent/internal/bootstrap/coordinator.go @@ -10,12 +10,22 @@ package bootstrap import ( "context" + "errors" "fmt" "log/slog" + "time" "github.com/Azure/unbounded/cmd/agent/internal/installstate" ) +// defaultLockWait bounds how long Run waits for another lifecycle operation to +// release the installation lock. On a reboot the daemon holds it briefly while +// it migrates the host on startup, and the first-boot unit runs start then. +const ( + defaultLockWait = 30 * time.Second + lockPollInterval = 250 * time.Millisecond +) + type Identity struct{ MachineName, ConfigFingerprint string } type Stages interface { @@ -51,16 +61,25 @@ type Coordinator struct { store *installstate.Store stages Stages reporter Reporter + lockWait time.Duration + lockPoll time.Duration } func New(log *slog.Logger, store *installstate.Store, stages Stages, reporter Reporter) *Coordinator { - return &Coordinator{log: log, store: store, stages: stages, reporter: reporter} + return &Coordinator{ + log: log, + store: store, + stages: stages, + reporter: reporter, + lockWait: defaultLockWait, + lockPoll: lockPollInterval, + } } type Outcome struct{ AlreadyComplete bool } func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { - lock, err := c.store.AcquireLock() + lock, err := c.acquireLock(ctx) if err != nil { return Outcome{}, err } @@ -91,18 +110,24 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { } if disposition == installstate.AlreadyComplete { - if err := c.stages.VerifyInstalled(ctx); err != nil { + verifyErr := c.stages.VerifyInstalled(ctx) + if verifyErr != nil { if err := c.stages.RepairDaemon(ctx); err != nil { - return Outcome{}, err + return Outcome{}, fmt.Errorf("repair daemon after %w: %w", verifyErr, err) } if err := c.stages.VerifyInstalled(ctx); err != nil { return Outcome{}, err } - } - if err := c.store.MarkComplete(r); err != nil { - return Outcome{}, err + // Only a repair can have changed anything, so only a repair needs + // to be committed. The record already says complete: rewriting it + // on a healthy host would be a durable write for no change, on + // every boot of every Ignition-provisioned node, since that unit + // has no completion condition and runs each time. + if err := c.store.MarkComplete(r); err != nil { + return Outcome{}, err + } } return Outcome{AlreadyComplete: true}, nil @@ -149,3 +174,29 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { return Outcome{}, nil } + +// acquireLock waits up to lockWait for the installation lock, and returns +// installstate.ErrLockHeld if it is still held after that. +func (c *Coordinator) acquireLock(ctx context.Context) (*installstate.Lock, error) { + deadline := time.Now().Add(c.lockWait) + logged := false + + for { + lock, err := c.store.AcquireLock() + if !errors.Is(err, installstate.ErrLockHeld) || !time.Now().Before(deadline) { + return lock, err + } + + if !logged { + c.log.Info("waiting for another lifecycle operation to release the installation lock", "timeout", c.lockWait) + + logged = true + } + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(c.lockPoll): + } + } +} diff --git a/cmd/agent/internal/bootstrap/coordinator_test.go b/cmd/agent/internal/bootstrap/coordinator_test.go index 5076aeda7..94273ba4d 100644 --- a/cmd/agent/internal/bootstrap/coordinator_test.go +++ b/cmd/agent/internal/bootstrap/coordinator_test.go @@ -7,8 +7,11 @@ import ( "context" "errors" "log/slog" + "os" "path/filepath" + "syscall" "testing" + "time" "github.com/stretchr/testify/require" @@ -22,7 +25,13 @@ type fakeStages struct { verifyErr error } -var errInjected = errors.New("injected stage failure") +var ( + errInjected = errors.New("injected stage failure") + // errRepairFailed is distinct from errInjected so a test can tell whether + // the reported error is the fault that triggered a repair or the failure of + // the repair itself. + errRepairFailed = errors.New("injected repair failure") +) func (f *fakeStages) run(name string) error { f.calls = append(f.calls, name) @@ -33,6 +42,10 @@ func (f *fakeStages) run(name string) error { } if name == f.fail { + if name == "repair" { + return errRepairFailed + } + return errInjected } @@ -161,7 +174,10 @@ func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { } stages := &fakeStages{store: store} - _, err = New(slog.New(slog.DiscardHandler), store, stages, nil).Run(t.Context(), id) + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + c.lockWait = 0 // waiting is covered by TestRunWaitsForTheInstallationLock + + _, err = c.Run(t.Context(), id) require.Error(t, err) require.Empty(t, stages.calls) }) @@ -189,3 +205,170 @@ func TestInterruptedRepairRemainsCompleteAndRetries(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"verify", "repair", "verify"}, stages.calls) } + +// recordInode identifies the record file itself rather than its contents. +// +// MarkComplete on an already-complete record writes the same bytes, so +// comparing content cannot tell a rewrite from a no-op. The store replaces the +// file atomically, so any write at all produces a new inode. +func recordInode(t *testing.T, store *installstate.Store) uint64 { + t.Helper() + + info, err := os.Stat(filepath.Join(store.Root(), "install-state.json")) + require.NoError(t, err) + + stat, ok := info.Sys().(*syscall.Stat_t) + require.True(t, ok, "inode is how this test distinguishes a rewrite from a no-op") + + return stat.Ino +} + +// TestHealthyCompletedInstallIsNotRewritten covers the cost of an Ignition unit +// that carries no completion condition. +// +// That unit runs on every boot and reaches this path each time. Rewriting the +// record when nothing changed would be a durable write per boot on every node, +// and a write is a chance to fail: a host that is entirely healthy would be +// taking one for no reason. +func TestHealthyCompletedInstallIsNotRewritten(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + before := recordInode(t, store) + + stages := &fakeStages{store: store} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + + outcome, err := c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.NoError(t, err) + require.True(t, outcome.AlreadyComplete) + require.Equal(t, []string{"verify"}, stages.calls, "a healthy host needs no repair") + + require.Equal(t, before, recordInode(t, store), + "nothing changed, so the record must not have been written at all") +} + +// TestRepairedInstallIsCommitted is the other half: when a repair did happen, +// the result has to be durable before the process exits. +func TestRepairedInstallIsCommitted(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + before := recordInode(t, store) + + stages := &fakeStages{store: store, verifyErr: errInjected} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + + _, err = c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.NoError(t, err) + require.Equal(t, []string{"verify", "repair", "verify"}, stages.calls) + + loaded, err := store.Load() + require.NoError(t, err) + require.Equal(t, installstate.Complete, loaded.Phase) + require.NotEqual(t, before, recordInode(t, store), + "a repair changed the host, so the result has to be made durable") +} + +// TestFailedRepairReportsWhatWasWrong pins that the original fault survives. +// +// The first verify says what is broken; the repair failure says only that +// fixing it did not work. Reporting the second alone sends an operator after +// the wrong thing. +func TestFailedRepairReportsWhatWasWrong(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + stages := &fakeStages{store: store, fail: "repair", verifyErr: errInjected} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + + _, err = c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.Error(t, err) + require.ErrorIs(t, err, errInjected, "the fault that triggered the repair must still be reported") + require.ErrorIs(t, err, errRepairFailed, "and so must the reason repairing it did not work") +} + +// TestRunWaitsForTheInstallationLock covers a reboot, where the daemon holds +// the lock while it migrates the host and the first-boot unit runs start at +// the same time. +func TestRunWaitsForTheInstallationLock(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + releaseIn time.Duration + lockWait time.Duration + cancel bool + wantErr error + wantCalls []string + }{ + { + name: "released within the wait", + releaseIn: 100 * time.Millisecond, + lockWait: 10 * time.Second, + wantCalls: []string{"verify"}, + }, + { + name: "still held at the deadline", + releaseIn: time.Hour, + lockWait: 100 * time.Millisecond, + wantErr: installstate.ErrLockHeld, + }, + { + name: "canceled while waiting", + releaseIn: time.Hour, + lockWait: 10 * time.Second, + cancel: true, + wantErr: context.Canceled, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + held, err := store.AcquireLock() + require.NoError(t, err) + + release := time.AfterFunc(tt.releaseIn, func() { _ = held.Release() }) + + t.Cleanup(func() { + release.Stop() + + _ = held.Release() + }) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + if tt.cancel { + time.AfterFunc(100*time.Millisecond, cancel) + } + + stages := &fakeStages{store: store} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + c.lockWait = tt.lockWait + c.lockPoll = 10 * time.Millisecond + + _, err = c.Run(ctx, Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) + } + + require.Equal(t, tt.wantCalls, stages.calls) + }) + } +} diff --git a/cmd/agent/internal/cmd/agentupgrade.go b/cmd/agent/internal/cmd/agentupgrade.go index a11eebc88..aba0f2a90 100644 --- a/cmd/agent/internal/cmd/agentupgrade.go +++ b/cmd/agent/internal/cmd/agentupgrade.go @@ -8,6 +8,7 @@ import ( _ "embed" "fmt" "io" + "log/slog" "os" "path/filepath" "text/template" @@ -35,9 +36,15 @@ type hostAgentUpgradeHandler struct { writer io.Writer executable func() (string, error) resolvedPath func() (goalstates.AgentUpgradePaths, error) + // plannedPath resolves the paths for preflight, which does not migrate + // the host root. Unset, preflight uses resolvedPath. + plannedPath func() (goalstates.AgentUpgradePaths, error) newService func(goalstates.AgentUpgradePaths) agentbinary.DaemonService geteuid func() int installation *installstate.Store + // migrate links the host root on a legacy host before paths are resolved. + // Preflight leaves the host alone and does not call it. + migrate func(*slog.Logger) error } func newCmdHostAgentUpgrade(cmdCtx *CommandContext) *cobra.Command { @@ -46,8 +53,10 @@ func newCmdHostAgentUpgrade(cmdCtx *CommandContext) *cobra.Command { writer: os.Stdout, executable: os.Executable, resolvedPath: goalstates.ResolvedAgentUpgradePaths, + plannedPath: goalstates.PlannedAgentUpgradePaths, geteuid: os.Geteuid, installation: installstate.DefaultStore(), + migrate: daemon.MigrateHostRoot, } handler.newService = func(paths goalstates.AgentUpgradePaths) agentbinary.DaemonService { return daemon.NewHostDaemonActivationService(handler.cmdCtx.Logger, paths) @@ -86,7 +95,27 @@ func (h *hostAgentUpgradeHandler) execute(ctx context.Context) error { return err } - paths, err := h.resolvedPath() + resolve := h.resolvedPath + + if h.preflight { + if h.plannedPath != nil { + resolve = h.plannedPath + } + } else { + // Before the migration, which cannot succeed without root either and + // would report it less plainly. + if h.geteuid() != 0 { + return fmt.Errorf("host agent upgrade requires root privileges") + } + + if h.migrate != nil { + if err := h.migrate(h.cmdCtx.Logger); err != nil { + return err + } + } + } + + paths, err := resolve() if err != nil { return fmt.Errorf("resolve agent binary paths: %w", err) } @@ -114,10 +143,6 @@ func (h *hostAgentUpgradeHandler) execute(ctx context.Context) error { return writeHostAgentUpgradePlan(h.writer, plan) } - if h.geteuid() != 0 { - return fmt.Errorf("host agent upgrade requires root privileges") - } - lock, err := h.installation.AcquireMutationLock() if err != nil { return err diff --git a/cmd/agent/internal/cmd/agentupgrade_test.go b/cmd/agent/internal/cmd/agentupgrade_test.go index 8545f3f1e..95e5e44cd 100644 --- a/cmd/agent/internal/cmd/agentupgrade_test.go +++ b/cmd/agent/internal/cmd/agentupgrade_test.go @@ -6,6 +6,8 @@ package cmd import ( "bytes" "context" + "errors" + "log/slog" "os" "path/filepath" "testing" @@ -100,6 +102,105 @@ func TestHostAgentUpgradeTakesInstallationLockBeforeActivation(t *testing.T) { require.ErrorIs(t, handler.execute(t.Context()), installstate.ErrLockHeld) } +// The handler tests are not parallel: execute sets up the process-wide logger. + +// TestHostAgentUpgradePreflightLeavesTheHostRootAlone covers a legacy host, +// where the resolved paths are not where the installation is until the +// migration has run. Preflight must not run it, so it plans against where the +// migration will put things instead. +func TestHostAgentUpgradePreflightLeavesTheHostRootAlone(t *testing.T) { + dir := t.TempDir() + paths := goalstates.AgentUpgradePaths{ + BinaryPath: filepath.Join(dir, "unbounded-agent"), + BluePath: filepath.Join(dir, "unbounded-agent-blue"), + GreenPath: filepath.Join(dir, "unbounded-agent-green"), + CurrentPath: filepath.Join(dir, "unbounded-agent-current"), + LastGoodPath: filepath.Join(dir, "unbounded-agent-last-good"), + } + require.NoError(t, os.WriteFile(paths.BinaryPath, []byte("#!/bin/sh\nexit 0\n"), 0o755)) + + candidatePath := filepath.Join(dir, "candidate") + require.NoError(t, os.WriteFile(candidatePath, []byte("#!/bin/sh\nexit 0\n# candidate\n"), 0o755)) + + var output bytes.Buffer + + handler := &hostAgentUpgradeHandler{ + cmdCtx: &CommandContext{LogFormat: "text"}, + preflight: true, + writer: &output, + executable: func() (string, error) { return candidatePath, nil }, + resolvedPath: func() (goalstates.AgentUpgradePaths, error) { + return goalstates.AgentUpgradePaths{}, errors.New("preflight must not use the resolved paths") + }, + plannedPath: func() (goalstates.AgentUpgradePaths, error) { return paths, nil }, + newService: func(goalstates.AgentUpgradePaths) agentbinary.DaemonService { return preflightOnlyDaemonService{} }, + geteuid: func() int { return 1000 }, + migrate: func(*slog.Logger) error { + t.Error("preflight must not migrate the host root") + return nil + }, + } + + require.NoError(t, handler.execute(t.Context())) + assert.Contains(t, output.String(), "Install target: "+paths.GreenPath) +} + +// TestHostAgentUpgradeMigratesBeforeResolvingPaths pins the order the rest of +// the upgrade depends on. Paths resolved before the migration name the new +// root on a legacy host, where the running daemon's slots are not. +func TestHostAgentUpgradeMigratesBeforeResolvingPaths(t *testing.T) { + var calls []string + + handler := &hostAgentUpgradeHandler{ + cmdCtx: &CommandContext{LogFormat: "text"}, + executable: func() (string, error) { return filepath.Join(t.TempDir(), "candidate"), nil }, + resolvedPath: func() (goalstates.AgentUpgradePaths, error) { + calls = append(calls, "resolve") + return goalstates.AgentUpgradePaths{}, errors.New("stop here") + }, + geteuid: func() int { return 0 }, + migrate: func(*slog.Logger) error { + calls = append(calls, "migrate") + return nil + }, + } + + require.ErrorContains(t, handler.execute(t.Context()), "stop here") + assert.Equal(t, []string{"migrate", "resolve"}, calls) +} + +func TestHostAgentUpgradeStopsOnAFailedMigration(t *testing.T) { + handler := &hostAgentUpgradeHandler{ + cmdCtx: &CommandContext{LogFormat: "text"}, + executable: func() (string, error) { return filepath.Join(t.TempDir(), "candidate"), nil }, + resolvedPath: func() (goalstates.AgentUpgradePaths, error) { + t.Error("paths must not be resolved after a failed migration") + return goalstates.AgentUpgradePaths{}, nil + }, + geteuid: func() int { return 0 }, + migrate: func(*slog.Logger) error { return errors.New("installed under both") }, + } + + require.ErrorContains(t, handler.execute(t.Context()), "installed under both") +} + +// TestHostAgentUpgradeRequiresRootBeforeMigrating keeps the error an operator +// sees plain. Without root the migration fails too, but on a permission error +// that does not say what is wrong. +func TestHostAgentUpgradeRequiresRootBeforeMigrating(t *testing.T) { + handler := &hostAgentUpgradeHandler{ + cmdCtx: &CommandContext{LogFormat: "text"}, + executable: func() (string, error) { return filepath.Join(t.TempDir(), "candidate"), nil }, + geteuid: func() int { return 1000 }, + migrate: func(*slog.Logger) error { + t.Error("a non-root upgrade must be refused before it migrates") + return nil + }, + } + + require.ErrorContains(t, handler.execute(t.Context()), "requires root privileges") +} + func TestRecordAgentUpgradeFailureSignalCommand(t *testing.T) { dir := t.TempDir() signalPath := filepath.Join(dir, "agent-upgrade-signal") diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index e00baf1a9..33df60a58 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -17,6 +17,7 @@ import ( "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/hostroot" "github.com/Azure/unbounded/pkg/agent/phases" "github.com/Azure/unbounded/pkg/agent/phases/host" "github.com/Azure/unbounded/pkg/agent/phases/nodestart" @@ -97,6 +98,10 @@ func (s *agentStages) ResolveInputs(ctx context.Context) error { } func (s *agentStages) PrepareHost(ctx context.Context) error { + if err := hostroot.Prepare(ctx, s.log, "bin", "libexec"); err != nil { + return err + } + if err := daemon.InstallBootstrapBinary(); err != nil { return err } @@ -107,7 +112,7 @@ func (s *agentStages) PrepareHost(ctx context.Context) error { return err } - return fsutil.SyncFilesystems("/etc", "/usr/local", installstate.DefaultDirectory) + return fsutil.SyncFilesystems("/etc", hostroot.Resolve(), installstate.DefaultDirectory) } // Credentials must be resolved on every unfinished attempt, but TPM prerequisites @@ -172,7 +177,7 @@ func (s *agentStages) PrepareRootFS(ctx context.Context) error { return err } - return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, "/usr/local", goalstates.SystemdSystemDir, goalstates.SystemdNSpawnDir) + return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, hostroot.Resolve(), goalstates.SystemdSystemDir, goalstates.SystemdNSpawnDir) } // nodeStartTask composes the work that brings the node up. @@ -239,7 +244,7 @@ func (s *agentStages) EnsureDaemonInstalled(ctx context.Context) error { return err } - return fsutil.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) + return fsutil.SyncFilesystems(hostroot.Resolve(), goalstates.AgentConfigDir, goalstates.SystemdSystemDir) } func (s *agentStages) VerifyInstalled(ctx context.Context) error { diff --git a/cmd/agent/internal/cmd/cmd.go b/cmd/agent/internal/cmd/cmd.go index 18f444fa1..6750f643b 100644 --- a/cmd/agent/internal/cmd/cmd.go +++ b/cmd/agent/internal/cmd/cmd.go @@ -33,6 +33,7 @@ func Run() { newCmdDaemon(cmdCtx), newCmdReset(cmdCtx), newCmdVersion(), + newCmdHostRoot(), newCmdNSpawnLifecycle(cmdCtx), newCmdHostAgentUpgrade(cmdCtx), newCmdRecordAgentUpgradeFailureSignal(), diff --git a/cmd/agent/internal/cmd/hostroot.go b/cmd/agent/internal/cmd/hostroot.go new file mode 100644 index 000000000..3988aa73c --- /dev/null +++ b/cmd/agent/internal/cmd/hostroot.go @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/Azure/unbounded/pkg/agent/goalstates" +) + +// newCmdHostRoot prints where this agent keeps its host-side files on this host, +// as it will once migrated. It does not migrate or otherwise change the host. +// +// Its existence is what the install script and AgentUpgrade check for: an +// agent without it predates the host root and still installs to the legacy +// root. +func newCmdHostRoot() *cobra.Command { + return &cobra.Command{ + Use: "host-root", + Short: "Print the directory that holds the agent's host-side files", + Hidden: true, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + _, err := fmt.Fprintln(cmd.OutOrStdout(), goalstates.PlannedHostPaths().Root) + return err + }, + } +} diff --git a/cmd/agent/internal/cmd/hostroot_test.go b/cmd/agent/internal/cmd/hostroot_test.go new file mode 100644 index 000000000..9fb6dfae4 --- /dev/null +++ b/cmd/agent/internal/cmd/hostroot_test.go @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/goalstates" +) + +// TestHostRootPrintsThePlannedRoot pins the output the install script and the +// AgentUpgrade downgrade guard read. The guard compares it with the root the +// running agent resolved, so anything but the bare path on one line refuses +// every upgrade. +func TestHostRootPrintsThePlannedRoot(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + cmd := newCmdHostRoot() + cmd.SetOut(&out) + cmd.SetArgs(nil) + + require.NoError(t, cmd.Execute()) + assert.Equal(t, goalstates.PlannedHostPaths().Root+"\n", out.String()) + assert.True(t, cmd.Hidden, "host-root is for the agent's own tooling, not for operators") +} + +func TestHostRootRejectsArguments(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + cmd := newCmdHostRoot() + cmd.SetOut(&out) + cmd.SetErr(&out) + cmd.SetArgs([]string{"extra"}) + + require.Error(t, cmd.Execute()) +} diff --git a/cmd/agent/internal/cmd/nspawn_lifecycle.go b/cmd/agent/internal/cmd/nspawn_lifecycle.go index e94e8e9c7..9093a0951 100644 --- a/cmd/agent/internal/cmd/nspawn_lifecycle.go +++ b/cmd/agent/internal/cmd/nspawn_lifecycle.go @@ -10,6 +10,7 @@ import ( "github.com/spf13/cobra" + "github.com/Azure/unbounded/cmd/agent/internal/daemon" "github.com/Azure/unbounded/pkg/agent/config" "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/nspawnlifecycle" @@ -48,6 +49,10 @@ func newCmdNSpawnLifecyclePhase( cmdCtx.Setup() + if err := daemon.MigrateHostRoot(cmdCtx.Logger); err != nil { + return err + } + return run(cmd.Context(), cmdCtx.Logger, args[0]) }, } diff --git a/cmd/agent/internal/cmd/reset.go b/cmd/agent/internal/cmd/reset.go index 6b6cebef7..2284758d7 100644 --- a/cmd/agent/internal/cmd/reset.go +++ b/cmd/agent/internal/cmd/reset.go @@ -35,6 +35,9 @@ Both possible nspawn machine names (kube1 and kube2) are stopped and removed.`, "commit", version.GitCommit, ) + // No host root migration: teardown sweeps both roots, so it works + // on a host the migration refuses, which is when an operator is + // told to run reset. return resetAgent(cmdCtx.Logger).Do(ctx) }, } diff --git a/cmd/agent/internal/cmd/start.go b/cmd/agent/internal/cmd/start.go index e28ee8fcd..63658915a 100644 --- a/cmd/agent/internal/cmd/start.go +++ b/cmd/agent/internal/cmd/start.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/Azure/unbounded/cmd/agent/internal/bootstrap" + "github.com/Azure/unbounded/cmd/agent/internal/daemon" "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/internal/version" @@ -34,6 +35,10 @@ func newCmdStart(cmdCtx *CommandContext) *cobra.Command { "commit", version.GitCommit, ) + if err := daemon.MigrateHostRoot(cmdCtx.Logger); err != nil { + return err + } + cfg, err := loadConfig(cmdCtx.Logger) if err != nil { return err diff --git a/cmd/agent/internal/daemon/daemon.go b/cmd/agent/internal/daemon/daemon.go index 61f612fe1..d7b9db5b7 100644 --- a/cmd/agent/internal/daemon/daemon.go +++ b/cmd/agent/internal/daemon/daemon.go @@ -102,6 +102,12 @@ func (o *runOptions) validate() error { // machine, builds a Kubernetes client, registers the Machine CR if needed, // and blocks until the context is canceled. func Run(ctx context.Context, log *slog.Logger) error { + // After an AgentUpgrade from a release that predates the host root, this is + // the first time the new agent runs on the host. + if err := MigrateHostRoot(log); err != nil { + return err + } + return run(ctx, log, runOptions{}) } diff --git a/cmd/agent/internal/daemon/hostroot.go b/cmd/agent/internal/daemon/hostroot.go new file mode 100644 index 000000000..6aa1627ba --- /dev/null +++ b/cmd/agent/internal/daemon/hostroot.go @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package daemon + +import ( + "log/slog" + + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/hostroot" +) + +// MigrateHostRoot links the host root to the legacy root on a host installed +// by an agent released before the host root. Commands that change the host +// call it before resolving any path; see hostroot.Migrate. +func MigrateHostRoot(log *slog.Logger) error { + return hostroot.Migrate(log, goalstates.HostRootMarkers()...) +} diff --git a/cmd/agent/internal/daemon/hostupgrade.go b/cmd/agent/internal/daemon/hostupgrade.go index 5decb613a..da8bb3aa3 100644 --- a/cmd/agent/internal/daemon/hostupgrade.go +++ b/cmd/agent/internal/daemon/hostupgrade.go @@ -191,7 +191,7 @@ func (s *HostDaemonActivationService) desiredAssets(currentBinaryPath string) (m return map[string]daemonAsset{ filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonUnit): {content: service, mode: 0o644}, filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonRecoveryUnit): {content: recoveryService, mode: 0o644}, - goalstates.DaemonRecoveryScriptPath: {content: recoveryScript, mode: 0o755}, + goalstates.ResolveHostPaths().DaemonRecoveryScript: {content: recoveryScript, mode: 0o755}, }, nil } diff --git a/cmd/agent/internal/daemon/lifecycle.go b/cmd/agent/internal/daemon/lifecycle.go index 8cd40302d..564fba5dc 100644 --- a/cmd/agent/internal/daemon/lifecycle.go +++ b/cmd/agent/internal/daemon/lifecycle.go @@ -20,6 +20,7 @@ import ( "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/hostroot" "github.com/Azure/unbounded/pkg/agent/phases" ) @@ -82,13 +83,15 @@ func (d *enableDaemon) Do(ctx context.Context) error { return fmt.Errorf("writing %s: %w", recoveryUnitPath, err) } + recoveryScriptPath := goalstates.ResolveHostPaths().DaemonRecoveryScript + recoveryScript, err := renderDaemonAsset("daemon-recovery-script", daemonRecoveryScriptContent) if err != nil { - return fmt.Errorf("rendering %s: %w", goalstates.DaemonRecoveryScriptPath, err) + return fmt.Errorf("rendering %s: %w", recoveryScriptPath, err) } - if err := writeFile(goalstates.DaemonRecoveryScriptPath, recoveryScript, 0o755); err != nil { - return fmt.Errorf("writing %s: %w", goalstates.DaemonRecoveryScriptPath, err) + if err := writeFile(recoveryScriptPath, recoveryScript, 0o755); err != nil { + return fmt.Errorf("writing %s: %w", recoveryScriptPath, err) } return activateDaemonUnit(ctx, d.log, executil.Systemctl()) @@ -132,8 +135,16 @@ func activateDaemonUnit(ctx context.Context, log *slog.Logger, sc func(context.C // host already has a usable daemon binary. The caller holds installation // ownership; existing binary layouts are retained and upgrades use their normal // activation path. +// +// The binary path comes from the resolved upgrade paths, so an environment +// override lands the binary where VerifyDaemonInstalled will look for it. func InstallBootstrapBinary() error { - if usableDaemonBinary(goalstates.DaemonBinaryPath) { + paths, err := goalstates.ResolvedAgentUpgradePaths() + if err != nil { + return err + } + + if usableDaemonBinary(paths.BinaryPath) { return nil } @@ -142,7 +153,7 @@ func InstallBootstrapBinary() error { return err } - return fsutil.InstallFile(source, goalstates.DaemonBinaryPath, 0o755) + return fsutil.InstallFile(source, paths.BinaryPath, 0o755) } // usableDaemonBinary resolves symlinks on purpose. The healthy layout reaches @@ -180,7 +191,7 @@ func renderDaemonAssetForPaths(name string, content []byte, paths goalstates.Age DaemonRecoveryUnit: goalstates.DaemonRecoveryUnit, DaemonBinaryCurrentPath: paths.CurrentPath, DaemonBinaryLastGoodPath: paths.LastGoodPath, - DaemonRecoveryScriptPath: goalstates.DaemonRecoveryScriptPath, + DaemonRecoveryScriptPath: goalstates.ResolveHostPaths().DaemonRecoveryScript, DaemonAgentUpgradeSignalPath: paths.SignalPath, DaemonDeferredExitCode: DeferredExitCode, } @@ -263,25 +274,98 @@ func disableAndRemoveDaemonUnit(ctx context.Context, log *slog.Logger) error { return err } - if err := removeOwnedFile(goalstates.DaemonRecoveryScriptPath); err != nil { + if err := removeOwnedFile(goalstates.ResolveHostPaths().DaemonRecoveryScript); err != nil { return err } return nil } +type removeFirstBootUnit struct { + log *slog.Logger +} + +// RemoveFirstBootBootstrapUnit returns a task that disables and removes the +// unit an Ignition config installs to bootstrap the agent. +func RemoveFirstBootBootstrapUnit(log *slog.Logger) phases.Task { + return &removeFirstBootUnit{log: log} +} + +func (t *removeFirstBootUnit) Name() string { return "remove-first-boot-unit" } + +func (t *removeFirstBootUnit) Do(ctx context.Context) error { + return removeFirstBootBootstrapUnit(ctx, t.log) +} + +// removeFirstBootBootstrapUnit disables and removes the unit an Ignition config +// installs to bootstrap the agent. +// +// Reset has to take this with it. The unit is installed into +// multi-user.target and carries no completion condition, so it runs on every +// boot and relies on the agent's ownership record to decide there is nothing to +// do. Reset removes that record, so a unit left behind would find a host with +// no installation and bootstrap it again, undoing the reset on the next boot. +// +// Absent on every host not provisioned through Ignition, which is the common +// case, so a missing unit is success rather than something to report. +func removeFirstBootBootstrapUnit(ctx context.Context, log *slog.Logger) error { + return removeFirstBootBootstrapUnitIn(ctx, log, goalstates.SystemdSystemDir) +} + +// removeFirstBootBootstrapUnitIn takes the unit directory so the sequence can +// be exercised without writing to /etc. +func removeFirstBootBootstrapUnitIn(ctx context.Context, log *slog.Logger, unitDir string) error { + unitPath := filepath.Join(unitDir, goalstates.FirstBootBootstrapUnit) + + if _, err := os.Lstat(unitPath); errors.Is(err, os.ErrNotExist) { + return nil + } + + log.Info("removing first-boot bootstrap unit", "unit", goalstates.FirstBootBootstrapUnit) + + // --now stops it as well as disabling it. The unit is a oneshot with + // RemainAfterExit=yes, so after it has run it stays active, and deleting + // the file does not change that: systemd keeps the loaded unit active until + // something stops it. A host provisioned again afterwards writes the unit + // back and starts it, systemd sees a unit that is already active and does + // nothing, and the agent never runs. Nothing reports an error, because + // nothing failed. + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", "--now", goalstates.FirstBootBootstrapUnit); err != nil { + // Disable removes the enablement symlink. If it failed but the unit + // file is already gone, there is nothing left to start. + if _, statErr := os.Lstat(unitPath); !errors.Is(statErr, os.ErrNotExist) { + return fmt.Errorf("disable %s: %w", goalstates.FirstBootBootstrapUnit, err) + } + } + + return removeOwnedFile(unitPath) +} + // --------------------------------------------------------------------------- // RemoveAgentArtifacts // --------------------------------------------------------------------------- type removeAgentArtifacts struct { log *slog.Logger + // files, dirs and removeRoot are resolved at construction so the task can + // be exercised against a temporary tree. Do removes real system paths, so a + // test that had to call the exported constructor could not run it at all. + files []string + dirs []string + removeRoot func() error } // RemoveAgentArtifacts returns a task that removes the agent binary, install -// script, legacy uninstall script, config directory, and temp files. +// script, legacy uninstall script, config directory, and temp files, and then +// the host root itself once it is empty, or the link to the legacy root on a +// migrated host. func RemoveAgentArtifacts(log *slog.Logger) phases.Task { - return &removeAgentArtifacts{log: log} + return &removeAgentArtifacts{ + log: log, + files: goalstates.OwnedHostFiles(), + dirs: []string{goalstates.AgentConfigDir, "/tmp/unbounded-agent"}, + removeRoot: func() error { return hostroot.Remove(log) }, + } } func (t *removeAgentArtifacts) Name() string { return "remove-agent-artifacts" } @@ -290,27 +374,14 @@ func (t *removeAgentArtifacts) Do(_ context.Context) error { t.log.Info("removing agent binaries and configuration") // Remove known file paths. - for _, path := range []string{ - goalstates.DaemonBinaryPath, - goalstates.DaemonBinaryBluePath, - goalstates.DaemonBinaryGreenPath, - goalstates.DaemonBinaryCurrentPath, - goalstates.DaemonBinaryLastGoodPath, - goalstates.NSpawnLifecycleBinaryPath, - goalstates.DaemonRecoveryScriptPath, - "/usr/local/bin/unbounded-agent-install.sh", - "/usr/local/bin/unbounded-agent-uninstall.sh", - } { + for _, path := range t.files { if err := removeOwnedFile(path); err != nil { return err } } // Remove directories. - for _, dir := range []string{ - "/etc/unbounded/agent", - "/tmp/unbounded-agent", - } { + for _, dir := range t.dirs { if err := os.RemoveAll(dir); err != nil { return err } @@ -324,11 +395,40 @@ func (t *removeAgentArtifacts) Do(_ context.Context) error { } } - return nil + // Last, so the files above are removed through a link to the legacy root + // before the link goes. + return t.removeRoot() } +// removeOwnedFile removes one of the agent's own files, tolerating its absence. +// +// The existence check is not an optimization. The installer scripts are +// removed from the legacy root on every host, and on an immutable host that is +// a read-only filesystem. Unlinking a path that is not there returns EROFS +// rather than ENOENT, because the kernel checks the parent directory for write +// permission before it resolves the final component, so an absent file there +// would fail a reset that had nothing to do. +// +// Lstat rather than Stat: a dangling symlink is still a file the agent left +// behind, and it has to be removed rather than read as absent. func removeOwnedFile(path string) error { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return removeOwnedFileWith(path, os.Lstat, os.Remove) +} + +// removeOwnedFileWith takes the two syscalls so the ordering between them can +// be tested. That ordering is the whole behavior, and it cannot be observed +// from the outside without a read-only mount, which a unit test has no way to +// arrange. +func removeOwnedFileWith( + path string, + lstat func(string) (os.FileInfo, error), + remove func(string) error, +) error { + if _, err := lstat(path); errors.Is(err, os.ErrNotExist) { + return nil + } + + if err := remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { return fmt.Errorf("remove owned artifact %s: %w", path, err) } @@ -350,7 +450,7 @@ func VerifyDaemonInstalled(ctx context.Context, log *slog.Logger) error { } } - for _, path := range []string{paths.CurrentPath, paths.LastGoodPath, paths.BinaryPath, goalstates.DaemonRecoveryScriptPath} { + for _, path := range []string{paths.CurrentPath, paths.LastGoodPath, paths.BinaryPath, goalstates.ResolveHostPaths().DaemonRecoveryScript} { info, err := os.Stat(path) if err != nil { return err @@ -396,5 +496,5 @@ func RepairDaemon(ctx context.Context, log *slog.Logger) error { return err } - return fsutil.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) + return fsutil.SyncFilesystems(hostroot.Resolve(), goalstates.AgentConfigDir, goalstates.SystemdSystemDir) } diff --git a/cmd/agent/internal/daemon/lifecycle_test.go b/cmd/agent/internal/daemon/lifecycle_test.go index 9528df2b9..3e3b4c230 100644 --- a/cmd/agent/internal/daemon/lifecycle_test.go +++ b/cmd/agent/internal/daemon/lifecycle_test.go @@ -4,10 +4,13 @@ package daemon import ( + "errors" + "fmt" "os" "path/filepath" "strconv" "strings" + "syscall" "testing" "github.com/stretchr/testify/assert" @@ -18,27 +21,45 @@ import ( "github.com/Azure/unbounded/pkg/agent/goalstates" ) +// TestRenderDaemonAsset renders the three daemon assets and checks every path +// they carry comes from the same host root. +// +// The recovery unit's ExecStart is the only reference to the recovery script, so +// a render that took the script from one root while the binaries came from +// another would produce a unit that points at a file that is not there. Nothing +// else would notice until recovery was needed. func TestRenderDaemonAsset(t *testing.T) { t.Parallel() - renderedBytes, err := renderDaemonAsset("daemon-service", daemonServiceContent) + paths, err := goalstates.ResolvedAgentUpgradePaths() require.NoError(t, err) - rendered := string(renderedBytes) + hostPaths := goalstates.ResolveHostPaths() + require.Equal(t, hostPaths.BinDir, filepath.Dir(paths.CurrentPath), "the binaries must be under the host root") + require.Equal(t, hostPaths.BinDir, filepath.Dir(hostPaths.DaemonRecoveryScript), "the recovery script must be under the host root") + + service := renderAsset(t, "daemon-service", daemonServiceContent) + assert.Contains(t, service, goalstates.DaemonRecoveryUnit) + assert.Contains(t, service, paths.CurrentPath+" daemon") + + recoveryUnit := renderAsset(t, "daemon-recovery-service", daemonRecoveryServiceContent) + assert.Contains(t, recoveryUnit, "ExecStart="+hostPaths.DaemonRecoveryScript) + + script := renderAsset(t, "daemon-recovery-script", daemonRecoveryScriptContent) + assert.Contains(t, script, paths.LastGoodPath) + assert.Contains(t, script, goalstates.DaemonUnit) + assert.Contains(t, script, goalstates.DaemonAgentUpgradeSignalPath) + assert.Contains(t, script, "record-agent-upgrade-failure-signal") +} - require.NotContains(t, rendered, "{{") - assert.Contains(t, rendered, goalstates.DaemonRecoveryUnit) - assert.Contains(t, rendered, goalstates.DaemonBinaryCurrentPath) +func renderAsset(t *testing.T, name string, content []byte) string { + t.Helper() - renderedRecoveryBytes, err := renderDaemonAsset("daemon-recovery-script", daemonRecoveryScriptContent) + rendered, err := renderDaemonAsset(name, content) require.NoError(t, err) + require.NotContains(t, string(rendered), "{{") - renderedRecovery := string(renderedRecoveryBytes) - require.NotContains(t, renderedRecovery, "{{") - assert.Contains(t, renderedRecovery, goalstates.DaemonBinaryLastGoodPath) - assert.Contains(t, renderedRecovery, goalstates.DaemonUnit) - assert.Contains(t, renderedRecovery, goalstates.DaemonAgentUpgradeSignalPath) - assert.Contains(t, renderedRecovery, "record-agent-upgrade-failure-signal") + return string(rendered) } func TestInstallBinaryStreamsAndReplacesAtomically(t *testing.T) { @@ -179,3 +200,225 @@ func TestActivateDaemonUnitToleratesDeniedResetFailed(t *testing.T) { require.NoError(t, activateDaemonUnit(t.Context(), discardLogger(), executil.Systemctl())) } + +// TestResetRemovesTheFirstBootBootstrapUnit covers the interaction between +// reset and an Ignition-provisioned host. +// +// The unit carries no completion condition and runs on every boot, deciding +// there is nothing to do from the agent's ownership record. Reset removes that +// record. A unit left behind would therefore find an uninstalled host on the +// next boot and bootstrap it, quietly undoing the reset. +func TestResetRemovesTheFirstBootBootstrapUnit(t *testing.T) { + dir := t.TempDir() + calls := filepath.Join(dir, "calls") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), + []byte("#!/bin/sh\necho \"$@\" >> \""+calls+"\"\n"), 0o755)) + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + + unitDir := t.TempDir() + unitPath := filepath.Join(unitDir, goalstates.FirstBootBootstrapUnit) + require.NoError(t, os.WriteFile(unitPath, []byte("[Unit]\n"), 0o644)) + + require.NoError(t, removeFirstBootBootstrapUnitIn(t.Context(), discardLogger(), unitDir)) + + require.NoFileExists(t, unitPath, "the unit file must be gone, or systemd can still start it") + + recorded, err := os.ReadFile(calls) + require.NoError(t, err) + + // --now is what stops it, and stopping it is the part that matters. The + // unit is a oneshot with RemainAfterExit=yes, so it stays active after it + // has run, and deleting the file does not change that. A host provisioned + // again afterwards writes the unit back and starts it, systemd finds it + // already active and does nothing, and the agent never runs. Nothing + // reports an error, so the reinstall looks like it succeeded and the node + // simply never appears. + require.Contains(t, string(recorded), "disable --now "+goalstates.FirstBootBootstrapUnit, + "disabling without stopping leaves the unit active, so a later start is a no-op") +} + +// TestFirstBootBootstrapUnitAbsentIsSuccess covers every host not provisioned +// through Ignition, which is the common case. There is nothing to remove and +// nothing to report. +func TestFirstBootBootstrapUnitAbsentIsSuccess(t *testing.T) { + t.Parallel() + + require.NoError(t, removeFirstBootBootstrapUnitIn(t.Context(), discardLogger(), t.TempDir())) +} + +// TestInstallBootstrapBinaryInstallsWhereTheDaemonLooks covers the first host +// mutation of a bootstrap. +// +// PrepareHost is the earliest stage that writes anything, and it writes the +// daemon binary. It has to land at the path the rest of the install resolves, +// including an environment override, or VerifyDaemonInstalled looks elsewhere. +func TestInstallBootstrapBinaryInstallsWhereTheDaemonLooks(t *testing.T) { + installed := filepath.Join(t.TempDir(), "bin", "unbounded-agent") + t.Setenv(goalstates.EnvDaemonBinary, installed) + + require.NoError(t, InstallBootstrapBinary()) + + info, err := os.Stat(installed) + require.NoError(t, err, "binary must land at the resolved path") + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm()) +} + +// TestInstallBootstrapBinaryKeepsAnExistingBinary pins the retention rule: a +// host that already has a usable binary keeps it, so a repair does not replace +// the slot an upgrade activated. +func TestInstallBootstrapBinaryKeepsAnExistingBinary(t *testing.T) { + installed := filepath.Join(t.TempDir(), "bin", "unbounded-agent") + t.Setenv(goalstates.EnvDaemonBinary, installed) + + require.NoError(t, os.MkdirAll(filepath.Dir(installed), 0o755)) + require.NoError(t, os.WriteFile(installed, []byte("incumbent"), 0o755)) + require.NoError(t, InstallBootstrapBinary()) + + data, err := os.ReadFile(installed) + require.NoError(t, err) + assert.Equal(t, "incumbent", string(data), "an existing usable binary must be left alone") +} + +// TestInstallBootstrapBinaryReplacesAnUnusableBinary is the other half: a +// present but non-executable file is the state a half-finished install leaves +// behind, and repair has to be able to get past it. +func TestInstallBootstrapBinaryReplacesAnUnusableBinary(t *testing.T) { + installed := filepath.Join(t.TempDir(), "bin", "unbounded-agent") + t.Setenv(goalstates.EnvDaemonBinary, installed) + + require.NoError(t, os.MkdirAll(filepath.Dir(installed), 0o755)) + require.NoError(t, os.WriteFile(installed, []byte("not executable"), 0o644)) + require.NoError(t, InstallBootstrapBinary()) + + data, err := os.ReadFile(installed) + require.NoError(t, err) + assert.NotEqual(t, "not executable", string(data), "an unusable binary must be replaced") +} + +// TestRemoveAgentArtifactsRemovesTheRootLast runs the teardown against a +// temporary tree. On a migrated host the files are reached through the link +// the root is, so removing the root first would leave them all behind. +func TestRemoveAgentArtifactsRemovesTheRootLast(t *testing.T) { + t.Parallel() + + root := t.TempDir() + + var files []string + for _, name := range []string{"bin/unbounded-agent", "bin/unbounded-agent-current", "libexec/unbounded-localdns-network"} { + files = append(files, filepath.Join(root, "opt", "unbounded", name)) + } + + files = append(files, filepath.Join(root, "usr", "local", "bin", "unbounded-agent-install.sh")) + + for _, path := range files { + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, []byte("installed"), 0o644)) + } + + configDir := filepath.Join(root, "etc", "unbounded", "agent") + require.NoError(t, os.MkdirAll(configDir, 0o755)) + + rootRemovals := 0 + task := &removeAgentArtifacts{ + log: discardLogger(), + files: files, + dirs: []string{configDir}, + removeRoot: func() error { + rootRemovals++ + + for _, path := range files { + if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("%s is still present when the root is removed", path) + } + } + + return nil + }, + } + require.NoError(t, task.Do(t.Context())) + assert.Equal(t, 1, rootRemovals, "the root must be removed") + + _, err := os.Stat(configDir) + assert.ErrorIs(t, err, os.ErrNotExist, "config directory must be removed") + + // Removing an already-absent file is the ordinary case on a partially + // provisioned host, so a second pass has to succeed. + require.NoError(t, task.Do(t.Context()), "teardown must be repeatable") +} + +// TestRemoveAgentArtifactsIsBuiltFromTheHostRoot pins the wiring between the +// exported constructor and the swept layout, which the test above cannot see +// because it supplies the list itself. +func TestRemoveAgentArtifactsIsBuiltFromTheHostRoot(t *testing.T) { + t.Parallel() + + task, ok := RemoveAgentArtifacts(discardLogger()).(*removeAgentArtifacts) + require.True(t, ok) + + assert.Contains(t, task.files, filepath.Join(goalstates.ResolveHostPaths().BinDir, "unbounded-agent")) + assert.Contains(t, task.files, "/usr/local/bin/unbounded-agent", "teardown must not depend on the migration having run") + assert.Contains(t, task.files, "/usr/local/bin/unbounded-agent-install.sh") + assert.Contains(t, task.dirs, goalstates.AgentConfigDir) + assert.NotNil(t, task.removeRoot) +} + +// TestRemoveOwnedFileSkipsTheUnlinkWhenTheFileIsAbsent covers the failure that +// stopped a reset on an immutable host. +// +// Teardown removes the installer scripts from the legacy root on every host, +// and on such a host that root is read-only. Unlinking a path that is not there returns +// EROFS rather than ENOENT, because the kernel checks the parent directory for +// write permission before it resolves the final component, so the ENOENT the +// old code tolerated never arrived and a reset failed over a file that had +// never existed. +// +// The unlink is asserted not to happen at all, rather than its error being +// tolerated. An unwritable directory is not a substitute: unlink returns ENOENT +// there, so a test built that way passes against the original bug. +func TestRemoveOwnedFileSkipsTheUnlinkWhenTheFileIsAbsent(t *testing.T) { + t.Parallel() + + called := false + remove := func(string) error { + called = true + + return syscall.EROFS + } + absent := func(string) (os.FileInfo, error) { return nil, os.ErrNotExist } + + require.NoError(t, removeOwnedFileWith("/usr/local/bin/unbounded-agent", absent, remove)) + assert.False(t, called, "an absent file must not be unlinked, whatever the filesystem would say") +} + +// TestRemoveOwnedFileReportsAFailedUnlink keeps the tolerance narrow. A file +// that is present and cannot be removed is still an error, because a teardown +// reporting success would leave an installation the next bootstrap refuses. +func TestRemoveOwnedFileReportsAFailedUnlink(t *testing.T) { + t.Parallel() + + present := func(string) (os.FileInfo, error) { return nil, nil } //nolint:nilnil // Only presence is read. + remove := func(string) error { return syscall.EROFS } + + err := removeOwnedFileWith("/usr/local/bin/unbounded-agent", present, remove) + + require.Error(t, err) + assert.Contains(t, err.Error(), "/usr/local/bin/unbounded-agent") +} + +// TestRemoveOwnedFileRemovesADanglingSymlink pins why the check uses Lstat. +// +// A dangling link is exactly what a partial install leaves behind, and it is +// still a file the agent owns. Stat would follow it, find nothing, and leave it +// on the host. +func TestRemoveOwnedFileRemovesADanglingSymlink(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + link := filepath.Join(dir, "unbounded-agent-current") + require.NoError(t, os.Symlink(filepath.Join(dir, "gone"), link)) + require.NoError(t, removeOwnedFile(link)) + + _, err := os.Lstat(link) + assert.ErrorIs(t, err, os.ErrNotExist, "a dangling link must be removed, not skipped") +} diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index 61a6cd435..d3963a33d 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -18,6 +18,7 @@ import ( "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/hostroot" "github.com/Azure/unbounded/pkg/agent/phases" "github.com/Azure/unbounded/pkg/agent/phases/reset" ) @@ -90,7 +91,25 @@ func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.S return err } - return durableReset(ctx, store, inner, []string{"/etc", "/var/lib/machines", "/usr/local", store.Root()}, unix.Syncfs) + return durableReset(ctx, store, inner, teardownSyncPaths(store.Root()), unix.Syncfs) +} + +// teardownSyncPaths returns the directories whose filesystems have to be +// persisted for a teardown to survive a crash part way through: those holding +// the agent's files, including the directory the host root link is in on a +// migrated host, and the legacy root, where the installer scripts are. They are +// resolved now, while the host root still leads to the files. A path that does +// not exist is not a problem, because durableReset walks up to the nearest +// existing ancestor before opening anything. +func teardownSyncPaths(storeRoot string) []string { + return []string{ + "/etc", + "/var/lib/machines", + filepath.Dir(hostroot.Path), + hostroot.Resolve(), + hostroot.LegacyPath, + storeRoot, + } } func stopRecoveryUnit(ctx context.Context, log *slog.Logger) error { @@ -168,6 +187,10 @@ func resetResources(log *slog.Logger) phases.Task { reset.RemoveBPFFSMount(log, goalstates.NSpawnMachineKube2), ), reset.CleanupNetwork(log), + // Before the artifacts, so a failure here stops the reset while the + // host is still recognizably installed. A unit that survived a reset + // would bootstrap the host again on the next boot. + RemoveFirstBootBootstrapUnit(log), RemoveAgentArtifacts(log), reset.ReloadSystemd(log), ) diff --git a/cmd/agent/internal/daemon/reset_test.go b/cmd/agent/internal/daemon/reset_test.go index 2ae2b4b5e..16676030c 100644 --- a/cmd/agent/internal/daemon/reset_test.go +++ b/cmd/agent/internal/daemon/reset_test.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/require" "github.com/Azure/unbounded/cmd/agent/internal/installstate" + "github.com/Azure/unbounded/pkg/agent/hostroot" ) func TestResetResourcesIncludesBPFFSMountCleanup(t *testing.T) { @@ -137,3 +138,41 @@ func TestTeardownKeepsAReadableRecord(t *testing.T) { require.Equal(t, "machine-1", r.MachineName) require.Equal(t, "fingerprint-1", r.ConfigFingerprint) } + +// TestResetRemovesTheFirstBootUnitBeforeArtifacts pins that reset actually runs +// the removal, not merely that the removal works. +// +// The unit runs on every boot and decides there is nothing to do from the +// ownership record that reset is about to delete. Left behind, it would find an +// uninstalled host and bootstrap it again, undoing the reset with nothing +// reporting why. Ordering it before the artifacts means a failure stops the +// reset while the host is still recognizably installed. +func TestResetRemovesTheFirstBootUnitBeforeArtifacts(t *testing.T) { + t.Parallel() + + taskName := resetResources(slog.New(slog.DiscardHandler)).Name() + + assert.Contains(t, taskName, "remove-first-boot-unit", + "reset must remove the Ignition bootstrap unit or the host re-bootstraps on next boot") + assert.Less(t, + strings.Index(taskName, "remove-first-boot-unit"), + strings.Index(taskName, "remove-agent-artifacts"), + "a failure here must stop the reset while the host is still recognizably installed") +} + +// TestTeardownSyncPathsCoverBothRoots pins what a teardown makes durable. +// +// A crash during reset could otherwise leave files the teardown had already +// removed present on the next boot, and those are exactly the files whose +// absence lets the host be provisioned again. On a migrated host they are under +// the legacy root and the link to it is in /opt; on every host the installer +// scripts are under the legacy root. +func TestTeardownSyncPathsCoverBothRoots(t *testing.T) { + t.Parallel() + + paths := teardownSyncPaths("/var/lib/unbounded") + + for _, want := range []string{"/etc", "/var/lib/machines", "/opt", hostroot.Resolve(), "/usr/local", "/var/lib/unbounded"} { + assert.Contains(t, paths, want) + } +} diff --git a/cmd/kubectl-unbounded/app/ignition.go b/cmd/kubectl-unbounded/app/ignition.go new file mode 100644 index 000000000..1237cba20 --- /dev/null +++ b/cmd/kubectl-unbounded/app/ignition.go @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "encoding/base64" + "fmt" + "net/url" + "strings" +) + +// Ignition configuration types, covering only the subset this command emits. +// +// These are hand-written rather than taken from github.com/coreos/ignition to +// avoid a dependency carrying the whole specification for the handful of fields +// used here. The schema version is pinned and asserted by tests. +const ignitionSpecVersion = "3.4.0" + +// File modes are serialized as decimal integers in an Ignition config. +const ( + ignitionModeConfig = 0o600 + ignitionModeScript = 0o755 + ignitionModeDir = 0o755 +) + +type ignitionConfig struct { + Ignition ignitionVersion `json:"ignition"` + Storage *ignitionStorage `json:"storage,omitempty"` + Systemd *ignitionSystemd `json:"systemd,omitempty"` +} + +type ignitionVersion struct { + Version string `json:"version"` +} + +type ignitionStorage struct { + Directories []ignitionDirectory `json:"directories,omitempty"` + Files []ignitionFile `json:"files,omitempty"` +} + +// ignitionDirectory declares a directory Ignition creates before writing files +// into it. Ignition creates parents implicitly, so this exists to pin the mode +// of the agent's bin directory rather than to make the write succeed. +type ignitionDirectory struct { + Path string `json:"path"` + Mode int `json:"mode,omitempty"` +} + +type ignitionFile struct { + Path string `json:"path"` + Mode int `json:"mode,omitempty"` + Overwrite *bool `json:"overwrite,omitempty"` + Contents ignitionContents `json:"contents"` +} + +type ignitionContents struct { + Source string `json:"source"` + Verification *ignitionVerification `json:"verification,omitempty"` +} + +type ignitionVerification struct { + // Hash is "-", for example "sha256-abc123...". + Hash string `json:"hash"` +} + +type ignitionSystemd struct { + Units []ignitionUnit `json:"units,omitempty"` +} + +type ignitionUnit struct { + Name string `json:"name"` + Enabled *bool `json:"enabled,omitempty"` + Contents string `json:"contents,omitempty"` +} + +// ignitionDataURL encodes content as a data URL, which is how Ignition carries +// inline file contents. +func ignitionDataURL(content string) string { + return "data:;base64," + base64.StdEncoding.EncodeToString([]byte(content)) +} + +// ignitionRemoteFetchable reports whether Ignition can fetch a source itself. +// +// Ignition can retrieve http, https, tftp, s3, arn and gs. It also understands +// data, but that is inline content rather than a fetch, so it is not listed +// below: a caller asking whether a source can be fetched remotely wants a no +// for a value it already holds. +// +// Ignition does not understand oci, which the agent resolves through its own +// artifact source. A source Ignition cannot fetch has to be left to the agent, +// which means the file lands after dbus has already started. +func ignitionRemoteFetchable(source string) bool { + parsed, err := url.Parse(strings.TrimSpace(source)) + if err != nil { + return false + } + + switch parsed.Scheme { + case "http", "https", "tftp", "s3", "arn", "gs": + return true + default: + return false + } +} + +// ignitionHashFromSHA256 converts a hex digest into the form Ignition expects. +func ignitionHashFromSHA256(hex string) (string, error) { + trimmed := strings.TrimSpace(hex) + + // Accept a plain digest or the first field of sha256sum output. + if fields := strings.Fields(trimmed); len(fields) > 0 { + trimmed = fields[0] + } + + if len(trimmed) != 64 { + return "", fmt.Errorf("sha256 digest must be 64 hex characters, got %d", len(trimmed)) + } + + for _, r := range trimmed { + isHex := (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') + if !isHex { + return "", fmt.Errorf("sha256 digest contains a non-hex character %q", r) + } + } + + return "sha256-" + strings.ToLower(trimmed), nil +} diff --git a/cmd/kubectl-unbounded/app/ignition_test.go b/cmd/kubectl-unbounded/app/ignition_test.go new file mode 100644 index 000000000..191db1ce2 --- /dev/null +++ b/cmd/kubectl-unbounded/app/ignition_test.go @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "encoding/base64" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestIgnitionDataURLRoundTrips covers how inline file contents reach the host. +// Ignition reads them from a data URL, so anything lost in the encoding is lost +// silently: the file appears, with the wrong bytes in it. +func TestIgnitionDataURLRoundTrips(t *testing.T) { + t.Parallel() + + for _, content := range []string{ + "", + "plain", + "{\n \"MachineName\": \"kube1\"\n}\n", + "trailing newline\n", + "unicode: \u00e9\u00e8\u00ea and emoji bytes", + "null\x00byte", + } { + t.Run(strings.SplitN(content, "\n", 2)[0], func(t *testing.T) { + t.Parallel() + + url := ignitionDataURL(content) + require.True(t, strings.HasPrefix(url, "data:;base64,"), "got %q", url) + + decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(url, "data:;base64,")) + require.NoError(t, err) + require.Equal(t, content, string(decoded)) + }) + } +} + +// TestIgnitionRemoteFetchable pins which sources Ignition can retrieve itself. +// +// This decides where a file lands in the boot. A fetchable source is written +// before dbus starts; anything else has to wait for the agent, which is after. +// Reading it the wrong way round produces a config Ignition rejects, or a file +// that silently arrives too late to be useful. +func TestIgnitionRemoteFetchable(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + source string + want bool + }{ + {"https://example.test/unbounded-agent", true}, + {"http://example.test/unbounded-agent", true}, + {"tftp://example.test/unbounded-agent", true}, + {"s3://bucket/unbounded-agent", true}, + {"arn:aws:s3:::bucket/unbounded-agent", true}, + {"gs://bucket/unbounded-agent", true}, + {" https://example.test/spaced ", true}, + + // oci is the one that matters: it is the agent's own artifact scheme, + // and Ignition has no idea what to do with it. + {"oci://ghcr.io/azure/unbounded-agent:v1", false}, + {"file:///tmp/unbounded-agent", false}, + {"ftp://example.test/unbounded-agent", false}, + {"/usr/local/bin/unbounded-agent", false}, + {"", false}, + {"://not a url", false}, + } { + t.Run(tc.source, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.want, ignitionRemoteFetchable(tc.source)) + }) + } +} + +// TestIgnitionHashFromSHA256 covers the digest conversion, including the +// sha256sum shape an operator is most likely to paste in. +func TestIgnitionHashFromSHA256(t *testing.T) { + t.Parallel() + + const digest = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + t.Run("plain digest", func(t *testing.T) { + t.Parallel() + + got, err := ignitionHashFromSHA256(digest) + require.NoError(t, err) + require.Equal(t, "sha256-"+digest, got) + }) + + t.Run("sha256sum output", func(t *testing.T) { + t.Parallel() + + got, err := ignitionHashFromSHA256(digest + " unbounded-agent\n") + require.NoError(t, err) + require.Equal(t, "sha256-"+digest, got) + }) + + t.Run("uppercase is normalized", func(t *testing.T) { + t.Parallel() + + got, err := ignitionHashFromSHA256(strings.ToUpper(digest)) + require.NoError(t, err) + require.Equal(t, "sha256-"+digest, got, "Ignition compares the hash as written") + }) + + for _, tc := range []struct{ name, input string }{ + {"empty", ""}, + {"too short", digest[:63]}, + {"too long", digest + "0"}, + {"non-hex", strings.Replace(digest, "9", "z", 1)}, + } { + t.Run("rejects "+tc.name, func(t *testing.T) { + t.Parallel() + + _, err := ignitionHashFromSHA256(tc.input) + require.Error(t, err, "a malformed digest must fail here, not on the host at first boot") + }) + } +} + +// TestIgnitionConfigOmitsEmptySections pins that the emitted document contains +// only what was asked for. +// +// Ignition validates the whole config before acting on any of it, so an empty +// section serialized as null or [] can reject a config that is otherwise fine, +// again on a host with nothing available to say so. +func TestIgnitionConfigOmitsEmptySections(t *testing.T) { + t.Parallel() + + encoded, err := json.Marshal(ignitionConfig{Ignition: ignitionVersion{Version: ignitionSpecVersion}}) + require.NoError(t, err) + + require.JSONEq(t, `{"ignition":{"version":"3.4.0"}}`, string(encoded)) + require.NotContains(t, string(encoded), "storage") + require.NotContains(t, string(encoded), "systemd") +} + +// TestIgnitionFileModesSerializeAsDecimal covers a trap in the format: Ignition +// file modes are decimal integers, and Go's octal literals are easy to read as +// if they were being emitted verbatim. +// +// A mode written as 600 rather than 0o600 is 0o1130 on disk, which for the +// agent config means credentials readable by everyone. +func TestIgnitionFileModesSerializeAsDecimal(t *testing.T) { + t.Parallel() + + encoded, err := json.Marshal(ignitionFile{ + Path: "/etc/unbounded/agent/config.json", + Mode: ignitionModeConfig, + Contents: ignitionContents{Source: ignitionDataURL("{}")}, + }) + require.NoError(t, err) + + // 0o600 is 384 decimal. Asserting the number rather than the constant is + // the point: it is what a reader of the emitted config would see. + require.Contains(t, string(encoded), `"mode":384`) +} diff --git a/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go b/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go index 6f0c740ec..53ac5cf97 100644 --- a/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go +++ b/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go @@ -15,6 +15,7 @@ import ( "log/slog" "net/url" "os" + "path" "path/filepath" "strings" "text/template" @@ -24,12 +25,15 @@ import ( "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/utils/ptr" unboundedv1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" "github.com/Azure/unbounded/internal/cloudprovider" "github.com/Azure/unbounded/internal/kube" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/config" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/hostroot" ) //go:embed assets/node-bootstrap/script.sh @@ -47,6 +51,12 @@ const ( // variantCloudInit produces a cloud-init user-data document. variantCloudInit bootstrapVariant = "cloud-init" + + // variantIgnition produces an Ignition config. It is the only mechanism + // image-based hosts such as Azure Container Linux consume: they ship no + // cloud-init at all, so a cloud-init payload passed as user data is never + // acted on and nothing reports an error. + variantIgnition bootstrapVariant = "ignition" ) func parseBootstrapVariant(s string) (bootstrapVariant, error) { @@ -55,8 +65,10 @@ func parseBootstrapVariant(s string) (bootstrapVariant, error) { return variantScript, nil case variantCloudInit: return variantCloudInit, nil + case variantIgnition: + return variantIgnition, nil default: - return "", fmt.Errorf("unknown variant %q (valid: script, cloud-init)", s) + return "", fmt.Errorf("unknown variant %q (valid: script, cloud-init, ignition)", s) } } @@ -103,8 +115,21 @@ type manualBootstrapHandler struct { // agentURL is a fully qualified override for the unbounded-agent download // URL. When set it takes precedence over agentVersion and agentBaseURL. + // + // The ignition variant requires it, and requires it to name the bare agent + // binary rather than the release tarball: Ignition fetches files, it does + // not extract archives. agentURL string + // agentSHA256 is the expected digest of the agent binary. Required by the + // ignition variant, which fetches the binary without a script that could + // verify it afterwards. + agentSHA256 string + + // agentHash is agentSHA256 in Ignition's form, set by validate for the + // ignition variant. + agentHash string + // agentBaseURL overrides the base URL used to construct the download URL // for the unbounded-agent. Useful for self-hosted release mirrors. Must // follow the same layout as GitHub releases @@ -186,6 +211,8 @@ func (h *manualBootstrapHandler) execute(ctx context.Context) error { switch bootstrapVariant(h.variant) { case variantCloudInit: output, err = h.renderCloudInit(cfg) + case variantIgnition: + output, err = h.renderIgnition(cfg) default: output, err = h.renderScript(cfg) } @@ -343,11 +370,54 @@ func parseAdditionalHostDevice(value string) (string, error) { return value, nil } +// validateIgnitionInput checks the rules that only apply to the Ignition +// variant, and returns the agent digest in Ignition's form. It runs before any +// cluster contact, so a mistake is reported without waiting on the cluster. +// +// Every input is required rather than defaulted, because this variant has no +// shell to fall back on. Ignition declares state; it cannot resolve a version, +// detect an architecture, or extract an archive at boot, so the artifact has to +// be named exactly, and a host that finds out otherwise has no way to report it. +func (h *manualBootstrapHandler) validateIgnitionInput() (string, error) { + source := strings.TrimSpace(h.agentURL) + if source == "" { + return "", fmt.Errorf("--agent-url is required with --variant %s, and must point at the bare agent binary rather than the release tarball, because Ignition cannot extract an archive", variantIgnition) + } + + if !ignitionRemoteFetchable(source) { + return "", fmt.Errorf("--agent-url %q cannot be fetched by Ignition; use an http, https, tftp, s3, arn, or gs URL", source) + } + + if isEmpty(h.agentSHA256) { + return "", fmt.Errorf("--agent-sha256 is required with --variant %s; the digest for each release binary is published in checksums.txt", variantIgnition) + } + + hash, err := ignitionHashFromSHA256(strings.TrimSpace(h.agentSHA256)) + if err != nil { + return "", fmt.Errorf("invalid --agent-sha256: %w", err) + } + + return hash, nil +} + func (h *manualBootstrapHandler) validate() error { if isEmpty(h.siteName) { return errors.New("site name is required") } + // Checked here, before any cluster contact, so a missing flag is reported + // immediately rather than after connecting and resolving a site. An + // unparseable variant is reported by parseBootstrapVariant later; this only + // adds rules for the one variant that has them. + if variant, err := parseBootstrapVariant(h.variant); err == nil && variant == variantIgnition { + hash, err := h.validateIgnitionInput() + if err != nil { + return err + } + + h.agentHash = hash + } + // The machine name is optional. When omitted, the unbounded-agent resolves // it at startup from the AGENT_MACHINE_NAME environment variable or the host // hostname, which lets a single bootstrap payload be reused across many @@ -481,6 +551,7 @@ func (h *manualBootstrapHandler) buildAgentConfig(ctx context.Context) (*provisi }) cfg.Kubelet.NodeIP = strings.TrimSpace(h.nodeIP) + if source := strings.TrimSpace(h.offlineArtifactsSource); source != "" { cfg.OfflineArtifacts = &provision.AgentOfflineArtifacts{Source: source} } @@ -710,9 +781,10 @@ Examples: cmd.Flags().StringArrayVar(&handler.additionalHostMounts, "additional-host-mount", nil, `Extra host bind-mount for the nspawn machine in "source[:target][:ro]" format (can be repeated). target defaults to source; append :ro for a read-only mount`) cmd.Flags().StringArrayVar(&handler.additionalHostDevices, "additional-host-device", nil, `Extra host device node or systemd device group specifier to expose in the nspawn machine (can be repeated). Accepts absolute /dev/* paths and systemd device group specifiers like char-input or block-*`) cmd.Flags().StringVar(&handler.kubernetesVersion, "kubernetes-version", "", "Override the Kubernetes version (default: auto-detected from API server)") - cmd.Flags().StringVar(&handler.variant, "variant", "script", "Output format: script or cloud-init") + cmd.Flags().StringVar(&handler.variant, "variant", "script", "Output format: script, cloud-init, or ignition") cmd.Flags().StringVar(&handler.agentVersion, "agent-version", "", "Pin the unbounded-agent release tag to download on the host (default: latest GitHub release)") - cmd.Flags().StringVar(&handler.agentURL, "agent-url", "", "Fully qualified download URL for the unbounded-agent tarball (overrides --agent-version and --agent-base-url)") + cmd.Flags().StringVar(&handler.agentURL, "agent-url", "", "Fully qualified download URL for the unbounded-agent tarball (overrides --agent-version and --agent-base-url). With --variant ignition this must name the bare binary, not the tarball") + cmd.Flags().StringVar(&handler.agentSHA256, "agent-sha256", "", "SHA-256 digest of the agent binary, published in checksums.txt. Required with --variant ignition") cmd.Flags().StringVar(&handler.agentBaseURL, "agent-base-url", "", "Base URL for unbounded-agent release downloads (default: https://github.com/Azure/unbounded/releases). Use this to self-host or mirror release assets") // Rootfs binary download overrides. See `kubectl unbounded machine register --help` @@ -803,3 +875,159 @@ func resolveBootstrapToken(ctx context.Context, logger *slog.Logger, kubeCli kub return nil, fmt.Errorf("no bootstrap token found for site %q and no tokens available in the cluster (run 'kubectl unbounded site init' first)", siteName) } + +// Paths the Ignition variant writes on the target host. The config path is the +// one `unbounded-agent start` reads from UNBOUNDED_AGENT_CONFIG_FILE. +const ( + ignitionAgentConfigPath = "/etc/unbounded/agent/config.json" + ignitionAgentBinaryName = "unbounded-agent" +) + +// renderIgnition emits an Ignition config that provisions the host with no +// shell and no operator present. +// +// Ignition is declarative and runs from the initramfs, so everything it writes +// is in place before any service starts. That is what lets the agent config, +// the agent binary and the bootstrap unit all be present on the first boot +// rather than fetched by something running on the host. +func (h *manualBootstrapHandler) renderIgnition(cfg *provision.UnboundedAgentConfig) (string, error) { + configJSON, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return "", fmt.Errorf("marshaling agent config: %w", err) + } + + config := ignitionConfig{ + Ignition: ignitionVersion{Version: ignitionSpecVersion}, + Storage: &ignitionStorage{ + Directories: []ignitionDirectory{{ + Path: ignitionAgentBinDir(), + Mode: ignitionModeDir, + }}, + Files: []ignitionFile{ + { + Path: ignitionAgentConfigPath, + Mode: ignitionModeConfig, + Overwrite: ptr.To(true), + Contents: ignitionContents{Source: ignitionDataURL(string(configJSON) + "\n")}, + }, + h.ignitionAgentBinaryFile(cfg), + }, + }, + Systemd: &ignitionSystemd{Units: []ignitionUnit{{ + Name: goalstates.FirstBootBootstrapUnit, + Enabled: ptr.To(true), + Contents: h.ignitionBootstrapUnitContents(cfg), + }}}, + } + + rendered, err := json.MarshalIndent(config, "", " ") + if err != nil { + return "", fmt.Errorf("marshaling ignition config: %w", err) + } + + return string(rendered) + "\n", nil +} + +// ignitionAgentBinDir returns the directory the agent binary is placed in, +// under the host root. It is not resolved: this runs on the workstation, and +// the host is a new installation whose host root is a real directory. +func ignitionAgentBinDir() string { + return path.Join(hostroot.Path, "bin") +} + +// ignitionAgentBinaryFile fetches the agent binary straight to its final +// location, verified against the digest validate parsed. +func (h *manualBootstrapHandler) ignitionAgentBinaryFile(cfg *provision.UnboundedAgentConfig) ignitionFile { + return ignitionFile{ + Path: ignitionAgentBinDir() + "/" + ignitionAgentBinaryName, + Mode: ignitionModeScript, + Overwrite: ptr.To(true), + Contents: ignitionContents{ + Source: strings.TrimSpace(h.agentURL), + Verification: &ignitionVerification{Hash: h.agentHash}, + }, + } +} + +// ignitionBootstrapUnitContents renders the oneshot unit that bootstraps the +// agent on first boot. +// +// The unit runs the agent directly rather than a shell script. Ignition has +// already placed and verified the binary, so a script here would only +// re-implement that imperatively. +// +// It carries no completion condition, and so runs on every boot. That is +// deliberate. A condition needs a marker file, and a marker is a second record +// of completion that can disagree with the ownership record the agent already +// keeps; the agent's own admission answers the same question from the record, +// which is written before the first host mutation and therefore cannot be +// missing on a host that started installing. Both commands below return +// immediately once that record says the installation is complete: preflight +// reports an empty result and start verifies the daemon and repairs it if it +// is not running, neither resolving artifacts nor touching the network. The +// cost is two short-lived processes per boot, and the benefit is that a node +// whose daemon was stopped or damaged comes back on reboot. +func (h *manualBootstrapHandler) ignitionBootstrapUnitContents(cfg *provision.UnboundedAgentConfig) string { + binary := ignitionAgentBinDir() + "/" + ignitionAgentBinaryName + + var b strings.Builder + + b.WriteString("[Unit]\n") + b.WriteString("Description=Bootstrap the unbounded agent\n") + b.WriteString("Wants=network-online.target\n") + // The agent downloads the node rootfs and the Kubernetes, CRI and CNI + // binaries, so it needs the network even though Ignition already fetched + // the agent itself. Ordering after systemd-sysext keeps any extension + // merged before the agent runs. + // + // On later boots the daemon unit starts in the same transaction, and is + // only active once the nspawn machine is up. Without ordering after it, + // start finds it not yet running and repairs a healthy host. On first boot + // the daemon unit does not exist yet, so this orders nothing. + b.WriteString("After=network-online.target nss-lookup.target systemd-sysext.service " + goalstates.DaemonUnit + "\n") + // Assert rather than Condition. A failed condition is not an error: systemd + // marks the unit inactive and moves on, so a host whose binary Ignition + // never placed sits there looking healthy and never bootstraps. A failed + // assertion puts the unit in the failed state, where `systemctl status` and + // any watchdog can see it. Neither one starts the service, so this changes + // only whether the reason is visible. + b.WriteString("AssertPathExists=" + binary + "\n") + // Retry indefinitely rather than giving up after systemd's default start + // limit. Bootstrap has no later opportunity to run, so a burst of early + // failures must not permanently disable it. + b.WriteString("StartLimitIntervalSec=0\n\n") + + b.WriteString("[Service]\n") + b.WriteString("Type=oneshot\n") + b.WriteString("RemainAfterExit=yes\n") + // network-online.target only means a link is configured, not that DNS + // resolves. On a first boot the agent can start before systemd-resolved is + // answering and fail with an unresolved host, so retry rather than ordering + // against something that does not carry that guarantee. Verified on systemd + // 255 that Type=oneshot honors Restart=. + b.WriteString("Restart=on-failure\n") + b.WriteString("RestartSec=10s\n") + // Back off towards a five minute ceiling instead of retrying every ten + // seconds forever. With no start limit to stop it, a host that cannot reach + // the network would otherwise spawn the agent several thousand times a day, + // and the journal that would explain why scrolls away. + // + // These need systemd 254. Older versions log an unknown key and carry on + // with the fixed RestartSec above, which is the behavior this replaces, so + // nothing is lost where they are not understood. + b.WriteString("RestartSteps=10\n") + b.WriteString("RestartMaxDelaySec=300\n") + // `unbounded-agent start` has no --config flag and reads this variable. + b.WriteString("Environment=UNBOUNDED_AGENT_CONFIG_FILE=" + ignitionAgentConfigPath + "\n") + + // Preflight runs as ExecStartPre so a failure is reported against this unit + // before any host mutation, and shows up in its status rather than being + // buried in a script's output. + b.WriteString("ExecStartPre=" + binary + " preflight\n") + b.WriteString("ExecStart=" + binary + " start\n\n") + + b.WriteString("[Install]\n") + b.WriteString("WantedBy=multi-user.target\n") + + return b.String() +} diff --git a/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go b/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go index 2ced9a19d..e6c58f14b 100644 --- a/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go +++ b/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go @@ -9,6 +9,7 @@ import ( "encoding/json" "os" "os/exec" + "path/filepath" "strings" "testing" @@ -20,6 +21,7 @@ import ( "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/config" + "github.com/Azure/unbounded/pkg/agent/goalstates" ) // --------------------------------------------------------------------------- @@ -1189,3 +1191,236 @@ func TestManualBootstrapHandler_BuildAgentConfig_AdditionalHostDevices(t *testin require.Equal(t, []string{"/dev/uinput", "char-input"}, cfg.AdditionalHostDevices) } + +// ignitionTestConfig returns an agent config shaped like one the command would +// build. +func ignitionTestConfig() *provision.UnboundedAgentConfig { + return &provision.UnboundedAgentConfig{ + AgentConfig: provision.AgentConfig{ + MachineName: "test-node", + Cluster: provision.AgentClusterConfig{ + CaCertBase64: "dGVzdA==", + ClusterDNS: "10.0.0.10", + Version: "v1.30.0", + }, + Kubelet: provision.AgentKubeletConfig{ + ApiServer: "https://api-server:6443", + Auth: provision.KubeletAuthInfo{BootstrapToken: "abc123.0123456789abcdef"}, + }, + }, + } +} + +const ignitionTestDigest = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + +// ignitionTestHandler returns a handler as validate leaves it. +func ignitionTestHandler() *manualBootstrapHandler { + return &manualBootstrapHandler{ + logger: discardLogger(), + agentURL: "https://example.test/unbounded-agent-linux-amd64", + agentSHA256: ignitionTestDigest, + agentHash: "sha256-" + ignitionTestDigest, + } +} + +// TestRenderIgnitionPlacesEverythingBeforeFirstBoot covers the property the +// whole variant exists for: on a host with no shell and no operator, every file +// the agent needs is already present when the unit starts. +func TestRenderIgnitionPlacesEverythingBeforeFirstBoot(t *testing.T) { + t.Parallel() + + out, err := ignitionTestHandler().renderIgnition(ignitionTestConfig()) + require.NoError(t, err) + + var cfg ignitionConfig + require.NoError(t, json.Unmarshal([]byte(out), &cfg), "emitted document must be valid JSON") + require.Equal(t, ignitionSpecVersion, cfg.Ignition.Version) + + require.NotNil(t, cfg.Storage) + + paths := map[string]ignitionFile{} + for _, f := range cfg.Storage.Files { + paths[f.Path] = f + } + + agentConfig, ok := paths[ignitionAgentConfigPath] + require.True(t, ok, "the agent config must be written, got %v", paths) + require.Equal(t, ignitionModeConfig, agentConfig.Mode, "the agent config carries a bootstrap token") + + binary, ok := paths["/opt/unbounded/bin/unbounded-agent"] + require.True(t, ok, "the agent binary must land under the host root, got %v", paths) + require.Equal(t, ignitionModeScript, binary.Mode) + require.Equal(t, "https://example.test/unbounded-agent-linux-amd64", binary.Contents.Source) + require.NotNil(t, binary.Contents.Verification, "an unattended host must not accept whatever the URL returns") + require.Equal(t, "sha256-"+ignitionTestDigest, binary.Contents.Verification.Hash) + + require.NotNil(t, cfg.Systemd) + require.Len(t, cfg.Systemd.Units, 1) + require.Equal(t, goalstates.FirstBootBootstrapUnit, cfg.Systemd.Units[0].Name) + require.NotNil(t, cfg.Systemd.Units[0].Enabled) + require.True(t, *cfg.Systemd.Units[0].Enabled, "an unenabled unit never runs and nothing reports it") +} + +// TestRenderIgnitionUsesTheHostRoot pins that every host-side path is under the +// host root. On a host that mounts /usr read-only, Ignition cannot write under +// /usr/local, and a unit pointing there could not start. +func TestRenderIgnitionUsesTheHostRoot(t *testing.T) { + t.Parallel() + + out, err := ignitionTestHandler().renderIgnition(ignitionTestConfig()) + require.NoError(t, err) + + var cfg ignitionConfig + require.NoError(t, json.Unmarshal([]byte(out), &cfg)) + require.NotNil(t, cfg.Storage) + require.Len(t, cfg.Storage.Directories, 1) + require.Equal(t, "/opt/unbounded/bin", cfg.Storage.Directories[0].Path) + + require.NotContains(t, out, "/usr/local", "nothing may be written or run from the legacy root") +} + +// TestIgnitionBootstrapUnitRunsOnEveryBoot pins the decision not to carry a +// completion condition. +// +// A condition needs a marker file, which is a second record of completion that +// can disagree with the ownership record the agent already keeps. Instead the +// unit runs every boot and the agent's own admission answers the question, +// returning immediately once the record says the installation is complete. The +// benefit is that a node whose daemon was stopped comes back on reboot. +func TestIgnitionBootstrapUnitRunsOnEveryBoot(t *testing.T) { + t.Parallel() + + unit := ignitionTestHandler().ignitionBootstrapUnitContents(ignitionTestConfig()) + + require.NotContains(t, unit, "ConditionPathExists=!", + "a completion marker would be a second source of truth beside the ownership record") + + // The one check that stays guards against running a binary Ignition failed + // to place. It asserts rather than conditions: a failed condition leaves the + // unit inactive and unremarkable, so a host that never bootstrapped would + // look no different from one that had nothing to do. + require.Contains(t, unit, "AssertPathExists=/opt/unbounded/bin/unbounded-agent") + require.NotContains(t, unit, "ConditionPathExists=/opt/unbounded/bin/unbounded-agent", + "a missing agent binary must fail visibly rather than skip silently") + + require.Contains(t, unit, "WantedBy=multi-user.target", "the unit has to be started on every boot for this to work") +} + +// TestIgnitionBootstrapUnitSurvivesEarlyBootRaces covers two failures seen on +// real hardware rather than reasoned about. +// +// network-online.target means a link is configured, not that DNS resolves: a +// unit can start in the same second the target is reached while +// systemd-resolved is still coming up, and fail on an unresolved host. And +// bootstrap has no later opportunity to run, so systemd's default start limit +// would turn a burst of early failures into a permanently disabled unit. +func TestIgnitionBootstrapUnitSurvivesEarlyBootRaces(t *testing.T) { + t.Parallel() + + unit := ignitionTestHandler().ignitionBootstrapUnitContents(ignitionTestConfig()) + + require.Contains(t, unit, "Restart=on-failure", "DNS may not answer yet on the first attempt") + require.Contains(t, unit, "StartLimitIntervalSec=0", "bootstrap gets no second chance if systemd gives up on it") + + // Retrying forever is the point, but retrying every ten seconds forever is + // not: with no start limit to stop it, an unreachable network would spawn + // the agent thousands of times a day and bury the reason in the journal. + require.Contains(t, unit, "RestartSteps=10") + require.Contains(t, unit, "RestartMaxDelaySec=300") + require.Contains(t, unit, "RestartSec=10s", "the first retry stays prompt") + require.Contains(t, unit, "Type=oneshot") + require.Contains(t, unit, "After=network-online.target nss-lookup.target systemd-sysext.service "+goalstates.DaemonUnit+"\n", + "on a reboot, start must not find the daemon still starting and repair it") + + // start reads the config path from the environment; it has no flag for it. + require.Contains(t, unit, "Environment=UNBOUNDED_AGENT_CONFIG_FILE="+ignitionAgentConfigPath) + + // Preflight runs before any host mutation and reports against this unit. + require.Contains(t, unit, "ExecStartPre=/opt/unbounded/bin/unbounded-agent preflight") + require.Contains(t, unit, "ExecStart=/opt/unbounded/bin/unbounded-agent start") +} + +// TestValidateRejectsIgnitionInputBeforeContactingTheCluster covers where the +// Ignition flag rules are enforced, not just that they are. +// +// validate runs before any Kubernetes client is built. Leaving these checks to +// the renderer meant an operator who forgot --agent-sha256 waited for a cluster +// connection and a site lookup before being told about a flag, and got that +// answer only if the connection succeeded at all. +func TestValidateRejectsIgnitionInputBeforeContactingTheCluster(t *testing.T) { + t.Parallel() + + base := func() *manualBootstrapHandler { + return &manualBootstrapHandler{ + siteName: "site-a", + variant: string(variantIgnition), + agentURL: "https://example.test/unbounded-agent", + agentSHA256: strings.Repeat("a", 64), + } + } + + // validate ends by requiring a readable kubeconfig, so the happy path needs + // one to reach that far. The failure cases below deliberately do not supply + // one: reporting a missing flag without it is the behavior being tested. + withKubeconfig := func(h *manualBootstrapHandler) *manualBootstrapHandler { + path := filepath.Join(t.TempDir(), "kubeconfig") + require.NoError(t, os.WriteFile(path, []byte("apiVersion: v1\n"), 0o600)) + h.kubeconfigPath = path + + return h + } + + valid := withKubeconfig(base()) + require.NoError(t, valid.validate(), "a complete ignition invocation must pass") + require.Equal(t, "sha256-"+strings.Repeat("a", 64), valid.agentHash, "the renderer uses the digest validate parsed") + + for name, tc := range map[string]struct { + mutate func(*manualBootstrapHandler) + wantErr string + }{ + "no agent url": { + mutate: func(h *manualBootstrapHandler) { h.agentURL = "" }, + wantErr: "--agent-url is required", + }, + "unfetchable agent url": { + mutate: func(h *manualBootstrapHandler) { h.agentURL = "oci://ghcr.io/azure/agent:v1" }, + wantErr: "cannot be fetched by Ignition", + }, + "no digest": { + mutate: func(h *manualBootstrapHandler) { h.agentSHA256 = "" }, + wantErr: "--agent-sha256 is required", + }, + "malformed digest": { + mutate: func(h *manualBootstrapHandler) { h.agentSHA256 = "not-a-digest" }, + wantErr: "invalid --agent-sha256", + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + h := base() + tc.mutate(h) + + // No kubeconfig on purpose. The point of checking these here is + // that an operator hears about a missing flag immediately, rather + // than after the tool has resolved a kubeconfig and built a client, + // so the flag error has to come first. + err := h.validate() + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + require.NotContains(t, err.Error(), "kubeconfig", + "the flag error must be reported before the kubeconfig is resolved") + }) + } + + // The other variants have no such requirements, and must not inherit them: + // they resolve the agent at runtime. + for _, variant := range []bootstrapVariant{variantScript, variantCloudInit} { + t.Run("no ignition rules for "+string(variant), func(t *testing.T) { + t.Parallel() + + h := withKubeconfig(&manualBootstrapHandler{siteName: "site-a", variant: string(variant)}) + require.NoError(t, h.validate()) + }) + } +} diff --git a/designs/agent-upgrade.md b/designs/agent-upgrade.md index 3e18888ce..c28da381c 100644 --- a/designs/agent-upgrade.md +++ b/designs/agent-upgrade.md @@ -24,7 +24,7 @@ The path set is represented by `goalstates.AgentUpgradePaths`. | Field | Purpose | |-------|---------| -| `BinaryPath` | Compatibility path, normally `/usr/local/bin/unbounded-agent`. | +| `BinaryPath` | Compatibility path, normally `/opt/unbounded/bin/unbounded-agent`. | | `BluePath` | First blue-green binary slot. | | `GreenPath` | Second blue-green binary slot. | | `CurrentPath` | Symlink used by the systemd daemon unit. | @@ -32,10 +32,18 @@ The path set is represented by `goalstates.AgentUpgradePaths`. | `SignalPath` | Single JSON signal file for pending and failure state. | | `CurrentTargetPath` | Resolved current binary target for one operation. | -`goalstates.ResolvedAgentUpgradePaths()` resolves environment overrides and -stores the resolved `CurrentPath` target in `CurrentTargetPath`. If -`CurrentPath` does not exist, the compatibility `BinaryPath` is used as the -current target. `NextTargetPath()` then chooses the inactive slot: +`goalstates.ResolvedAgentUpgradePaths()` resolves the slots under the host +root, applies environment overrides, and stores the resolved `CurrentPath` +target in `CurrentTargetPath`. If `CurrentPath` does not exist, the +compatibility `BinaryPath` is used as the current target. + +The host root is `/opt/unbounded`, resolved through symlinks before any path is +built from it. On a host installed by a release before the host root, the agent +links `/opt/unbounded` to `/usr/local`, where that release put its files, before +it resolves anything. The slots then resolve to the paths that release wrote, +so the resolved current target still compares equal to one of them. + +`NextTargetPath()` chooses the inactive slot: ```text current target == BluePath -> next target = GreenPath @@ -112,7 +120,9 @@ Without `--preflight`, the command performs one transactional activation: activated. 3. Inspect the current binary layout and validate path safety, destination entry types, collisions, and unsafe aliases. -4. Verify the pinned candidate snapshot by running its `version` command. +4. Verify the pinned candidate snapshot by running its `version` command, + and, unless the host root is linked to `/usr/local`, its `host-root` + command, which must print the same host root. 5. If the managed layout is not initialized, preserve the existing single-path daemon binary in one slot and establish `CurrentPath` and `LastGoodPath`. @@ -260,7 +270,7 @@ Logs and errors omit URL query and fragment data. 2. Download the tarball within the configured size bound. 3. Require the archive to contain only the exact `unbounded-agent` entry. 4. Bound decompression and atomically install the inactive slot. -5. Run `unbounded-agent version` against the staged binary without exposing output. +5. Run `unbounded-agent version` against the staged binary without exposing output, and check its host root as above. 6. If the inactive slot is last-good, protect the running binary through `LastGoodPath` before replacing it; otherwise defer the last-good update until candidate verification succeeds. 7. Atomically update `CurrentPath` to the staged binary. diff --git a/docs/content/guides/agent.md b/docs/content/guides/agent.md index e8dc3d751..117a0162e 100644 --- a/docs/content/guides/agent.md +++ b/docs/content/guides/agent.md @@ -196,6 +196,52 @@ runcmd: - export AGENT_MACHINE_NAME=my-custom-node ``` +### Where the agent installs + +The agent keeps its own host-side files under `/opt/unbounded`: the daemon +binaries and helper scripts in `/opt/unbounded/bin`, and the LocalDNS network +helper in `/opt/unbounded/libexec`. Paths inside the nspawn machine are always +relative to the machine directory, and `/etc/unbounded/agent` and +`/var/lib/unbounded` are separate. + +Earlier releases installed these files under `/usr/local`. On a host installed +by one of them, the first command of a newer agent that changes the host links +`/opt/unbounded` to `/usr/local`. That happens when the daemon starts after an +AgentUpgrade, or when `start` or `agent-upgrade` runs. The files stay where +they are and the units that run them are unchanged, so the host can still be +returned to the earlier release. `unbounded-agent reset` removes the agent's +files from both locations, and the link. + +The agent refuses to run on a host that has an installation under both +locations, or an installation under `/usr/local` beside an existing +`/opt/unbounded` directory. Run `unbounded-agent reset` first. On a host +installed under `/opt/unbounded`, an AgentUpgrade to an earlier release is +refused, because that release would look for its files under `/usr/local`. + +### Immutable hosts (read-only /usr) + +Some images mount `/usr` read-only and provide no package manager, so there is +no shell-based provisioning path at first boot. Azure Container Linux is one +such image. `/opt`, and with it the agent's files, is on the writable root +filesystem there. + +For these hosts, generate an Ignition config: + +```bash +curl -fsSLO https://github.com/Azure/unbounded/releases/download/v0.8.1/checksums.txt +kubectl unbounded machine manual-bootstrap my-node --site mysite \ + --variant ignition \ + --agent-url https://github.com/Azure/unbounded/releases/download/v0.8.1/unbounded-agent-linux-amd64 \ + --agent-sha256 "$(grep ' unbounded-agent-linux-amd64$' checksums.txt)" \ + > config.ign +``` + +Ignition declares state rather than running commands, so this variant cannot +resolve a version, detect an architecture, or extract an archive at boot. It +therefore requires `--agent-url` pointing at the *bare agent binary* rather +than the release tarball, and `--agent-sha256` to verify it. The digest for +each release binary is published in `checksums.txt`. + ### Customizing the agent download By default the bootstrap script downloads the latest published diff --git a/docs/content/reference/agent/nspawn.md b/docs/content/reference/agent/nspawn.md index e0439de68..af21de0f3 100644 --- a/docs/content/reference/agent/nspawn.md +++ b/docs/content/reference/agent/nspawn.md @@ -191,7 +191,7 @@ The configuration is written to these files on the host before the machine boots | nspawn config | `/etc/systemd/nspawn/.nspawn` | | Service override | `/etc/systemd/system/systemd-nspawn@.service.d/override.conf` | | Config regeneration unit | `/etc/systemd/system/unbounded-agent-regenerate-config@.service` | -| Rollback-stable lifecycle helper | `/usr/local/bin/unbounded-agent-nspawn-lifecycle` | +| Rollback-stable lifecycle helper | `/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle` | ### Customization points @@ -346,7 +346,7 @@ The container operates in the host's network namespace (`VirtualEthernet=no`): | `/etc/systemd/nspawn/.nspawn` | nspawn configuration file. | | `/etc/systemd/system/systemd-nspawn@.service.d/override.conf` | Systemd service override. | | `/etc/systemd/system/unbounded-agent-regenerate-config@.service` | Host-side retrying oneshot unit that regenerates host-side configuration before machine start. | -| `/usr/local/bin/unbounded-agent-nspawn-lifecycle` | Lifecycle command binary retained across daemon binary rollback. | +| `/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle` | Lifecycle command binary retained across daemon binary rollback. On a host installed by an earlier release, `/opt/unbounded` is a link to `/usr/local`. | | `/run/host-nvidia//` | (Inside container) Read-only bind-mount of host NVIDIA library directories. | ## See Also diff --git a/hack/agent/e2e-kind/README.md b/hack/agent/e2e-kind/README.md index cea35f1bc..99a45c914 100644 --- a/hack/agent/e2e-kind/README.md +++ b/hack/agent/e2e-kind/README.md @@ -1,7 +1,8 @@ # Agent e2e: local and CI -`e2e.py` defines shared `setup`, `lifecycle`, `configuration`, and `fresh-bootstrap` -suites. Run `e2e.py list-suite --suite lifecycle` to inspect the exact sequence. +`e2e.py` defines shared `setup`, `lifecycle`, `configuration`, `fresh-bootstrap`, +`bootstrap-recovery`, and `migration` suites. Run +`e2e.py list-suite --suite lifecycle` to inspect the exact sequence. The host lifecycle includes existing upgrade/rollback, reset/reinstall, and repave operations plus an unassisted host reboot with fresh node identity and workload/DNS. Persistent DNS failure is fatal after bounded convergence retries. @@ -18,6 +19,63 @@ Focused configuration creates only the bridge rather than a colliding default VM Use matching cluster/VM/subnet variables when invoking commands or cleanup on a preserved environment. Same-disk reinstall checks host boot identity. +## Azure Container Linux (immutable hosts) + +`HOST_BASE_OS=acl` boots an immutable host: `/usr` is a read-only dm-verity +image with no package manager, so nothing is installed at boot and the image +must already carry what the agent needs. It does. + +`/usr/local` is a real directory inside that read-only `/usr` rather than a +symlink to somewhere writable, so an agent released before the host root cannot +be installed there. The current agent installs under `/opt/unbounded` on every +host, which is on the writable root filesystem here. + +Provisioning is Ignition rather than cloud-init, which inverts the usual order. +An Ignition config is applied before the host boots and has to carry the +bootstrap token and the API server address, so `create-vm` acquires the image +and stops; `run-agent` renders the config and launches the VM. Nothing is +delivered over SSH: Ignition places the binary and the agent config, and a +first-boot unit runs preflight and bootstrap. + +The image is resolved from the manifest published alongside it, so a refreshed +build is picked up without a code change. It is fetched with a federated Azure +login, because the storage account holding it disables anonymous access and +shared keys alike. The image can be chosen in other ways: + +- `ACL_IMAGE_MANIFEST_URL` reads a different manifest. +- `ACL_IMAGE_URL`, `ACL_IMAGE_SHA256` and `ACL_IMAGE_BUILD_ID`, set together, + pin a build and skip the manifest. `e2e.py resolve-host-image` prints them for + the current build; CI runs it once per job. +- `ACL_IMAGE_BUILD_ID` alone fails the run unless the manifest publishes that + build. +- `HOST_IMAGE_PATH` boots a local file with no Azure login at all: + +```sh +HOST_BASE_OS=acl E2E_SUITE=lifecycle HOST_IMAGE_PATH="$PWD/acl.qcow2" \ + bash hack/agent/e2e-kind/run-local.sh +``` + +The configuration suite does not run on this host: its scenarios supply their +own agent, and the Ignition path only boots the agent it staged itself. + +Running this locally needs `ovmf` and `qemu-nbd` in addition to the usual +prerequisites. The host boots through its own UEFI bootloader, and the Ignition +config URL is appended to the kernel command line by patching a UKI addon on +the EFI system partition; see `ukiboot.py` for why the boot chain is extended +rather than replaced. + +In CI this entry is skipped unless a federated Azure login is configured, and +on pull requests from forks, because GitHub withholds secrets from +fork-triggered workflows. It is left out of the matrix rather than added and +failed, so it appears on its own once `ACL_IMAGE_CLIENT_ID`, +`ACL_IMAGE_TENANT_ID` and `ACL_IMAGE_SUBSCRIPTION_ID` exist as repository +secrets. They have to be repository secrets rather than environment ones: the +`azure-ci` environment requires a reviewer, which would put a manual approval +in front of every pull request. + +Every other host downloads from a public mirror and runs normally in all of +these cases. + Cloud-init preparation is fail-fast, with the success marker last. EL10 hosts install `kernel-modules-extra-$(uname -r)` and load the netfilter modules required by kube-proxy. Completion and marker are verified before bootstrap. Fedora's @@ -31,3 +89,19 @@ additional batches from consuming memory. Commands have bounded execution and CI's monitor records host resources before the suite deadline, leaving time for diagnostic collection and upload. These tests exercise main's existing lifecycle; they do not assert resumable bootstrap or introduce new recovery operations. + +## Host root migration + +The `migration` suite starts from a host installed by the last release before +the host root, `LEGACY_AGENT_VERSION` (default `v0.8.0`), fetched from its +GitHub release by the install script. An AgentUpgrade to this build must link +`/opt/unbounded` to `/usr/local` and leave that release's layout and units as +they were. The host then reboots, upgrades again, returns to the older release, +and upgrades once more before a reset, which must remove the link along with +the files. The older release cannot be installed on an immutable host, so the +suite needs a cloud-init host: + +```sh +HOST_BASE_OS=ubuntu2404 E2E_SUITE=migration KEEP_ENV=1 \ + bash hack/agent/e2e-kind/run-local.sh +``` diff --git a/hack/agent/e2e-kind/e2e.py b/hack/agent/e2e-kind/e2e.py index e6bd4a641..4064f2197 100755 --- a/hack/agent/e2e-kind/e2e.py +++ b/hack/agent/e2e-kind/e2e.py @@ -59,6 +59,7 @@ import argparse import base64 import concurrent.futures +import functools import hashlib import json import os @@ -70,12 +71,16 @@ import sys import textwrap import time +import urllib.parse +import urllib.request from dataclasses import dataclass, field, replace from http.server import HTTPServer, SimpleHTTPRequestHandler from pathlib import Path from threading import Thread from typing import Any, Callable +import ukiboot + # --------------------------------------------------------------------------- # Paths and defaults # --------------------------------------------------------------------------- @@ -99,6 +104,13 @@ AGENT_MACHINE_NAME = os.environ.get("AGENT_MACHINE_NAME", "agent-e2e") AGENT_DEBUG = os.environ.get("AGENT_DEBUG", "") OFFLINE_BOOTSTRAP = os.environ.get("OFFLINE_BOOTSTRAP", "").lower() in ("1", "true", "yes") +OFFLINE_ARTIFACTS_DIR = "/var/lib/unbounded-e2e/artifacts" + +# The last release before the host root. The migration suite installs it, and +# returns to it after moving to this build. +LEGACY_AGENT_VERSION = os.environ.get("LEGACY_AGENT_VERSION", "v0.8.0") +LEGACY_AGENT_RELEASE_URL = f"https://github.com/Azure/unbounded/releases/download/{LEGACY_AGENT_VERSION}" +LEGACY_AGENT_TARBALL = "unbounded-agent-linux-amd64.tar.gz" # Site name used when generating the bootstrap script via kubectl-unbounded. E2E_SITE_NAME = os.environ.get("E2E_SITE_NAME", "e2e") @@ -145,11 +157,17 @@ UNBOUNDED_NS = "unbounded-system" E2E_WORKLOAD_IMAGE = "docker.io/library/busybox:1.36" MACHINE_CONFIG_NAME = f"{AGENT_MACHINE_NAME}-config" -DAEMON_BINARY = "/usr/local/bin/unbounded-agent" -DAEMON_BINARY_BLUE = "/usr/local/bin/unbounded-agent-blue" -DAEMON_BINARY_GREEN = "/usr/local/bin/unbounded-agent-green" -DAEMON_BINARY_CURRENT = "/usr/local/bin/unbounded-agent-current" -DAEMON_BINARY_LAST_GOOD = "/usr/local/bin/unbounded-agent-last-good" +# The agent's host root, and where agents released before it installed their +# files. A host installed by an older agent keeps its files under the legacy +# root, and the current agent links the host root to it. +HOST_ROOT = "/opt/unbounded" +LEGACY_HOST_ROOT = "/usr/local" +DAEMON_BIN_DIR = f"{HOST_ROOT}/bin" +DAEMON_BINARY = f"{DAEMON_BIN_DIR}/unbounded-agent" +DAEMON_BINARY_BLUE = f"{DAEMON_BIN_DIR}/unbounded-agent-blue" +DAEMON_BINARY_GREEN = f"{DAEMON_BIN_DIR}/unbounded-agent-green" +DAEMON_BINARY_CURRENT = f"{DAEMON_BIN_DIR}/unbounded-agent-current" +DAEMON_BINARY_LAST_GOOD = f"{DAEMON_BIN_DIR}/unbounded-agent-last-good" BPFFS_SENTINEL = "unbounded-e2e-bpffs-sentinel" DEVICE_REFRESH_PATH = "/dev/infiniband/unbounded-e2e-zero" DEVICE_REFRESH_TMPFILES_PATH = "/etc/tmpfiles.d/unbounded-e2e-device.conf" @@ -206,18 +224,87 @@ def run_quiet(args: list[str], **kw: Any) -> subprocess.CompletedProcess[str]: ) -def download_file(url: str, destination: Path) -> None: - run([ - "curl", - "-fsSL", - "--connect-timeout", "30", - "--retry", "5", - "--retry-delay", "5", - "--retry-all-errors", - "--remove-on-error", - "-o", str(destination), - url, +def download_file(url: str, destination: Path, auth: str = "") -> None: + config = curl_auth_config(auth) + try: + run([ + "curl", + "-fsSL", + "--connect-timeout", "30", + "--retry", "5", + "--retry-delay", "5", + "--retry-all-errors", + "--remove-on-error", + *(["--config", "-"] if config else []), + "-o", str(destination), + url, + ], input=config, text=True) + except subprocess.CalledProcessError as exc: + die(f"downloading {url} failed (curl exit {exc.returncode})") + + +def http_get(url: str, auth: str = "") -> str: + config = curl_auth_config(auth) + try: + return capture([ + "curl", "-fsSL", "--connect-timeout", "30", + "--retry", "3", "--retry-delay", "2", "--retry-all-errors", + *(["--config", "-"] if config else []), url, + ], input=config) + except subprocess.CalledProcessError as exc: + die(f"fetching {url} failed (curl exit {exc.returncode})") + raise AssertionError("unreachable") from exc + + +def curl_auth_config(auth: str) -> str: + """Return a curl config that authenticates to a protected source. + + Azure Blob Storage is reached with an AAD bearer token rather than a shared + key or a SAS: the account that publishes the ACL image disables both, so + there is no static credential to hold and nothing useful to put in a secret + beyond the federated identity itself. + + The token goes to curl on stdin rather than in its arguments, which a + failed command prints. GitHub does not know to mask a token minted during + the job, so it is registered as a mask as well. + """ + if not auth: + return "" + + if auth != "azure-storage": + die(f"unknown auth mode {auth!r}") + + token = capture([ + "az", "account", "get-access-token", + "--resource", "https://storage.azure.com/", + "--query", "accessToken", "-o", "tsv", ]) + if os.environ.get("GITHUB_ACTIONS") == "true": + print(f"::add-mask::{token}", flush=True) + + return f'header = "Authorization: Bearer {token}"\nheader = "x-ms-version: 2021-12-02"\n' + + +def verify_sha256(path: Path, expected: str) -> None: + """Fail unless a downloaded file matches its published digest. + + The image is fetched over the network and then booted as the host under + test, so a truncated or substituted file would surface as an unexplained + boot failure rather than as a download problem. + """ + got = file_sha256(path) + if got != expected.lower(): + path.unlink(missing_ok=True) + die(f"{path.name} sha256 {got} does not match the published {expected}") + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + + return digest.hexdigest() def capture(args: list[str], **kw: Any) -> str: @@ -1246,16 +1333,47 @@ def wait_for_node_reboot_event(node_name: str, boot_id: str, timeout_secs: int = die(f"Timed out waiting for Node Rebooted event for '{node_name}' boot ID '{boot_id}'") +def resolve_on_host(path: str) -> str: + """Return *path* on the VM with every symlink resolved, missing parts kept.""" + + return ssh_capture(f"readlink -m {shlex.quote(path)}").strip() + + +def host_root_state() -> str: + """Return what the host root is on the VM: absent, dir, link:, or other.""" + + return ssh_capture( + f"sudo sh -c 'r={HOST_ROOT}; " + 'if [ -L "$r" ]; then echo "link:$(readlink "$r")"; ' + 'elif [ -d "$r" ]; then echo dir; ' + 'elif [ -e "$r" ]; then echo other; ' + "else echo absent; fi'" + ).strip() + + +def _resolve_daemon_link(path: str) -> str: + """Return a command that resolves *path*, one of the daemon links. + + On a host installed by a release before the host root, the host root does + not exist until this build first runs there, and until then the link is + found under the legacy root. + """ + + legacy = LEGACY_HOST_ROOT + path.removeprefix(HOST_ROOT) + return "sudo sh -c " + shlex.quote( + f"if [ -e {HOST_ROOT} ]; then readlink -f {path}; else readlink -f {legacy}; fi") + + def read_daemon_current_target() -> str: """Return the target path of the host daemon current binary symlink.""" - return ssh_capture(f"sudo readlink -f {DAEMON_BINARY_CURRENT}").strip() + return ssh_capture(_resolve_daemon_link(DAEMON_BINARY_CURRENT)).strip() def read_daemon_last_good_target() -> str: """Return the target path of the host daemon last-good binary symlink.""" - return ssh_capture(f"sudo readlink -f {DAEMON_BINARY_LAST_GOOD}").strip() + return ssh_capture(_resolve_daemon_link(DAEMON_BINARY_LAST_GOOD)).strip() def wait_for_daemon_current_target(expected_target: str, timeout_secs: int = 180) -> None: @@ -1266,8 +1384,7 @@ def wait_for_daemon_current_target(expected_target: str, timeout_secs: int = 180 last_target = "" while elapsed < timeout_secs: result = subprocess.run( - ["ssh", *SSH_OPTS, SSH_TARGET, - f"sudo readlink -f {DAEMON_BINARY_CURRENT}"], + ["ssh", *SSH_OPTS, SSH_TARGET, _resolve_daemon_link(DAEMON_BINARY_CURRENT)], capture_output=True, text=True, ) if result.returncode == 0: @@ -1309,6 +1426,23 @@ def wait_for_daemon_active(timeout_secs: int = 180) -> None: die(f"Timed out waiting for daemon to become active; last status={last_status!r}") +def check_reset_failed() -> None: + """Reset the daemon's start-limit budget between upgrade scenarios. + + On Azure Container Linux reset-failed, a privileged D-Bus call, is refused + for a sudo'd SSH session even though the agent's own systemctl calls succeed + from its service context. Losing the isolation there only risks a scenario + inheriting a start-limit budget, so it is a warning. Every other host is + expected to allow it. + """ + reset = ssh_capture_quiet("sudo systemctl reset-failed unbounded-agent-daemon.service") + if reset.returncode != 0: + if host_image().provisioning != "ignition": + die(f"could not reset the daemon start-limit budget: {reset.stderr.strip()}") + log("WARNING: could not reset the daemon start-limit budget " + f"({reset.stderr.strip()}); scenarios may share it") + + def _serve_agent_upgrade_tarball(tarball: Path, operation_name: str, expect_complete: bool = True) -> dict[str, Any]: """Serve *tarball* to the VM, create AgentUpgrade, and wait for it.""" @@ -1326,7 +1460,7 @@ def _serve_agent_upgrade_tarball(tarball: Path, operation_name: str, expect_comp # Each scenario intentionally restarts or fails the daemon. Isolate its # systemd start-limit budget so the candidate under test gets the # configured retries before recovery runs. - ssh_cmd("sudo systemctl reset-failed unbounded-agent-daemon.service") + check_reset_failed() run_quiet([KUBECTL, "delete", _machine_operation_resource(), operation_name, "--ignore-not-found"], check=False) create_machine_operation( @@ -1367,7 +1501,12 @@ def _build_failing_agent_tarball(tarball: Path) -> None: def _build_daemon_failing_agent_tarball(tarball: Path) -> None: - """Package an executable that passes preflight but fails as the daemon.""" + """Package an executable that passes preflight but fails as the daemon. + + It answers host-root the way a current agent does, resolving the host root + through any symlink, so that verification accepts it and the failure comes + from the daemon. + """ _build_script_agent_tarball( tarball, @@ -1377,11 +1516,35 @@ def _build_daemon_failing_agent_tarball(tarball: Path) -> None: " echo unbounded-agent e2e-daemon-failing\n" " exit 0\n" "fi\n" + "if [ \"${1:-}\" = \"host-root\" ]; then\n" + f" readlink -m {HOST_ROOT}\n" + " exit 0\n" + "fi\n" "echo failing upgraded agent daemon >&2\n" "exit 42\n", ) +def _build_legacy_agent_tarball(tarball: Path) -> None: + """Package an executable that behaves like an agent released before the host root. + + It answers version and has no host-root command, which is all verification + can see of the difference. + """ + + _build_script_agent_tarball( + tarball, + "agent-upgrade-legacy", + "#!/bin/sh\n" + "if [ \"${1:-}\" = \"version\" ]; then\n" + " echo unbounded-agent e2e-legacy\n" + " exit 0\n" + "fi\n" + "echo \"unknown command \\\"${1:-}\\\"\" >&2\n" + "exit 1\n", + ) + + def _build_script_agent_tarball(tarball: Path, build_name: str, script: str) -> None: """Package script content as the agent binary in an upgrade tarball.""" @@ -1434,8 +1597,33 @@ class HostImage: write_files: str = "" pre_marker_commands: list[str] | None = None + # The account the harness connects as. Cloud images conventionally carry a + # distro-named user; an image provisioned by Ignition gets whichever user + # its own config creates. + ssh_user: str = "ubuntu" + + # How the host is configured before it is reachable. "cloud-init" seeds a + # NoCloud ISO as a second drive. "ignition" boots the image's own UKI and + # seeds an Ignition config, which is the only first-boot mechanism the + # immutable Flatcar-derived images implement. + provisioning: str = "cloud-init" + + # Published digest of the image, verified after download. Empty for the + # public mirrors, which publish no digest alongside the image. + sha256: str = "" + + # Credential needed to read the image, for sources that are not public. + auth: str = "" + def host_image() -> HostImage: + """Return the selected host image. + + Deliberately not cached. Tests select a host by patching HOST_BASE_OS, and + a cache here silently returns whichever image was resolved first, so they + render the wrong distro and fail somewhere unrelated. The expensive part is + the manifest lookup, which is cached where it happens. + """ if HOST_BASE_OS == "ubuntu2404": return HostImage( url=HOST_IMAGE_URL @@ -1492,13 +1680,142 @@ def host_image() -> HostImage: network_interface="eth0" if version == "9" else "ens3", ) + if HOST_BASE_OS == "acl": + return acl_host_image() + die( f"Unsupported HOST_BASE_OS {HOST_BASE_OS!r}; " "expected ubuntu2404, ubuntu2604, fedora, almalinux9, almalinux10, " - "centosstream9, or centosstream10" + "centosstream9, centosstream10, or acl" ) +# Azure Container Linux publishes no image to a public mirror, so the harness +# resolves one from a manifest in the storage account that builds it. The +# manifest names the blob, its size and its sha256, which is what lets the +# download be verified and cached by build. +ACL_IMAGE_MANIFEST_URL = os.environ.get( + "ACL_IMAGE_MANIFEST_URL", + "https://aksflexaclimagestme.blob.core.windows.net/images/latest.json", +) +ACL_IMAGE_BUILD_ID = os.environ.get("ACL_IMAGE_BUILD_ID", "") +# Set together, these name the image directly and the manifest is not read. +# resolve-host-image exports them, so CI resolves the manifest once per job. +ACL_IMAGE_URL = os.environ.get("ACL_IMAGE_URL", "") +ACL_IMAGE_SHA256 = os.environ.get("ACL_IMAGE_SHA256", "") +ACL_BUILD_ID_PATTERN = re.compile(r"[A-Za-z0-9._-]+") + + +def acl_host_image() -> HostImage: + """Return the Azure Container Linux host image. + + /usr is a read-only dm-verity image with no package manager, so nothing can + be installed at boot and the image has to already carry everything the agent + needs. It does: systemd-nspawn, machinectl and systemd-machined are present + and usable on the first boot, with the dbus policy baked into /usr so + machined can take its bus name before anything asks for it. + + /usr/local is a real directory inside that read-only /usr rather than a + symlink to somewhere writable, so an agent released before the host root + cannot be installed at all. /opt, where the host root is, is on the writable + root filesystem. + """ + path = os.environ.get("HOST_IMAGE_PATH", "") + if path: + # A local file wins, so a developer can run against an image that is not + # published yet without editing anything. + if not Path(path).is_file(): + die(f"HOST_IMAGE_PATH does not exist: {path}") + url, file_name, digest = f"file://{Path(path).resolve()}", Path(path).name, "" + else: + # Left empty and filled in by resolved_host_image. Naming the blob means + # reading the published manifest, which is a network call and an Azure + # token, and host_image is called for the ssh user far more often than + # for the image itself, including at import. + url, file_name, digest = "", "", "" + + return HostImage( + url=url, + file_name=file_name, + backing_format="qcow2", + # The image's own Ignition config creates core and puts it in sudo. + sudo_group="sudo", + ssh_user="core", + packages=[], + provisioning="ignition", + sha256=digest, + auth="" if path else "azure-storage", + ) + + +@functools.cache +def acl_image_from_manifest() -> tuple[str, str, str]: + """Resolve the image URL, file name, and digest. + + By default the published manifest is followed, so a refreshed image is + picked up without a code change. ACL_IMAGE_URL, ACL_IMAGE_SHA256 and + ACL_IMAGE_BUILD_ID together pin a build instead and skip the manifest. + ACL_IMAGE_BUILD_ID alone only checks that the manifest still publishes that + build. + """ + pinned = [ACL_IMAGE_URL, ACL_IMAGE_SHA256] + if any(pinned): + if not all(pinned) or not ACL_IMAGE_BUILD_ID: + die("ACL_IMAGE_URL, ACL_IMAGE_SHA256 and ACL_IMAGE_BUILD_ID must be set together") + _check_acl_build_id(ACL_IMAGE_BUILD_ID, "ACL_IMAGE_BUILD_ID") + return ACL_IMAGE_URL, f"acl-{ACL_IMAGE_BUILD_ID}.qcow2", ACL_IMAGE_SHA256 + + manifest = json.loads(http_get(ACL_IMAGE_MANIFEST_URL, auth="azure-storage")) + qcow2 = manifest.get("qcow2", {}) + + build = manifest.get("build_id") + _check_acl_build_id(build, f"{ACL_IMAGE_MANIFEST_URL} build_id") + if ACL_IMAGE_BUILD_ID and ACL_IMAGE_BUILD_ID != build: + die(f"ACL_IMAGE_BUILD_ID={ACL_IMAGE_BUILD_ID} but the manifest publishes {build!r}; " + "pin that build with ACL_IMAGE_URL and ACL_IMAGE_SHA256, or clear the override") + + url = qcow2.get("url", "") + digest = qcow2.get("sha256", "") + if not url or not digest: + die(f"{ACL_IMAGE_MANIFEST_URL} does not name a qcow2 url and sha256") + + # Named for the build so a refreshed image does not reuse a cached file, and + # so a cache key can be derived from the name alone. + log(f"Azure Container Linux build {build} ({qcow2.get('size', 0)} bytes, sha256 {digest[:12]})") + + return url, f"acl-{build}.qcow2", digest + + +def _check_acl_build_id(build: object, source: str) -> None: + """The build names the cached image file, so it has to be a plain name.""" + if not isinstance(build, str) or not ACL_BUILD_ID_PATTERN.fullmatch(build): + die(f"{source} {build!r} is not a build id") + + +def resolve_host_image() -> None: + """Resolve the Azure Container Linux image once and export it. + + In GitHub Actions it is written to $GITHUB_ENV, so every later e2e.py + process in the job uses the same build instead of reading the manifest + again, and the build is also written to $GITHUB_OUTPUT for the cache key. + """ + if HOST_BASE_OS != "acl": + die("resolve-host-image only applies to HOST_BASE_OS=acl") + + url, file_name, digest = acl_image_from_manifest() + build = file_name.removeprefix("acl-").removesuffix(".qcow2") + exports = f"ACL_IMAGE_URL={url}\nACL_IMAGE_SHA256={digest}\nACL_IMAGE_BUILD_ID={build}\n" + + github_env = os.environ.get("GITHUB_ENV", "") + if github_env: + with open(github_env, "a", encoding="utf-8") as env_file: + env_file.write(exports) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"build={build}\n") + else: + print(exports, end="") + + def ubuntu_netplan_write_files() -> str: return textwrap.dedent(f"""\ write_files: @@ -1521,6 +1838,32 @@ def ubuntu_netplan_write_files() -> str: """) +def resolved_host_image() -> HostImage: + """Return the host image with its download location filled in. + + Only the two places that actually fetch or open the image need this. Every + other caller wants the ssh user or the provisioning mechanism, and making them + resolve a manifest to get those would put an Azure round trip behind + importing this module. + """ + image = host_image() + if image.url: + return image + + url, file_name, digest = acl_image_from_manifest() + + return replace(image, url=url, file_name=file_name, sha256=digest) + + +# The SSH user is a property of the image, but SSH_TARGET is referenced as a +# module constant throughout. Rebind it once the image is known, rather than +# threading an image argument through every call site that needs it. +# +# This sits below host_image and everything it calls, because it runs at import. +VM_SSH_USER = os.environ.get("VM_SSH_USER", "") or host_image().ssh_user +SSH_TARGET = f"{VM_SSH_USER}@{VM_IP}" + + def yaml_list(items: list[str], indent: str) -> str: return "\n".join(f"{indent}- {item}" for item in items) @@ -1567,16 +1910,12 @@ def _launch_vm(ssh_pub_key: str) -> None: Networking (bridge, TAP, NAT) must already be configured. """ - image = host_image() + image = resolved_host_image() image_file = VM_DIR / image.file_name if not image_file.exists(): die(f"Base cloud image not found: {image_file}. Run create-vm first.") - # Create VM disk - vm_disk = VM_DIR / f"{VM_NAME}.qcow2" - log(f"Creating snapshot disk: {vm_disk}") - run(["qemu-img", "create", "-f", "qcow2", "-b", str(image_file), - "-F", image.backing_format, str(vm_disk), VM_DISK_SIZE]) + vm_disk = _create_vm_disk(image_file, image) # cloud-init configuration log("Generating cloud-init configuration...") @@ -1632,12 +1971,73 @@ def _launch_vm(ssh_pub_key: str) -> None: log(f" Log: {qemu_log}") log("============================================") + qemu_pid = _start_qemu( + vm_disk, mac_address, pid_file, qemu_log, + extra_drives=["-drive", f"file={seed_iso},format=raw,if=virtio"], + ) + _wait_for_ssh(qemu_pid, qemu_log) + + +def _create_vm_disk(image_file: Path, image: HostImage) -> Path: + """Create the overlay the VM boots from, never smaller than its backing file. + + VM_DISK_SIZE is a floor rather than the size. Azure Container Linux is a + 31.4 GiB image whose root partition runs to the end of the disk, so a 20 GiB + overlay truncates it and the guest waits in the initramfs forever for a root + that cannot be found. qcow2 is sparse, so the larger figure costs nothing + until it is written to. + """ + vm_disk = VM_DIR / f"{VM_NAME}.qcow2" + + info = json.loads(capture([ + "qemu-img", "info", "-f", image.backing_format, + "--output=json", str(image_file), + ])) + backing_size = int(info["virtual-size"]) + + size = max(_parse_size(VM_DISK_SIZE), backing_size) + if size > _parse_size(VM_DISK_SIZE): + log(f"Growing overlay to the backing image's {size} bytes") + + log(f"Creating snapshot disk: {vm_disk}") + run(["qemu-img", "create", "-f", "qcow2", "-b", str(image_file), + "-F", image.backing_format, str(vm_disk), str(size)]) + + return vm_disk + + +def _parse_size(text: str) -> int: + """Parse a qemu-img size such as "20G" into bytes.""" + units = {"K": 1 << 10, "M": 1 << 20, "G": 1 << 30, "T": 1 << 40} + trimmed = text.strip().upper() + if trimmed and trimmed[-1] in units: + return int(float(trimmed[:-1]) * units[trimmed[-1]]) + + return int(trimmed) + + +def _start_qemu( + vm_disk: Path, + mac_address: str, + pid_file: Path, + qemu_log: Path, + extra_drives: list[str] | None = None, + boot_args: list[str] | None = None, +) -> str: + """Start QEMU in the background and return its PID. + + boot_args carries the firmware selection for images that need one. The + cloud-init hosts boot with the default SeaBIOS; an image whose boot chain is + a UKI loaded by shim and systemd-boot needs OVMF and a writable copy of its + variables store. + """ qemu_args = [ "qemu-system-x86_64", "-cpu", "host", "-accel", "kvm", "-m", VM_MEMORY, "-smp", VM_CPUS, + *(boot_args or []), "-drive", f"file={vm_disk},format=qcow2,if=virtio", - "-drive", f"file={seed_iso},format=raw,if=virtio", + *(extra_drives or []), "-netdev", f"tap,id=net0,ifname={TAP_NAME},script=no,downscript=no", "-device", f"virtio-net-pci,netdev=net0,mac={mac_address}", "-daemonize", "-pidfile", str(pid_file), @@ -1649,7 +2049,10 @@ def _launch_vm(ssh_pub_key: str) -> None: qemu_pid = pid_file.read_text().strip() log(f"VM started in background (PID: {qemu_pid})") - # Wait for SSH + return qemu_pid + + +def _wait_for_ssh(qemu_pid: str, qemu_log: Path) -> None: log(f"Waiting for SSH to become available on {VM_IP}...") max_attempts = 120 for attempt in range(1, max_attempts + 1): @@ -1676,12 +2079,252 @@ def _launch_vm(ssh_pub_key: str) -> None: log(f"VM is ready at {VM_IP}") +# --------------------------------------------------------------------------- +# Ignition provisioning +# +# An image that provisions with Ignition is configured before it boots, not +# after. That inverts the harness's usual order, in which the VM comes up first +# and the bootstrap script is delivered over SSH afterwards: here the config has +# to carry the bootstrap token and the API server address, so the VM cannot be +# launched until the cluster exists. run-agent launches it. +# --------------------------------------------------------------------------- +IGNITION_BOOTSTRAP_UNIT = "unbounded-agent-bootstrap.service" +IGNITION_NETWORK_UNIT = "10-e2e-static.network" +IGNITION_CONFIG_NAME = "config.ign" +# The name the initramfs gives the virtio NIC. Observed on this image; the +# real root uses predictable names, which is why that side matches on MAC. +IGNITION_INITRAMFS_INTERFACE = "eth0" + + +def ignition_data_url(content: str) -> str: + return "data:;base64," + base64.b64encode(content.encode()).decode() + + +def _decode_ignition_source(source: str) -> str | None: + """Return the inline content of a data URL, or None if it is not one.""" + if not source.startswith("data:"): + return None + _header, _, payload = source.partition(",") + if ";base64" in _header: + return base64.b64decode(payload).decode("utf-8", "replace") + return urllib.parse.unquote(payload) + + +def rewrite_ignition_api_server(doc: dict, old: str, new: str) -> dict: + """Replace the API server URL everywhere it appears in an Ignition config. + + The agent config rides inside a data URL, so the plain text substitution the + script variant uses would silently do nothing here and leave the VM pointed + at a loopback address it cannot reach. + """ + if old == new: + return doc + + for entry in doc.get("storage", {}).get("files", []): + contents = entry.get("contents", {}) + decoded = _decode_ignition_source(contents.get("source", "")) + if decoded is None or old not in decoded: + continue + contents["source"] = ignition_data_url(decoded.replace(old, new)) + # The digest no longer matches once the body changes, and Ignition + # verifies before writing. + contents.pop("verification", None) + + for unit in doc.get("systemd", {}).get("units", []): + if old in unit.get("contents", ""): + unit["contents"] = unit["contents"].replace(old, new) + + return doc + + +def static_network_unit(mac_address: str) -> str: + """Return the systemd-networkd unit that gives the VM its static address. + + Matched on MAC alone. Every condition in [Match] has to hold, and the + interface name depends on the machine type, so naming it as well would make + the unit silently not apply and leave the VM on DHCP. + """ + return textwrap.dedent(f"""\ + [Match] + MACAddress={mac_address} + + [Network] + Address={VM_IP}/24 + Gateway={VM_GATEWAY} + DNS=8.8.8.8 + DNS=8.8.4.4 + """) + + +def initramfs_ip_karg() -> str: + """Return the dracut ip= argument that configures networking in the initramfs. + + Ignition runs from the initramfs and fetches both its own config and the + agent binary from the harness, so it needs an address before the real root + exists. bootengine's parse-ip-for-networkd turns this into a networkd unit. + + The interface has to be named. With the device field empty that script + writes Name=*, which matches loopback first and quietly assigns the address + and gateway to lo, so the guest dials itself and every fetch fails with + connection refused without a single packet reaching the wire. + """ + return (f"ip={VM_IP}::{VM_GATEWAY}:24:{VM_NAME}:" + f"{IGNITION_INITRAMFS_INTERFACE}:none:8.8.8.8:8.8.4.4") + + +def add_ignition_harness_access(doc: dict, ssh_pub_key: str, mac_address: str) -> dict: + """Add what the harness needs to drive the VM, and keep the image's own intent. + + This config arrives as Ignition's *user* config, which puts it above the + config the image ships on its OEM partition rather than merged into it: the + OEM config's own systemd section does not take effect once a user config is + present. So anything the image expected to be true has to be restated here, + or the host is configured differently from a stock boot. + + Two things follow. The login is specified in full rather than by name alone, + because a bare name leaves usermod with nothing to apply and the account + keeps its /sbin/nologin shell. And the Azure agent is masked, which is what + the image's own config does, since local provisioning replaces it. + """ + passwd = doc.setdefault("passwd", {}) + users = passwd.setdefault("users", []) + for user in users: + if user.get("name") == VM_SSH_USER: + keys = user.setdefault("sshAuthorizedKeys", []) + if ssh_pub_key not in keys: + keys.append(ssh_pub_key) + break + else: + users.append({ + "name": VM_SSH_USER, + "shell": "/bin/bash", + "groups": ["sudo", "systemd-journal"], + "sshAuthorizedKeys": [ssh_pub_key], + }) + + units = doc.setdefault("systemd", {}).setdefault("units", []) + if not any(unit.get("name") == "waagent.service" for unit in units): + units.append({"name": "waagent.service", "enabled": False, "mask": True}) + + files = doc.setdefault("storage", {}).setdefault("files", []) + + # The node registers under the host's hostname, and this image leaves it as + # "localhost": it masks the metadata hostname service and has no cloud-init + # to apply NoCloud's local-hostname. The cloud-init hosts get VM_NAME, so + # set the same thing here or the node joins under the wrong name. + files.append({ + "path": "/etc/hostname", + "mode": 0o644, + "overwrite": True, + "contents": {"source": ignition_data_url(f"{VM_NAME}\n")}, + }) + # The ip= karg only configures the initramfs. The real root gets its address + # from this unit, which outranks the DHCP default the image ships. + files.append({ + "path": f"/etc/systemd/network/{IGNITION_NETWORK_UNIT}", + "mode": 0o644, + "overwrite": True, + "contents": {"source": ignition_data_url(static_network_unit(mac_address))}, + }) + + return doc + +def ovmf_firmware() -> tuple[Path, Path]: + """Locate the OVMF code and variables images. + + The non-Secure-Boot build is used deliberately. shim is happy without it, + and enabling it would mean enrolling keys for an image the harness patches. + """ + for code, template in ( + (Path("/usr/share/OVMF/OVMF_CODE.fd"), Path("/usr/share/OVMF/OVMF_VARS.fd")), + (Path("/usr/share/OVMF/OVMF_CODE_4M.fd"), Path("/usr/share/OVMF/OVMF_VARS_4M.fd")), + (Path("/usr/share/edk2/ovmf/OVMF_CODE.fd"), Path("/usr/share/edk2/ovmf/OVMF_VARS.fd")), + (Path("/usr/share/qemu/ovmf-x86_64-code.bin"), Path("/usr/share/qemu/ovmf-x86_64-vars.bin")), + ): + if code.is_file() and template.is_file(): + return code, template + + die("OVMF firmware not found. Install the 'ovmf' (or 'edk2-ovmf') package; " + "an Ignition host boots through its own UEFI bootloader.") + raise AssertionError("unreachable") + + +def launch_ignition_vm(ignition_json: str) -> None: + """Boot the VM through its own bootloader with an Ignition config in place. + + The image's boot chain is left intact rather than replaced, because + Ignition's once-only behavior depends on it: systemd-boot appends + flatcar.first_boot only while firstboot.addon.efi exists, and + ignition-quench.service deletes that addon after a successful first boot. + Booting the kernel directly with a fixed -append makes every boot look like + a first boot, so Ignition re-runs, re-fetches from a file server that is no + longer listening, and the guest isolates to emergency.target instead of + coming back. + + So the config source and the initramfs address are appended to the command + line by patching a UKI addon on the ESP, in place, in the overlay. + """ + image = resolved_host_image() + image_file = VM_DIR / image.file_name + if not image_file.exists(): + die(f"Base image not found: {image_file}. Run create-vm first.") + + vm_disk = _create_vm_disk(image_file, image) + + config_path = VM_DIR / IGNITION_CONFIG_NAME + config_path.write_text(ignition_json) + config_path.chmod(0o600) + serve_base = os.environ.get("IGNITION_SERVE_BASE", f"http://{VM_GATEWAY}:{SERVE_PORT}") + config_url = f"{serve_base}/{IGNITION_CONFIG_NAME}" + + log(f"Patching the ESP boot command line in {vm_disk}...") + patched = ukiboot.patch_uki_cmdline_addon( + vm_disk, f"ignition.config.url={config_url} {initramfs_ip_karg()}") + log(f"Patched {patched.addon} ({patched.used}/{patched.capacity} bytes)") + + code, vars_template = ovmf_firmware() + vars_file = VM_DIR / f"{VM_NAME}-OVMF_VARS.fd" + shutil.copyfile(vars_template, vars_file) + + pid_file = VM_DIR / f"{VM_NAME}.pid" + qemu_log = VM_DIR / f"{VM_NAME}.log" + + log("============================================") + log(f" Launching VM: {VM_NAME} (Ignition)") + log(f" Host OS: {HOST_BASE_OS}") + log(f" Config: {config_url}") + log(f" Disk: {vm_disk}") + log(f" IP: {VM_IP}") + log(f" Log: {qemu_log}") + log("============================================") + + qemu_pid = _start_qemu( + vm_disk, qemu_mac_address(), pid_file, qemu_log, + boot_args=[ + "-machine", "q35", + "-drive", f"if=pflash,format=raw,readonly=on,file={code}", + "-drive", f"if=pflash,format=raw,file={vars_file}", + ], + ) + _wait_for_ssh(qemu_pid, qemu_log) + + # --------------------------------------------------------------------------- # create-vm # --------------------------------------------------------------------------- def _check_vm_prereqs() -> None: # Pre-flight - for cmd in ("qemu-system-x86_64", "qemu-img", "genisoimage"): + required = ["qemu-system-x86_64", "qemu-img"] + + if host_image().provisioning == "ignition": + # The boot command line is patched through NBD rather than a loop + # mount, which is what keeps that step unprivileged. + required.append("qemu-nbd") + ovmf_firmware() + else: + required.append("genisoimage") + + for cmd in required: if shutil.which(cmd) is None: die(f"{cmd} is required but not found in PATH") if not os.access("/dev/kvm", os.R_OK): @@ -1733,16 +2376,61 @@ def launch_vm() -> None: run(["sudo", "ip", "link", "set", TAP_NAME, "up"]) _nm_unmanage(TAP_NAME) - image = host_image() + image = resolved_host_image() + acquire_host_image(image) + + # An Ignition host is configured before it boots, and its config has to + # carry the bootstrap token and the API server address. Neither exists yet, + # so there is nothing to boot into: run-agent launches this VM instead. + if image.provisioning == "ignition": + log("Ignition host: VM launch deferred to run-agent") + return + + _launch_vm(ssh_pub_key) + + +def acquire_host_image(image: HostImage) -> Path: + """Place the base image in VM_DIR, verified, and return its path. + + A file:// source is symlinked rather than copied. The ACL image is 31 GiB + virtual and a copy per run is pure cost, and the overlay the VM boots from + is created separately, so the base is never written to. + """ image_file = VM_DIR / image.file_name - if not image_file.exists(): - log(f"Downloading {HOST_BASE_OS} cloud image...") - download_file(image.url, image_file) - else: + + if image.url.startswith("file://"): + source = Path(image.url[len("file://"):]) + if not image_file.exists(): + image_file.symlink_to(source) + log(f"Using local image: {source}") + elif image_file.exists() and _existing_image_is_intact(image_file, image): log(f"Using existing image: {image_file}") + else: + log(f"Downloading {HOST_BASE_OS} host image...") + # Downloaded under another name and renamed only once verified, so an + # interrupted download never sits under the name that is trusted. + partial = image_file.with_name(image_file.name + ".part") + partial.unlink(missing_ok=True) + download_file(image.url, partial, auth=image.auth) + if image.sha256: + verify_sha256(partial, image.sha256) + partial.replace(image_file) + run(["qemu-img", "info", "-f", image.backing_format, str(image_file)]) - _launch_vm(ssh_pub_key) + return image_file + + +def _existing_image_is_intact(image_file: Path, image: HostImage) -> bool: + """Check an image left by an earlier run or restored from the CI cache. Its + name says which build it should be, not that it is complete. One that does + not match is removed so it is downloaded again.""" + if not image.sha256 or file_sha256(image_file) == image.sha256.lower(): + return True + + log(f"Existing image {image_file.name} does not match its published digest; downloading it again") + image_file.unlink() + return False def create_vm() -> None: @@ -1845,6 +2533,14 @@ def block_external_network() -> None: def prepare_blocked_network_vm() -> None: """Install host packages that are outside the bootstrap artifact bundle.""" + # An image-managed host has no package manager and a read-only /usr, so + # there is nothing to install and nowhere to install it. Its prerequisites + # ship in the image; that is the premise the host entry rests on. + if host_image().provisioning == "ignition": + log("Image-managed host: prerequisites must be present in the image; " + "no preboot package installation") + return + log("Preparing VM host packages before blocking external egress...") wait_for_cloud_init() ssh_cmd(r""" @@ -2034,9 +2730,24 @@ def configure_kind_node_ip() -> None: # --------------------------------------------------------------------------- # run-agent # --------------------------------------------------------------------------- -def run_agent(node_config: NodeConfig) -> None: +def run_agent(node_config: NodeConfig, *, reinstall: bool = False) -> None: """Build agent, generate bootstrap script, and run it on the VM.""" + # The offline bootstrap path delivers an artifact bundle over SSH before the + # agent runs. An Ignition host has no such window: it is configured before + # it boots and the agent starts itself. A scenario naming an OCI reference + # is how that host takes artifacts offline. + if OFFLINE_BOOTSTRAP and host_image().provisioning == "ignition": + die("OFFLINE_BOOTSTRAP=1 is not supported with Ignition; " + "use an explicit offlineArtifactsOCIRef scenario") + + # The Ignition path serves the agent binary it stages itself, and points + # the config at that server. An agent from elsewhere, as the configuration + # scenarios pass, is never staged or served. + if os.environ.get("AGENT_URL") and host_image().provisioning == "ignition": + die("AGENT_URL is not supported with Ignition, so neither is the configuration suite; " + "run E2E_SUITE=lifecycle") + if not SSH_KEY.exists(): die(f"SSH key not found: {SSH_KEY}. Run create-vm first.") for cmd in (KUBECTL,): @@ -2045,7 +2756,7 @@ def run_agent(node_config: NodeConfig) -> None: agent_url_override = os.environ.get("AGENT_URL", "") if agent_url_override: - _run_agent_inner(agent_url_override, node_config) + _run_agent_inner(agent_url_override, node_config, reinstall=reinstall) log("Agent bootstrap completed") return @@ -2058,7 +2769,7 @@ def run_agent(node_config: NodeConfig) -> None: log(f"Agent download URL: {agent_url}") try: - _run_agent_inner(agent_url, node_config) + _run_agent_inner(agent_url, node_config, reinstall=reinstall) finally: httpd.shutdown() @@ -2103,6 +2814,11 @@ def prepare_agent_artifacts() -> str: run(["tar", "-czf", str(agent_tarball), "-C", str(REPO_ROOT / "bin"), "unbounded-agent"]) log(f"Agent tarball: {agent_tarball}") + # Ignition writes files declaratively and cannot extract an archive, so the + # bare binary is served alongside the tarball. It is staged unconditionally + # rather than per host, so that the two paths serve the same build. + shutil.copy2(agent_bin, VM_DIR / "unbounded-agent") + # Serve the tarball over HTTP runner_ip = VM_GATEWAY agent_url = f"http://{runner_ip}:{SERVE_PORT}/unbounded-agent-linux-amd64.tar.gz" @@ -2199,10 +2915,12 @@ def prepare_offline_bootstrap_artifacts(node_config: NodeConfig) -> str: log("Copying offline artifact bundle to VM...") scp_cmd(str(tarball), f"{SSH_TARGET}:/tmp/offline-bootstrap-artifacts.tar.gz") - ssh_cmd("sudo rm -rf /opt/unbounded/artifacts && sudo mkdir -p /opt/unbounded/artifacts") - ssh_cmd("sudo tar -xzf /tmp/offline-bootstrap-artifacts.tar.gz -C /opt/unbounded/artifacts") + # Not under the agent's host root: reset removes that once it is empty, and + # files the harness left there would keep it. + ssh_cmd(f"sudo rm -rf {OFFLINE_ARTIFACTS_DIR} && sudo mkdir -p {OFFLINE_ARTIFACTS_DIR}") + ssh_cmd(f"sudo tar -xzf /tmp/offline-bootstrap-artifacts.tar.gz -C {OFFLINE_ARTIFACTS_DIR}") - source = f"file:///opt/unbounded/artifacts/{kube_version}" + source = f"file://{OFFLINE_ARTIFACTS_DIR}/{kube_version}" log(f"Offline artifact source installed on VM: {source}") return source @@ -2466,7 +3184,221 @@ def log_message(self, format: str, *args: Any) -> None: # noqa: A002 return Handler -def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: +def agent_binary_url_and_digest() -> tuple[str, str]: + """Return the URL and SHA-256 of the bare agent binary served to the VM. + + Ignition writes files declaratively and cannot extract an archive, so the + bare binary is served alongside the tarball the other hosts download, and + its digest is what Ignition verifies before writing it. + """ + binary = VM_DIR / "unbounded-agent" + if not binary.exists(): + die(f"Agent binary not staged: {binary}. Run prepare_agent_artifacts first.") + + digest = hashlib.sha256(binary.read_bytes()).hexdigest() + serve_base = os.environ.get("IGNITION_SERVE_BASE", f"http://{VM_GATEWAY}:{SERVE_PORT}") + + return f"{serve_base}/unbounded-agent", digest + + +def _bootstrap_via_ignition(node_config: NodeConfig, api_server: str, + local_api_server: str, *, reinstall: bool = False) -> None: + """Render an Ignition config, boot the VM with it, and wait for bootstrap. + + Nothing is delivered over SSH here. Ignition places the agent binary and its + config, and a systemd unit runs preflight and bootstrap on first boot, which + is the path an Ignition-provisioned host uses in production. SSH is only + used afterwards, to report what happened. + """ + ssh_pub_key = _ensure_vm_ssh_key() + binary_url, binary_digest = agent_binary_url_and_digest() + + if node_config.block_external_network and not node_config.offline_artifacts_oci_ref: + die("blocked-network Ignition bootstrap requires a prepared local artifact bundle") + + args = [ + KUBECTL_UNBOUNDED, "machine", "manual-bootstrap", + AGENT_MACHINE_NAME, + "--site", E2E_SITE_NAME, + "--variant", "ignition", + "--agent-url", binary_url, + "--agent-sha256", binary_digest, + *node_config_bootstrap_args(node_config), + ] + if node_config.offline_artifacts_oci_ref: + args.extend(["--offline-artifacts-source", node_config.offline_artifacts_oci_ref]) + + log("Generating Ignition config with kubectl-unbounded machine manual-bootstrap...") + log_active_node_config(node_config) + doc = json.loads(capture(args)) + + # The kubeconfig names a loopback address the VM cannot reach. The agent + # config is base64 inside a data URL here, so this has to rewrite the + # decoded body rather than the rendered document. + doc = rewrite_ignition_api_server(doc, local_api_server, api_server) + if node_config.kubelet_configuration: + for item in doc["storage"]["files"]: + if item["path"] == "/etc/unbounded/agent/config.json": + cfg = json.loads(_decode_ignition_source(item["contents"]["source"])) + cfg.setdefault("Kubelet", {})["Configuration"] = node_config.kubelet_configuration + item["contents"]["source"] = ignition_data_url(json.dumps(cfg)) + doc = add_ignition_harness_access(doc, ssh_pub_key, qemu_mac_address()) + + previous_invocation = "" + + if reinstall: + # Same disk, same boot. Reinstall exists to prove a reset host can be + # provisioned again from what is already there, so replacing the disk + # would answer a different question and the caller checks the boot id + # to make sure it was not. + previous_invocation = _reinstall_ignition_payload(doc) + else: + # Stop whatever is running on this disk and discard it. The cloud-init + # path reaches a fresh VM through create-vm, but an Ignition host defers + # its launch to here, so nothing else has cleared the previous one and + # the overlay it still holds open cannot be recreated underneath it. + destroy_vm() + launch_ignition_vm(json.dumps(doc, indent=2)) + + _wait_for_ignition_bootstrap(previous_invocation) + + + +def destroy_vm() -> None: + """Stop the VM and discard its disk and firmware state. + + The firmware variables are removed along with the disk. They record the boot + entries of the disk that is being discarded, so keeping them across a fresh + provision leaves the new VM's firmware describing a disk that no longer + exists. + """ + _stop_qemu() + + for path in (VM_DIR / f"{VM_NAME}.qcow2", VM_DIR / f"{VM_NAME}-OVMF_VARS.fd"): + if path.exists(): + log(f"Removing {path}") + path.unlink() + + +def _reinstall_ignition_payload(doc: dict[str, Any]) -> str: + """Explicitly install agent payloads on a reset host; do not rerun Ignition. + + Only the agent binary, config and bootstrap unit are delivered. Guest + identity, networking, filesystem, boot state and SSH access must survive + reset; recreating those would hide cleanup/reinstallation defects. + + The config is expected to hold exactly those plus the harness's own access + additions. Anything else is a change in what bootstrap installs, and stops + the run rather than being skipped. + """ + agent_files = { + DAEMON_BINARY, + "/etc/unbounded/agent/config.json", + } + harness_files = {"/etc/hostname", f"/etc/systemd/network/{IGNITION_NETWORK_UNIT}"} + files = {item["path"]: item for item in doc["storage"]["files"]} + if set(files) - harness_files != agent_files: + die(f"Ignition payload paths {sorted(set(files) - harness_files)} are not the agent's {sorted(agent_files)}") + units = {u["name"]: u for u in doc["systemd"]["units"]} + if set(units) - {"waagent.service"} != {IGNITION_BOOTSTRAP_UNIT}: + die(f"Ignition units {sorted(units)} are not the bootstrap unit and the harness's own") + + for index, destination in enumerate(sorted(agent_files)): + item = files[destination] + content = _decode_ignition_source(item["contents"]["source"]) + if content is not None: + local = VM_DIR / f"reinstall-{index}" + local.write_text(content) + local.chmod(0o600) + elif destination.endswith("/bin/unbounded-agent"): + local = VM_DIR / "unbounded-agent" + else: + die(f"unsupported reinstall payload source for {destination}") + remote = f"/var/tmp/unbounded-reinstall-{index}" + scp_cmd(str(local), f"{SSH_TARGET}:{remote}") + ssh_cmd(f"sudo install -D -m {item['mode']:o} {remote} {destination} && rm {remote}") + + # Checked where it was installed, against the digest Ignition would + # have enforced, so what the host runs is tied to the build under test. + verification = item["contents"].get("verification") + if verification: + installed = ssh_capture(f"sudo sha256sum {destination}").split()[0] + if f"sha256-{installed}" != verification["hash"]: + die(f"{destination} on the VM does not match the rendered Ignition digest") + + unit = units[IGNITION_BOOTSTRAP_UNIT] + local = VM_DIR / "reinstall-bootstrap.service" + local.write_text(unit["contents"]) + + # Read before starting. Reset is supposed to have stopped this unit, and if + # it did not, starting it again does nothing and the wait below would + # otherwise accept the previous boot's run as this one's. + previous = ignition_bootstrap_invocation() + + scp_cmd(str(local), f"{SSH_TARGET}:/var/tmp/unbounded-reinstall.service") + ssh_cmd(f"sudo install -m 0644 /var/tmp/unbounded-reinstall.service " + f"/etc/systemd/system/{IGNITION_BOOTSTRAP_UNIT} && " + "rm /var/tmp/unbounded-reinstall.service && " + "sudo systemctl daemon-reload && " + f"sudo systemctl enable --now --no-block {IGNITION_BOOTSTRAP_UNIT}") + + return previous + + +def ignition_bootstrap_invocation() -> str: + """Return the current invocation id of the first-boot bootstrap unit. + + systemd assigns a new one each time a unit is started, so comparing it is + how the harness tells a run that just happened from one that happened + before. An empty string means the unit has never run, or is not loaded. + """ + result = bounded_ssh( + f"systemctl show {IGNITION_BOOTSTRAP_UNIT} -p InvocationID --value", + time.monotonic() + 30, + ) + + return result.stdout.strip() if result.returncode == 0 else "" + + +def _wait_for_ignition_bootstrap(previous_invocation: str = "") -> None: + """Wait for the first-boot bootstrap unit to finish, and report if it fails. + + The unit retries indefinitely by design, so a failure shows up as a unit + that never leaves activating rather than one that stops. Report its journal + either way: on this path there is no bootstrap script output to read. + + A run that has already finished is not a run. The unit is a oneshot with + RemainAfterExit=yes, so it stays active afterwards, and starting an active + unit does nothing. Waiting only for "active" therefore returns instantly + against the previous boot's run and reports success for an agent that never + executed. The invocation id has to change as well. + """ + unit = IGNITION_BOOTSTRAP_UNIT + log(f"Waiting for {unit} to complete...") + + deadline = time.monotonic() + 1200 + state = "" + while time.monotonic() < deadline: + result = bounded_ssh(f"systemctl show {unit} -p ActiveState --value", deadline) + state = result.stdout.strip() if result.returncode == 0 else "unreachable" + if state in ("active", "failed") and ignition_bootstrap_invocation() != previous_invocation: + break + time.sleep(10) + + diagnostics_deadline = time.monotonic() + 30 + result = bounded_ssh(f"systemctl show {unit} -p Result --value", diagnostics_deadline).stdout.strip() + journal = bounded_ssh(f"sudo journalctl -u {unit} --no-pager -n 80", diagnostics_deadline).stdout + (VM_DIR / "ignition-bootstrap.log").write_text(journal) + + if state != "active" or result not in ("success", ""): + log(journal) + die(f"{unit} did not complete: ActiveState={state or 'unknown'} Result={result}") + + log(f"{unit} completed") + + + +def _run_agent_inner(agent_url: str, node_config: NodeConfig, *, reinstall: bool = False) -> None: """Core logic for run-agent (after HTTP server is up).""" # Determine the Kind control-plane IP so connectivity checks have the @@ -2527,6 +3459,14 @@ def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: if not local_api_server: die("Could not determine local API server URL from kubeconfig") + # An Ignition host is configured before it boots, so there is no VM yet to + # wait for, nothing to deliver over SSH, and no cloud-init to complete. The + # config carries the binary and the agent config, and a first-boot unit runs + # preflight and bootstrap, which is the path such a host uses in production. + if host_image().provisioning == "ignition": + _bootstrap_via_ignition(node_config, api_server, local_api_server, reinstall=reinstall) + return + # Wait for cloud-init and verify connectivity before preparing optional # offline artifacts because preparing them copies files to the VM. log("Waiting for cloud-init to complete on VM...") @@ -2595,7 +3535,7 @@ def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: inject = ( "for i in $(seq 1 600); do " "if test -f /var/lib/unbounded/agent/install-state.json; then " - "mkdir -p /usr/local/bin/unbounded-agent-daemon-recovery.sh; exit 0; fi; " + f"mkdir -p {DAEMON_BIN_DIR}/unbounded-agent-daemon-recovery.sh; exit 0; fi; " "sleep 1; done; exit 1" ) ssh_cmd("sudo systemd-run --unit=p6-bootstrap-injection --collect /bin/bash -c " + shlex.quote(inject)) @@ -2644,7 +3584,7 @@ def add_retry_label(agent_config: dict) -> None: scp_cmd(str(retry_script_path), f"{SSH_TARGET}:/tmp/bootstrap-retry.sh") ssh_cmd("chmod +x /tmp/bootstrap-retry.sh") - ssh_cmd("sudo rmdir /usr/local/bin/unbounded-agent-daemon-recovery.sh") + ssh_cmd(f"sudo rmdir {DAEMON_BIN_DIR}/unbounded-agent-daemon-recovery.sh") run(["timeout", "1200", "ssh", *SSH_OPTS, SSH_TARGET, f"sudo {env_prefix} /tmp/bootstrap-retry.sh"]) after_text = bounded_ssh( @@ -3352,6 +4292,9 @@ def patch_kind_control_plane_node_ip() -> None: def validate_node_config_scenarios() -> None: """Discover node config scenarios and validate them in parallel.""" + # Refused before any mirroring or VM work; see the same check in run_agent. + if host_image().provisioning == "ignition": + die("the configuration suite is not supported with Ignition; run E2E_SUITE=lifecycle") workers = int(os.environ.get("CONFIG_SCENARIO_WORKERS", "2")) if workers < 1: die("CONFIG_SCENARIO_WORKERS must be positive") @@ -3848,11 +4791,71 @@ def reset_agent() -> None: die(f"nspawn machine '{nspawn_name}' is still running after reset") log(f"nspawn machine '{nspawn_name}' is not running") + validate_reset_cleanup() + log("============================================") log(" Agent reset PASSED") log("============================================") +def validate_reset_cleanup() -> None: + """Assert reset removed the agent's own files from the host. + + A reset that reports success while leaving an installation on disk orphans + the files, and some of them, such as the recovery script, make the next + bootstrap's existing-deployment preflight refuse a host the operator was + just told is clean. Preflight checks only that subset, since Ignition places + the agent binary before it runs; this checks everything reset should remove. + + Both the host root and the legacy root are checked, and the host root has to + be gone as well: removed once empty on a host installed under it, and the + link removed on a host migrated from the legacy root. Reset does not depend + on the migration having run, so a check that only looked where this run + installed would not notice the other being left behind. + """ + log(f"Verifying reset removed the agent's files under {HOST_ROOT} and {LEGACY_HOST_ROOT}...") + + must_be_absent = [HOST_ROOT] + for root in (HOST_ROOT, LEGACY_HOST_ROOT): + must_be_absent.extend([ + f"{root}/bin/unbounded-agent", + f"{root}/bin/unbounded-agent-blue", + f"{root}/bin/unbounded-agent-green", + f"{root}/bin/unbounded-agent-current", + f"{root}/bin/unbounded-agent-last-good", + f"{root}/bin/unbounded-agent-nspawn-lifecycle", + f"{root}/bin/unbounded-agent-daemon-recovery.sh", + f"{root}/libexec/unbounded-localdns-network", + ]) + + must_be_absent.extend([ + "/etc/systemd/system/unbounded-agent-daemon.service", + "/etc/systemd/system/unbounded-agent-daemon-recovery.service", + "/etc/unbounded/agent", + ]) + + # A first-boot unit that survived a reset would bootstrap the host again on + # the next boot, undoing the reset without anyone asking. + if host_image().provisioning == "ignition": + must_be_absent.append("/etc/systemd/system/unbounded-agent-bootstrap.service") + + # -e follows symlinks, so a dangling link reads as absent. Test the link + # itself as well, because a leftover symlink is still a leftover file and + # is exactly what a partial cleanup leaves behind. + checks = " ; ".join( + f'if [ -e "{path}" ] || [ -L "{path}" ]; then echo "{path}"; fi' + for path in must_be_absent + ) + remaining = ssh_capture(f"sudo sh -c '{checks}'").strip() + + if remaining: + for path in remaining.splitlines(): + log(f" still present: {path}") + die(f"reset left {len(remaining.splitlines())} agent artifacts on the host") + + log(f"Reset removed all {len(must_be_absent)} agent artifacts") + + # --------------------------------------------------------------------------- # install-machine-crd # --------------------------------------------------------------------------- @@ -4285,10 +5288,13 @@ def validate_host_agent_upgrade() -> None: wait_for_daemon_active() after_current = read_daemon_current_target() last_good = read_daemon_last_good_target() - if after_current != DAEMON_BINARY_GREEN: - die(f"host-driven current target mismatch: got {after_current!r}, expected {DAEMON_BINARY_GREEN!r}") - if last_good != DAEMON_BINARY_BLUE: - die(f"host-driven last-good target mismatch: got {last_good!r}, expected {DAEMON_BINARY_BLUE!r}") + # The targets are read resolved, so compare them with the slots resolved the + # same way. + green, blue = resolve_on_host(DAEMON_BINARY_GREEN), resolve_on_host(DAEMON_BINARY_BLUE) + if after_current != green: + die(f"host-driven current target mismatch: got {after_current!r}, expected {green!r}") + if last_good != blue: + die(f"host-driven last-good target mismatch: got {last_good!r}, expected {blue!r}") preserved_digest = ssh_capture(f"sudo sha256sum {DAEMON_BINARY_BLUE} | awk '{{print $1}}'").strip() if preserved_digest != legacy_digest: @@ -4316,6 +5322,211 @@ def validate_host_agent_upgrade() -> None: log("============================================") +# --------------------------------------------------------------------------- +# validate-host-root / migration suite +# --------------------------------------------------------------------------- +LEGACY_LAYOUT = [ + "unbounded-agent", "unbounded-agent-blue", "unbounded-agent-green", + "unbounded-agent-current", "unbounded-agent-last-good", +] + + +def _daemon_unit_runs(binary_dir: str) -> None: + """Assert the daemon unit starts the current link in *binary_dir*.""" + + want = f"{binary_dir}/unbounded-agent-current daemon" + unit = ssh_capture("sudo cat /etc/systemd/system/unbounded-agent-daemon.service") + if want not in unit: + die(f"daemon unit does not run {want!r}:\n{unit}") + + +def _log_selinux_denials() -> None: + """Print SELinux denials that mention the agent, without failing on them.""" + + denials = ssh_capture_quiet( + "if command -v selinuxenabled >/dev/null 2>&1 && selinuxenabled; then " + "sudo journalctl -b -k --no-pager | grep 'avc: denied' | grep -i unbounded; fi" + ).stdout.strip() + if denials: + log("SELinux denials mentioning the agent:") + for line in denials.splitlines(): + log(f" {line}") + + +def validate_host_root() -> None: + """Assert a host installed by this build keeps the agent under the host root. + + Nothing may be installed under the legacy root: the install script seeds it + only for an agent released before the host root, and a fresh host that + carried the legacy layout would be migrated to it by the next agent. + + On an SELinux host the directories the agent creates start with the label of + /opt, which is not the one policy gives them, so they are checked against + policy. + """ + + state = host_root_state() + if state != "dir": + die(f"{HOST_ROOT} is {state!r}; a host installed by this build must have a real directory there") + + bin_dir = resolve_on_host(DAEMON_BIN_DIR) + legacy = " ".join(f"{LEGACY_HOST_ROOT}/bin/{name}" for name in LEGACY_LAYOUT) + script = textwrap.dedent(f""" + set -eu + for d in {HOST_ROOT} {HOST_ROOT}/bin {HOST_ROOT}/libexec; do + got=$(stat -c '%a %U' "$d") + [ "$got" = "755 root" ] || {{ echo "$d is $got, expected 755 root"; exit 1; }} + done + current=$(readlink -f {DAEMON_BINARY_CURRENT}) + case "$current" in + {bin_dir}/*) ;; + *) echo "current binary $current is not under {bin_dir}"; exit 1 ;; + esac + for f in {legacy}; do + if [ -e "$f" ] || [ -L "$f" ]; then echo "$f exists under the legacy root"; exit 1; fi + done + if command -v selinuxenabled >/dev/null 2>&1 && selinuxenabled && command -v restorecon >/dev/null 2>&1; then + relabel=$(restorecon -nvR {HOST_ROOT}) + if [ -n "$relabel" ]; then echo "labels differ from policy:"; echo "$relabel"; exit 1; fi + fi + """) + result = ssh_capture_quiet("sudo bash -c " + shlex.quote(script)) + if result.returncode != 0: + die(f"host root check failed: {(result.stdout + result.stderr).strip()}") + + _daemon_unit_runs(bin_dir) + _log_selinux_denials() + log(f"Agent is installed under {HOST_ROOT}, and nothing under {LEGACY_HOST_ROOT}") + + +def validate_host_root_legacy() -> None: + """Assert the host carries the layout of an agent released before the host root.""" + + state = host_root_state() + if state != "absent": + die(f"{HOST_ROOT} is {state!r}; an agent released before the host root must not create it") + + current = resolve_on_host(f"{LEGACY_HOST_ROOT}/bin/unbounded-agent-current") + if not current.startswith(f"{LEGACY_HOST_ROOT}/bin/"): + die(f"legacy current binary resolves to {current!r}") + + _daemon_unit_runs(f"{LEGACY_HOST_ROOT}/bin") + log(f"Legacy layout is installed under {LEGACY_HOST_ROOT}") + + +def validate_host_root_migrated() -> None: + """Assert the host root is linked to the legacy root and the layout is unchanged. + + The link is all the migration adds. The units and the recovery script keep + naming the legacy paths, so an older agent rolled back to still finds its + files, and the slots compare equal to the targets the older agent wrote. + """ + + state = host_root_state() + if state != f"link:{LEGACY_HOST_ROOT}": + die(f"{HOST_ROOT} is {state!r}; a migrated host must link it to {LEGACY_HOST_ROOT}") + + current = read_daemon_current_target() + if not current.startswith(f"{LEGACY_HOST_ROOT}/bin/"): + die(f"current binary resolves to {current!r}, not under {LEGACY_HOST_ROOT}/bin") + + _daemon_unit_runs(f"{LEGACY_HOST_ROOT}/bin") + _log_selinux_denials() + log(f"{HOST_ROOT} links to {LEGACY_HOST_ROOT}, and the legacy layout is unchanged") + + +def run_legacy_agent(node_config: NodeConfig) -> None: + """Install the last release before the host root. + + It is fetched by the install script from the published release, the same + way a host installed before the host root got it. + """ + + if host_image().provisioning == "ignition": + die("the migration suite needs an agent released before the host root, which cannot be " + "installed on an immutable host; run it on a cloud-init host") + if not re.fullmatch(r"v\d+\.\d+\.\d+", LEGACY_AGENT_VERSION): + die(f"LEGACY_AGENT_VERSION must be a release tag such as v0.8.0, got {LEGACY_AGENT_VERSION!r}") + + # The bootstrap payload is still rendered by this build's kubectl-unbounded, + # and the upgrades that follow serve this build's agent. + prepare_agent_artifacts() + + previous = os.environ.get("AGENT_URL") + os.environ["AGENT_URL"] = f"{LEGACY_AGENT_RELEASE_URL}/{LEGACY_AGENT_TARBALL}" + try: + run_agent(node_config) + finally: + if previous is None: + os.environ.pop("AGENT_URL", None) + else: + os.environ["AGENT_URL"] = previous + + +def _download_legacy_agent_tarball() -> Path: + """Return an AgentUpgrade archive holding the legacy release's agent binary. + + The release tarball is checked against the release checksums, then its + binary is repackaged alone: AgentUpgrade accepts an archive holding only + the agent, and the release also ships its license files. + """ + + tarball = VM_DIR / f"unbounded-agent-{LEGACY_AGENT_VERSION}.tar.gz" + with urllib.request.urlopen(f"{LEGACY_AGENT_RELEASE_URL}/checksums.txt", timeout=60) as response: + checksums = response.read().decode() + want = next((line.split()[0] for line in checksums.splitlines() + if line.split()[1:] == [LEGACY_AGENT_TARBALL]), "") + if not want: + die(f"{LEGACY_AGENT_VERSION} publishes no checksum for {LEGACY_AGENT_TARBALL}") + + with urllib.request.urlopen(f"{LEGACY_AGENT_RELEASE_URL}/{LEGACY_AGENT_TARBALL}", timeout=300) as response: + tarball.write_bytes(response.read()) + got = hashlib.sha256(tarball.read_bytes()).hexdigest() + if got != want: + die(f"{LEGACY_AGENT_TARBALL} from {LEGACY_AGENT_VERSION} has digest {got}, expected {want}") + + build_dir = VM_DIR / "agent-upgrade-legacy-release" + shutil.rmtree(build_dir, ignore_errors=True) + build_dir.mkdir(parents=True) + run(["tar", "-xzf", str(tarball), "-C", str(build_dir), "unbounded-agent"]) + upgrade = VM_DIR / f"unbounded-agent-{LEGACY_AGENT_VERSION}-upgrade.tar.gz" + run(["tar", "-czf", str(upgrade), "-C", str(build_dir), "unbounded-agent"]) + + return upgrade + + +def validate_agent_downgrade_to_legacy() -> None: + """Validate AgentUpgrade back to the last release before the host root. + + On a migrated host the host root is the legacy root, which every agent finds, + so this is an ordinary upgrade. What it proves is that the migration left + nothing an older agent cannot run with. + """ + + before_current = read_daemon_current_target() + tarball = _download_legacy_agent_tarball() + operation_name = f"e2e-agent-downgrade-{int(time.time())}" + _serve_agent_upgrade_tarball(tarball, operation_name) + + wait_for_daemon_active() + after_current = read_daemon_current_target() + last_good = read_daemon_last_good_target() + if after_current == before_current: + die(f"downgrade did not switch the daemon current symlink (still points to {after_current})") + if last_good != before_current: + die(f"last-good symlink mismatch: got {last_good!r}, expected {before_current!r}") + + version_output = ssh_capture(f"sudo {DAEMON_BINARY_CURRENT} version") + if LEGACY_AGENT_VERSION.lstrip("v") not in version_output: + die(f"current daemon is not {LEGACY_AGENT_VERSION}: {version_output!r}") + + validate_host_root_migrated() + wait_for_node_ready(AGENT_MACHINE_NAME) + log("============================================") + log(f" Downgrade to {LEGACY_AGENT_VERSION} validation PASSED") + log("============================================") + + # --------------------------------------------------------------------------- # validate-agent-upgrade-operation # --------------------------------------------------------------------------- @@ -4376,6 +5587,22 @@ def validate_agent_upgrade_rollback() -> None: if read_daemon_current_target() != previous_good: die("broken AgentUpgrade changed current daemon binary symlink") + # An agent released before the host root would look for its files under the + # legacy root, where a host installed under the host root has none. On a + # host linked to the legacy root every agent finds them, so the check only + # applies here. + if host_root_state() == "dir": + legacy_operation_name = f"e2e-agent-upgrade-legacy-{int(time.time())}" + legacy_tarball = VM_DIR / "unbounded-agent-upgrade-legacy.tar.gz" + _build_legacy_agent_tarball(legacy_tarball) + legacy_operation = _serve_agent_upgrade_tarball( + legacy_tarball, legacy_operation_name, expect_complete=False) + legacy_message = legacy_operation.get("status", {}).get("message", "") + if "predates the host root" not in legacy_message: + die(f"AgentUpgrade to an agent without host-root was not refused: {legacy_message!r}") + if read_daemon_current_target() != previous_good: + die("refused AgentUpgrade changed current daemon binary symlink") + operation_name = f"e2e-agent-upgrade-rollback-{int(time.time())}" tarball = VM_DIR / "unbounded-agent-upgrade-daemon-bad.tar.gz" _build_daemon_failing_agent_tarball(tarball) @@ -4609,9 +5836,17 @@ def ssh_log(name: str, command: str) -> None: _write_command_log(logs_dir / f"{prefix}{name}", ["ssh", *ssh_opts, ssh_target, command]) ssh_log("vm-journal.log", "sudo journalctl --no-pager -l") - ssh_log("vm-cloud-init.log", "sudo cat /var/log/cloud-init.log") - ssh_log("vm-cloud-init-output.log", "sudo cat /var/log/cloud-init-output.log") - ssh_log("vm-cloud-init-status.json", "sudo cloud-init status --format json") + + if host_image().provisioning == "ignition": + # No cloud-init to report on. Ignition records what it did in the + # journal, which is already collected above, and asking anyway leaves + # three empty files that read as a host where cloud-init failed. + ssh_log("vm-ignition.log", "sudo journalctl -u ignition-\\* --no-pager -l") + else: + ssh_log("vm-cloud-init.log", "sudo cat /var/log/cloud-init.log") + ssh_log("vm-cloud-init-output.log", "sudo cat /var/log/cloud-init-output.log") + ssh_log("vm-cloud-init-status.json", "sudo cloud-init status --format json") + ssh_log("vm-unbounded-agent.log", "sudo journalctl -u unbounded-agent --no-pager -l") ssh_log("vm-unbounded-agent-daemon.log", "sudo journalctl -u unbounded-agent-daemon --no-pager -l") ssh_log("vm-systemd-machined.log", "sudo journalctl -u systemd-machined --no-pager -l") @@ -4805,11 +6040,59 @@ def reboot_host_and_wait() -> str: die("host did not return with a new boot ID within 300s") +INSTALL_RECORD = "/var/lib/unbounded/agent/install-state.json" + + +def install_record_stamp(deadline: float) -> str: + """Identify the install record's current contents. Every write replaces the + file, so the inode changes even within one second.""" + return bounded_ssh(f"sudo stat -c '%i %Y' {INSTALL_RECORD}", deadline, check=True).stdout.strip() + + +def ignition_reboot_problems(state: str, restarts: str, journal: str, + record_before: str, record_after: str) -> list[str]: + """What the first-boot unit did on a reboot that it should not have. + + The unit runs start on every boot. On a healthy host that only verifies: + it must not fail and retry, repair the daemon, or rewrite the record. + """ + problems = [] + if state != "active": + problems.append(f"ActiveState={state or 'unknown'}") + if restarts != "0": + problems.append(f"NRestarts={restarts or 'unknown'}") + if "daemon unit started" in journal: + problems.append("start repaired the daemon") + if record_after != record_before: + problems.append("the install record was rewritten") + return problems + + +def validate_ignition_reboot(record_before: str, deadline: float) -> None: + unit = IGNITION_BOOTSTRAP_UNIT + state = "" + while time.monotonic() < deadline: + state = bounded_ssh(f"systemctl show {unit} -p ActiveState --value", deadline).stdout.strip() + if state in ("active", "failed"): + break + time.sleep(5) + + restarts = bounded_ssh(f"systemctl show {unit} -p NRestarts --value", deadline).stdout.strip() + journal = bounded_ssh(f"sudo journalctl -b -u {unit} --no-pager", deadline).stdout + problems = ignition_reboot_problems(state, restarts, journal, record_before, install_record_stamp(deadline)) + if problems: + log(journal) + die(f"{unit} after a reboot: " + "; ".join(problems)) + log(f"{unit} verified the host after the reboot without repairing it") + + def validate_host_reboot() -> None: """Require fresh host/node identity and networking without repairing components.""" previous = node_boot_id(AGENT_MACHINE_NAME) if not previous: die("node boot identity is absent before host reboot") + ignition = host_image().provisioning == "ignition" + record_before = install_record_stamp(time.monotonic() + 30) if ignition else "" reboot_host_and_wait() deadline = time.monotonic() + 300 while time.monotonic() < deadline: @@ -4820,6 +6103,8 @@ def validate_host_reboot() -> None: ready = any(c.get("type") == "Ready" and c.get("status") == "True" for c in node.get("status", {}).get("conditions", [])) if boot and boot != previous and ready: bounded_ssh("systemctl is-active unbounded-agent-daemon.service", deadline, check=True) + if ignition: + validate_ignition_reboot(record_before, deadline) validate_workload() log("Host reboot and fresh workload/DNS passed without component repair") return @@ -4829,7 +6114,7 @@ def validate_host_reboot() -> None: def reinstall_agent(node_config: NodeConfig) -> None: before = bounded_ssh("cat /proc/sys/kernel/random/boot_id", time.monotonic() + 15, check=True).stdout.strip() - run_agent(node_config) + run_agent(node_config, reinstall=True) after = bounded_ssh("cat /proc/sys/kernel/random/boot_id", time.monotonic() + 15, check=True).stdout.strip() if not before or after != before: die("same-disk reinstall changed host boot identity") @@ -4844,7 +6129,7 @@ def validate_bootstrap_repair() -> None: before=$(sha256sum /etc/unbounded/agent/kube2-applied-config.json) node_pid=$(systemctl show systemd-nspawn@kube2.service --property=MainPID --value) test "$node_pid" -gt 0 - cp /usr/local/bin/unbounded-agent-current /tmp/p6-repair-agent + cp @DAEMON_BINARY_CURRENT@ /tmp/p6-repair-agent chmod 0755 /tmp/p6-repair-agent python3 - <<'PY' from pathlib import Path @@ -4867,7 +6152,7 @@ def validate_bootstrap_repair() -> None: test "$node_pid" = "$(systemctl show systemd-nspawn@kube2.service --property=MainPID --value)" grep -q '"phase": "complete"' /var/lib/unbounded/agent/install-state.json systemctl is-active unbounded-agent-daemon.service - """) + """).replace("@DAEMON_BINARY_CURRENT@", DAEMON_BINARY_CURRENT) bounded_ssh("sudo bash -c " + shlex.quote(script), time.monotonic() + 180, check=True) validate_workload() log("Completed-install repair preserved the repaved slot and workload") @@ -4876,15 +6161,25 @@ def validate_bootstrap_repair() -> None: SUITES: dict[str, list[str]] = { "setup": ["configure-kind-kube-proxy", "install-machine-crd", "deploy-unbounded-net-controller", "start-machina-controller", "validate-machina-controller", "validate-controllers-healthy"], - "lifecycle": ["run-agent", "wait-for-node", "validate-host-nspawn-distro", "validate-controllers-healthy", + "lifecycle": ["run-agent", "wait-for-node", "validate-host-root", "validate-host-nspawn-distro", "validate-controllers-healthy", "validate-node-config", "dump-persisted-agent-config", "validate-kube-proxy", "validate-machine-cr-created", "validate-node-reboot-operation", "validate-host-agent-upgrade", "validate-agent-upgrade-operation", "validate-agent-upgrade-rollback", "validate-workload", "validate-host-reboot", "reset-agent", - "delete-machine-cr", "ensure-kind-bridge", "reinstall-agent", "wait-for-node", "validate-host-nspawn-distro", + "delete-machine-cr", "ensure-kind-bridge", "reinstall-agent", "wait-for-node", "validate-host-root", + "validate-host-nspawn-distro", "validate-controllers-healthy", "dump-persisted-agent-config", "validate-kube-proxy", "validate-machine-cr-created", "validate-node-reboot-operation", "validate-workload", "validate-node-repave-upgrade"], "configuration": ["validate-node-configs"], "fresh-bootstrap": ["run-agent", "wait-for-node", "validate-workload"], + # A host installed before the host root: the first upgrade links the host + # root to the legacy root, the host keeps working across a reboot, upgrades + # and a return to the older release, and reset removes the link with the + # files. + "migration": ["run-legacy-agent", "wait-for-node", "validate-host-root-legacy", + "validate-agent-upgrade-operation", "validate-host-root-migrated", "validate-host-reboot", + "validate-agent-upgrade-operation", "validate-host-root-migrated", + "validate-agent-downgrade-to-legacy", "validate-agent-upgrade-operation", + "validate-host-root-migrated", "reset-agent"], "bootstrap-recovery": ["run-agent-recovery", "wait-for-node", "validate-workload", "validate-node-repave-upgrade", "validate-bootstrap-repair"], } @@ -4932,6 +6227,7 @@ def command(_node_config: NodeConfig) -> None: "retire-lifecycle-vm": _without_node_config(retire_lifecycle_vm), "collect-logs": _without_node_config(collect_logs), "create-vm-bridge": _without_node_config(create_vm_bridge), + "resolve-host-image": _without_node_config(resolve_host_image), "create-vm": _without_node_config(create_vm), "prepare-blocked-network-vm": _without_node_config(prepare_blocked_network_vm), "block-external-network": _without_node_config(block_external_network), @@ -4964,6 +6260,11 @@ def command(_node_config: NodeConfig) -> None: "validate-node-repave-upgrade": validate_node_repave_upgrade, "validate-node-configs": _without_node_config(validate_node_config_scenarios), "reset-agent": _without_node_config(reset_agent), + "validate-host-root": _without_node_config(validate_host_root), + "validate-host-root-legacy": _without_node_config(validate_host_root_legacy), + "validate-host-root-migrated": _without_node_config(validate_host_root_migrated), + "run-legacy-agent": run_legacy_agent, + "validate-agent-downgrade-to-legacy": _without_node_config(validate_agent_downgrade_to_legacy), "cleanup": _without_node_config(cleanup), } diff --git a/hack/agent/e2e-kind/test_host_image.py b/hack/agent/e2e-kind/test_host_image.py new file mode 100644 index 000000000..90adcae5d --- /dev/null +++ b/hack/agent/e2e-kind/test_host_image.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for host image selection and acquisition. + +These cover the decisions the harness makes before a VM exists: which image to +boot, where to get it, and which paths the agent will use on it. Getting any of +them wrong produces a run that fails somewhere else entirely, usually as a guest +that will not boot or an assertion against a path nothing ever wrote to. +""" +import json +import os +import unittest +from pathlib import Path +from unittest.mock import patch + +import e2e + + +def clear_image_pin(test: unittest.TestCase) -> None: + """Resolve from the manifest regardless of the environment. CI exports a + pinned build for the job, which these tests must not inherit.""" + for name in ("ACL_IMAGE_URL", "ACL_IMAGE_SHA256", "ACL_IMAGE_BUILD_ID"): + patcher = patch.object(e2e, name, "") + patcher.start() + test.addCleanup(patcher.stop) + + +class TestHostImageSelection(unittest.TestCase): + """The per-OS differences that the rest of the harness reads.""" + + def setUp(self): + # The manifest lookup is cached so a real run fetches it once. These + # feed it different manifests, so each starts from a clear cache. + e2e.acl_image_from_manifest.cache_clear() + clear_image_pin(self) + + def test_conventional_hosts_use_cloud_init(self): + """Every pre-existing host must keep the behavior it had. + + The provisioning field was added for one image. If it changed the + answer for any other, the change would show up as a different bootstrap + path on hosts that were working. + """ + for base_os in ("ubuntu2404", "ubuntu2604", "fedora", "almalinux9", + "almalinux10", "centosstream9", "centosstream10"): + with self.subTest(base_os=base_os): + with patch.object(e2e, "HOST_BASE_OS", base_os): + image = e2e.host_image() + + self.assertEqual(image.provisioning, "cloud-init") + self.assertEqual(image.ssh_user, "ubuntu") + self.assertEqual(image.auth, "") + self.assertTrue(image.packages, "a host with a package manager installs prerequisites") + + def test_acl_declares_an_immutable_host(self): + """The properties that make ACL different, asserted together. + + They are not independent. Ignition provisioning is why there is no + package installation step, and no package installation is why the image + has to carry the tools. A change to one of them without the others + describes a host that does not exist. + """ + with patch.dict(os.environ, {"HOST_IMAGE_PATH": __file__}): + with patch.object(e2e, "HOST_BASE_OS", "acl"): + image = e2e.host_image() + + self.assertEqual(image.provisioning, "ignition") + self.assertEqual(image.ssh_user, "core") + self.assertEqual(image.packages, []) + + def test_acl_from_the_manifest_carries_download_credentials(self): + """The blob needs a token too, not just the manifest. + + These are two separate requests, and only the manifest fetch names its + credential inline. The image download reads it off the HostImage, so an + image resolved from the manifest has to carry one or the download gets a + 401 after the manifest has already succeeded. + """ + manifest = json.dumps(TestACLImageResolution.MANIFEST) + + with patch.dict(os.environ, {"HOST_IMAGE_PATH": ""}): + with patch.object(e2e, "HOST_BASE_OS", "acl"): + with patch.object(e2e, "http_get", return_value=manifest): + unresolved = e2e.host_image() + image = e2e.resolved_host_image() + + self.assertEqual(image.auth, "azure-storage") + self.assertEqual(image.sha256, TestACLImageResolution.MANIFEST["qcow2"]["sha256"]) + self.assertEqual(image.url, TestACLImageResolution.MANIFEST["qcow2"]["url"]) + + # The unresolved form names no blob. Resolving it reads the published + # manifest, and host_image is called for the ssh user and provisioning far + # more often than for the image, including at import, so it must not + # drag a network call along with it. + self.assertEqual(unresolved.url, "") + self.assertEqual(unresolved.provisioning, "ignition") + + def test_a_local_image_needs_no_credentials(self): + """HOST_IMAGE_PATH is the developer path and must not require an Azure + login to boot a file already on disk.""" + with patch.dict(os.environ, {"HOST_IMAGE_PATH": __file__}): + with patch.object(e2e, "HOST_BASE_OS", "acl"): + image = e2e.host_image() + + self.assertEqual(image.auth, "") + self.assertEqual(image.sha256, "", "a local file has no published digest to check") + self.assertTrue(image.url.startswith("file://")) + + def test_unsupported_host_names_the_supported_ones(self): + with patch.object(e2e, "HOST_BASE_OS", "windows"): + with self.assertRaises(SystemExit): + e2e.host_image() + + + +class TestACLImageResolution(unittest.TestCase): + """Resolving the image from the published manifest.""" + + def setUp(self): + e2e.acl_image_from_manifest.cache_clear() + clear_image_pin(self) + + MANIFEST = { + "build_id": "2026091817", + "qcow2": { + "url": "https://example.test/images/2026091817/acl.qcow2", + "sha256": "7c45558dac005626c06d40594567964739fc96eefab22fdb62e7275191231f45", + "size": 661192704, + }, + } + + def test_file_name_is_derived_from_the_build(self): + """A refreshed image must not be masked by a cached file. + + The harness reuses an image already in VM_DIR rather than downloading + again. If every build landed under the same name, a machine that had run + the suite before would silently keep booting the old one. + """ + with patch.object(e2e, "http_get", return_value=json.dumps(self.MANIFEST)): + url, file_name, digest = e2e.acl_image_from_manifest() + + self.assertEqual(url, self.MANIFEST["qcow2"]["url"]) + self.assertEqual(file_name, "acl-2026091817.qcow2") + self.assertEqual(digest, self.MANIFEST["qcow2"]["sha256"]) + + def test_manifest_is_read_with_storage_credentials(self): + """The account disables anonymous access and shared keys alike, so the + manifest is unreadable without a bearer token.""" + with patch.object(e2e, "http_get", return_value=json.dumps(self.MANIFEST)) as get: + e2e.acl_image_from_manifest() + + get.assert_called_once() + self.assertEqual(get.call_args.kwargs.get("auth"), "azure-storage") + + def test_build_override_must_match_the_manifest(self): + """On its own the build id checks the manifest rather than pinning it. + Silently ignoring it when the manifest has moved on would leave the run + on a build it was not expecting.""" + with patch.object(e2e, "http_get", return_value=json.dumps(self.MANIFEST)): + with patch.object(e2e, "ACL_IMAGE_BUILD_ID", "2026010101"): + with self.assertRaises(SystemExit): + e2e.acl_image_from_manifest() + + with patch.object(e2e, "ACL_IMAGE_BUILD_ID", "2026091817"): + _url, file_name, _digest = e2e.acl_image_from_manifest() + + self.assertEqual(file_name, "acl-2026091817.qcow2") + + def test_a_malformed_build_id_is_refused(self): + """The build names the cached file, so an empty or odd one could make + different builds share a name, or a path.""" + for build in ("", None, 2026, "../x", "a b"): + with self.subTest(build=build): + e2e.acl_image_from_manifest.cache_clear() + manifest = dict(self.MANIFEST, build_id=build) + with patch.object(e2e, "http_get", return_value=json.dumps(manifest)): + with self.assertRaises(SystemExit): + e2e.acl_image_from_manifest() + + def test_a_pinned_build_skips_the_manifest(self): + with patch.object(e2e, "ACL_IMAGE_URL", "https://example.test/old.qcow2"), \ + patch.object(e2e, "ACL_IMAGE_SHA256", "ab" * 32), \ + patch.object(e2e, "ACL_IMAGE_BUILD_ID", "2026010101"), \ + patch.object(e2e, "http_get") as get: + self.assertEqual(e2e.acl_image_from_manifest(), + ("https://example.test/old.qcow2", "acl-2026010101.qcow2", "ab" * 32)) + get.assert_not_called() + + def test_a_partial_pin_is_refused(self): + for url, digest, build in (("u", "", "b"), ("", "d", "b"), ("u", "d", "")): + with self.subTest(url=url, digest=digest, build=build): + e2e.acl_image_from_manifest.cache_clear() + with patch.object(e2e, "ACL_IMAGE_URL", url), \ + patch.object(e2e, "ACL_IMAGE_SHA256", digest), \ + patch.object(e2e, "ACL_IMAGE_BUILD_ID", build), \ + patch.object(e2e, "http_get", return_value=json.dumps(self.MANIFEST)): + with self.assertRaises(SystemExit): + e2e.acl_image_from_manifest() + + def test_resolve_host_image_exports_a_pin_that_reads_back(self): + """What resolve-host-image writes for later steps has to resolve to the + same image without reading the manifest.""" + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + env_file, output_file = Path(tmp) / "env", Path(tmp) / "output" + with patch.object(e2e, "HOST_BASE_OS", "acl"), \ + patch.object(e2e, "http_get", return_value=json.dumps(self.MANIFEST)), \ + patch.dict(os.environ, {"GITHUB_ENV": str(env_file), "GITHUB_OUTPUT": str(output_file)}): + e2e.resolve_host_image() + + exported = dict(line.split("=", 1) for line in env_file.read_text().splitlines()) + self.assertEqual(output_file.read_text(), "build=2026091817\n") + + e2e.acl_image_from_manifest.cache_clear() + with patch.object(e2e, "ACL_IMAGE_URL", exported["ACL_IMAGE_URL"]), \ + patch.object(e2e, "ACL_IMAGE_SHA256", exported["ACL_IMAGE_SHA256"]), \ + patch.object(e2e, "ACL_IMAGE_BUILD_ID", exported["ACL_IMAGE_BUILD_ID"]), \ + patch.object(e2e, "http_get") as get: + self.assertEqual(e2e.acl_image_from_manifest(), ( + self.MANIFEST["qcow2"]["url"], "acl-2026091817.qcow2", self.MANIFEST["qcow2"]["sha256"])) + get.assert_not_called() + + def test_manifest_without_a_digest_is_refused(self): + """An unverified image is the one thing worse than no image: it boots, + and whatever goes wrong afterwards looks like a product bug.""" + for qcow2 in ({}, {"url": "https://example.test/a.qcow2"}, {"sha256": "abc"}): + with self.subTest(qcow2=qcow2): + manifest = json.dumps({"build_id": "b", "qcow2": qcow2}) + with patch.object(e2e, "http_get", return_value=manifest): + with self.assertRaises(SystemExit): + e2e.acl_image_from_manifest() + + +class TestVerifySHA256(unittest.TestCase): + def test_mismatch_removes_the_file_and_fails(self): + """The file is deleted so the next run downloads again rather than + reusing a bad image that is now sitting under the expected name.""" + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "image.qcow2" + target.write_bytes(b"not the image") + + with self.assertRaises(SystemExit): + e2e.verify_sha256(target, "0" * 64) + + self.assertFalse(target.exists(), "a corrupt download must not be left in place") + + def test_matching_digest_keeps_the_file(self): + import hashlib + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "image.qcow2" + target.write_bytes(b"contents") + + e2e.verify_sha256(target, hashlib.sha256(b"contents").hexdigest()) + + self.assertTrue(target.exists()) + + +class TestAcquireHostImage(unittest.TestCase): + """Only a verified image is kept under the name later runs trust.""" + + GOOD = b"the image" + + def _image(self): + import hashlib + + return e2e.HostImage(url="https://example.test/acl.qcow2", file_name="acl-b.qcow2", + backing_format="qcow2", sudo_group="sudo", packages=[], + ssh_user="core", provisioning="ignition", + sha256=hashlib.sha256(self.GOOD).hexdigest(), auth="") + + def _acquire(self, tmp, download_writes): + downloads = [] + + def download(url, destination, auth=""): + downloads.append(destination) + destination.write_bytes(download_writes) + + with patch.object(e2e, "VM_DIR", Path(tmp)), \ + patch.object(e2e, "download_file", side_effect=download), \ + patch.object(e2e, "run"): + e2e.acquire_host_image(self._image()) + return downloads + + def test_an_intact_existing_image_is_reused(self): + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + (Path(tmp) / "acl-b.qcow2").write_bytes(self.GOOD) + self.assertEqual(self._acquire(tmp, self.GOOD), []) + + def test_a_damaged_existing_image_is_downloaded_again(self): + """A cache restore or an interrupted download can leave the right name + on the wrong bytes.""" + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) / "acl-b.qcow2" + target.write_bytes(b"truncated") + self.assertEqual(len(self._acquire(tmp, self.GOOD)), 1) + self.assertEqual(target.read_bytes(), self.GOOD) + + def test_a_download_is_renamed_only_once_verified(self): + import tempfile + + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaises(SystemExit): + downloads = self._acquire(tmp, b"wrong bytes") + self.assertEqual(sorted(p.name for p in Path(tmp).iterdir()), [], + "neither the partial file nor the trusted name may be left behind") + + downloads = self._acquire(tmp, self.GOOD) + self.assertEqual([p.name for p in downloads], ["acl-b.qcow2.part"]) + self.assertEqual((Path(tmp) / "acl-b.qcow2").read_bytes(), self.GOOD) + self.assertFalse((Path(tmp) / "acl-b.qcow2.part").exists()) + + +class TestCurlAuthConfig(unittest.TestCase): + """The storage token must not reach curl's arguments, which a failed + command prints.""" + + def test_the_token_goes_to_stdin_and_is_masked(self): + calls = [] + + def fake_run(args, **kw): + calls.append((args, kw)) + + with patch.object(e2e, "capture", return_value="secret-token"), \ + patch.object(e2e, "run", side_effect=fake_run), \ + patch.dict(os.environ, {"GITHUB_ACTIONS": "true"}), \ + patch("builtins.print") as printed: + e2e.download_file("https://example.test/x", Path("/tmp/x"), auth="azure-storage") + + args, kw = calls[0] + self.assertNotIn("secret-token", " ".join(args)) + self.assertIn("--config", args) + self.assertIn('header = "Authorization: Bearer secret-token"', kw["input"]) + printed.assert_any_call("::add-mask::secret-token", flush=True) + + def test_a_failed_download_does_not_print_the_command(self): + import subprocess + + def failing_run(args, **kw): + raise subprocess.CalledProcessError(22, args) + + with patch.object(e2e, "capture", return_value="secret-token"), \ + patch.object(e2e, "run", side_effect=failing_run), \ + patch.object(e2e, "die", side_effect=SystemExit) as died: + with self.assertRaises(SystemExit): + e2e.download_file("https://example.test/x", Path("/tmp/x"), auth="azure-storage") + + self.assertNotIn("secret-token", died.call_args.args[0]) + + def test_no_auth_needs_no_config(self): + self.assertEqual(e2e.curl_auth_config(""), "") + + +if __name__ == "__main__": + unittest.main() diff --git a/hack/agent/e2e-kind/test_host_root.py b/hack/agent/e2e-kind/test_host_root.py new file mode 100644 index 000000000..bc02b4ec5 --- /dev/null +++ b/hack/agent/e2e-kind/test_host_root.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +"""The harness side of the host root and of migrating to it. + +A host installed by this build keeps the agent under /opt/unbounded. A host +installed by a release before that keeps it under /usr/local, and the current +agent links /opt/unbounded there. These tests cover what the harness asserts +about each, and the fixtures that stand in for agents it cannot build. +""" +import hashlib +import io +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import e2e + + +def _run_script(script: str, *args: str) -> subprocess.CompletedProcess[str]: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "unbounded-agent" + path.write_text(script) + path.chmod(0o755) + return subprocess.run([str(path), *args], capture_output=True, text=True, check=False) + + +def _fixture(builder) -> str: + captured = {} + with patch.object(e2e, "_build_script_agent_tarball", + side_effect=lambda _tarball, _name, script: captured.setdefault("script", script)): + builder(Path("unused.tar.gz")) + return captured["script"] + + +class TestFixtures(unittest.TestCase): + def test_daemon_failing_agent_gets_past_verification(self): + """AgentUpgrade verifies a candidate by asking for its version and its + host root. The rollback scenario needs the failure to come from the + daemon, so the fixture has to answer both the way a current agent + does.""" + script = _fixture(e2e._build_daemon_failing_agent_tarball) + + self.assertEqual(_run_script(script, "version").returncode, 0) + host_root = _run_script(script, "host-root") + self.assertEqual(host_root.returncode, 0) + self.assertEqual(host_root.stdout.strip(), os.path.realpath(e2e.HOST_ROOT)) + self.assertEqual(_run_script(script, "daemon").returncode, 42) + + def test_legacy_agent_has_no_host_root(self): + """An agent released before the host root answers version and nothing + else, which is all verification can tell it apart by.""" + script = _fixture(e2e._build_legacy_agent_tarball) + + self.assertEqual(_run_script(script, "version").returncode, 0) + self.assertNotEqual(_run_script(script, "host-root").returncode, 0) + + +class TestResetCleanup(unittest.TestCase): + def test_both_roots_and_the_root_itself_are_checked(self): + """Reset does not depend on the host root having been migrated, so it + sweeps both roots, and it removes the host root last. A check that only + looked where this run installed would not see the other left behind.""" + with patch.object(e2e, "ssh_capture", return_value="") as capture, \ + patch.object(e2e, "host_image") as image: + image.return_value.provisioning = "cloud-init" + e2e.validate_reset_cleanup() + + checks = capture.call_args.args[0] + self.assertIn(f'[ -L "{e2e.HOST_ROOT}" ]', checks) + for root in (e2e.HOST_ROOT, e2e.LEGACY_HOST_ROOT): + self.assertIn(f"{root}/bin/unbounded-agent-current", checks) + self.assertIn(f"{root}/libexec/unbounded-localdns-network", checks) + + def test_a_leftover_fails_the_step(self): + with patch.object(e2e, "ssh_capture", return_value=e2e.HOST_ROOT + "\n"), \ + patch.object(e2e, "host_image") as image: + image.return_value.provisioning = "cloud-init" + with self.assertRaises(SystemExit): + e2e.validate_reset_cleanup() + + +class TestHostRootStates(unittest.TestCase): + """Each validation accepts only the layout it names.""" + + LEGACY_BIN = f"{e2e.LEGACY_HOST_ROOT}/bin" + + # For each validation: the state it accepts, and what the host reports + # otherwise, chosen so that the state is the only thing that can fail it. + CASES = { + "validate_host_root": ("dir", e2e.DAEMON_BIN_DIR, e2e.DAEMON_BIN_DIR), + "validate_host_root_legacy": ("absent", LEGACY_BIN, LEGACY_BIN), + "validate_host_root_migrated": (f"link:{e2e.LEGACY_HOST_ROOT}", LEGACY_BIN, LEGACY_BIN), + } + + def _check(self, name, state, bin_dir, unit_dir): + unit = f"ExecStart={unit_dir}/unbounded-agent-current daemon\n" + with patch.object(e2e, "host_root_state", return_value=state), \ + patch.object(e2e, "resolve_on_host", + side_effect=lambda path: bin_dir if path.endswith("/bin") else f"{bin_dir}/unbounded-agent-blue"), \ + patch.object(e2e, "read_daemon_current_target", return_value=f"{bin_dir}/unbounded-agent-blue"), \ + patch.object(e2e, "ssh_capture", return_value=unit), \ + patch.object(e2e, "ssh_capture_quiet") as quiet: + quiet.return_value.returncode = 0 + quiet.return_value.stdout = "" + quiet.return_value.stderr = "" + getattr(e2e, name)() + + def test_the_named_state_passes(self): + for name, (accepted, bin_dir, unit_dir) in self.CASES.items(): + with self.subTest(validation=name): + self._check(name, accepted, bin_dir, unit_dir) + + def test_every_other_state_is_refused(self): + states = ["absent", "dir", f"link:{e2e.LEGACY_HOST_ROOT}", "link:/elsewhere", "other"] + for name, (accepted, bin_dir, unit_dir) in self.CASES.items(): + for state in states: + if state == accepted: + continue + with self.subTest(validation=name, state=state): + with self.assertRaises(SystemExit): + self._check(name, state, bin_dir, unit_dir) + + def test_the_daemon_unit_must_name_the_expected_root(self): + """A fresh host's unit runs the agent under the host root. A legacy or + migrated host's unit keeps naming the legacy path an older agent wrote, + which rolling back to that agent depends on.""" + for name, (accepted, bin_dir, unit_dir) in self.CASES.items(): + other = self.LEGACY_BIN if unit_dir == e2e.DAEMON_BIN_DIR else e2e.DAEMON_BIN_DIR + with self.subTest(validation=name): + with self.assertRaises(SystemExit): + self._check(name, accepted, bin_dir, other) + + +class TestLegacyAgent(unittest.TestCase): + def test_refused_on_an_immutable_host(self): + config = e2e.NodeConfig(name="default", node_labels={}, register_with_taints=[]) + with patch.object(e2e, "host_image") as image, \ + patch.object(e2e, "prepare_agent_artifacts") as prepare: + image.return_value.provisioning = "ignition" + with self.assertRaises(SystemExit): + e2e.run_legacy_agent(config) + prepare.assert_not_called() + + def test_version_must_be_a_release_tag(self): + """It is interpolated into a URL and a shell command line.""" + config = e2e.NodeConfig(name="default", node_labels={}, register_with_taints=[]) + with patch.object(e2e, "host_image") as image, \ + patch.object(e2e, "LEGACY_AGENT_VERSION", "v0.8.0; true"), \ + patch.object(e2e, "prepare_agent_artifacts") as prepare: + image.return_value.provisioning = "cloud-init" + with self.assertRaises(SystemExit): + e2e.run_legacy_agent(config) + prepare.assert_not_called() + + def test_installs_the_release_and_restores_the_environment(self): + config = e2e.NodeConfig(name="default", node_labels={}, register_with_taints=[]) + seen = {} + + def run_agent(_config): + seen["url"] = os.environ.get("AGENT_URL") + + with patch.dict(os.environ, {"AGENT_URL": "http://previous"}), \ + patch.object(e2e, "host_image") as image, \ + patch.object(e2e, "prepare_agent_artifacts"), \ + patch.object(e2e, "run_agent", side_effect=run_agent): + image.return_value.provisioning = "cloud-init" + e2e.run_legacy_agent(config) + after = os.environ.get("AGENT_URL") + + self.assertEqual(seen["url"], f"{e2e.LEGACY_AGENT_RELEASE_URL}/{e2e.LEGACY_AGENT_TARBALL}") + self.assertEqual(after, "http://previous") + + @staticmethod + def _release(content: bytes) -> bytes: + """A release tarball: the agent and the license files beside it.""" + import tarfile + + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + for name, data in (("LICENSE", b"license"), ("NOTICE", b"notice"), ("unbounded-agent", content)): + info = tarfile.TarInfo(name) + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + return buffer.getvalue() + + def test_downloaded_release_is_checked_against_its_checksums(self): + good = self._release(b"the release") + + def urlopen(url, timeout): + if url.endswith("/checksums.txt"): + body = f"{hashlib.sha256(good).hexdigest()} {e2e.LEGACY_AGENT_TARBALL}\n".encode() + else: + body = urlopen.tarball + return io.BytesIO(body) + + for tarball, ok in ((good, True), (self._release(b"something else"), False)): + urlopen.tarball = tarball + with self.subTest(ok=ok), tempfile.TemporaryDirectory() as tmp, \ + patch.object(e2e, "VM_DIR", Path(tmp)), \ + patch.object(e2e.urllib.request, "urlopen", side_effect=urlopen): + if ok: + import tarfile + + # AgentUpgrade takes an archive holding the agent alone. + with tarfile.open(e2e._download_legacy_agent_tarball()) as archive: + self.assertEqual(archive.getnames(), ["unbounded-agent"]) + self.assertEqual(archive.extractfile("unbounded-agent").read(), b"the release") + else: + with self.assertRaises(SystemExit): + e2e._download_legacy_agent_tarball() + + +class TestDaemonLinks(unittest.TestCase): + def test_links_are_found_under_the_legacy_root_before_migration(self): + """The first upgrade in the migration suite starts on a host the + current agent has not run on yet, where the host root does not exist.""" + with tempfile.TemporaryDirectory() as tmp: + root, legacy = Path(tmp, "opt", "unbounded"), Path(tmp, "usr", "local") + (legacy / "bin").mkdir(parents=True) + (legacy / "bin" / "unbounded-agent-blue").write_text("") + (legacy / "bin" / "unbounded-agent-current").symlink_to(legacy / "bin" / "unbounded-agent-blue") + + def resolve(): + with patch.object(e2e, "HOST_ROOT", str(root)), \ + patch.object(e2e, "LEGACY_HOST_ROOT", str(legacy)): + command = e2e._resolve_daemon_link(f"{root}/bin/unbounded-agent-current") + return subprocess.run(command.removeprefix("sudo "), shell=True, + capture_output=True, text=True, check=True).stdout.strip() + + self.assertEqual(resolve(), str(legacy / "bin" / "unbounded-agent-blue")) + + root.parent.mkdir(parents=True) + root.symlink_to(legacy) + self.assertEqual(resolve(), str(legacy / "bin" / "unbounded-agent-blue"), + "through the link once migrated") + + +class TestSuites(unittest.TestCase): + def test_migration_starts_legacy_and_ends_with_the_current_agent_resetting(self): + """Reset is performed by the daemon. An older one leaves the link, so + the suite returns to this build before resetting.""" + steps = e2e.SUITES["migration"] + + self.assertEqual(steps[0], "run-legacy-agent") + self.assertEqual(steps[-1], "reset-agent") + downgrade = steps.index("validate-agent-downgrade-to-legacy") + self.assertIn("validate-agent-upgrade-operation", steps[downgrade:]) + self.assertLess(steps.index("validate-host-root-legacy"), steps.index("validate-host-root-migrated")) + e2e.validate_suites() + + def test_lifecycle_checks_the_host_root_after_each_install(self): + steps = e2e.SUITES["lifecycle"] + + installs = [i for i, step in enumerate(steps) if step in ("run-agent", "reinstall-agent")] + self.assertEqual(len(installs), 2) + for index in installs: + self.assertIn("validate-host-root", steps[index:index + 3]) + + +if __name__ == "__main__": + unittest.main() diff --git a/hack/agent/e2e-kind/test_ignition.py b/hack/agent/e2e-kind/test_ignition.py new file mode 100644 index 000000000..41c883c99 --- /dev/null +++ b/hack/agent/e2e-kind/test_ignition.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the Ignition config the harness hands an immutable host. + +An Ignition config is applied once, before anything is reachable. A mistake in +it does not produce an error at the point it is made: the guest boots, does the +wrong thing quietly, and the harness fails later against a host that is +configured differently from the one the test meant to describe. These cover the +manipulations where that has actually happened. +""" +import base64 +import json +import unittest +from unittest.mock import patch + +import e2e + + +def _data_url(text: str) -> str: + return "data:;base64," + base64.b64encode(text.encode()).decode() + + +class TestRewriteAPIServer(unittest.TestCase): + """The agent config travels base64 inside a data URL.""" + + def test_rewrites_inside_an_encoded_file(self): + """A plain-text substitution over the rendered document finds nothing + here, and silently leaves the VM pointed at a loopback address it + cannot reach.""" + config = json.dumps({"Kubelet": {"ApiServer": "https://127.0.0.1:6443"}}) + doc = {"storage": {"files": [{ + "path": "/etc/unbounded/agent/config.json", + "contents": {"source": _data_url(config), "verification": {"hash": "sha256-old"}}, + }]}} + + e2e.rewrite_ignition_api_server(doc, "https://127.0.0.1:6443", "https://10.0.0.2:6443") + + source = doc["storage"]["files"][0]["contents"]["source"] + rewritten = json.loads(base64.b64decode(source.partition(",")[2])) + self.assertEqual(rewritten["Kubelet"]["ApiServer"], "https://10.0.0.2:6443") + + def test_drops_the_digest_it_invalidates(self): + """Ignition verifies before writing, so a rewritten body with its + original digest fails the whole config rather than the one file.""" + doc = {"storage": {"files": [{ + "path": "/etc/unbounded/agent/config.json", + "contents": {"source": _data_url("server=old"), "verification": {"hash": "sha256-old"}}, + }]}} + + e2e.rewrite_ignition_api_server(doc, "old", "new") + + self.assertNotIn("verification", doc["storage"]["files"][0]["contents"]) + + def test_leaves_untouched_files_verified(self): + """The agent binary is fetched by URL and its digest is the only thing + proving the host got the build under test.""" + doc = {"storage": {"files": [{ + "path": "/opt/unbounded/bin/unbounded-agent", + "contents": {"source": "https://example.test/agent", "verification": {"hash": "sha256-abc"}}, + }]}} + + e2e.rewrite_ignition_api_server(doc, "old", "new") + + self.assertEqual( + doc["storage"]["files"][0]["contents"]["verification"]["hash"], "sha256-abc") + + def test_rewrites_unit_contents(self): + doc = {"systemd": {"units": [{"name": "u.service", "contents": "ExecStart=x --server old"}]}} + e2e.rewrite_ignition_api_server(doc, "old", "new") + self.assertIn("new", doc["systemd"]["units"][0]["contents"]) + + def test_no_change_is_a_no_op(self): + doc = {"storage": {"files": [{ + "path": "/f", "contents": {"source": _data_url("same"), "verification": {"hash": "h"}}, + }]}} + + e2e.rewrite_ignition_api_server(doc, "same", "same") + + self.assertIn("verification", doc["storage"]["files"][0]["contents"], + "an unchanged body keeps a digest that is still correct") + + +class TestHarnessAccess(unittest.TestCase): + """What the harness adds so it can drive the host.""" + + def test_a_new_user_is_specified_in_full(self): + """A bare name leaves usermod nothing to apply. + + The image ships this account with /sbin/nologin. Adding only a name and + a key produces a host that accepts the key and then refuses the session + with "This account is currently not available", which looks like an SSH + problem rather than a config one. + """ + doc = e2e.add_ignition_harness_access({}, "ssh-ed25519 AAAA", "52:54:00:12:34:56") + + user = doc["passwd"]["users"][0] + self.assertEqual(user["name"], e2e.VM_SSH_USER) + self.assertEqual(user["shell"], "/bin/bash") + self.assertIn("sudo", user["groups"]) + self.assertIn("ssh-ed25519 AAAA", user["sshAuthorizedKeys"]) + + def test_an_existing_user_keeps_its_record(self): + doc = {"passwd": {"users": [{"name": e2e.VM_SSH_USER, "shell": "/bin/zsh"}]}} + + e2e.add_ignition_harness_access(doc, "ssh-ed25519 KEY", "52:54:00:12:34:56") + + user = doc["passwd"]["users"][0] + self.assertEqual(user["shell"], "/bin/zsh", "an explicit record must not be overwritten") + self.assertIn("ssh-ed25519 KEY", user["sshAuthorizedKeys"]) + + def test_hostname_is_set(self): + """The image masks the metadata hostname service and has no cloud-init, + so it stays "localhost" and the node registers under the wrong name.""" + doc = e2e.add_ignition_harness_access({}, "key", "52:54:00:12:34:56") + + names = {f["path"] for f in doc["storage"]["files"]} + self.assertIn("/etc/hostname", names) + + def test_network_unit_matches_on_mac_only(self): + """Every condition in [Match] has to hold. The interface name depends on + the machine type, so naming it as well makes the unit silently not + apply and leaves the VM on DHCP.""" + doc = e2e.add_ignition_harness_access({}, "key", "52:54:00:12:34:56") + + unit = next(f for f in doc["storage"]["files"] + if f["path"].endswith(e2e.IGNITION_NETWORK_UNIT)) + body = base64.b64decode(unit["contents"]["source"].partition(",")[2]).decode() + + self.assertIn("MACAddress=52:54:00:12:34:56", body) + self.assertNotIn("Name=", body) + + def test_waagent_is_masked_once(self): + """The image's own config masks it, but a user config displaces that + section entirely, so the harness has to restate it.""" + doc = e2e.add_ignition_harness_access({}, "key", "52:54:00:12:34:56") + doc = e2e.add_ignition_harness_access(doc, "key", "52:54:00:12:34:56") + + masked = [u for u in doc["systemd"]["units"] if u["name"] == "waagent.service"] + self.assertEqual(len(masked), 1) + self.assertTrue(masked[0]["mask"]) + + +class TestInitramfsNetworking(unittest.TestCase): + def test_the_interface_is_named(self): + """With the device field empty, bootengine's parse-ip-for-networkd + writes Name=*, which matches loopback first and assigns the address to + lo. Every fetch then fails with connection refused without a packet + reaching the wire. + """ + karg = e2e.initramfs_ip_karg() + + self.assertTrue(karg.startswith("ip=")) + fields = karg[len("ip="):].split(":") + self.assertEqual(fields[5], e2e.IGNITION_INITRAMFS_INTERFACE) + self.assertNotEqual(fields[5], "", "an empty device field assigns the address to lo") + + def test_it_carries_address_gateway_and_dns(self): + fields = e2e.initramfs_ip_karg()[len("ip="):].split(":") + + self.assertEqual(fields[0], e2e.VM_IP) + self.assertEqual(fields[2], e2e.VM_GATEWAY) + self.assertTrue(fields[7], "Ignition fetches by URL and needs a resolver") + + +class TestDataURLs(unittest.TestCase): + def test_round_trip(self): + self.assertEqual(e2e._decode_ignition_source(e2e.ignition_data_url("hello")), "hello") + + def test_percent_encoded_is_understood(self): + """Not every producer base64s, and a config written by hand commonly + does not.""" + self.assertEqual(e2e._decode_ignition_source("data:,line%0A"), "line\n") + + def test_a_remote_source_is_not_inline(self): + self.assertIsNone(e2e._decode_ignition_source("https://example.test/f")) + + + +class TestIgnitionHostBoundaries(unittest.TestCase): + """Paths that assume a host the harness can prepare before it boots.""" + + @staticmethod + def _ignition_image(): + return e2e.HostImage(url="file:///x", file_name="x.qcow2", backing_format="qcow2", + sudo_group="sudo", packages=[], ssh_user="core", + provisioning="ignition") + + def test_blocked_network_preparation_installs_nothing(self): + """There is no package manager and /usr is read-only, so the apt/dnf + path would fail on a host whose prerequisites are in the image by + design. Reaching it at all means the premise of the host entry is + wrong, so it returns before any SSH.""" + with patch.object(e2e, "host_image", return_value=self._ignition_image()), \ + patch.object(e2e, "wait_for_cloud_init") as waited, \ + patch.object(e2e, "ssh_cmd") as ssh: + e2e.prepare_blocked_network_vm() + + waited.assert_not_called() + ssh.assert_not_called() + + def test_offline_bootstrap_is_refused_before_anything_is_built(self): + """The offline path delivers a bundle over SSH before the agent runs. + An Ignition host has no such window, and finding out later costs an + agent build and a VM boot first.""" + with patch.object(e2e, "host_image", return_value=self._ignition_image()), \ + patch.object(e2e, "OFFLINE_BOOTSTRAP", True), \ + patch.object(e2e, "prepare_agent_artifacts") as prepared: + with self.assertRaises(SystemExit): + e2e.run_agent(e2e.NodeConfig(name="n", node_labels={}, register_with_taints=[])) + + prepared.assert_not_called() + + def test_an_outside_agent_is_refused_before_anything_is_built(self): + """The configuration scenarios pass their own AGENT_URL. The Ignition + path only serves the binary it staged, so such a run would die later, + after minting a bootstrap token.""" + with patch.object(e2e, "host_image", return_value=self._ignition_image()), \ + patch.dict(e2e.os.environ, {"AGENT_URL": "http://runner/unbounded-agent.tar.gz"}), \ + patch.object(e2e, "prepare_agent_artifacts") as prepared, \ + patch.object(e2e, "_run_agent_inner") as ran: + with self.assertRaises(SystemExit): + e2e.run_agent(e2e.NodeConfig(name="n", node_labels={}, register_with_taints=[])) + + prepared.assert_not_called() + ran.assert_not_called() + + def test_the_configuration_suite_is_refused_up_front(self): + with patch.object(e2e, "host_image", return_value=self._ignition_image()), \ + patch.object(e2e, "patch_kind_control_plane_node_ip") as patched, \ + patch.object(e2e, "discover_node_configs") as discovered: + with self.assertRaises(SystemExit): + e2e.validate_node_config_scenarios() + + patched.assert_not_called() + discovered.assert_not_called() + + def test_reset_failed_is_only_optional_on_an_ignition_host(self): + """A refused reset-failed is expected on Azure Container Linux. Elsewhere + it means something is wrong, and scenarios would share a start-limit + budget without anyone noticing.""" + import subprocess + + refused = subprocess.CompletedProcess(["ssh"], 1, "", "Access denied") + for provisioning, should_die in (("ignition", False), ("cloud-init", True)): + with self.subTest(provisioning=provisioning): + image = self._ignition_image() + image = e2e.replace(image, provisioning=provisioning) + with patch.object(e2e, "host_image", return_value=image), \ + patch.object(e2e, "ssh_capture_quiet", return_value=refused), \ + patch.object(e2e, "die", side_effect=SystemExit) as died: + try: + e2e.check_reset_failed() + except SystemExit: + pass + self.assertEqual(died.called, should_die) + + +class TestIgnitionReboot(unittest.TestCase): + """What counts as the first-boot unit repairing a healthy host on reboot.""" + + def test_a_verify_only_run_passes(self): + self.assertEqual(e2e.ignition_reboot_problems("active", "0", "verified\n", "1 2", "1 2"), []) + + def test_each_sign_of_a_repair_is_reported(self): + cases = { + "failed unit": (("failed", "0", "", "1 2", "1 2"), "ActiveState=failed"), + "retried": (("active", "1", "", "1 2", "1 2"), "NRestarts=1"), + "unknown restarts": (("active", "", "", "1 2", "1 2"), "NRestarts=unknown"), + "daemon repaired": (("active", "0", 'msg="daemon unit started"', "1 2", "1 2"), + "start repaired the daemon"), + "record rewritten": (("active", "0", "", "1 2", "3 4"), "the install record was rewritten"), + } + for name, (args, want) in cases.items(): + with self.subTest(name): + self.assertEqual(e2e.ignition_reboot_problems(*args), [want]) + + +if __name__ == "__main__": + unittest.main() diff --git a/hack/agent/e2e-kind/test_reinstall.py b/hack/agent/e2e-kind/test_reinstall.py new file mode 100644 index 000000000..6b1daa8d1 --- /dev/null +++ b/hack/agent/e2e-kind/test_reinstall.py @@ -0,0 +1,272 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +"""Reinstalling on an Ignition host must reuse the disk it already has. + +Reinstall exists to prove a reset host can be provisioned again from what is +already on it. Replacing the disk would answer a different and easier question, +and would quietly turn the same-disk assertion in reinstall_agent into a +fresh-install one, since a new disk boots with a new boot id. +""" +import hashlib +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import e2e + + +class TestReinstallUsesTheSameDisk(unittest.TestCase): + def test_reinstall_asks_for_the_same_disk(self): + """The flag is the whole mechanism. + + Without it the Ignition path destroys the disk and boots a fresh VM, + which is exactly what the caller then fails on. + """ + config = e2e.NodeConfig(name="default", node_labels={}, register_with_taints=[]) + + with patch.object(e2e, "run_agent") as run, \ + patch.object(e2e, "destroy_vm") as destroy, \ + patch.object(e2e, "bounded_ssh") as ssh: + ssh.return_value.stdout = "boot-a" + ssh.return_value.returncode = 0 + e2e.reinstall_agent(config) + + run.assert_called_once_with(config, reinstall=True) + destroy.assert_not_called() + + +class TestReinstallPayload(unittest.TestCase): + """What a reinstall is allowed to touch.""" + + @staticmethod + def _doc(agent_config: str) -> dict: + return { + "storage": {"files": [ + {"path": e2e.DAEMON_BINARY, "mode": 0o755, + "contents": {"source": "http://runner/unbounded-agent", "verification": { + "hash": "sha256-" + hashlib.sha256(b"test-binary").hexdigest()}}}, + {"path": "/etc/unbounded/agent/config.json", "mode": 0o600, + "contents": {"source": e2e.ignition_data_url(agent_config)}}, + {"path": "/etc/hostname", "contents": {"source": "ignored"}}, + ]}, + "systemd": {"units": [ + {"name": "unbounded-agent-bootstrap.service", "contents": "[Service]\n"}, + {"name": "waagent.service", "mask": True}, + ]}, + } + + def test_delivers_only_the_agent_payloads(self): + """Identity, networking and boot state have to survive a reset. + + Rewriting /etc/hostname or remasking waagent would recreate host state + that reset is supposed to have left alone, hiding exactly the cleanup + defects this step exists to find. + """ + agent_config = json.dumps({"MachineName": "agent-e2e"}) + + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(e2e, "VM_DIR", Path(tmp)), \ + patch.object(e2e, "host_image") as image, \ + patch.object(e2e, "scp_cmd") as scp, \ + patch.object(e2e, "ssh_cmd") as ssh, \ + patch.object(e2e, "ssh_capture", return_value=self._installed_digest(b"test-binary")), \ + patch.object(e2e, "ignition_bootstrap_invocation", return_value="inv-before"): + (Path(tmp) / "unbounded-agent").write_bytes(b"test-binary") + + previous = e2e._reinstall_ignition_payload(self._doc(agent_config)) + + self.assertEqual(previous, "inv-before", "the wait needs the run from before the reinstall") + self.assertEqual(scp.call_count, 3, "binary, agent config, bootstrap unit") + + commands = "\n".join(str(call) for call in ssh.call_args_list) + self.assertNotIn("/etc/hostname", commands) + self.assertNotIn("waagent", commands) + self.assertIn("enable --now --no-block", commands) + + @staticmethod + def _installed_digest(content: bytes) -> str: + return hashlib.sha256(content).hexdigest() + " " + e2e.DAEMON_BINARY + + def test_the_installed_binary_is_checked_against_the_rendered_digest(self): + """The binary is fetched by URL in the Ignition path, so its digest is + the only thing tying what gets installed to the build under test. A + reinstall that delivers a different binary would pass every later + assertion while testing the wrong artifact, so the copy on the VM is + what gets checked.""" + doc = self._doc(json.dumps({"MachineName": "agent-e2e"})) + + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(e2e, "VM_DIR", Path(tmp)), \ + patch.object(e2e, "host_image") as image, \ + patch.object(e2e, "scp_cmd"), patch.object(e2e, "ssh_cmd"), \ + patch.object(e2e, "ssh_capture", return_value=self._installed_digest(b"a different binary")), \ + patch.object(e2e, "ignition_bootstrap_invocation", return_value=""): + (Path(tmp) / "unbounded-agent").write_bytes(b"test-binary") + + with self.assertRaises(SystemExit): + e2e._reinstall_ignition_payload(doc) + + def test_an_unexpected_payload_is_refused(self): + """The payload set is asserted rather than filtered. A file or unit + appearing here that the test does not know about is a change in what + bootstrap installs, and it should stop the run rather than be skipped + silently.""" + def moved_binary(doc): + doc["storage"]["files"][0]["path"] = "/somewhere/else/unbounded-agent" + + def extra_file(doc): + doc["storage"]["files"].append({"path": "/etc/unbounded/agent/extra", "contents": {"source": "x"}}) + + def extra_unit(doc): + doc["systemd"]["units"].append({"name": "surprise.service", "contents": "[Service]\n"}) + + for name, change in (("moved binary", moved_binary), ("extra file", extra_file), ("extra unit", extra_unit)): + with self.subTest(name), tempfile.TemporaryDirectory() as tmp, \ + patch.object(e2e, "VM_DIR", Path(tmp)), \ + patch.object(e2e, "host_image") as image, \ + patch.object(e2e, "scp_cmd") as scp, patch.object(e2e, "ssh_cmd"), \ + patch.object(e2e, "ssh_capture", return_value=self._installed_digest(b"test-binary")), \ + patch.object(e2e, "ignition_bootstrap_invocation", return_value=""): + (Path(tmp) / "unbounded-agent").write_bytes(b"test-binary") + doc = self._doc(json.dumps({"MachineName": "agent-e2e"})) + change(doc) + + with self.assertRaises(SystemExit): + e2e._reinstall_ignition_payload(doc) + scp.assert_not_called() + + + +class TestBootstrapChoosesThePath(unittest.TestCase): + """The branch that connects the flag to the delivery. + + The flag and the payload delivery are each covered above, but neither says + the two are wired together. Patching only the expensive parts leaves that + decision exercised. + """ + + def _run(self, *, reinstall: bool): + config = e2e.NodeConfig(name="default", node_labels={}, register_with_taints=[]) + doc = json.dumps({"storage": {"files": []}, "systemd": {"units": []}}) + + # host_image is patched because these run inside every matrix job with + # that job's HOST_BASE_OS set. Under acl the real one resolves a + # published manifest, which would consume the patched capture below and + # fail on a machine that has nothing to do with this branch. + image = e2e.HostImage(url="file:///x", file_name="x.qcow2", backing_format="qcow2", + sudo_group="sudo", packages=[], ssh_user="core", + provisioning="ignition") + + with patch.object(e2e, "host_image", return_value=image), \ + patch.object(e2e, "_ensure_vm_ssh_key", return_value="ssh-ed25519 AAAA"), \ + patch.object(e2e, "agent_binary_url_and_digest", return_value=("http://x/a", "d" * 64)), \ + patch.object(e2e, "node_config_bootstrap_args", return_value=[]), \ + patch.object(e2e, "log_active_node_config"), \ + patch.object(e2e, "capture", return_value=doc), \ + patch.object(e2e, "qemu_mac_address", return_value="52:54:00:12:34:56"), \ + patch.object(e2e, "add_ignition_harness_access", side_effect=lambda d, *a: d), \ + patch.object(e2e, "_wait_for_ignition_bootstrap") as wait, \ + patch.object(e2e, "destroy_vm") as destroy, \ + patch.object(e2e, "launch_ignition_vm") as launch, \ + patch.object(e2e, "_reinstall_ignition_payload", + return_value="inv-before") as payload: + e2e._bootstrap_via_ignition(config, "https://api:6443", "https://127.0.0.1:6443", + reinstall=reinstall) + + return destroy, launch, payload, wait + + def test_fresh_provisioning_replaces_the_disk(self): + destroy, launch, payload, wait = self._run(reinstall=False) + + destroy.assert_called_once() + launch.assert_called_once() + payload.assert_not_called() + + # A fresh VM has no previous run to distinguish this one from. + wait.assert_called_once_with("") + + def test_reinstall_keeps_the_disk(self): + """Destroying it here would change the boot id that reinstall_agent + checks, so the same-disk assertion would pass against a fresh host.""" + destroy, launch, payload, wait = self._run(reinstall=True) + + payload.assert_called_once() + destroy.assert_not_called() + launch.assert_not_called() + + # The invocation the payload step read before starting the unit has to + # reach the wait, or the wait has nothing to compare against and accepts + # the previous run as this one. + wait.assert_called_once_with("inv-before") + + +class TestBootstrapCompletionIsFresh(unittest.TestCase): + """A run that already finished is not a run. + + The unit is a oneshot with RemainAfterExit=yes, so it stays active after it + has run and starting an active unit does nothing. Accepting "active" on its + own reports success for an agent that never executed, and the node simply + never appears with nothing in any log to say why. + """ + + @staticmethod + def _ssh(invocations): + """Answer the three questions the wait asks, invocation id last.""" + def ssh(command, _deadline, **_kwargs): + if "InvocationID" in command: + out = next(invocations) + elif "ActiveState" in command: + out = "active" + elif "-p Result" in command: + out = "success" + else: + out = "journal" + + return subprocess.CompletedProcess([], 0, out, "") + + return ssh + + def test_a_stale_active_unit_is_not_accepted(self): + invocations = iter(["inv-old", "inv-old", "inv-old", "inv-new", "inv-new"]) + + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(e2e, "bounded_ssh", side_effect=self._ssh(invocations)), \ + patch.object(e2e.time, "sleep") as slept, \ + patch.object(e2e, "VM_DIR", Path(tmp)): + e2e._wait_for_ignition_bootstrap("inv-old") + + # It waited while the unit reported the previous run, and stopped only + # once systemd reported a new one. + self.assertEqual(slept.call_count, 3) + + def test_a_fresh_invocation_completes_immediately(self): + invocations = iter(["inv-new"] * 4) + + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(e2e, "bounded_ssh", side_effect=self._ssh(invocations)), \ + patch.object(e2e.time, "sleep") as slept, \ + patch.object(e2e, "VM_DIR", Path(tmp)): + e2e._wait_for_ignition_bootstrap("inv-old") + + slept.assert_not_called() + + def test_a_first_boot_has_no_previous_invocation(self): + """Fresh provisioning passes an empty id, so any real one is new.""" + invocations = iter(["inv-first"] * 4) + + with tempfile.TemporaryDirectory() as tmp, \ + patch.object(e2e, "bounded_ssh", side_effect=self._ssh(invocations)), \ + patch.object(e2e.time, "sleep") as slept, \ + patch.object(e2e, "VM_DIR", Path(tmp)): + e2e._wait_for_ignition_bootstrap() + + slept.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/hack/agent/e2e-kind/test_reliability.py b/hack/agent/e2e-kind/test_reliability.py index 7e8ac798c..d28d4b5bb 100644 --- a/hack/agent/e2e-kind/test_reliability.py +++ b/hack/agent/e2e-kind/test_reliability.py @@ -102,7 +102,11 @@ def test_same_disk_reinstall_checks_host_identity(self): e2e.reinstall_agent(cfg) else: e2e.reinstall_agent(cfg) - run.assert_called_once_with(cfg) + # reinstall=True is what keeps this on the same disk. Without + # it an Ignition host replaces the disk and reboots, which + # changes the boot id this test is checking and turns a + # same-disk assertion into a fresh-install one. + run.assert_called_once_with(cfg, reinstall=True) def test_recovered_hostname_needs_done_correct_host_and_marker(self): warning = f"Failed to set the hostname to {e2e.VM_NAME} ({e2e.VM_NAME})" @@ -168,7 +172,8 @@ def scenario(cfg, index, url): active -= 1 if fail and index == 0: raise RuntimeError("failed guest") - with patch.dict(e2e.os.environ, {"CONFIG_SCENARIO_WORKERS":"2"}), patch.object(e2e, "VM_DIR", Path(directory)), patch.object(e2e, "patch_kind_control_plane_node_ip"), patch.object(e2e, "discover_node_configs", return_value=configs), patch.object(e2e, "mirror_oci_refs_to_local_registry", side_effect=lambda x:x), patch.object(e2e, "prepare_agent_artifacts", return_value="url"), patch.object(e2e, "HTTPServer"), patch.object(e2e, "validate_kube_proxy"), patch.object(e2e, "_validate_node_config_scenario", side_effect=scenario): + # The configuration suite does not run on an Ignition host. + with patch.object(e2e, "HOST_BASE_OS", "ubuntu2404"), patch.dict(e2e.os.environ, {"CONFIG_SCENARIO_WORKERS":"2"}), patch.object(e2e, "VM_DIR", Path(directory)), patch.object(e2e, "patch_kind_control_plane_node_ip"), patch.object(e2e, "discover_node_configs", return_value=configs), patch.object(e2e, "mirror_oci_refs_to_local_registry", side_effect=lambda x:x), patch.object(e2e, "prepare_agent_artifacts", return_value="url"), patch.object(e2e, "HTTPServer"), patch.object(e2e, "validate_kube_proxy"), patch.object(e2e, "_validate_node_config_scenario", side_effect=scenario): if fail: with self.assertRaises(SystemExit): e2e.validate_node_config_scenarios() diff --git a/hack/agent/e2e-kind/test_ukiboot.py b/hack/agent/e2e-kind/test_ukiboot.py new file mode 100644 index 000000000..0c4d55fbc --- /dev/null +++ b/hack/agent/e2e-kind/test_ukiboot.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the PE parsing ukiboot uses to extend a UKI cmdline addon in place. + +The disk-facing half of ukiboot needs qemu-nbd and a real image and is covered +by running the ACL host in the e2e suite. What is worth pinning here is the +header arithmetic underneath it, because a wrong offset does not fail loudly: +it writes plausible bytes into the wrong part of an EFI executable, and the +first sign of trouble is a guest that will not boot. +""" +import io +import os +import struct +import unittest +from pathlib import Path +from unittest.mock import patch + +import ukiboot + + +def _pe(sections: list[tuple[str, int, int, int, int]], opt_size: int = 240) -> bytes: + """Build a PE header with the given (name, vsize, vaddr, rsize, rptr) sections. + + Only the fields ukiboot reads are populated. The point is to be able to + state the expected offsets independently of the code under test. + """ + lfanew = 0x80 + out = bytearray(4096) + out[0:2] = b"MZ" + struct.pack_into(" bytes: + buf = b"" + while len(buf) < n: + chunk = self.sock.recv(n - len(buf)) + if not chunk: + raise EOFError("NBD connection closed") + buf += chunk + return buf + + def _handshake(self) -> int: + if self._recv(8) != b"NBDMAGIC": + raise RuntimeError("not an NBD server") + if self._recv(8) != b"IHAVEOPT": + raise RuntimeError("server does not speak fixed newstyle NBD") + self._recv(2) # handshake flags + self.sock.sendall(struct.pack(">I", NBD_FLAG_C_FIXED_NEWSTYLE)) + + payload = struct.pack(">I", 0) + struct.pack(">H", 0) # default export, no info requests + self.sock.sendall(b"IHAVEOPT" + struct.pack(">II", NBD_OPT_GO, len(payload)) + payload) + + size = 0 + while True: + magic, option, rep_type, length = struct.unpack(">QIII", self._recv(20)) + if magic != NBD_OPT_REPLY_MAGIC: + raise RuntimeError(f"bad NBD option reply magic {magic:#x}") + data = self._recv(length) if length else b"" + if rep_type == NBD_REP_INFO and len(data) >= 10: + if struct.unpack(">H", data[:2])[0] == NBD_INFO_EXPORT: + size = struct.unpack(">Q", data[2:10])[0] + elif rep_type == NBD_REP_ACK: + return size + elif rep_type & NBD_REP_ERROR_BIT: + raise RuntimeError(f"NBD option {option} rejected ({rep_type:#x}): {data!r}") + + def _request(self, cmd: int, offset: int, length: int, data: bytes = b"") -> None: + self._handle += 1 + self.sock.sendall(struct.pack( + ">IHHQQI", NBD_REQUEST_MAGIC, 0, cmd, self._handle, offset, length)) + if data: + self.sock.sendall(data) + + def _reply(self, offset: int) -> None: + magic, error, _handle = struct.unpack(">IIQ", self._recv(16)) + if magic != NBD_SIMPLE_REPLY_MAGIC: + raise RuntimeError(f"bad NBD reply magic {magic:#x}") + if error: + raise RuntimeError(f"NBD error {error} at offset {offset}") + + def read(self, offset: int, length: int) -> bytes: + out = bytearray() + while length > 0: + n = min(length, 4 << 20) + self._request(NBD_CMD_READ, offset, n) + self._reply(offset) + out += self._recv(n) + offset += n + length -= n + return bytes(out) + + def write(self, offset: int, data: bytes) -> None: + view = memoryview(data) + while view: + chunk = view[: 4 << 20] + self._request(NBD_CMD_WRITE, offset, len(chunk), bytes(chunk)) + self._reply(offset) + offset += len(chunk) + view = view[len(chunk):] + + def flush(self) -> None: + self._request(NBD_CMD_FLUSH, 0, 0) + self._reply(0) + + def close(self) -> None: + try: + self.sock.close() + except OSError: + # Best effort: the caller is already tearing down, and the export + # has been flushed by this point. A failure to close a socket that + # is about to be discarded is not worth masking the reason the + # caller is unwinding. + pass + + +class NbdServer: + """qemu-nbd serving a disk image on a unix socket in a temporary directory.""" + + def __init__(self, image: str, image_format: str = "qcow2", writable: bool = False): + self._dir = tempfile.mkdtemp(prefix="ukiboot-") + self.sock_path = os.path.join(self._dir, "nbd.sock") + self.proc: subprocess.Popen[bytes] | None = None + + args = ["qemu-nbd", "--persistent", "--format", image_format, + "--socket", self.sock_path] + if not writable: + args.append("--read-only") + args.append(image) + + # __exit__ does not run when the constructor raises, so every failure + # here cleans up itself. + try: + self.proc = subprocess.Popen(args, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + + deadline = time.time() + 15 + while time.time() < deadline: + if os.path.exists(self.sock_path): + return + if self.proc.poll() is not None: + err = self.proc.stderr.read().decode("utf-8", "replace") if self.proc.stderr else "" + raise RuntimeError(f"qemu-nbd exited: {err}") + time.sleep(0.05) + raise RuntimeError("qemu-nbd did not create its socket in time") + except BaseException: + self.close() + raise + + def close(self) -> None: + if self.proc is not None: + self.proc.terminate() + try: + self.proc.wait(timeout=10) + except subprocess.TimeoutExpired: + self.proc.kill() + # Reap it: without this the killed qemu-nbd stays a zombie for the + # lifetime of the harness, which can outlast many VM cycles. + self.proc.wait() + for cleanup in (lambda: os.unlink(self.sock_path), lambda: os.rmdir(self._dir)): + try: + cleanup() + except FileNotFoundError: + # Already gone, which is the ordinary case when qemu-nbd + # removed its own socket on exit. + pass + except OSError as exc: + # Report rather than raise. close() runs from __exit__ and from + # the startup failure path above, so raising here would replace + # the reason the caller is unwinding with a cleanup detail, + # which is how the actual failure gets lost. + print(f"warning: ukiboot cleanup failed: {exc}", file=sys.stderr) + + def __enter__(self) -> "NbdServer": + return self + + def __exit__(self, *_exc: object) -> None: + self.close() + + +@dataclass(frozen=True) +class Partition: + name: str + type_guid: str + first_lba: int + last_lba: int + + @property + def offset(self) -> int: + return self.first_lba * 512 + + +def read_partitions(dev: NbdClient) -> list[Partition]: + header = dev.read(512, 512) + if header[:8] != b"EFI PART": + raise RuntimeError("disk has no GPT") + entries_lba = struct.unpack_from(" str: + d1, d2, d3 = struct.unpack_from(" bytes: + return self.dev.read(self.base + offset, length) + + def _cluster_runs(self, cluster: int) -> list[tuple[int, int]]: + """Collapse a cluster chain into contiguous (partition offset, length) runs.""" + chain = [] + while 2 <= cluster < 0x0FFFFFF8: + chain.append(cluster) + cluster = struct.unpack_from(" list[tuple[int, int]]: + """Map a byte range of a file to absolute (disk offset, length) ranges. + + A file is not necessarily contiguous, so a range can span several runs. + Returning them lets a caller read or write the range without assuming + it lies in one piece. + """ + out: list[tuple[int, int]] = [] + remaining = length + pos = 0 + for run_start, run_len in self._cluster_runs(cluster): + if remaining <= 0: + break + run_end = pos + run_len + if run_end > offset: + skip = max(0, offset - pos) + take = min(run_len - skip, remaining) + out.append((self.base + run_start + skip, take)) + remaining -= take + pos = run_end + if remaining > 0: + raise RuntimeError("range extends past the end of the file") + return out + + def read_file(self, cluster: int, size: int, offset: int = 0, + length: int | None = None) -> bytes: + """Read a byte range of a file without materializing the whole file.""" + if length is None: + length = size - offset + length = max(0, min(length, size - offset)) + if length == 0: + return b"" + return b"".join(self.dev.read(start, count) + for start, count in self.map_ranges(cluster, offset, length)) + + def write_file(self, cluster: int, size: int, offset: int, data: bytes) -> None: + """Overwrite a byte range of a file in place. + + The file keeps its length and its clusters; only the bytes change. That + is what makes this safe without a FAT allocator. + """ + if offset + len(data) > size: + raise RuntimeError("in-place write would extend the file") + view = memoryview(data) + for start, count in self.map_ranges(cluster, offset, len(data)): + self.dev.write(start, bytes(view[:count])) + view = view[count:] + + def list_dir(self, cluster: int) -> list[tuple[str, int, int, int]]: + """Return (name, attributes, start cluster, size) for each entry.""" + data = b"".join(self.dev.read(self.base + start, length) + for start, length in self._cluster_runs(cluster)) + entries: list[tuple[str, int, int, int]] = [] + long_name: list[tuple[int, str]] = [] + for i in range(0, len(data), 32): + entry = data[i:i + 32] + if len(entry) < 32 or entry[0] == 0: + break + if entry[0] == 0xE5: + long_name = [] + continue + if entry[11] == 0x0F: + text = (entry[1:11] + entry[14:26] + entry[28:32]).decode("utf-16-le", "ignore") + long_name.append((entry[0] & 0x3F, text.split("\x00")[0])) + continue + if long_name: + name = "".join(text for _, text in sorted(long_name)) + else: + stem = entry[0:8].decode("ascii", "replace").rstrip() + ext = entry[8:11].decode("ascii", "replace").rstrip() + name = f"{stem}.{ext}" if ext else stem + long_name = [] + start = ((struct.unpack_from(" tuple[int, int] | None: + """Resolve a path to (start cluster, size). FAT lookups ignore case.""" + cluster = self.root_cluster + parts = [p for p in path.split("/") if p] + for index, part in enumerate(parts): + for name, _attr, start, size in self.list_dir(cluster): + if name.lower() != part.lower(): + continue + if index == len(parts) - 1: + return start, size + cluster = start + break + else: + return None + return None + + def list_names(self, path: str) -> list[str]: + found = self.lookup(path) + if not found: + return [] + return [name for name, _attr, _start, _size in self.list_dir(found[0]) + if name not in (".", "..")] + + +def pe_section_table_offset(header: bytes) -> tuple[int, int, int]: + """Return (section table offset, section count, optional header size).""" + lfanew = struct.unpack_from(" dict[str, tuple[int, int, int, int]]: + """Return {section: (virtual size, virtual address, raw size, raw pointer)}.""" + table, count, _opt = pe_section_table_offset(header) + out = {} + for i in range(count): + entry = header[table + i * 40: table + (i + 1) * 40] + out[entry[0:8].rstrip(b"\x00").decode()] = struct.unpack_from(" int: + """Return the byte offset of a section's VirtualSize field in the header. + + systemd-stub reads VirtualSize bytes from the .cmdline section, so a longer + command line is truncated unless this field is updated to match. It is the + one write ukiboot makes outside the section body, and the only one that + lands in a PE header, where being four bytes out overwrites the section's + VirtualAddress instead and produces an executable that loads its command + line from nowhere. + """ + table, _count, _opt = pe_section_table_offset(header) + + return table + _pe_section_index(header, name) * 40 + 8 + + +def _pe_section_index(header: bytes, name: str) -> int: + table, count, _opt = pe_section_table_offset(header) + for i in range(count): + entry = header[table + i * 40: table + (i + 1) * 40] + if entry[0:8].rstrip(b"\x00").decode() == name: + return i + raise RuntimeError(f"PE image has no {name} section") + + +@dataclass(frozen=True) +class PatchedAddon: + """Where the patch landed, for logging and verification.""" + + addon: str + cmdline: str + used: int + capacity: int + + +def single_uki(names: list[str], image: Path) -> str: + """Return the one UKI under /EFI/Linux. + + systemd-boot picks among several by its own rules, so choosing one here + could patch an image that is not the one that boots, and Ignition would + then get no config URL. + """ + ukis = sorted(n for n in names if n.lower().endswith(".efi")) + if len(ukis) != 1: + raise RuntimeError(f"{image} has {len(ukis)} UKIs under /EFI/Linux, expected one: {ukis}") + return ukis[0] + + +def fit_cmdline(current: str, extra: str, raw_size: int) -> bytes | None: + """Return the merged command line encoded, or None if it and its NUL + terminator do not fit in a section of raw_size bytes. Sizes are in encoded + bytes, since that is what the section holds.""" + encoded = f"{current} {extra}".strip().encode() + if len(encoded) + 1 > raw_size: + return None + return encoded + + +def patch_uki_cmdline_addon(image: Path, extra_args: str, + image_format: str = "qcow2") -> PatchedAddon: + """Append kernel command line arguments to a UKI addon on the image's ESP. + + Chooses the largest .cmdline addon that can hold the addition, appends to + its existing contents, and updates the section's VirtualSize so + systemd-stub reads the longer string. firstboot.addon.efi is never chosen: + ignition-quench.service deletes it after a successful first boot, which is + exactly the mechanism that stops Ignition re-running, and the addition has + to survive that. + """ + with NbdServer(str(image), image_format, writable=True) as server: + dev = NbdClient(server.sock_path) + try: + esp = next((p for p in read_partitions(dev) + if p.type_guid == EFI_SYSTEM_PARTITION_TYPE), None) + if esp is None: + raise RuntimeError(f"{image} has no EFI system partition") + fat = Fat32(dev, esp.offset) + + addon_dir = f"/EFI/Linux/{single_uki(fat.list_names('/EFI/Linux'), image)}.extra.d" + + best = None + for addon in sorted(fat.list_names(addon_dir)): + if not addon.lower().endswith(".efi") or addon == "firstboot.addon.efi": + continue + entry = fat.lookup(f"{addon_dir}/{addon}") + if entry is None: + continue + cluster, size = entry + header = fat.read_file(cluster, size, 0, min(size, 8192)) + sections = pe_sections(header) + if ".cmdline" not in sections: + continue + vsize, _vaddr, rsize, rptr = sections[".cmdline"] + current = fat.read_file(cluster, size, rptr, min(vsize, rsize) if vsize else rsize) + current = current.split(b"\x00")[0].decode("utf-8", "replace").strip() + merged = fit_cmdline(current, extra_args, rsize) + if merged is None: + continue + if best is None or rsize > best[0]: + best = (rsize, addon, cluster, size, header, rptr, merged) + + if best is None: + raise RuntimeError( + f"no addon under {addon_dir} has room for {len(extra_args)} more bytes") + + rsize, addon, cluster, size, header, rptr, merged = best + + # Rewrite the section body, NUL-padded to its full raw size so no + # remnant of the previous contents is left behind. + body = merged + b"\x00" * (rsize - len(merged)) + fat.write_file(cluster, size, rptr, body) + + # systemd-stub reads VirtualSize bytes, so a longer string is + # truncated unless the header agrees. + vsize_offset = pe_section_vsize_offset(header, ".cmdline") + fat.write_file(cluster, size, vsize_offset, struct.pack(" str: + """Return the command line systemd-stub would assemble for the image's UKI. + + This is the UKI's own .cmdline plus every addon in its .extra.d directory, + in the order systemd-stub reads them. Used to check what a patch produced. + """ + with NbdServer(str(image), image_format) as server: + dev = NbdClient(server.sock_path) + try: + esp = next((p for p in read_partitions(dev) + if p.type_guid == EFI_SYSTEM_PARTITION_TYPE), None) + if esp is None: + raise RuntimeError(f"{image} has no EFI system partition") + fat = Fat32(dev, esp.offset) + + uki_name = single_uki(fat.list_names("/EFI/Linux"), image) + + parts: list[str] = [] + + def section_text(cluster: int, size: int) -> str | None: + header = fat.read_file(cluster, size, 0, min(size, 8192)) + sections = pe_sections(header) + if ".cmdline" not in sections: + return None + vsize, _vaddr, rsize, rptr = sections[".cmdline"] + raw = fat.read_file(cluster, size, rptr, min(vsize, rsize) if vsize else rsize) + return raw.split(b"\x00")[0].decode("utf-8", "replace").strip() + + entry = fat.lookup(f"/EFI/Linux/{uki_name}") + if entry: + text = section_text(*entry) + if text: + parts.append(text) + + addon_dir = f"/EFI/Linux/{uki_name}.extra.d" + for addon in sorted(fat.list_names(addon_dir)): + if not addon.lower().endswith(".efi"): + continue + found = fat.lookup(f"{addon_dir}/{addon}") + if found is None: + continue + text = section_text(*found) + if text: + parts.append(text) + + return re.sub(r"\s+", " ", " ".join(parts)).strip() + finally: + dev.close() + + +def main() -> None: + import argparse + + parser = argparse.ArgumentParser(description=__doc__.split("\n", maxsplit=1)[0]) + parser.add_argument("image", type=Path) + parser.add_argument("--append", help="kernel command line arguments to add") + parser.add_argument("--show", action="store_true", help="print the assembled command line") + args = parser.parse_args() + + if args.append: + result = patch_uki_cmdline_addon(args.image, args.append) + print(f"patched: {result.addon} ({result.used}/{result.capacity} bytes)") + print(f"cmdline: {result.cmdline}") + + if args.show or not args.append: + print(f"assembled: {read_uki_cmdline(args.image)}") + + +if __name__ == "__main__": + main() diff --git a/internal/provision/assets/unbounded-agent-install.sh b/internal/provision/assets/unbounded-agent-install.sh index 0fb3d4f59..9c39e7ec6 100644 --- a/internal/provision/assets/unbounded-agent-install.sh +++ b/internal/provision/assets/unbounded-agent-install.sh @@ -85,23 +85,28 @@ curl -fsSL "${AGENT_URL}" | tar -xz -C "${tmp_dir}" unbounded-agent AGENT_BIN="${tmp_dir}/unbounded-agent" chmod 0755 "${AGENT_BIN}" -# Seed the daemon binary path when nothing usable is there yet. The agent -# version is selected independently of this script - by AGENT_VERSION, by -# AGENT_URL, or by the default of tracking the latest published release - so an -# installer that relied on the agent to install its own binary would silently -# break every agent released before that behavior existed. Such an agent never -# writes the binary, and bootstrap then fails at daemon setup with no indication -# that the installer and the agent disagree. +# Seed the daemon binary path for an agent released before the host root. The +# agent version is selected independently of this script - by AGENT_VERSION, by +# AGENT_URL, or by the default of tracking the latest published release - so it +# may be one that never writes its own binary and looks for it at +# /usr/local/bin. An agent that answers host-root installs itself under the host +# root, and seeding /usr/local/bin for it would make a fresh host look like one +# installed by an older agent. # # The test follows symlinks on purpose. On a host this installation already owns # the path resolves through the compatibility symlink to a live blue-green slot, # so it is left untouched and admission still runs from the staged executable # above. A dangling link resolves to nothing and is replaced, because install # would otherwise write through it to a stale location. -AGENT_BIN_TARGET="/usr/local/bin/unbounded-agent" -if [ ! -x "${AGENT_BIN_TARGET}" ]; then - rm -f "${AGENT_BIN_TARGET}" - install -m 0755 "${AGENT_BIN}" "${AGENT_BIN_TARGET}" +if ! "${AGENT_BIN}" host-root >/dev/null 2>&1; then + AGENT_BIN_TARGET="/usr/local/bin/unbounded-agent" + if [ ! -x "${AGENT_BIN_TARGET}" ]; then + rm -f "${AGENT_BIN_TARGET}" + if ! install -m 0755 "${AGENT_BIN}" "${AGENT_BIN_TARGET}"; then + echo "unbounded-agent ${_version_desc} predates /opt/unbounded and needs a writable /usr/local/bin; use a newer release" >&2 + exit 1 + fi + fi fi _START_ARGS="" diff --git a/internal/provision/script_test.go b/internal/provision/script_test.go index a9cb86a55..bd440100a 100644 --- a/internal/provision/script_test.go +++ b/internal/provision/script_test.go @@ -45,14 +45,23 @@ func TestUnboundedAgentInstallScript(t *testing.T) { require.Contains(t, script, "preflight ${_START_ARGS}") require.Contains(t, script, "0|false|no|FALSE|NO|False|No") - // The installer must place the agent binary itself. The agent version is - // selected independently of this script, including the default of tracking - // the latest published release, so an installer that relies on the agent to - // install its own binary breaks every agent released before that behavior - // existed. The uninstall script removes this same path. + // The installer must place the agent binary itself for an agent released + // before the host root. The agent version is selected independently of this + // script, including the default of tracking the latest published release, + // and such an agent never installs its own binary and looks for it at + // /usr/local/bin. require.Contains(t, script, `AGENT_BIN_TARGET="/usr/local/bin/unbounded-agent"`) require.Contains(t, script, `install -m 0755 "${AGENT_BIN}" "${AGENT_BIN_TARGET}"`) + // Only for such an agent. One that answers host-root installs itself under + // the host root, and a binary seeded at /usr/local/bin would make the fresh + // host look like one installed by an older agent, which it migrates as one. + require.Contains(t, script, `if ! "${AGENT_BIN}" host-root >/dev/null 2>&1; then`) + require.Less(t, + strings.Index(script, `if ! "${AGENT_BIN}" host-root`), + strings.Index(script, `AGENT_BIN_TARGET="/usr/local/bin/unbounded-agent"`), + "the seeding must be inside the host-root check") + // It must not clobber a live binary. The test follows symlinks so a host // this installation already owns resolves through the compatibility symlink // to a live slot and is skipped, which keeps admission running from the diff --git a/pkg/agent/agentbinary/activation_test.go b/pkg/agent/agentbinary/activation_test.go index 1128b96cc..4a04d7b48 100644 --- a/pkg/agent/agentbinary/activation_test.go +++ b/pkg/agent/agentbinary/activation_test.go @@ -14,6 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/hostroot" ) type fakeDaemonService struct { @@ -101,7 +103,7 @@ func TestPreflightHostDaemonActivationInitialLayoutDoesNotMutate(t *testing.T) { dir := t.TempDir() layout := testActivationLayout(dir) legacy := []byte("#!/bin/sh\nexit 0\n") - candidate := []byte("#!/bin/sh\n[ \"$1\" = version ]\n") + candidate := []byte(candidateAgent()) require.NoError(t, os.WriteFile(layout.BinaryPath, legacy, 0o755)) @@ -134,7 +136,7 @@ func TestPreflightHostDaemonActivationInitialLayoutDoesNotMutate(t *testing.T) { func TestPreflightHostDaemonActivationRejectsUnstagedCandidate(t *testing.T) { dir := t.TempDir() layout := testActivationLayout(dir) - writeExecutable(t, layout.BinaryPath, "#!/bin/sh\n[ \"$1\" = version ]\n") + writeExecutable(t, layout.BinaryPath, candidateAgent()) _, err := PreflightHostDaemonActivation(context.Background(), ActivationOptions{ Layout: layout, @@ -155,7 +157,7 @@ func TestPreflightHostDaemonActivationRejectsAliasedTargetSlot(t *testing.T) { require.NoError(t, os.Symlink(layout.CurrentPath, layout.BinaryPath)) candidatePath := filepath.Join(dir, "candidate") - writeExecutable(t, candidatePath, "#!/bin/sh\n[ \"$1\" = version ]\n") + writeExecutable(t, candidatePath, candidateAgent()) _, err := PreflightHostDaemonActivation(context.Background(), ActivationOptions{ Layout: layout, @@ -170,7 +172,7 @@ func TestActivateHostDaemonInitializesAndActivatesCandidate(t *testing.T) { dir := t.TempDir() layout := testActivationLayout(dir) legacy := []byte("#!/bin/sh\nexit 0\n") - candidate := []byte("#!/bin/sh\n[ \"$1\" = version ]\n") + candidate := []byte(candidateAgent()) require.NoError(t, os.WriteFile(layout.BinaryPath, legacy, 0o755)) @@ -202,7 +204,7 @@ func TestActivateHostDaemonAdoptsCurrentLinkToSingleBinary(t *testing.T) { dir := t.TempDir() layout := testActivationLayout(dir) legacy := []byte("#!/bin/sh\nexit 0\n") - candidate := []byte("#!/bin/sh\n[ \"$1\" = version ]\n") + candidate := []byte(candidateAgent()) require.NoError(t, os.WriteFile(layout.BinaryPath, legacy, 0o755)) require.NoError(t, os.Symlink(layout.BinaryPath, layout.CurrentPath)) @@ -230,7 +232,7 @@ func TestActivateHostDaemonRepairsMissingCurrentLink(t *testing.T) { dir := t.TempDir() layout := testActivationLayout(dir) legacy := []byte("#!/bin/sh\nexit 0\n") - candidate := []byte("#!/bin/sh\n[ \"$1\" = version ]\n") + candidate := []byte(candidateAgent()) require.NoError(t, os.WriteFile(layout.BluePath, legacy, 0o755)) require.NoError(t, os.Symlink(layout.BluePath, layout.LastGoodPath)) @@ -263,7 +265,7 @@ func TestActivateHostDaemonSwitchesExistingLayout(t *testing.T) { require.NoError(t, os.Symlink(layout.CurrentPath, layout.BinaryPath)) candidatePath := filepath.Join(dir, "candidate") - writeExecutable(t, candidatePath, "#!/bin/sh\n[ \"$1\" = version ]\n") + writeExecutable(t, candidatePath, candidateAgent()) service := &fakeDaemonService{} result, err := ActivateHostDaemon(context.Background(), discardLogger(), ActivationOptions{ @@ -292,7 +294,7 @@ func TestActivateHostDaemonRejectsDirectoryDestinationDuringPreflight(t *testing require.NoError(t, os.Mkdir(layout.BinaryPath, 0o755)) candidatePath := filepath.Join(dir, "candidate") - writeExecutable(t, candidatePath, "#!/bin/sh\n[ \"$1\" = version ]\n") + writeExecutable(t, candidatePath, candidateAgent()) service := &fakeDaemonService{} _, err := ActivateHostDaemon(context.Background(), discardLogger(), ActivationOptions{ @@ -317,7 +319,7 @@ func TestActivateHostDaemonAllowsMissingCompatibilityPath(t *testing.T) { require.NoError(t, os.Symlink(layout.BluePath, layout.LastGoodPath)) candidatePath := filepath.Join(dir, "candidate") - writeExecutable(t, candidatePath, "#!/bin/sh\n[ \"$1\" = version ]\n") + writeExecutable(t, candidatePath, candidateAgent()) result, err := ActivateHostDaemon(context.Background(), discardLogger(), ActivationOptions{ Layout: layout, @@ -334,7 +336,7 @@ func TestActivateHostDaemonAllowsMissingCompatibilityPath(t *testing.T) { func TestActivateHostDaemonIdenticalCandidatePreservesLastGood(t *testing.T) { dir := t.TempDir() layout := testActivationLayout(dir) - active := "#!/bin/sh\n[ \"$1\" = version ]\n" + active := candidateAgent() writeExecutable(t, layout.BluePath, active) writeExecutable(t, layout.GreenPath, "#!/bin/sh\nexit 0\n") require.NoError(t, os.Symlink(layout.BluePath, layout.CurrentPath)) @@ -360,7 +362,7 @@ func TestActivateHostDaemonIdenticalCandidatePreservesLastGood(t *testing.T) { func TestActivateHostDaemonIdenticalCandidateRepairsMissingLastGood(t *testing.T) { dir := t.TempDir() layout := testActivationLayout(dir) - active := "#!/bin/sh\n[ \"$1\" = version ]\n" + active := candidateAgent() writeExecutable(t, layout.BluePath, active) require.NoError(t, os.Symlink(layout.BluePath, layout.CurrentPath)) require.NoError(t, os.Symlink(layout.CurrentPath, layout.BinaryPath)) @@ -390,7 +392,7 @@ func TestActivateHostDaemonRollsBackUnhealthyCandidate(t *testing.T) { require.NoError(t, os.Symlink(layout.CurrentPath, layout.BinaryPath)) candidatePath := filepath.Join(dir, "candidate") - writeExecutable(t, candidatePath, "#!/bin/sh\n[ \"$1\" = version ]\n") + writeExecutable(t, candidatePath, candidateAgent()) ctx, cancel := context.WithCancel(context.Background()) service := &fakeDaemonService{ @@ -420,7 +422,7 @@ func TestActivateHostDaemonReportsUnsuccessfulRollback(t *testing.T) { require.NoError(t, os.Symlink(layout.CurrentPath, layout.BinaryPath)) candidatePath := filepath.Join(dir, "candidate") - writeExecutable(t, candidatePath, "#!/bin/sh\n[ \"$1\" = version ]\n") + writeExecutable(t, candidatePath, candidateAgent()) service := &fakeDaemonService{restartErr: errors.New("restart failed")} result, err := ActivateHostDaemon(context.Background(), discardLogger(), ActivationOptions{ @@ -471,3 +473,9 @@ func mustReadFile(t *testing.T, path string) []byte { func discardLogger() *slog.Logger { return slog.New(slog.NewTextHandler(io.Discard, nil)) } + +// candidateAgent is a fake agent that passes Verify: it answers version, and +// host-root with the root this host resolves. +func candidateAgent() string { + return "#!/bin/sh\n" + hostRootAnswer(hostroot.Resolve()) + "[ \"$1\" = version ]\n" +} diff --git a/pkg/agent/agentbinary/agentbinary.go b/pkg/agent/agentbinary/agentbinary.go index 6d696f487..859b3e277 100644 --- a/pkg/agent/agentbinary/agentbinary.go +++ b/pkg/agent/agentbinary/agentbinary.go @@ -12,10 +12,12 @@ import ( "os" "os/exec" "path/filepath" + "strings" "syscall" "time" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/hostroot" "github.com/Azure/unbounded/pkg/agent/internal/utilio" ) @@ -116,7 +118,8 @@ func initialDaemonBinaryTarget(paths goalstates.AgentUpgradePaths) (string, erro return paths.BluePath, nil } -// Verify runs the installed agent binary's version command. +// Verify runs the installed agent binary's version command, and checks that it +// uses the same host root as this agent. func Verify(ctx context.Context, path string) error { verifyCtx, cancel := context.WithTimeout(ctx, verifyTimeout) defer cancel() @@ -124,7 +127,7 @@ func Verify(ctx context.Context, path string) error { for { err := exec.CommandContext(verifyCtx, path, "version").Run() if err == nil { - return nil + return verifyHostRoot(verifyCtx, path, hostroot.Resolve()) } if errors.Is(err, syscall.ETXTBSY) { @@ -139,3 +142,25 @@ func Verify(ctx context.Context, path string) error { return fmt.Errorf("verify agent binary %s: %w", path, err) } } + +// verifyHostRoot refuses an agent that would look for its files somewhere +// other than root. An agent released before the host root looks under the +// legacy root, so on a host installed under the new one it would find nothing, +// and on a writable /usr it would start a second layout there. On a host that +// still uses the legacy root there is nothing to check: every agent finds it. +func verifyHostRoot(ctx context.Context, path, root string) error { + if root == hostroot.LegacyPath { + return nil + } + + out, err := exec.CommandContext(ctx, path, "host-root").Output() + if err != nil { + return fmt.Errorf("verify agent binary %s: it predates the host root %s and cannot run on this host: %w", path, root, err) + } + + if got := strings.TrimSpace(string(out)); got != root { + return fmt.Errorf("verify agent binary %s: it uses the host root %s, but this host uses %s", path, got, root) + } + + return nil +} diff --git a/pkg/agent/agentbinary/agentbinary_test.go b/pkg/agent/agentbinary/agentbinary_test.go index fa25bb39f..c13a460af 100644 --- a/pkg/agent/agentbinary/agentbinary_test.go +++ b/pkg/agent/agentbinary/agentbinary_test.go @@ -23,6 +23,7 @@ import ( "github.com/stretchr/testify/require" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/hostroot" ) func TestInstallFromTarGzVerifiesInstalledBinary(t *testing.T) { @@ -195,7 +196,74 @@ func writeTestAgentArchive(w io.Writer, binary []byte) error { } func testAgentScript(version string, exitCode int) []byte { - return []byte(fmt.Sprintf("#!/bin/sh\nprintf '%%s\\n' %s\nexit %d\n", posixShellQuote(version), exitCode)) + return []byte(fmt.Sprintf("#!/bin/sh\n%sprintf '%%s\\n' %s\nexit %d\n", hostRootAnswer(hostroot.Resolve()), posixShellQuote(version), exitCode)) +} + +// hostRootAnswer is the start of a fake agent that answers host-root with +// root. Verify asks every candidate, and a current agent answers with the root +// this host resolves. +func hostRootAnswer(root string) string { + return fmt.Sprintf("if [ \"$1\" = host-root ]; then printf '%%s\\n' %s; exit 0; fi\n", posixShellQuote(root)) +} + +// TestVerifyHostRoot covers the guard against activating an agent that looks +// for its files somewhere other than where this host keeps them. An agent +// released before the host root answers host-root with an error, because it +// has no such command. +func TestVerifyHostRoot(t *testing.T) { + t.Parallel() + + const root = "/opt/unbounded" + + tests := []struct { + name string + script string + root string + wantErr string + }{ + {name: "same root", script: hostRootAnswer(root) + "exit 1\n", root: root}, + {name: "trailing whitespace is ignored", script: "printf '%s \\n\\n' " + root + "\n", root: root}, + {name: "older agent", script: "echo unknown command >&2\nexit 1\n", root: root, wantErr: "predates the host root"}, + {name: "different root", script: hostRootAnswer("/usr/local"), root: root, wantErr: "uses the host root /usr/local, but this host uses /opt/unbounded"}, + {name: "empty answer", script: "exit 0\n", root: root, wantErr: "uses the host root , but"}, + // Every agent finds the legacy root, so a migrated host accepts any + // candidate without asking it. + {name: "legacy root accepts an older agent", script: "exit 1\n", root: hostroot.LegacyPath}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "unbounded-agent") + require.NoError(t, os.WriteFile(path, []byte("#!/bin/sh\n"+tt.script), 0o755)) + + err := verifyHostRoot(t.Context(), path, tt.root) + if tt.wantErr == "" { + require.NoError(t, err) + + return + } + + require.ErrorContains(t, err, tt.wantErr) + }) + } +} + +// TestVerifyAsksTheCandidateForItsHostRoot pins that Verify runs the host-root +// check, not only that the check works. An agent that passes version but has no +// host-root is exactly an older release. +func TestVerifyAsksTheCandidateForItsHostRoot(t *testing.T) { + t.Parallel() + + if hostroot.Resolve() == hostroot.LegacyPath { + t.Skip("this host's root is linked to the legacy root, where every agent is accepted") + } + + path := filepath.Join(t.TempDir(), "unbounded-agent") + require.NoError(t, os.WriteFile(path, []byte("#!/bin/sh\n[ \"$1\" = version ]\n"), 0o755)) + + require.ErrorContains(t, Verify(t.Context(), path), "predates the host root") } func posixShellQuote(value string) string { diff --git a/pkg/agent/agentbinary/upgrade_test.go b/pkg/agent/agentbinary/upgrade_test.go index 55e1c076d..35f260a79 100644 --- a/pkg/agent/agentbinary/upgrade_test.go +++ b/pkg/agent/agentbinary/upgrade_test.go @@ -19,6 +19,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/Azure/unbounded/pkg/agent/hostroot" ) func TestInstallAndSwitchFromTarGzWithOptions(t *testing.T) { @@ -37,7 +39,7 @@ func TestInstallAndSwitchFromTarGzWithOptions(t *testing.T) { t.Fatalf("symlink last-good: %v", err) } - payload := secureUpgradeArchive(t, "custom-agent", []byte("#!/bin/sh\nexit 0\n")) + payload := secureUpgradeArchive(t, "custom-agent", []byte("#!/bin/sh\n"+hostRootAnswer(hostroot.Resolve())+"exit 0\n")) server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(payload) })) @@ -425,7 +427,7 @@ func TestInstallAndSwitchFromTarGzWithOptionsAllowsHTTPRedirect(t *testing.T) { t.Parallel() paths := secureUpgradeReadyPaths(t) - payload := secureUpgradeArchive(t, "custom-agent", []byte("#!/bin/sh\nexit 0\n")) + payload := secureUpgradeArchive(t, "custom-agent", []byte("#!/bin/sh\n"+hostRootAnswer(hostroot.Resolve())+"exit 0\n")) insecure := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(payload) })) diff --git a/pkg/agent/goalstates/agentupgrade.go b/pkg/agent/goalstates/agentupgrade.go index 9fb64dc30..ebc160511 100644 --- a/pkg/agent/goalstates/agentupgrade.go +++ b/pkg/agent/goalstates/agentupgrade.go @@ -22,15 +22,31 @@ type AgentUpgradePaths struct { CurrentTargetPath string } -// ResolvedAgentUpgradePaths returns the host-side agent binary paths after -// applying environment overrides. +// ResolvedAgentUpgradePaths returns the host-side agent binary paths under the +// resolved host root, after applying environment overrides. Overrides name a +// specific file and so take precedence over the root. +// +// The AgentUpgrade signal path is not under the host root. It is state about +// an upgrade rather than part of the installed layout, and lives in the agent +// config directory. func ResolvedAgentUpgradePaths() (AgentUpgradePaths, error) { + return agentUpgradePathsIn(ResolveHostPaths().BinDir) +} + +// PlannedAgentUpgradePaths returns the paths ResolvedAgentUpgradePaths will +// return once the host root is migrated, without migrating it; see +// PlannedHostPaths. +func PlannedAgentUpgradePaths() (AgentUpgradePaths, error) { + return agentUpgradePathsIn(PlannedHostPaths().BinDir) +} + +func agentUpgradePathsIn(binDir string) (AgentUpgradePaths, error) { paths := AgentUpgradePaths{ - BinaryPath: resolveDaemonBinaryPath(EnvDaemonBinary, DaemonBinaryPath), - BluePath: resolveDaemonBinaryPath(EnvDaemonBinaryBlue, DaemonBinaryBluePath), - GreenPath: resolveDaemonBinaryPath(EnvDaemonBinaryGreen, DaemonBinaryGreenPath), - CurrentPath: resolveDaemonBinaryPath(EnvDaemonBinaryCurrent, DaemonBinaryCurrentPath), - LastGoodPath: resolveDaemonBinaryPath(EnvDaemonBinaryLastGood, DaemonBinaryLastGoodPath), + BinaryPath: resolveDaemonBinaryPath(EnvDaemonBinary, filepath.Join(binDir, daemonBinaryName)), + BluePath: resolveDaemonBinaryPath(EnvDaemonBinaryBlue, filepath.Join(binDir, daemonBinaryBlueName)), + GreenPath: resolveDaemonBinaryPath(EnvDaemonBinaryGreen, filepath.Join(binDir, daemonBinaryGreenName)), + CurrentPath: resolveDaemonBinaryPath(EnvDaemonBinaryCurrent, filepath.Join(binDir, daemonBinaryCurrentName)), + LastGoodPath: resolveDaemonBinaryPath(EnvDaemonBinaryLastGood, filepath.Join(binDir, daemonBinaryLastGoodName)), SignalPath: resolveDaemonBinaryPath(EnvDaemonAgentUpgradeSignalPath, DaemonAgentUpgradeSignalPath), } diff --git a/pkg/agent/goalstates/agentupgrade_test.go b/pkg/agent/goalstates/agentupgrade_test.go index 0a50521ed..45b0ae9e1 100644 --- a/pkg/agent/goalstates/agentupgrade_test.go +++ b/pkg/agent/goalstates/agentupgrade_test.go @@ -59,9 +59,10 @@ func TestResolvedAgentUpgradePaths_UsesDefaultsForBlankOverrides(t *testing.T) { paths, err := ResolvedAgentUpgradePaths() require.NoError(t, err) - assert.Equal(t, DaemonBinaryPath, paths.BinaryPath) - assert.Equal(t, DaemonBinaryBluePath, paths.BluePath) - assert.Equal(t, DaemonAgentUpgradeSignalPath, paths.SignalPath) + binDir := ResolveHostPaths().BinDir + assert.Equal(t, filepath.Join(binDir, "unbounded-agent"), paths.BinaryPath) + assert.Equal(t, filepath.Join(binDir, "unbounded-agent-blue"), paths.BluePath) + assert.Equal(t, DaemonAgentUpgradeSignalPath, paths.SignalPath, "the signal is state in the config directory, not part of the layout") } func TestAgentUpgradePathsNextTargetPathUsesGreenWhenCurrentIsBlue(t *testing.T) { diff --git a/pkg/agent/goalstates/constants.go b/pkg/agent/goalstates/constants.go index fcfc18043..d57c37f86 100644 --- a/pkg/agent/goalstates/constants.go +++ b/pkg/agent/goalstates/constants.go @@ -23,16 +23,41 @@ const ( // DaemonUnit is the systemd unit name for the unbounded-agent daemon. DaemonUnit = "unbounded-agent-daemon.service" + // NFTablesFlushUnit clears stale firewall rules before the nspawn machines + // start. + NFTablesFlushUnit = "nftables-flush.service" + // DaemonRecoveryUnit is the systemd recovery unit for the agent daemon. DaemonRecoveryUnit = "unbounded-agent-daemon-recovery.service" - DaemonBinaryPath = "/usr/local/bin/unbounded-agent" - DaemonBinaryBluePath = "/usr/local/bin/unbounded-agent-blue" - DaemonBinaryGreenPath = "/usr/local/bin/unbounded-agent-green" - DaemonBinaryCurrentPath = "/usr/local/bin/unbounded-agent-current" - DaemonBinaryLastGoodPath = "/usr/local/bin/unbounded-agent-last-good" - NSpawnLifecycleBinaryPath = "/usr/local/bin/unbounded-agent-nspawn-lifecycle" - DaemonRecoveryScriptPath = "/usr/local/bin/unbounded-agent-daemon-recovery.sh" + // FirstBootBootstrapUnit is the unit an Ignition config installs to bootstrap + // the agent on boot. + // + // Named here rather than in the command that writes it because reset has to + // remove it, and the two live in packages that cannot import each other. A + // name that drifted between them would leave the unit enabled on a host that + // had been reset, which re-bootstraps it on the next boot. + FirstBootBootstrapUnit = "unbounded-agent-bootstrap.service" + + // The agent's host-side files under hostroot.LegacyPath, where agents + // released before hostroot.Path installed them. + // + // Deprecated: on hosts installed since, these files are elsewhere. Use + // ResolvedAgentUpgradePaths and ResolveHostPaths, which follow the host root. + DaemonBinaryPath = "/usr/local/bin/unbounded-agent" + // Deprecated: use ResolvedAgentUpgradePaths. + DaemonBinaryBluePath = "/usr/local/bin/unbounded-agent-blue" + // Deprecated: use ResolvedAgentUpgradePaths. + DaemonBinaryGreenPath = "/usr/local/bin/unbounded-agent-green" + // Deprecated: use ResolvedAgentUpgradePaths. + DaemonBinaryCurrentPath = "/usr/local/bin/unbounded-agent-current" + // Deprecated: use ResolvedAgentUpgradePaths. + DaemonBinaryLastGoodPath = "/usr/local/bin/unbounded-agent-last-good" + // Deprecated: use ResolveHostPaths. + NSpawnLifecycleBinaryPath = "/usr/local/bin/unbounded-agent-nspawn-lifecycle" + // Deprecated: use ResolveHostPaths. + DaemonRecoveryScriptPath = "/usr/local/bin/unbounded-agent-daemon-recovery.sh" + DaemonAgentUpgradeSignalPath = AgentConfigDir + "/agent-upgrade-signal" DaemonAgentUpgradeLockPath = "/run/unbounded-agent-upgrade.lock" diff --git a/pkg/agent/goalstates/hostpaths.go b/pkg/agent/goalstates/hostpaths.go new file mode 100644 index 000000000..8e5ba736a --- /dev/null +++ b/pkg/agent/goalstates/hostpaths.go @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package goalstates + +import ( + "path/filepath" + + "github.com/Azure/unbounded/pkg/agent/hostroot" +) + +// Base names of the agent's own host-side files, joined with the resolved host +// root. +const ( + daemonBinaryName = "unbounded-agent" + daemonBinaryBlueName = "unbounded-agent-blue" + daemonBinaryGreenName = "unbounded-agent-green" + daemonBinaryCurrentName = "unbounded-agent-current" + daemonBinaryLastGoodName = "unbounded-agent-last-good" + nspawnLifecycleName = "unbounded-agent-nspawn-lifecycle" + daemonRecoveryScriptName = "unbounded-agent-daemon-recovery.sh" + localDNSNetworkHelperName = "unbounded-localdns-network" +) + +// HostPaths is the host-side layout of the agent's own files under the +// resolved host root; see the hostroot package. +// +// These are paths on the host. Files inside the nspawn machine are always +// resolved relative to the machine directory. +type HostPaths struct { + // Root is the resolved host root. + Root string + // BinDir is /bin. + BinDir string + // LibexecDir is /libexec. + LibexecDir string + + // NSpawnLifecycleBinary is the rollback-stable helper invoked by the + // generated nspawn hook units. + NSpawnLifecycleBinary string + // DaemonRecoveryScript is executed by the daemon recovery unit. + DaemonRecoveryScript string + // LocalDNSNetworkHelper backs unbounded-localdns-network.service. + LocalDNSNetworkHelper string +} + +// ResolveHostPaths returns the agent's host-side layout on this host. +func ResolveHostPaths() HostPaths { + return hostPathsUnder(hostroot.Resolve()) +} + +// PlannedHostPaths returns the layout ResolveHostPaths will return once the +// host root is migrated, without migrating it. It is for code that must not +// change the host, such as preflight. +func PlannedHostPaths() HostPaths { + return hostPathsUnder(hostroot.Planned(HostRootMarkers()...)) +} + +// LegacyHostPaths returns the layout under hostroot.LegacyPath, for the few +// checks that have to find an installation that has not been migrated yet. +func LegacyHostPaths() HostPaths { + return hostPathsUnder(hostroot.LegacyPath) +} + +func hostPathsUnder(root string) HostPaths { + binDir := filepath.Join(root, "bin") + libexecDir := filepath.Join(root, "libexec") + + return HostPaths{ + Root: root, + BinDir: binDir, + LibexecDir: libexecDir, + NSpawnLifecycleBinary: filepath.Join(binDir, nspawnLifecycleName), + DaemonRecoveryScript: filepath.Join(binDir, daemonRecoveryScriptName), + LocalDNSNetworkHelper: filepath.Join(libexecDir, localDNSNetworkHelperName), + } +} + +// HostRootMarkers returns the files, relative to the host root, whose presence +// under hostroot.LegacyPath identifies an unbounded-agent installation from +// before the host root; pass them to hostroot.Migrate. +// +// They are the daemon's binary layout only. The installer scripts are written +// under the legacy root on fresh hosts too, and a helper left behind by an +// older reset is not an installation. +func HostRootMarkers() []string { + return []string{ + filepath.Join("bin", daemonBinaryName), + filepath.Join("bin", daemonBinaryBlueName), + filepath.Join("bin", daemonBinaryGreenName), + filepath.Join("bin", daemonBinaryCurrentName), + filepath.Join("bin", daemonBinaryLastGoodName), + } +} + +// Base names of the installer scripts. The cloud-init variant and netboot write +// the install script under the legacy root on every host, and older versions +// left the uninstall script there, so teardown removes both from there. +const ( + agentInstallScriptName = "unbounded-agent-install.sh" + agentUninstallScriptName = "unbounded-agent-uninstall.sh" +) + +// OwnedHostFiles returns every host file outside the config directory that +// teardown removes: the agent's files under the host root and, when that is not +// the legacy root, under the legacy root as well, and the installer scripts +// under the legacy root. +// +// The legacy layout is swept on every host so teardown does not depend on the +// host root having been migrated. Reset is what an operator runs when the +// migration refuses, and it has to leave the host clean then too. +// +// The existing-deployment preflight deliberately checks only a subset: the +// daemon units and the recovery script. The install script and Ignition both +// put the agent binary in place before preflight runs, so a preflight that +// checked this whole list would refuse every fresh host. +// +// Environment overrides are deliberately not applied. These are the paths the +// agent installs to as a matter of layout, and teardown needs to find them on a +// host whose environment no longer resembles the one that provisioned it. +func OwnedHostFiles() []string { + return ownedHostFilesUnder(hostroot.Resolve(), hostroot.LegacyPath) +} + +func ownedHostFilesUnder(root, legacy string) []string { + files := layoutFilesUnder(root) + if root != legacy { + files = append(files, layoutFilesUnder(legacy)...) + } + + return append(files, + filepath.Join(legacy, "bin", agentInstallScriptName), + filepath.Join(legacy, "bin", agentUninstallScriptName), + ) +} + +func layoutFilesUnder(root string) []string { + paths := hostPathsUnder(root) + + return []string{ + filepath.Join(paths.BinDir, daemonBinaryName), + filepath.Join(paths.BinDir, daemonBinaryBlueName), + filepath.Join(paths.BinDir, daemonBinaryGreenName), + filepath.Join(paths.BinDir, daemonBinaryCurrentName), + filepath.Join(paths.BinDir, daemonBinaryLastGoodName), + paths.NSpawnLifecycleBinary, + paths.DaemonRecoveryScript, + paths.LocalDNSNetworkHelper, + } +} diff --git a/pkg/agent/goalstates/hostpaths_test.go b/pkg/agent/goalstates/hostpaths_test.go new file mode 100644 index 000000000..54dce846f --- /dev/null +++ b/pkg/agent/goalstates/hostpaths_test.go @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package goalstates + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestHostPathsUnder(t *testing.T) { + t.Parallel() + + paths := hostPathsUnder("/opt/unbounded") + + assert.Equal(t, HostPaths{ + Root: "/opt/unbounded", + BinDir: "/opt/unbounded/bin", + LibexecDir: "/opt/unbounded/libexec", + NSpawnLifecycleBinary: "/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle", + DaemonRecoveryScript: "/opt/unbounded/bin/unbounded-agent-daemon-recovery.sh", + LocalDNSNetworkHelper: "/opt/unbounded/libexec/unbounded-localdns-network", + }, paths) +} + +// TestLegacyHostPathsMatchTheReleasedLayout pins the layout under the legacy +// root to the paths released agents used. A migrated host keeps those files, +// and the units and recovery script an older agent wrote name them. +func TestLegacyHostPathsMatchTheReleasedLayout(t *testing.T) { + t.Parallel() + + legacy := LegacyHostPaths() + + assert.Equal(t, NSpawnLifecycleBinaryPath, legacy.NSpawnLifecycleBinary) //nolint:staticcheck // The released value is what is being pinned. + assert.Equal(t, DaemonRecoveryScriptPath, legacy.DaemonRecoveryScript) //nolint:staticcheck // The released value is what is being pinned. + assert.Equal(t, "/usr/local/libexec/unbounded-localdns-network", legacy.LocalDNSNetworkHelper) + + for _, pinned := range []string{ + DaemonBinaryPath, //nolint:staticcheck // The released value is what is being pinned. + DaemonBinaryBluePath, //nolint:staticcheck // The released value is what is being pinned. + DaemonBinaryGreenPath, //nolint:staticcheck // The released value is what is being pinned. + DaemonBinaryCurrentPath, //nolint:staticcheck // The released value is what is being pinned. + DaemonBinaryLastGoodPath, //nolint:staticcheck // The released value is what is being pinned. + } { + rel, err := filepath.Rel("/usr/local", pinned) + assert.NoError(t, err) + assert.Contains(t, HostRootMarkers(), rel, "a released binary path must identify a legacy installation") + } +} + +// TestHostRootMarkersAreTheBinaryLayout keeps files that a fresh host also has +// under the legacy root out of the markers. Counting one would link a fresh +// host's root to the legacy root and install the agent there. +func TestHostRootMarkersAreTheBinaryLayout(t *testing.T) { + t.Parallel() + + markers := HostRootMarkers() + + assert.Len(t, markers, 5) + assert.NotContains(t, markers, filepath.Join("bin", agentInstallScriptName), "cloud-init writes it on fresh hosts") + assert.NotContains(t, markers, filepath.Join("bin", nspawnLifecycleName), "an older reset can leave it behind") +} + +func TestOwnedHostFilesUnder(t *testing.T) { + t.Parallel() + + layout := func(root string) []string { + return []string{ + root + "/bin/unbounded-agent", + root + "/bin/unbounded-agent-blue", + root + "/bin/unbounded-agent-green", + root + "/bin/unbounded-agent-current", + root + "/bin/unbounded-agent-last-good", + root + "/bin/unbounded-agent-nspawn-lifecycle", + root + "/bin/unbounded-agent-daemon-recovery.sh", + root + "/libexec/unbounded-localdns-network", + } + } + // Written under the legacy root by cloud-init and netboot on every host. + scripts := []string{ + "/usr/local/bin/unbounded-agent-install.sh", + "/usr/local/bin/unbounded-agent-uninstall.sh", + } + + t.Run("new root sweeps the legacy layout too", func(t *testing.T) { + t.Parallel() + + // Reset does not migrate, so on a host the migration refused the + // installation is under the legacy root while the root is a real + // directory. + want := append(append(layout("/opt/unbounded"), layout("/usr/local")...), scripts...) + assert.ElementsMatch(t, want, ownedHostFilesUnder("/opt/unbounded", "/usr/local")) + }) + + t.Run("migrated root is swept once", func(t *testing.T) { + t.Parallel() + + want := append(layout("/usr/local"), scripts...) + assert.ElementsMatch(t, want, ownedHostFilesUnder("/usr/local", "/usr/local")) + }) +} diff --git a/pkg/agent/hostroot/hostroot.go b/pkg/agent/hostroot/hostroot.go new file mode 100644 index 000000000..a06e9dd07 --- /dev/null +++ b/pkg/agent/hostroot/hostroot.go @@ -0,0 +1,334 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Package hostroot locates the directory that holds the agent's own host-side +// files: its binaries and the helpers systemd units run. It does not cover the +// config, state, logs, units, or anything inside the nspawn machine. +// +// New installations use Path, which is writable on every supported host, +// including those that mount /usr read-only such as Azure Container Linux. +// Hosts installed by an agent released before Path keep their files under +// LegacyPath, and Migrate points Path at them with a symlink. Every path is +// resolved through Resolve, so on such a host it names the files where they +// already are, and the units and recovery script the older agent wrote stay +// valid for it as well as for the new one. +package hostroot + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "syscall" +) + +const ( + // Path is where the agent's host-side files live. + Path = "/opt/unbounded" + + // LegacyPath is where agents released before Path installed them. + LegacyPath = "/usr/local" +) + +// Resolve returns the directory Path refers to on this host, with symlinks +// resolved: LegacyPath on a migrated host, and Path itself on any other. +// +// Paths built from it are compared with symlink targets, which are resolved, +// so they have to be resolved too. Building them from an unresolved Path on a +// migrated host would name /opt/unbounded/bin/unbounded-agent-blue while the +// current link resolves to /usr/local/bin/unbounded-agent-blue, and the two +// would never compare equal. +func Resolve() string { + return canonical(Path) +} + +// Planned returns the directory Resolve will return once Migrate has run with +// the same markers. It changes nothing, so scripts that place files before the +// agent runs can ask where to put them. +func Planned(markers ...string) string { + return planned(Path, LegacyPath, markers) +} + +func planned(root, legacy string, markers []string) string { + if _, err := os.Lstat(root); errors.Is(err, os.ErrNotExist) && holdsAny(legacy, markers) { + return canonical(legacy) + } + + return canonical(root) +} + +// canonical resolves symlinks in the longest leading part of path that exists +// and appends the rest. It gives the same answer before and after the missing +// part is created, so paths resolved before a first install still match the +// ones resolved after it. +func canonical(path string) string { + path = filepath.Clean(path) + + missing := "" + + for current := path; ; current = filepath.Dir(current) { + if resolved, err := filepath.EvalSymlinks(current); err == nil { + return filepath.Join(resolved, missing) + } + + parent := filepath.Dir(current) + if parent == current { + return path + } + + missing = filepath.Join(filepath.Base(current), missing) + } +} + +// Migrate points Path at LegacyPath on a host whose agent installation is +// under LegacyPath. markers are paths relative to the root whose presence +// under LegacyPath identifies such an installation: the product's own binary +// layout, not files a fresh installation also creates there. +// +// It is idempotent and does nothing on a host without a legacy installation. +// It refuses a host with a legacy installation where Path is also a directory, +// because either could be the live one, and a symlink left by an older agent's +// reset is removed so a fresh installation gets a real directory. +// +// Commands that change the host call it first, before any path is resolved: a +// path resolved on an unmigrated legacy host names Path, where nothing is +// installed. Reset is the exception. It removes the files under both roots, so +// it works on a host this refuses, which is when an operator is told to run it. +func Migrate(log *slog.Logger, markers ...string) error { + return migrate(log, Path, LegacyPath, markers) +} + +func migrate(log *slog.Logger, root, legacy string, markers []string) error { + info, err := os.Lstat(root) + + switch { + case errors.Is(err, os.ErrNotExist): + if !holdsAny(legacy, markers) { + return nil + } + + return linkLegacy(log, root, legacy) + case err != nil: + return fmt.Errorf("inspect %s: %w", root, err) + case info.Mode()&os.ModeSymlink != 0: + target, err := os.Readlink(root) + if err != nil { + return fmt.Errorf("read %s: %w", root, err) + } + + // Only the link this package creates is ours to remove. A link an + // operator made, to put the root on another filesystem, is theirs. + if target != legacy || holdsAny(legacy, markers) { + return nil + } + + // An older agent's reset removes the files but not the link it never + // knew about. Left in place, it would put a fresh installation back + // under LegacyPath, which is read-only on some hosts. + log.Info("removing a host root link with no installation behind it", "path", root, "target", target) + + if err := removeAndSync(root); err != nil { + return err + } + + return nil + case !info.IsDir(): + return fmt.Errorf("%s is not a directory", root) + case !holdsAny(legacy, markers): + return nil + case holdsAny(root, markers): + return fmt.Errorf("the agent is installed under both %s and %s; run reset, then install again", legacy, root) + default: + return fmt.Errorf("the agent is installed under %s, but %s also exists; remove %s if nothing uses it, or run reset", legacy, root, root) + } +} + +// linkLegacy creates root as a symlink to legacy. It is built under a +// temporary name and renamed into place, so root is never half-made, and a +// concurrent migration renames an identical link over it. +func linkLegacy(log *slog.Logger, root, legacy string) error { + parent := filepath.Dir(root) + if err := os.MkdirAll(parent, 0o755); err != nil { + return fmt.Errorf("create %s: %w", parent, err) + } + + temp := fmt.Sprintf("%s.migrating-%d", root, os.Getpid()) + _ = os.Remove(temp) //nolint:errcheck // Leftover from an interrupted migration by this PID; absence is expected. + + if err := os.Symlink(legacy, temp); err != nil { + return fmt.Errorf("link %s to %s: %w", root, legacy, err) + } + + if err := os.Rename(temp, root); err != nil { + _ = os.Remove(temp) //nolint:errcheck // Best-effort cleanup; the rename error is returned. + return fmt.Errorf("link %s to %s: %w", root, legacy, err) + } + + if err := syncDir(parent); err != nil { + return err + } + + log.Info("linked the host root to the existing installation", "path", root, "target", legacy) + + return nil +} + +func holdsAny(root string, markers []string) bool { + for _, marker := range markers { + if _, err := os.Lstat(filepath.Join(root, marker)); err == nil { + return true + } + } + + return false +} + +// Prepare creates the root and the given subdirectories as a new installation +// needs them, with mode 0755 regardless of the umask, and restores their +// SELinux labels where the policy tools are present. On a migrated host the +// root is the existing installation and is left as it is. +// +// The labels matter because a directory takes its parent's label when it is +// created. Under /opt that is usr_t, while the policy expects bin_t under +// /opt/*/bin; files created later inherit the directory's label. +func Prepare(ctx context.Context, log *slog.Logger, subdirs ...string) error { + return prepare(ctx, log, Path, subdirs, restoreLabels) +} + +func prepare( + ctx context.Context, + log *slog.Logger, + root string, + subdirs []string, + relabel func(context.Context, *slog.Logger, string), +) error { + if info, err := os.Lstat(root); err == nil && info.Mode()&os.ModeSymlink != 0 { + return nil + } + + for _, dir := range append([]string{root}, prefixed(root, subdirs)...) { + if err := mkdirMode(dir, 0o755); err != nil { + return err + } + } + + relabel(ctx, log, root) + + return nil +} + +func prefixed(root string, subdirs []string) []string { + out := make([]string, 0, len(subdirs)) + for _, dir := range subdirs { + out = append(out, filepath.Join(root, dir)) + } + + return out +} + +// mkdirMode creates dir, and its parents, and sets its mode. An existing +// directory keeps its mode, which may have been chosen by whoever made it. +func mkdirMode(dir string, mode os.FileMode) error { + if _, err := os.Stat(dir); err == nil { + return nil + } + + if err := os.MkdirAll(dir, mode); err != nil { + return fmt.Errorf("create %s: %w", dir, err) + } + + if err := os.Chmod(dir, mode); err != nil { + return fmt.Errorf("set mode of %s: %w", dir, err) + } + + return nil +} + +func restoreLabels(ctx context.Context, log *slog.Logger, root string) { + restorecon, err := exec.LookPath("restorecon") + if err != nil { + return + } + + if out, err := exec.CommandContext(ctx, restorecon, "-R", root).CombinedOutput(); err != nil { //nolint:gosec // Fixed tool and the package's own root. + log.Warn("could not restore SELinux labels on the host root", "path", root, "error", err, "output", string(out)) + } +} + +// Remove removes the host root once reset has removed the files in it. A +// link this package created is removed, and a real directory is removed along +// with its now empty subdirectories. Anything not empty is left in place, and +// a link pointing somewhere else is left for whoever made it. +func Remove(log *slog.Logger) error { + return remove(log, Path, LegacyPath) +} + +func remove(log *slog.Logger, root, legacy string) error { + info, err := os.Lstat(root) + + switch { + case errors.Is(err, os.ErrNotExist): + return nil + case err != nil: + return fmt.Errorf("inspect %s: %w", root, err) + case info.Mode()&os.ModeSymlink != 0: + target, err := os.Readlink(root) + if err != nil { + return fmt.Errorf("read %s: %w", root, err) + } + + if target != legacy { + return nil + } + + log.Info("removing the host root link", "path", root) + + return removeAndSync(root) + case !info.IsDir(): + return nil + } + + entries, err := os.ReadDir(root) + if err != nil { + return fmt.Errorf("read %s: %w", root, err) + } + + for _, entry := range entries { + if entry.IsDir() { + if err := removeIfEmpty(filepath.Join(root, entry.Name())); err != nil { + return err + } + } + } + + return removeIfEmpty(root) +} + +func removeIfEmpty(dir string) error { + err := os.Remove(dir) + if err == nil || errors.Is(err, os.ErrNotExist) || errors.Is(err, syscall.ENOTEMPTY) || errors.Is(err, syscall.EEXIST) { + return nil + } + + return fmt.Errorf("remove %s: %w", dir, err) +} + +func removeAndSync(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove %s: %w", path, err) + } + + return syncDir(filepath.Dir(path)) +} + +func syncDir(dir string) error { + f, err := os.Open(dir) //nolint:gosec // The package's own directory. + if err != nil { + return fmt.Errorf("open %s: %w", dir, err) + } + + return errors.Join(f.Sync(), f.Close()) +} diff --git a/pkg/agent/hostroot/hostroot_test.go b/pkg/agent/hostroot/hostroot_test.go new file mode 100644 index 000000000..4feade803 --- /dev/null +++ b/pkg/agent/hostroot/hostroot_test.go @@ -0,0 +1,362 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package hostroot + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var testMarkers = []string{"bin/unbounded-agent", "bin/unbounded-agent-current"} + +type layout struct { + root, legacy string +} + +func newLayout(t *testing.T) layout { + t.Helper() + + dir := t.TempDir() + l := layout{root: filepath.Join(dir, "opt", "unbounded"), legacy: filepath.Join(dir, "usr", "local")} + require.NoError(t, os.MkdirAll(filepath.Join(l.legacy, "bin"), 0o755)) + + return l +} + +func touch(t *testing.T, path string) { + t.Helper() + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o755)) + require.NoError(t, os.WriteFile(path, nil, 0o755)) +} + +func discard() *slog.Logger { return slog.New(slog.DiscardHandler) } + +func TestCanonical(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + real := filepath.Join(dir, "real") + require.NoError(t, os.MkdirAll(real, 0o755)) + require.NoError(t, os.Symlink(real, filepath.Join(dir, "link"))) + + tests := []struct { + name, path, want string + }{ + {name: "existing path", path: real, want: real}, + {name: "symlink is resolved", path: filepath.Join(dir, "link"), want: real}, + // Paths resolved before the directory exists must match those resolved + // after, so the missing tail is kept under the resolved parent. + {name: "missing tail under a symlink", path: filepath.Join(dir, "link", "unbounded", "bin"), want: filepath.Join(real, "unbounded", "bin")}, + {name: "unclean input", path: filepath.Join(dir, "link") + "/./x/", want: filepath.Join(real, "x")}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, canonical(tt.path)) + }) + } +} + +func TestMigrate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + setup func(t *testing.T, l layout) + wantErr string + wantLink bool + wantDir bool + }{ + { + name: "fresh host is left alone", + setup: func(*testing.T, layout) {}, + }, + { + name: "legacy installation is linked", + setup: func(t *testing.T, l layout) { touch(t, filepath.Join(l.legacy, "bin/unbounded-agent")) }, + wantLink: true, + }, + { + name: "a dangling legacy link still counts as an installation", + setup: func(t *testing.T, l layout) { + require.NoError(t, os.Symlink("missing", filepath.Join(l.legacy, "bin/unbounded-agent-current"))) + }, + wantLink: true, + }, + { + // The cloud-init variant writes the install script under the + // legacy root on fresh hosts too. + name: "files that are not markers are not an installation", + setup: func(t *testing.T, l layout) { touch(t, filepath.Join(l.legacy, "bin/unbounded-agent-install.sh")) }, + }, + { + name: "a new installation is left alone", + setup: func(t *testing.T, l layout) { + touch(t, filepath.Join(l.root, "bin/unbounded-agent")) + }, + wantDir: true, + }, + { + name: "installations under both roots are refused", + setup: func(t *testing.T, l layout) { + touch(t, filepath.Join(l.root, "bin/unbounded-agent")) + touch(t, filepath.Join(l.legacy, "bin/unbounded-agent")) + }, + wantErr: "installed under both", + wantDir: true, + }, + { + name: "a legacy installation beside an empty root directory is refused", + setup: func(t *testing.T, l layout) { + require.NoError(t, os.MkdirAll(l.root, 0o755)) + touch(t, filepath.Join(l.legacy, "bin/unbounded-agent")) + }, + wantErr: "also exists", + wantDir: true, + }, + { + name: "a migrated host stays migrated", + setup: func(t *testing.T, l layout) { + touch(t, filepath.Join(l.legacy, "bin/unbounded-agent")) + require.NoError(t, os.MkdirAll(filepath.Dir(l.root), 0o755)) + require.NoError(t, os.Symlink(l.legacy, l.root)) + }, + wantLink: true, + }, + { + // An older agent's reset removes the files but not the link. + name: "a link with no installation behind it is removed", + setup: func(t *testing.T, l layout) { + require.NoError(t, os.MkdirAll(filepath.Dir(l.root), 0o755)) + require.NoError(t, os.Symlink(l.legacy, l.root)) + }, + }, + { + name: "a link someone else made is kept", + setup: func(t *testing.T, l layout) { + elsewhere := filepath.Join(filepath.Dir(l.legacy), "data") + require.NoError(t, os.MkdirAll(elsewhere, 0o755)) + require.NoError(t, os.MkdirAll(filepath.Dir(l.root), 0o755)) + require.NoError(t, os.Symlink(elsewhere, l.root)) + }, + wantLink: true, + }, + { + name: "a file at the root is refused", + setup: func(t *testing.T, l layout) { + touch(t, l.root) + }, + wantErr: "not a directory", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + l := newLayout(t) + tt.setup(t, l) + + err := migrate(discard(), l.root, l.legacy, testMarkers) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + } else { + require.NoError(t, err) + require.NoError(t, migrate(discard(), l.root, l.legacy, testMarkers), "migration must be idempotent") + } + + info, err := os.Lstat(l.root) + + switch { + case tt.wantLink: + require.NoError(t, err) + assert.NotZero(t, info.Mode()&os.ModeSymlink, "root must be a link") + case tt.wantDir: + require.NoError(t, err) + assert.True(t, info.IsDir(), "root must stay a directory") + case tt.wantErr == "": + assert.ErrorIs(t, err, os.ErrNotExist, "root must not exist") + } + }) + } +} + +// TestMigratedPathsMatchTheLegacyLayout is why the root is resolved: the +// current link a legacy agent wrote resolves to the legacy root, and a path +// built from the new root has to compare equal to it. +func TestMigratedPathsMatchTheLegacyLayout(t *testing.T) { + t.Parallel() + + l := newLayout(t) + blue := filepath.Join(l.legacy, "bin/unbounded-agent-blue") + touch(t, blue) + require.NoError(t, os.Symlink(blue, filepath.Join(l.legacy, "bin/unbounded-agent-current"))) + + require.NoError(t, migrate(discard(), l.root, l.legacy, testMarkers)) + + current, err := filepath.EvalSymlinks(filepath.Join(l.root, "bin/unbounded-agent-current")) + require.NoError(t, err) + assert.Equal(t, filepath.Join(canonical(l.root), "bin/unbounded-agent-blue"), current) +} + +func TestPlanned(t *testing.T) { + t.Parallel() + + t.Run("fresh host", func(t *testing.T) { + t.Parallel() + + l := newLayout(t) + assert.Equal(t, canonical(l.root), planned(l.root, l.legacy, testMarkers)) + _, err := os.Lstat(l.root) + assert.ErrorIs(t, err, os.ErrNotExist, "planning must not change the host") + }) + + t.Run("unmigrated legacy host", func(t *testing.T) { + t.Parallel() + + l := newLayout(t) + touch(t, filepath.Join(l.legacy, "bin/unbounded-agent")) + assert.Equal(t, canonical(l.legacy), planned(l.root, l.legacy, testMarkers)) + _, err := os.Lstat(l.root) + assert.ErrorIs(t, err, os.ErrNotExist, "planning must not change the host") + }) + + t.Run("migrated host", func(t *testing.T) { + t.Parallel() + + l := newLayout(t) + touch(t, filepath.Join(l.legacy, "bin/unbounded-agent")) + require.NoError(t, migrate(discard(), l.root, l.legacy, testMarkers)) + assert.Equal(t, canonical(l.legacy), planned(l.root, l.legacy, testMarkers)) + }) +} + +// TestPrepareIgnoresTheUmask is not parallel because the umask belongs to the +// process. Parallel tests are paused while it runs. +func TestPrepareIgnoresTheUmask(t *testing.T) { + old := syscall.Umask(0o077) + + t.Cleanup(func() { syscall.Umask(old) }) + + l := newLayout(t) + relabeled := "" + + require.NoError(t, prepare(t.Context(), discard(), l.root, []string{"bin", "libexec"}, + func(_ context.Context, _ *slog.Logger, root string) { relabeled = root })) + + for _, dir := range []string{l.root, filepath.Join(l.root, "bin"), filepath.Join(l.root, "libexec")} { + info, err := os.Stat(dir) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0o755), info.Mode().Perm(), dir) + } + + assert.Equal(t, l.root, relabeled, "new directories take their parent's SELinux label until restored") +} + +func TestPrepareLeavesAMigratedHostAlone(t *testing.T) { + t.Parallel() + + l := newLayout(t) + touch(t, filepath.Join(l.legacy, "bin/unbounded-agent")) + require.NoError(t, migrate(discard(), l.root, l.legacy, testMarkers)) + + relabeled := false + + require.NoError(t, prepare(t.Context(), discard(), l.root, []string{"libexec"}, + func(context.Context, *slog.Logger, string) { relabeled = true })) + + _, err := os.Stat(filepath.Join(l.legacy, "libexec")) + assert.ErrorIs(t, err, os.ErrNotExist, "the legacy root is not ours to arrange") + assert.False(t, relabeled, "the legacy root is not ours to relabel") +} + +func TestRemove(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + setup func(t *testing.T, l layout) + wantRoot bool + wantLegacy bool + }{ + {name: "absent root", setup: func(*testing.T, layout) {}, wantLegacy: true}, + { + name: "migration link is removed, and only the link", + setup: func(t *testing.T, l layout) { + touch(t, filepath.Join(l.legacy, "bin/other-tool")) + require.NoError(t, os.MkdirAll(filepath.Dir(l.root), 0o755)) + require.NoError(t, os.Symlink(l.legacy, l.root)) + }, + wantLegacy: true, + }, + { + name: "a link someone else made is kept", + setup: func(t *testing.T, l layout) { + elsewhere := filepath.Join(filepath.Dir(l.legacy), "data") + require.NoError(t, os.MkdirAll(elsewhere, 0o755)) + require.NoError(t, os.MkdirAll(filepath.Dir(l.root), 0o755)) + require.NoError(t, os.Symlink(elsewhere, l.root)) + }, + wantRoot: true, + wantLegacy: true, + }, + { + name: "emptied root directory is removed", + setup: func(t *testing.T, l layout) { + require.NoError(t, os.MkdirAll(filepath.Join(l.root, "bin"), 0o755)) + require.NoError(t, os.MkdirAll(filepath.Join(l.root, "libexec"), 0o755)) + }, + wantLegacy: true, + }, + { + name: "a root with files left in it is kept", + setup: func(t *testing.T, l layout) { + require.NoError(t, os.MkdirAll(filepath.Join(l.root, "bin"), 0o755)) + touch(t, filepath.Join(l.root, "lib/other/keep")) + }, + wantRoot: true, + wantLegacy: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + l := newLayout(t) + tt.setup(t, l) + + require.NoError(t, remove(discard(), l.root, l.legacy)) + + _, err := os.Lstat(l.root) + assert.Equal(t, tt.wantRoot, err == nil, "root present") + + _, err = os.Stat(filepath.Join(l.legacy, "bin")) + assert.Equal(t, tt.wantLegacy, err == nil, "legacy root untouched") + }) + } +} + +func TestRemoveKeepsNonEmptySubdirectories(t *testing.T) { + t.Parallel() + + l := newLayout(t) + require.NoError(t, os.MkdirAll(filepath.Join(l.root, "bin"), 0o755)) + touch(t, filepath.Join(l.root, "lib/keep")) + + require.NoError(t, remove(discard(), l.root, l.legacy)) + + _, err := os.Stat(filepath.Join(l.root, "bin")) + assert.ErrorIs(t, err, os.ErrNotExist, "an empty subdirectory is removed") + _, err = os.Stat(filepath.Join(l.root, "lib/keep")) + assert.NoError(t, err, "a file left in the root is kept") +} diff --git a/pkg/agent/phases/host/assets/nftables-flush.service b/pkg/agent/phases/host/assets/nftables-flush.service index 8ce2a8e5e..638673890 100644 --- a/pkg/agent/phases/host/assets/nftables-flush.service +++ b/pkg/agent/phases/host/assets/nftables-flush.service @@ -1,6 +1,8 @@ [Unit] Description=Flush nftables rules to a clean state -Before=systemd-nspawn@.service +# The machines order themselves after this unit in their service override. A +# Before= on the template here would not reach them: systemd fills in the +# missing instance with this unit's own name. # Some hosts load a default firewall at boot. Azure Container Linux enables # iptables.service, which installs an INPUT policy of drop; without ordering # the two start within milliseconds of each other and the firewall is applied diff --git a/pkg/agent/phases/host/configure_nftables.go b/pkg/agent/phases/host/configure_nftables.go index f3cf603b9..ca973ec54 100644 --- a/pkg/agent/phases/host/configure_nftables.go +++ b/pkg/agent/phases/host/configure_nftables.go @@ -20,7 +20,7 @@ import ( ) const ( - nftablesFlushUnit = "nftables-flush.service" + nftablesFlushUnit = goalstates.NFTablesFlushUnit nftablesClearPath = goalstates.ConfigDir + "/nftables-clear.nft" ) diff --git a/pkg/agent/phases/host/configure_nftables_test.go b/pkg/agent/phases/host/configure_nftables_test.go index 90fd9684e..15ad3494f 100644 --- a/pkg/agent/phases/host/configure_nftables_test.go +++ b/pkg/agent/phases/host/configure_nftables_test.go @@ -4,11 +4,14 @@ package host import ( + "bytes" "context" "errors" "log/slog" + "strings" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -59,3 +62,52 @@ func TestFlushFailsClosedOnUninspectableHost(t *testing.T) { require.ErrorIs(t, err, injected) require.False(t, start) } + +// TestNFTablesFlushUnitOutranksTheImageFirewall pins the boot ordering that +// makes the flush mean anything on an image with a firewall of its own. +// +// The flush hands a clean ruleset to a node that has not started yet. That only +// holds if nothing reinstalls rules after it. Azure Container Linux enables +// iptables.service, which loads an INPUT policy of DROP, and it was starting +// after the flush: the flush ran, iptables.service put the policy back, and the +// node came up with kubelet unreachable from the control plane on every boot. +// +// This was found by running the agent on that host, not by reading the unit, +// because nothing fails at install time and the node still reaches Ready. The +// ordering itself arrived with the host capability work; this pins it, so that +// a later edit to the unit cannot quietly drop it again. +func TestNFTablesFlushUnitOutranksTheImageFirewall(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + require.NoError(t, nftablesFlushServiceTemplate.Execute(&buf, map[string]string{ + "NFTablesClearPath": nftablesClearPath, + })) + + unit := buf.String() + + after := "" + + for line := range strings.SplitSeq(unit, "\n") { + if strings.HasPrefix(line, "After=") { + after = line + } + } + + require.NotEmpty(t, after, "the unit must order itself after the image's firewall units") + + for _, other := range []string{"iptables.service", "ip6tables.service", "nftables.service"} { + assert.Contains(t, after, other, + "a firewall unit starting after the flush undoes it") + } + + // Ordering only, never a dependency: pulling these in would start a + // firewall on a host that had deliberately disabled one. + assert.NotContains(t, unit, "Wants=iptables.service") + assert.NotContains(t, unit, "Requires=iptables.service") + + // The machines order themselves after the flush; see the rootfs service + // override. Before= on the bare template would name this unit's own + // instance and order nothing. + assert.NotContains(t, unit, "Before=systemd-nspawn@.service") +} diff --git a/pkg/agent/phases/host/preflight_existing_deployment.go b/pkg/agent/phases/host/preflight_existing_deployment.go index 7e34cf616..5578608f7 100644 --- a/pkg/agent/phases/host/preflight_existing_deployment.go +++ b/pkg/agent/phases/host/preflight_existing_deployment.go @@ -127,8 +127,14 @@ func existingDeploymentMachineArtifacts(machineName string) []existingDeployment } } +// existingDeploymentHostArtifacts returns the host files whose presence means +// this host already carries a deployment. +// +// The recovery script is looked for under the legacy root as well as the host +// root. Preflight does not migrate, so on a host installed by an older agent +// the host root does not yet lead to its files. func existingDeploymentHostArtifacts() []existingDeploymentArtifact { - return []existingDeploymentArtifact{ + artifacts := []existingDeploymentArtifact{ { description: "agent daemon unit", path: filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonUnit), @@ -137,11 +143,21 @@ func existingDeploymentHostArtifacts() []existingDeploymentArtifact { description: "agent daemon recovery unit", path: filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonRecoveryUnit), }, - { + } + + scripts := []string{goalstates.ResolveHostPaths().DaemonRecoveryScript} + if legacy := goalstates.LegacyHostPaths().DaemonRecoveryScript; legacy != scripts[0] { + scripts = append(scripts, legacy) + } + + for _, script := range scripts { + artifacts = append(artifacts, existingDeploymentArtifact{ description: "agent daemon recovery script", - path: goalstates.DaemonRecoveryScriptPath, - }, + path: script, + }) } + + return artifacts } func appendExistingDeploymentArtifactResult( diff --git a/pkg/agent/phases/host/preflight_host.go b/pkg/agent/phases/host/preflight_host.go index c87144bbc..72b91bdee 100644 --- a/pkg/agent/phases/host/preflight_host.go +++ b/pkg/agent/phases/host/preflight_host.go @@ -221,10 +221,11 @@ func checkHostOSConfiguration(log *slog.Logger, deps hostCheckDeps) preflight.Ch } // agentInstallDirs returns the host directories the agent writes its own files -// into. Derived from the binary path rather than restated, so the check cannot -// drift from where the agent actually installs. +// into. Derived from the host layout rather than restated, so the check cannot +// drift from where the agent actually installs. Planned rather than resolved, +// because preflight does not migrate the host root. func agentInstallDirs() []string { - return []string{filepath.Dir(goalstates.DaemonBinaryPath)} + return []string{goalstates.PlannedHostPaths().BinDir} } // installDirResults verifies the agent can write its own host-side files. diff --git a/pkg/agent/phases/host/preflight_host_test.go b/pkg/agent/phases/host/preflight_host_test.go index 807343487..df1825522 100644 --- a/pkg/agent/phases/host/preflight_host_test.go +++ b/pkg/agent/phases/host/preflight_host_test.go @@ -15,6 +15,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/preflight" @@ -112,9 +113,16 @@ func TestAgentInstallDirsProbeIsCreatable(t *testing.T) { func TestAgentInstallDirsTracksTheBinaryPath(t *testing.T) { t.Parallel() + if goalstates.PlannedHostPaths() != goalstates.ResolveHostPaths() { + t.Skip("this host has an agent installation that has not been migrated to the host root") + } + + paths, err := goalstates.ResolvedAgentUpgradePaths() + require.NoError(t, err) + dirs := agentInstallDirs() assert.Len(t, dirs, 1) - assert.Equal(t, filepath.Dir(goalstates.DaemonBinaryPath), dirs[0]) + assert.Equal(t, filepath.Dir(paths.BinaryPath), dirs[0]) } func TestCheckExistingDeploymentCleanHost(t *testing.T) { @@ -127,6 +135,33 @@ func TestCheckExistingDeploymentCleanHost(t *testing.T) { assert.Equal(t, preflight.SeverityOK, results[0].Severity) } +// TestCheckExistingDeploymentFindsTheRecoveryScriptUnderEitherRoot covers a +// host installed by an older release. Preflight does not migrate the host root, +// so the recovery script is found under the legacy root, not through the new +// one. +func TestCheckExistingDeploymentFindsTheRecoveryScriptUnderEitherRoot(t *testing.T) { + t.Parallel() + + for _, script := range []string{ + goalstates.ResolveHostPaths().DaemonRecoveryScript, + "/usr/local/bin/unbounded-agent-daemon-recovery.sh", + } { + t.Run(script, func(t *testing.T) { + t.Parallel() + + deps := defaultHostCheckDeps() + deps.stat = statOnlyExists(script) + deps.outputCmd = outputWith("", errors.New("not found")) + + results := checkExistingDeployment(slog.New(slog.DiscardHandler), deps).Check(context.Background()) + + require.Len(t, results, 1) + assert.Equal(t, preflight.SeverityError, results[0].Severity) + assert.Contains(t, results[0].Message, script) + }) + } +} + func TestCheckExistingDeploymentDetectsMachineRegistration(t *testing.T) { deps := defaultHostCheckDeps() deps.stat = statNotExist() diff --git a/pkg/agent/phases/nodestart/assets/unbounded-localdns-network.service b/pkg/agent/phases/nodestart/assets/unbounded-localdns-network.service index a270e1092..54f15115e 100644 --- a/pkg/agent/phases/nodestart/assets/unbounded-localdns-network.service +++ b/pkg/agent/phases/nodestart/assets/unbounded-localdns-network.service @@ -5,4 +5,4 @@ Before=systemd-nspawn@{{.MachineName}}.service [Service] Type=oneshot -ExecStart=/usr/local/libexec/unbounded-localdns-network +ExecStart={{.NetworkHelper}} diff --git a/pkg/agent/phases/nodestart/localdns.go b/pkg/agent/phases/nodestart/localdns.go index a3cd816ea..655420352 100644 --- a/pkg/agent/phases/nodestart/localdns.go +++ b/pkg/agent/phases/nodestart/localdns.go @@ -112,10 +112,14 @@ func (s *setupLocalDNSNetwork) Do(ctx context.Context) error { return nil } + // The unit names the helper, so both are written from one resolution. + helper := goalstates.ResolveHostPaths().LocalDNSNetworkHelper + data := map[string]string{ "MachineName": s.goalState.MachineName, "NodeListenerIP": s.goalState.LocalDNS.NodeListenerIP.String(), "ClusterListenerIP": s.goalState.LocalDNS.ClusterListenerIP.String(), + "NetworkHelper": helper, } var script bytes.Buffer @@ -123,7 +127,7 @@ func (s *setupLocalDNSNetwork) Do(ctx context.Context) error { return fmt.Errorf("render LocalDNS network script: %w", err) } - if err := utilio.WriteFile("/usr/local/libexec/unbounded-localdns-network", script.Bytes(), 0o755); err != nil { + if err := utilio.WriteFile(helper, script.Bytes(), 0o755); err != nil { return fmt.Errorf("write LocalDNS network script: %w", err) } diff --git a/pkg/agent/phases/reset/network.go b/pkg/agent/phases/reset/network.go index 5044903de..99d2b2461 100644 --- a/pkg/agent/phases/reset/network.go +++ b/pkg/agent/phases/reset/network.go @@ -118,7 +118,7 @@ func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { for _, path := range []string{ filepath.Join(goalstates.SystemdSystemDir, goalstates.LocalDNSNetworkUnit), - "/usr/local/libexec/unbounded-localdns-network", + goalstates.ResolveHostPaths().LocalDNSNetworkHelper, } { if err := removeFileIfExists(t.log, path); err != nil { return err diff --git a/pkg/agent/phases/rootfs/assets/service-override.conf b/pkg/agent/phases/rootfs/assets/service-override.conf index 4136cad48..c3b5ebc9f 100644 --- a/pkg/agent/phases/rootfs/assets/service-override.conf +++ b/pkg/agent/phases/rootfs/assets/service-override.conf @@ -30,10 +30,15 @@ # and is a bpffs mount before the .nspawn Bind= directive exposes it inside the # machine. eBPF CNIs need a usable bpffs at /sys/fs/bpf to create and share BPF # maps, but each nspawn machine should get its own pinned-object namespace. +# +# nftables-flush.service clears stale firewall rules, and has to finish before +# the machine starts for the node to come up with a clean ruleset. Its +# [Install] section makes every machine require it, which does not order them. [Unit] Requires={{.ConfigRegenerationUnit}} After={{.ConfigRegenerationUnit}} +After=nftables-flush.service StartLimitIntervalSec=0 [Service] diff --git a/pkg/agent/phases/rootfs/lifecycle_helper.go b/pkg/agent/phases/rootfs/lifecycle_helper.go index 3916c7b88..d6ad4dfdd 100644 --- a/pkg/agent/phases/rootfs/lifecycle_helper.go +++ b/pkg/agent/phases/rootfs/lifecycle_helper.go @@ -32,7 +32,7 @@ func (e *ensureNSpawnLifecycleHelper) Do(_ context.Context) error { return fmt.Errorf("resolve running agent executable: %w", err) } - return installNSpawnLifecycleHelper(sourcePath, goalstates.NSpawnLifecycleBinaryPath) + return installNSpawnLifecycleHelper(sourcePath, goalstates.ResolveHostPaths().NSpawnLifecycleBinary) } func installNSpawnLifecycleHelper(sourcePath, targetPath string) (retErr error) { diff --git a/pkg/agent/phases/rootfs/nspawn.go b/pkg/agent/phases/rootfs/nspawn.go index 30a5cbe52..d4413207f 100644 --- a/pkg/agent/phases/rootfs/nspawn.go +++ b/pkg/agent/phases/rootfs/nspawn.go @@ -167,7 +167,7 @@ func writeNSpawnConfigs(log *slog.Logger, goalState *goalstates.RootFS) error { AMDGPUDevicePaths: amdGPUDevicePaths, AMDSysFSPaths: goalState.AMD.SysFSPaths, ConfigRegenerationUnit: goalstates.ConfigRegenerationUnit(machineName), - AgentBinaryPath: goalstates.NSpawnLifecycleBinaryPath, + AgentBinaryPath: goalstates.ResolveHostPaths().NSpawnLifecycleBinary, } if len(hostDevicePaths) > 0 { diff --git a/pkg/agent/phases/rootfs/nspawn_render_test.go b/pkg/agent/phases/rootfs/nspawn_render_test.go index 2d20f077c..6d5879b17 100644 --- a/pkg/agent/phases/rootfs/nspawn_render_test.go +++ b/pkg/agent/phases/rootfs/nspawn_render_test.go @@ -309,6 +309,21 @@ func TestServiceOverride_ConfigRegenerationDependency(t *testing.T) { require.Less(t, strings.Index(out, "After=unbounded-agent-regenerate-config@kube1.service"), strings.Index(out, "[Service]")) } +// TestServiceOverride_OrdersAfterTheNFTablesFlush pins the ordering that gives +// the machine a clean ruleset. The flush unit makes every machine require it, +// but only this line orders them. +func TestServiceOverride_OrdersAfterTheNFTablesFlush(t *testing.T) { + t.Parallel() + + var buf bytes.Buffer + require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", defaultNSpawnTemplateData("kube1"))) + + out := buf.String() + require.Contains(t, out, "\nAfter="+goalstates.NFTablesFlushUnit+"\n") + require.Less(t, strings.Index(out, "[Unit]"), strings.Index(out, "After="+goalstates.NFTablesFlushUnit)) + require.Less(t, strings.Index(out, "After="+goalstates.NFTablesFlushUnit), strings.Index(out, "[Service]")) +} + func TestConfigRegenerationUnit(t *testing.T) { t.Parallel() @@ -320,7 +335,7 @@ func TestConfigRegenerationUnit(t *testing.T) { require.Contains(t, out, "Wants=systemd-udev-settle.service") require.Contains(t, out, "After=systemd-udev-settle.service") require.Contains(t, out, "Type=oneshot") - require.Contains(t, out, "ExecStart=/usr/local/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle pre-start kube1") + require.Contains(t, out, "ExecStart=/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle pre-start kube1") require.NotContains(t, out, "ExecStart=-") require.NotContains(t, out, "if [ ! -x") require.Contains(t, out, "Restart=on-failure") @@ -334,7 +349,7 @@ func TestServiceOverride_NVIDIAReconcilesOnEveryStart(t *testing.T) { var buf bytes.Buffer require.NoError(t, nspawnTemplates.ExecuteTemplate(&buf, "service-override.conf", data)) - require.Contains(t, buf.String(), "ExecStartPost=/usr/local/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1") + require.Contains(t, buf.String(), "ExecStartPost=/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1") require.NotContains(t, buf.String(), "ExecStartPost=-") require.NotContains(t, buf.String(), "if [ ! -x") } @@ -354,7 +369,7 @@ func defaultNSpawnTemplateData(machineName string) nspawnTemplateData { ContainerImageArchiveDir: goalstates.ContainerImageArchiveDir, ContainerImageArchiveHostDir: goalstates.ContainerImageArchiveHostDir, ConfigRegenerationUnit: goalstates.ConfigRegenerationUnit(machineName), - AgentBinaryPath: goalstates.NSpawnLifecycleBinaryPath, + AgentBinaryPath: "/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle", } } @@ -484,3 +499,44 @@ func TestAdditionalHostMounts_ConfigToNSpawn(t *testing.T) { // The writable mount must not appear as a BindReadOnly entry. require.NotContains(t, out, "BindReadOnly=/var/lib/data") } + +// TestWriteNSpawnConfigsInvokeTheHelperUnderTheHostRoot writes the generated +// units and checks they invoke the helper where the host root puts it. +// +// These units are the only callers of the helper. If they name the legacy +// location while the helper is installed under the host root, nothing fails +// until systemd starts the machine and the hook cannot exec. +// +// This goes through writeNSpawnConfigs rather than rendering the templates from +// hand-built data, because the defect being guarded against is the population +// step reverting to the constant. A test that supplies its own template data +// passes either way. +func TestWriteNSpawnConfigsInvokeTheHelperUnderTheHostRoot(t *testing.T) { + t.Parallel() + + helper := goalstates.ResolveHostPaths().NSpawnLifecycleBinary + if helper == goalstates.LegacyHostPaths().NSpawnLifecycleBinary { + t.Skip("this host's root is linked to the legacy root, so the two cannot be told apart") + } + + dir := t.TempDir() + goalState := &goalstates.RootFS{ + MachineDir: filepath.Join(dir, "machines", "kube1"), + NSpawnConfigFile: filepath.Join(dir, "kube1.nspawn"), + ServiceOverrideFile: filepath.Join(dir, "override.conf"), + ConfigRegenerationFile: filepath.Join(dir, "config-regeneration.service"), + } + + require.NoError(t, writeNSpawnConfigs(slog.New(slog.DiscardHandler), goalState)) + + for _, path := range []string{goalState.ServiceOverrideFile, goalState.ConfigRegenerationFile} { + content, err := os.ReadFile(path) + require.NoError(t, err) + + rendered := string(content) + require.Contains(t, rendered, helper+" nspawn-lifecycle", + "%s must invoke the helper under the host root", filepath.Base(path)) + require.NotContains(t, rendered, goalstates.LegacyHostPaths().NSpawnLifecycleBinary, + "%s must not fall back to the legacy root", filepath.Base(path)) + } +} diff --git a/pkg/agent/phases/rootfs/testdata/render/cpu-only.service-override.conf.golden b/pkg/agent/phases/rootfs/testdata/render/cpu-only.service-override.conf.golden index c8eef969c..3ae5fe0ce 100644 --- a/pkg/agent/phases/rootfs/testdata/render/cpu-only.service-override.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/render/cpu-only.service-override.conf.golden @@ -30,10 +30,15 @@ # and is a bpffs mount before the .nspawn Bind= directive exposes it inside the # machine. eBPF CNIs need a usable bpffs at /sys/fs/bpf to create and share BPF # maps, but each nspawn machine should get its own pinned-object namespace. +# +# nftables-flush.service clears stale firewall rules, and has to finish before +# the machine starts for the node to come up with a clean ruleset. Its +# [Install] section makes every machine require it, which does not order them. [Unit] Requires=unbounded-agent-regenerate-config@kube1.service After=unbounded-agent-regenerate-config@kube1.service +After=nftables-flush.service StartLimitIntervalSec=0 [Service] @@ -42,7 +47,7 @@ RestartSec=10s ExecStartPre=-/usr/bin/machinectl terminate kube1 ExecStartPre=/usr/bin/mkdir -p /run/bpffs/kube1 ExecStartPre=/bin/sh -c '/usr/bin/mountpoint -q /run/bpffs/kube1 || /usr/bin/mount -t bpf bpf /run/bpffs/kube1' -ExecStartPost=/usr/local/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1 +ExecStartPost=/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1 Environment=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 Environment=SYSTEMD_NSPAWN_API_VFS_WRITABLE=network diff --git a/pkg/agent/phases/rootfs/testdata/render/nvidia-all-helpers.service-override.conf.golden b/pkg/agent/phases/rootfs/testdata/render/nvidia-all-helpers.service-override.conf.golden index b01108858..c84aee659 100644 --- a/pkg/agent/phases/rootfs/testdata/render/nvidia-all-helpers.service-override.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/render/nvidia-all-helpers.service-override.conf.golden @@ -30,10 +30,15 @@ # and is a bpffs mount before the .nspawn Bind= directive exposes it inside the # machine. eBPF CNIs need a usable bpffs at /sys/fs/bpf to create and share BPF # maps, but each nspawn machine should get its own pinned-object namespace. +# +# nftables-flush.service clears stale firewall rules, and has to finish before +# the machine starts for the node to come up with a clean ruleset. Its +# [Install] section makes every machine require it, which does not order them. [Unit] Requires=unbounded-agent-regenerate-config@kube1.service After=unbounded-agent-regenerate-config@kube1.service +After=nftables-flush.service StartLimitIntervalSec=0 [Service] @@ -42,7 +47,7 @@ RestartSec=10s ExecStartPre=-/usr/bin/machinectl terminate kube1 ExecStartPre=/usr/bin/mkdir -p /run/bpffs/kube1 ExecStartPre=/bin/sh -c '/usr/bin/mountpoint -q /run/bpffs/kube1 || /usr/bin/mount -t bpf bpf /run/bpffs/kube1' -ExecStartPost=/usr/local/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1 +ExecStartPost=/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1 Environment=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 Environment=SYSTEMD_NSPAWN_API_VFS_WRITABLE=network diff --git a/pkg/agent/phases/rootfs/testdata/render/nvidia-gb300-rack-full.service-override.conf.golden b/pkg/agent/phases/rootfs/testdata/render/nvidia-gb300-rack-full.service-override.conf.golden index b6dd717cf..4388598b9 100644 --- a/pkg/agent/phases/rootfs/testdata/render/nvidia-gb300-rack-full.service-override.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/render/nvidia-gb300-rack-full.service-override.conf.golden @@ -30,10 +30,15 @@ # and is a bpffs mount before the .nspawn Bind= directive exposes it inside the # machine. eBPF CNIs need a usable bpffs at /sys/fs/bpf to create and share BPF # maps, but each nspawn machine should get its own pinned-object namespace. +# +# nftables-flush.service clears stale firewall rules, and has to finish before +# the machine starts for the node to come up with a clean ruleset. Its +# [Install] section makes every machine require it, which does not order them. [Unit] Requires=unbounded-agent-regenerate-config@kube1.service After=unbounded-agent-regenerate-config@kube1.service +After=nftables-flush.service StartLimitIntervalSec=0 [Service] @@ -42,7 +47,7 @@ RestartSec=10s ExecStartPre=-/usr/bin/machinectl terminate kube1 ExecStartPre=/usr/bin/mkdir -p /run/bpffs/kube1 ExecStartPre=/bin/sh -c '/usr/bin/mountpoint -q /run/bpffs/kube1 || /usr/bin/mount -t bpf bpf /run/bpffs/kube1' -ExecStartPost=/usr/local/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1 +ExecStartPost=/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1 Environment=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 Environment=SYSTEMD_NSPAWN_API_VFS_WRITABLE=network diff --git a/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden b/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden index c8eef969c..3ae5fe0ce 100644 --- a/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/service-override-kube1.conf.golden @@ -30,10 +30,15 @@ # and is a bpffs mount before the .nspawn Bind= directive exposes it inside the # machine. eBPF CNIs need a usable bpffs at /sys/fs/bpf to create and share BPF # maps, but each nspawn machine should get its own pinned-object namespace. +# +# nftables-flush.service clears stale firewall rules, and has to finish before +# the machine starts for the node to come up with a clean ruleset. Its +# [Install] section makes every machine require it, which does not order them. [Unit] Requires=unbounded-agent-regenerate-config@kube1.service After=unbounded-agent-regenerate-config@kube1.service +After=nftables-flush.service StartLimitIntervalSec=0 [Service] @@ -42,7 +47,7 @@ RestartSec=10s ExecStartPre=-/usr/bin/machinectl terminate kube1 ExecStartPre=/usr/bin/mkdir -p /run/bpffs/kube1 ExecStartPre=/bin/sh -c '/usr/bin/mountpoint -q /run/bpffs/kube1 || /usr/bin/mount -t bpf bpf /run/bpffs/kube1' -ExecStartPost=/usr/local/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1 +ExecStartPost=/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube1 Environment=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 Environment=SYSTEMD_NSPAWN_API_VFS_WRITABLE=network diff --git a/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden b/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden index b01624797..edc6a3047 100644 --- a/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden +++ b/pkg/agent/phases/rootfs/testdata/service-override-kube2.conf.golden @@ -30,10 +30,15 @@ # and is a bpffs mount before the .nspawn Bind= directive exposes it inside the # machine. eBPF CNIs need a usable bpffs at /sys/fs/bpf to create and share BPF # maps, but each nspawn machine should get its own pinned-object namespace. +# +# nftables-flush.service clears stale firewall rules, and has to finish before +# the machine starts for the node to come up with a clean ruleset. Its +# [Install] section makes every machine require it, which does not order them. [Unit] Requires=unbounded-agent-regenerate-config@kube2.service After=unbounded-agent-regenerate-config@kube2.service +After=nftables-flush.service StartLimitIntervalSec=0 [Service] @@ -42,7 +47,7 @@ RestartSec=10s ExecStartPre=-/usr/bin/machinectl terminate kube2 ExecStartPre=/usr/bin/mkdir -p /run/bpffs/kube2 ExecStartPre=/bin/sh -c '/usr/bin/mountpoint -q /run/bpffs/kube2 || /usr/bin/mount -t bpf bpf /run/bpffs/kube2' -ExecStartPost=/usr/local/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube2 +ExecStartPost=/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle nspawn-lifecycle post-start kube2 Environment=SYSTEMD_NSPAWN_UNIFIED_HIERARCHY=1 Environment=SYSTEMD_NSPAWN_API_VFS_WRITABLE=network