diff --git a/.github/workflows/agent-e2e-kind.yaml b/.github/workflows/agent-e2e-kind.yaml index bdebf97c5..19db28bd4 100644 --- a/.github/workflows/agent-e2e-kind.yaml +++ b/.github/workflows/agent-e2e-kind.yaml @@ -121,6 +121,40 @@ jobs: if: always() run: python3 ./hack/agent/e2e-kind/e2e.py --verbose cleanup + agent-bootstrap-recovery: + name: agent bootstrap recovery (Ubuntu) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + KIND_CLUSTER_NAME: agent-bootstrap-recovery + VM_NAME: agent-bootstrap-recovery + VM_SUBNET: "192.168.100" + VM_IP: "192.168.100.10" + AGENT_MACHINE_NAME: agent-bootstrap-recovery + HOST_BASE_OS: ubuntu2404 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up test control plane + uses: ./.github/actions/agent-e2e-kind-control-plane + with: + cluster-name: ${{ env.KIND_CLUSTER_NAME }} + vm-subnet: ${{ env.VM_SUBNET }} + - name: Set up machina resources + uses: ./.github/actions/agent-e2e-machina-setup + - name: Retry late bootstrap and repair after ordinary repave + run: python3 ./hack/agent/e2e-kind/e2e.py --verbose run-suite --suite bootstrap-recovery + - name: Verify strict reset + run: python3 ./hack/agent/e2e-kind/e2e.py --verbose reset-agent + - name: Collect logs + if: always() + uses: ./.github/actions/agent-e2e-kind-logs + with: + artifact-name: agent-bootstrap-recovery-logs + - name: Cleanup + if: always() + run: python3 ./hack/agent/e2e-kind/e2e.py --verbose cleanup + agent-config-e2e: name: agent config e2e runs-on: ubuntu-24.04 diff --git a/NOTICE b/NOTICE index 72686913b..db6bd30d1 100644 --- a/NOTICE +++ b/NOTICE @@ -350,6 +350,14 @@ notices: license: - name: Apache License, Version 2.0 link: https://github.com/go-logr/logr/blob/v1.4.4/LICENSE + - dependency: github.com/gofrs/flock + ecosystem: go + copyright: + - Copyright (c) 2018-2024, The Gofrs + - Copyright (c) 2015-2020, Tim Heckman + license: + - name: BSD 3-Clause License + link: https://github.com/gofrs/flock/blob/v0.10.0/LICENSE - dependency: github.com/golang-jwt/jwt/v5 ecosystem: go copyright: diff --git a/cmd/agent/internal/bootstrap/coordinator.go b/cmd/agent/internal/bootstrap/coordinator.go new file mode 100644 index 000000000..a810e2a6f --- /dev/null +++ b/cmd/agent/internal/bootstrap/coordinator.go @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Package bootstrap reapplies the stages of an owned initial installation. +// +// Every stage runs on every attempt. Each decides what to do by looking at the +// host rather than at a record of what a previous attempt claimed to have done, +// so a host that changed in between converges instead of being skipped. +package bootstrap + +import ( + "context" + "fmt" + "log/slog" + + "github.com/Azure/unbounded/cmd/agent/internal/installstate" +) + +// Identity is what makes one installation distinguishable from another. +// +// HostPrefix is the resolved installation prefix. It is carried here so the +// record written before the first host mutation knows where this installation +// puts its files, which is the only thing teardown can consult after a +// bootstrap that failed before the node started. +type Identity struct { + MachineName string + ConfigFingerprint string + HostPrefix string +} + +type Stages interface { + EnsureHostClean(context.Context) error + ResolveInputs(context.Context) error + PrepareHost(context.Context) error + PrepareRootFS(context.Context) error + EnsureNodeStarted(context.Context) error + EnsureDaemonInstalled(context.Context) error + RepairDaemon(context.Context) error + VerifyInstalled(context.Context) error +} + +// Stage names the work being reported on. It is a label for status reporting +// and logs, deliberately not persisted: writing down which stage was reached is +// what lets a record disagree with the host. +type Stage string + +const ( + StagePrepareHost Stage = "preparing-host" + StagePrepareRootFS Stage = "preparing-rootfs" + StageStartNode Stage = "starting-node" + StageInstallDaemon Stage = "installing-daemon" +) + +type Reporter interface { + StageStarted(context.Context, Stage) + StageFailed(context.Context, Stage, error) +} + +type Coordinator struct { + log *slog.Logger + store *installstate.Store + stages Stages + reporter Reporter +} + +func New(log *slog.Logger, store *installstate.Store, stages Stages, reporter Reporter) *Coordinator { + return &Coordinator{log: log, store: store, stages: stages, reporter: reporter} +} + +type Outcome struct{ AlreadyComplete bool } + +func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { + lock, err := c.store.AcquireLock() + if err != nil { + return Outcome{}, err + } + defer func() { + if err := lock.Release(); err != nil { + c.log.Error("release installation lock", "error", err) + } + }() + + r, disposition, err := installstate.Admit(c.store, id.MachineName, id.ConfigFingerprint) + if err != nil { + return Outcome{}, err + } + + if disposition == installstate.Fresh { + if err := c.stages.EnsureHostClean(ctx); err != nil { + return Outcome{}, err + } + + r, err = installstate.NewRecord(id.MachineName, id.ConfigFingerprint, id.HostPrefix) + if err != nil { + return Outcome{}, err + } + + if err := c.store.Save(r); err != nil { + return Outcome{}, err + } + } + + if disposition == installstate.AlreadyComplete { + verifyErr := c.stages.VerifyInstalled(ctx) + if verifyErr != nil { + if err := c.stages.RepairDaemon(ctx); err != nil { + return Outcome{}, fmt.Errorf("repair daemon after %w: %w", verifyErr, err) + } + + if err := c.stages.VerifyInstalled(ctx); err != nil { + return Outcome{}, err + } + + // Only a repair can have changed anything, so only a repair needs + // to be committed. The record already says complete: rewriting it + // on a healthy host would be a durable write for no change, on + // every boot of every Ignition-provisioned node, since that unit + // has no completion condition and runs each time. + if err := c.store.MarkComplete(r); err != nil { + return Outcome{}, err + } + } + + return Outcome{AlreadyComplete: true}, nil + } + + if err := c.stages.ResolveInputs(ctx); err != nil { + return Outcome{}, fmt.Errorf("resolve bootstrap inputs: %w", err) + } + + // Every stage runs, in order, on every attempt. Each one decides from the + // host what it still has to do: host preparation leaves a live nftables + // ruleset alone, the rootfs is left in place when a machine is registered + // from it, an already running machine is not restarted, and node services + // are restarted only when their configuration actually changed. + for _, stage := range []struct { + name Stage + run func(context.Context) error + }{ + {StagePrepareHost, c.stages.PrepareHost}, + {StagePrepareRootFS, c.stages.PrepareRootFS}, + {StageStartNode, c.stages.EnsureNodeStarted}, + {StageInstallDaemon, c.stages.EnsureDaemonInstalled}, + } { + if err := ctx.Err(); err != nil { + return Outcome{}, err + } + + if c.reporter != nil { + c.reporter.StageStarted(ctx, stage.name) + } + + if err := stage.run(ctx); err != nil { + if c.reporter != nil { + c.reporter.StageFailed(ctx, stage.name, err) + } + + return Outcome{}, fmt.Errorf("%s: %w", stage.name, err) + } + } + + if err := c.store.MarkComplete(r); err != nil { + return Outcome{}, err + } + + return Outcome{}, nil +} diff --git a/cmd/agent/internal/bootstrap/coordinator_test.go b/cmd/agent/internal/bootstrap/coordinator_test.go new file mode 100644 index 000000000..2708be64c --- /dev/null +++ b/cmd/agent/internal/bootstrap/coordinator_test.go @@ -0,0 +1,291 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package bootstrap + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/cmd/agent/internal/installstate" +) + +type fakeStages struct { + store *installstate.Store + calls []string + fail string + verifyErr error +} + +var ( + errInjected = errors.New("injected stage failure") + // errRepairFailed is distinct from errInjected so a test can tell whether + // the reported error is the fault that triggered a repair or the failure of + // the repair itself. + errRepairFailed = errors.New("injected repair failure") +) + +func (f *fakeStages) run(name string) error { + f.calls = append(f.calls, name) + if name != "clean" { + if _, err := f.store.Load(); err != nil { + return err + } + } + + if name == f.fail { + if name == "repair" { + return errRepairFailed + } + + return errInjected + } + + return nil +} +func (f *fakeStages) EnsureHostClean(context.Context) error { return f.run("clean") } +func (f *fakeStages) ResolveInputs(context.Context) error { return f.run("resolve") } +func (f *fakeStages) PrepareHost(context.Context) error { return f.run("host") } +func (f *fakeStages) PrepareRootFS(context.Context) error { return f.run("rootfs") } +func (f *fakeStages) EnsureNodeStarted(context.Context) error { return f.run("node") } +func (f *fakeStages) EnsureDaemonInstalled(context.Context) error { return f.run("daemon") } +func (f *fakeStages) RepairDaemon(context.Context) error { f.verifyErr = nil; return f.run("repair") } + +func (f *fakeStages) VerifyInstalled(context.Context) error { + if err := f.run("verify"); err != nil { + return err + } + + return f.verifyErr +} + +// TestEveryStageRunsOnEveryAttempt is the core of the reapply model. +// +// An earlier design recorded which stage had been reached and skipped anything +// before it. That made the record a claim about the host, and a host changed in +// between would be skipped past rather than repaired. Every stage now runs every +// time and decides from the host what it still has to do, so a retry after a +// failure at any point does the same thing: all of them, in order. +func TestEveryStageRunsOnEveryAttempt(t *testing.T) { + t.Parallel() + + all := []string{"resolve", "host", "rootfs", "node", "daemon"} + + for _, failAt := range []string{"host", "rootfs", "node", "daemon"} { + t.Run(failAt, func(t *testing.T) { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + stages := &fakeStages{store: store, fail: failAt} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + id := Identity{MachineName: "machine", ConfigFingerprint: "fingerprint"} + + _, err := c.Run(t.Context(), id) + require.ErrorIs(t, err, errInjected) + + record, err := store.Load() + require.NoError(t, err) + require.Equal(t, installstate.Installing, record.Phase, + "an unfinished installation records only that it is under way") + + stages.calls = nil + stages.fail = "" + + outcome, err := c.Run(t.Context(), id) + require.NoError(t, err) + require.False(t, outcome.AlreadyComplete) + require.Equal(t, all, stages.calls, "the retry reapplies every stage regardless of where it failed") + + complete, err := store.Load() + require.NoError(t, err) + require.Equal(t, record.InstallID, complete.InstallID, "the retry is the same installation") + require.Equal(t, installstate.Complete, complete.Phase) + }) + } +} + +func TestCompletedRecoveryDoesNotResolveRetiredBootstrapInputs(t *testing.T) { + t.Parallel() + + for _, repair := range []bool{false, true} { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + r, err := installstate.NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + + r.Phase = installstate.Complete + require.NoError(t, store.Save(r)) + + stages := &fakeStages{store: store, fail: "resolve"} + if repair { + stages.verifyErr = errInjected + } + + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + outcome, err := c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.NoError(t, err) + require.True(t, outcome.AlreadyComplete) + + want := []string{"verify"} + if repair { + want = append(want, "repair", "verify") + } + + require.Equal(t, want, stages.calls) + + complete, err := store.Load() + require.NoError(t, err) + require.Equal(t, installstate.Complete, complete.Phase) + } +} + +func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { + t.Parallel() + + for _, mode := range []string{"different-intent", "resetting", "locked"} { + t.Run(mode, func(t *testing.T) { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + r, err := installstate.NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + + if mode == "resetting" { + r.Phase = installstate.Resetting + } + + require.NoError(t, store.Save(r)) + + id := Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint} + if mode == "different-intent" { + id.ConfigFingerprint = "different" + } + + if mode == "locked" { + lock, err := store.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + } + + stages := &fakeStages{store: store} + _, err = New(slog.New(slog.DiscardHandler), store, stages, nil).Run(t.Context(), id) + require.Error(t, err) + require.Empty(t, stages.calls) + }) + } +} + +func TestInterruptedRepairRemainsCompleteAndRetries(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + stages := &fakeStages{store: store, fail: "repair", verifyErr: errInjected} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + id := Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint} + _, err = c.Run(t.Context(), id) + require.ErrorIs(t, err, errInjected) + loaded, err := store.Load() + require.NoError(t, err) + require.Equal(t, installstate.Complete, loaded.Phase) + + stages.fail = "" + stages.verifyErr = errInjected + stages.calls = nil + _, err = c.Run(t.Context(), id) + require.NoError(t, err) + require.Equal(t, []string{"verify", "repair", "verify"}, stages.calls) +} + +// recordInode identifies the record file itself rather than its contents. +// +// MarkComplete on an already-complete record writes the same bytes, so +// comparing content cannot tell a rewrite from a no-op. The store replaces the +// file atomically, so any write at all produces a new inode. +func recordInode(t *testing.T, store *installstate.Store) uint64 { + t.Helper() + + info, err := os.Stat(filepath.Join(store.Root(), "install-state.json")) + require.NoError(t, err) + + stat, ok := info.Sys().(*syscall.Stat_t) + require.True(t, ok, "inode is how this test distinguishes a rewrite from a no-op") + + return stat.Ino +} + +// TestHealthyCompletedInstallIsNotRewritten covers the cost of an Ignition unit +// that carries no completion condition. +// +// That unit runs on every boot and reaches this path each time. Rewriting the +// record when nothing changed would be a durable write per boot on every node, +// and a write is a chance to fail: a host that is entirely healthy would be +// taking one for no reason. +func TestHealthyCompletedInstallIsNotRewritten(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + before := recordInode(t, store) + + stages := &fakeStages{store: store} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + + outcome, err := c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.NoError(t, err) + require.True(t, outcome.AlreadyComplete) + require.Equal(t, []string{"verify"}, stages.calls, "a healthy host needs no repair") + + require.Equal(t, before, recordInode(t, store), + "nothing changed, so the record must not have been written at all") +} + +// TestRepairedInstallIsCommitted is the other half: when a repair did happen, +// the result has to be durable before the process exits. +func TestRepairedInstallIsCommitted(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + before := recordInode(t, store) + + stages := &fakeStages{store: store, verifyErr: errInjected} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + + _, err = c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.NoError(t, err) + require.Equal(t, []string{"verify", "repair", "verify"}, stages.calls) + + loaded, err := store.Load() + require.NoError(t, err) + require.Equal(t, installstate.Complete, loaded.Phase) + require.NotEqual(t, before, recordInode(t, store), + "a repair changed the host, so the result has to be made durable") +} + +// TestFailedRepairReportsWhatWasWrong pins that the original fault survives. +// +// The first verify says what is broken; the repair failure says only that +// fixing it did not work. Reporting the second alone sends an operator after +// the wrong thing. +func TestFailedRepairReportsWhatWasWrong(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + stages := &fakeStages{store: store, fail: "repair", verifyErr: errInjected} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + + _, err = c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.Error(t, err) + require.ErrorIs(t, err, errInjected, "the fault that triggered the repair must still be reported") + require.ErrorIs(t, err, errRepairFailed, "and so must the reason repairing it did not work") +} diff --git a/cmd/agent/internal/cmd/agentupgrade.go b/cmd/agent/internal/cmd/agentupgrade.go index 02ad6e766..fad1f5bb2 100644 --- a/cmd/agent/internal/cmd/agentupgrade.go +++ b/cmd/agent/internal/cmd/agentupgrade.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "github.com/Azure/unbounded/cmd/agent/internal/daemon" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -36,15 +37,24 @@ type hostAgentUpgradeHandler struct { resolvedPath func() (goalstates.AgentUpgradePaths, error) newService func(goalstates.AgentUpgradePaths) agentbinary.DaemonService geteuid func() int + installation *installstate.Store } func newCmdHostAgentUpgrade(cmdCtx *CommandContext) *cobra.Command { handler := &hostAgentUpgradeHandler{ - cmdCtx: cmdCtx, - writer: os.Stdout, - executable: os.Executable, - resolvedPath: goalstates.ResolvedAgentUpgradePaths, + cmdCtx: cmdCtx, + writer: os.Stdout, + executable: os.Executable, + // Wrapped rather than referenced directly so the prefix is read when + // the command runs, not when it is constructed. This runs on the host + // rather than under systemd, but the applied config is still the + // authority: the prefix belongs to the installation, not to whatever + // environment happens to be invoking the upgrade. + resolvedPath: func() (goalstates.AgentUpgradePaths, error) { + return goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) + }, geteuid: os.Geteuid, + installation: installstate.DefaultStore(), } handler.newService = func(paths goalstates.AgentUpgradePaths) agentbinary.DaemonService { return daemon.NewHostDaemonActivationService(handler.cmdCtx.Logger, paths) @@ -115,6 +125,17 @@ func (h *hostAgentUpgradeHandler) execute(ctx context.Context) error { return fmt.Errorf("host agent upgrade requires root privileges") } + lock, err := h.installation.AcquireMutationLock() + if err != nil { + return err + } + + defer func() { + if err := lock.Release(); err != nil { + h.cmdCtx.Logger.Error("release installation lock", "error", err) + } + }() + result, err := agentbinary.ActivateHostDaemon(ctx, h.cmdCtx.Logger, options, service) if err != nil { return err diff --git a/cmd/agent/internal/cmd/agentupgrade_test.go b/cmd/agent/internal/cmd/agentupgrade_test.go index 702804c1b..8545f3f1e 100644 --- a/cmd/agent/internal/cmd/agentupgrade_test.go +++ b/cmd/agent/internal/cmd/agentupgrade_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -82,6 +83,23 @@ func TestWriteHostAgentUpgradePlanOmitsUnchangedLastGood(t *testing.T) { assert.NotContains(t, output.String(), "Last-good link:") } +func TestHostAgentUpgradeTakesInstallationLockBeforeActivation(t *testing.T) { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + lock, err := store.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + + handler := &hostAgentUpgradeHandler{ + cmdCtx: &CommandContext{LogFormat: "text"}, installation: store, + executable: func() (string, error) { return filepath.Join(dir, "candidate"), nil }, + resolvedPath: func() (goalstates.AgentUpgradePaths, error) { return goalstates.AgentUpgradePaths{}, nil }, + newService: func(goalstates.AgentUpgradePaths) agentbinary.DaemonService { return preflightOnlyDaemonService{} }, + geteuid: func() int { return 0 }, + } + require.ErrorIs(t, handler.execute(t.Context()), installstate.ErrLockHeld) +} + func TestRecordAgentUpgradeFailureSignalCommand(t *testing.T) { dir := t.TempDir() signalPath := filepath.Join(dir, "agent-upgrade-signal") diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go new file mode 100644 index 000000000..deb7e1b41 --- /dev/null +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -0,0 +1,303 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "context" + "encoding/json" + "log/slog" + "net/url" + "strings" + + "github.com/Azure/unbounded/cmd/agent/internal/attest" + "github.com/Azure/unbounded/cmd/agent/internal/bootstrap" + "github.com/Azure/unbounded/cmd/agent/internal/daemon" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" + "github.com/Azure/unbounded/internal/fsutil" + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/phases" + "github.com/Azure/unbounded/pkg/agent/phases/host" + "github.com/Azure/unbounded/pkg/agent/phases/nodestart" + "github.com/Azure/unbounded/pkg/agent/phases/reset" + "github.com/Azure/unbounded/pkg/agent/phases/rootfs" +) + +type agentStages struct { + log *slog.Logger + cfg *provision.UnboundedAgentConfig + gs *goalstates.MachineGoalState + archives *goalstates.ContainerImageArchiveStaging + reporter *daemon.BootstrapStatusReporter + credentialsReady bool +} + +// canonicalImageIdentity reduces an OCI image reference to the part that +// determines which image gets installed. HTTPS archive references carry +// expiring signed query parameters, so a refreshed signature points at the same +// artifact and must not read as a different installation. Registry and +// oci-layout references carry no such credentials and are used as-is. +// +// Trailing path slashes are trimmed to match how parseHTTPSArchiveReference +// normalizes the reference before fetching it. +func canonicalImageIdentity(image string) string { + if !strings.HasPrefix(image, "https://") { + return image + } + + parsed, err := url.Parse(image) + if err != nil { + // Unparseable references fail later at acquire time with a better + // message. Hash the original so identity stays deterministic. + return image + } + + parsed.RawQuery = "" + parsed.ForceQuery = false + parsed.Path = strings.TrimRight(parsed.Path, "/") + parsed.RawPath = strings.TrimRight(parsed.RawPath, "/") + + return parsed.String() +} + +func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity, error) { + // Keep identity tied to the cluster and installed rootfs, while allowing + // credentials and artifact locations to be refreshed for a retry. + // + // HostPrefix enters the hash only when it resolves somewhere other than the + // default, and carries omitempty so that at the default it contributes + // nothing at all. Every host already in the field was fingerprinted without + // this input; if the default hashed as a value, each of them would read as a + // different installation and demand an explicit reset on upgrade, for a + // field they never set. TestBootstrapV1CompatibilityFixtures catches that. + // + // It is the resolved prefix that matters, not how it was written. Leaving it + // unset and naming /usr/local explicitly put the files in the same place, so + // they are the same installation and must hash alike. + // + // A prefix that resolves elsewhere does belong in the identity. The agent's + // own files live under it, so starting with a different one is not a retry: + // it would leave the first installation behind and build a second one + // beside it. + resolvedPrefix := goalstates.HostPrefixOrDefault(cfg.HostPrefix) + + fingerprintedPrefix := resolvedPrefix + if fingerprintedPrefix == goalstates.DefaultHostPrefix { + fingerprintedPrefix = "" + } + + data, err := json.Marshal(struct { + KubernetesVersion string + OCIImage string + APIServer string + HostPrefix string `json:",omitempty"` + }{ + strings.TrimPrefix(cfg.Cluster.Version, "v"), + canonicalImageIdentity(cfg.OCIImage), + cfg.Kubelet.ApiServer, + fingerprintedPrefix, + }) + if err != nil { + return bootstrap.Identity{}, err + } + + return bootstrap.Identity{ + MachineName: cfg.MachineName, + ConfigFingerprint: installstate.Fingerprint(data), + // Resolved rather than configured, so the record names a real directory + // instead of an empty string meaning "wherever the default was at the + // time", which is what teardown would have to guess from. + HostPrefix: resolvedPrefix, + }, nil +} + +func (s *agentStages) EnsureHostClean(ctx context.Context) error { + return host.EnsureNoExistingDeployment(ctx, s.log) +} + +func (s *agentStages) ResolveInputs(ctx context.Context) error { + downloads, archives, err := provision.ResolveDownloadOverridesWithOfflineArtifacts(ctx, s.cfg) + if err != nil { + return err + } + + s.archives = archives + + s.gs, err = goalstates.ResolveMachine(s.log, &s.cfg.AgentConfig, goalstates.NSpawnMachineKube1, downloads) + if err != nil { + return err + } + + return nil +} + +func (s *agentStages) PrepareHost(ctx context.Context) error { + if err := daemon.InstallBootstrapBinary(); err != nil { + return err + } + + if err := phases.Serial(s.log, host.InstallPackages(s.log), phases.Parallel(s.log, + host.ConfigureOS(s.log), host.ConfigureNFTables(s.log), phases.Serial(s.log, host.DisableDocker(s.log), host.ConfigureDocker(s.log)), + host.DisableContainerd(s.log), host.DisableKubelet(s.log), host.DisableSwap(s.log), host.HardenAPT(s.log))).Do(ctx); err != nil { + return err + } + + return fsutil.SyncFilesystems("/etc", "/usr/local", installstate.DefaultDirectory) +} + +// Credentials must be resolved on every unfinished attempt, but TPM prerequisites +// must first be installed on a fresh host. This stage always precedes node work. +func (s *agentStages) prepareCredentials(ctx context.Context) error { + if s.credentialsReady { + return nil + } + + if err := attest.ApplyAttestation(s.log, s.cfg.Attest, s.cfg.MachineName, s.gs.NodeStart).Do(ctx); err != nil { + return err + } + + syncAttestedKubeletConfig(&s.cfg.AgentConfig, s.gs.NodeStart) + + // The reporter is built here rather than in the constructor because it + // captures credentials at construction: an empty bootstrap token makes it a + // permanent no-op, and it registers the Machine over the API. On an attested + // host the token does not exist until ApplyAttestation has run just above, + // so constructing it earlier would silently disable status reporting for the + // whole bootstrap and issue the registration call before admission. + if s.reporter == nil { + s.reporter = daemon.NewBootstrapStatusReporter(ctx, s.log, &s.cfg.AgentConfig) + s.reporter.Running(ctx) + } + + s.credentialsReady = true + + return nil +} + +func (s *agentStages) PrepareRootFS(ctx context.Context) error { + // ProvisionOwned rebuilds the rootfs in place and must never be pointed at + // a slot that has started a node, which would pull the filesystem out from + // under a running one. A registered machine means the rootfs this stage + // would build is already built and in use, so the requirement is met and + // there is nothing to do. + // + // This is a property of the host, not of how far a previous attempt got. It + // holds whether the machine was started by an earlier attempt of this + // installation or independently afterwards. + // + // Asked of the slot this bootstrap manages rather than of either slot. + // Bootstrap only ever builds gs.NodeStart.MachineName, so a machine in the + // other slot says nothing about whether this one needs a rootfs. + registered, err := reset.RegisteredMachine(ctx, s.log, s.gs.NodeStart.MachineName) + if err != nil { + return err + } + + if registered { + s.log.Info("nspawn machine is registered; leaving its rootfs in place", "machine", s.gs.NodeStart.MachineName) + + return nil + } + + if err := s.prepareCredentials(ctx); err != nil { + return err + } + + if err := phases.Serial(s.log, rootfs.DownloadContainerImageArchives(s.log, s.archives), rootfs.ProvisionOwned(s.log, s.gs.RootFS)).Do(ctx); err != nil { + return err + } + + return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, "/usr/local", goalstates.SystemdSystemDir, goalstates.SystemdNSpawnDir) +} + +// nodeStartTask composes the work that brings the node up. +// +// The applied config records what the running node was built from, and the +// daemon compares it against the desired config to decide whether the node has +// drifted far enough to need a repave. Only an attempt that actually built the +// node may write it. nodeAlreadyBuilt says a machine was already registered +// when this stage began, so this attempt found the node standing rather than +// raising it. +// +// Writing it anyway would be a claim the host cannot support. Node labels reach +// a node through kubelet's --node-labels at registration; restarting kubelet +// under an already-registered node does not revise them. Recording a label the +// node never took would leave the applied config matching the desired config, +// which reads as no drift, which is precisely what suppresses the repave that +// would have delivered it. Leaving the record alone keeps the difference +// visible and lets the daemon resolve it. +func (s *agentStages) nodeStartTask(nodeAlreadyBuilt bool) phases.Task { + tasks := []phases.Task{ + nodestart.StartNode(s.log, s.gs.NodeStart), + nodestart.WaitForKubeletBootstrap(s.log, s.gs.NodeStart.MachineName), + } + + if !nodeAlreadyBuilt { + tasks = append(tasks, daemon.PersistAppliedConfig(s.log, s.gs.NodeStart.MachineName, &s.cfg.AgentConfig)) + } + + return phases.Serial(s.log, tasks...) +} + +func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { + if err := s.prepareCredentials(ctx); err != nil { + return err + } + + // Asked before the stage runs, because afterwards every answer is yes, and + // asked of the slot this bootstrap manages: a machine in the other slot was + // not built by this attempt either, but it is not the node being started. + registered, err := reset.RegisteredMachine(ctx, s.log, s.gs.NodeStart.MachineName) + if err != nil { + return err + } + + if err := s.nodeStartTask(registered).Do(ctx); err != nil { + return err + } + + // AgentConfigDir holds the applied config written above, so it must reach + // disk before this stage reports success. + return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, goalstates.AgentConfigDir, goalstates.SystemdSystemDir) +} + +func (s *agentStages) daemonInstallTask() phases.Task { + return phases.Serial(s.log, daemon.EnableDaemon(s.log)) +} + +func (s *agentStages) EnsureDaemonInstalled(ctx context.Context) error { + if err := s.prepareCredentials(ctx); err != nil { + return err + } + + if err := s.daemonInstallTask().Do(ctx); err != nil { + return err + } + + return fsutil.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) +} + +func (s *agentStages) VerifyInstalled(ctx context.Context) error { + return daemon.VerifyDaemonInstalled(ctx, s.log) +} + +func (s *agentStages) RepairDaemon(ctx context.Context) error { return daemon.RepairDaemon(ctx, s.log) } + +func (s *agentStages) StageStarted(_ context.Context, stage bootstrap.Stage) { + s.log.Info("bootstrap stage", "stage", stage) +} + +func (s *agentStages) StageFailed(ctx context.Context, stage bootstrap.Stage, err error) { + reason := "Failed" + if stage == bootstrap.StagePrepareRootFS { + reason = "RootFSFailed" + } + + if stage == bootstrap.StageStartNode { + reason = classifyNodeStartFailure(err) + } + + // Safe before the reporter exists: it reports through a nil-receiver check. + s.reporter.Failed(ctx, reason, err) +} diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go new file mode 100644 index 000000000..41cfb0738 --- /dev/null +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -0,0 +1,323 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/cmd/agent/internal/installstate" + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +// TestBootstrapV1CompatibilityFixtures pins the on-disk ownership format at +// schema version 1. The fixtures hold records written by this package's own +// config loader, normalizer, fingerprint and installstate.Store, from the +// synthetic input.json alongside them; only the random install ID is fixed. +// +// A later release must still admit an installation created by an earlier one +// when given the same original input, so a mismatch here is a compatibility +// break rather than a fixture to refresh. Record.Validate pins these files to +// the package's schema version, so bumping it fails loudly; a new schema version +// gets its own fixture directory rather than regenerated files. +func TestBootstrapV1CompatibilityFixtures(t *testing.T) { + dir := filepath.Join("testdata", "bootstrap-v1") + cfg, err := loadConfigFromFile(filepath.Join(dir, "input.json")) + require.NoError(t, err) + require.NoError(t, normalizeConfig(slog.New(slog.DiscardHandler), cfg)) + require.NoError(t, cfg.Validate()) + id, err := bootstrapIdentity(cfg) + require.NoError(t, err) + + for _, phase := range []installstate.Phase{installstate.Installing, installstate.Complete, installstate.Resetting} { + data, err := os.ReadFile(filepath.Join(dir, string(phase)+".json")) + require.NoError(t, err) + + var record installstate.Record + require.NoError(t, json.Unmarshal(data, &record)) + require.NoError(t, record.Validate()) + require.Equal(t, id.MachineName, record.MachineName) + require.Equal(t, id.ConfigFingerprint, record.ConfigFingerprint) + + // Admit through a store so the fixture also proves it survives a + // load round-trip, not just an in-memory classification. + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + require.NoError(t, store.Save(record)) + + loaded, disposition, err := installstate.Admit(store, id.MachineName, id.ConfigFingerprint) + if phase == installstate.Resetting { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, record, loaded) + + want := installstate.Resume + if phase == installstate.Complete { + want = installstate.AlreadyComplete + } + + require.Equal(t, want, disposition) + } + } + + cfg.OCIImage = "example.test/unbounded/node:changed" + changed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.NotEqual(t, id.ConfigFingerprint, changed.ConfigFingerprint) +} + +func TestBootstrapFingerprintAllowsCredentialAndDownloadRefresh(t *testing.T) { + cfg, err := loadConfigFromFile(filepath.Join("testdata", "bootstrap-v1", "input.json")) + require.NoError(t, err) + original, err := bootstrapIdentity(cfg) + require.NoError(t, err) + + cfg.Kubelet.Auth.BootstrapToken = "rotated-token" + cfg.Cluster.CaCertBase64 = "rotated-ca" + cfg.Downloads = &provision.AgentDownloads{Kubernetes: &provision.AgentDownloadSource{BaseURL: "https://mirror.example.test"}} + refreshed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.Equal(t, original, refreshed) + + for _, change := range []func(){ + func() { cfg.Cluster.Version = "1.35.0" }, + func() { cfg.OCIImage = "other-image" }, + func() { cfg.Kubelet.ApiServer = "https://other-cluster" }, + } { + version, image, endpoint := cfg.Cluster.Version, cfg.OCIImage, cfg.Kubelet.ApiServer + + change() + + changed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.NotEqual(t, original.ConfigFingerprint, changed.ConfigFingerprint) + + cfg.Cluster.Version, cfg.OCIImage, cfg.Kubelet.ApiServer = version, image, endpoint + } +} + +func TestBootstrapFingerprintIgnoresSignedImageQuery(t *testing.T) { + t.Parallel() + + // A signed archive URL carries an expiring signature. Refreshing it points + // at the same artifact, so a retry must stay the same installation rather + // than being rejected as a different one. + const base = "https://artifacts.example.test/node/rootfs.oci.tar.gz" + + cfg, err := loadConfigFromFile(filepath.Join("testdata", "bootstrap-v1", "input.json")) + require.NoError(t, err) + + cfg.OCIImage = base + "?sp=r&sv=2022-11-02&sig=first-signature" + original, err := bootstrapIdentity(cfg) + require.NoError(t, err) + + for _, equivalent := range []string{ + base + "?sp=r&sv=2022-11-02&sig=second-signature", + base + "?", + base + "/", + base, + } { + cfg.OCIImage = equivalent + + refreshed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.Equal(t, original.ConfigFingerprint, refreshed.ConfigFingerprint, "reference %q", equivalent) + } + + // The path and host still identify the artifact, so they must not be + // collapsed away with the credential. + for _, different := range []string{ + "https://artifacts.example.test/node/other-rootfs.oci.tar.gz", + "https://other-host.example.test/node/rootfs.oci.tar.gz", + } { + cfg.OCIImage = different + + changed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.NotEqual(t, original.ConfigFingerprint, changed.ConfigFingerprint, "reference %q", different) + } +} + +func TestCanonicalImageIdentityLeavesNonHTTPSReferencesAlone(t *testing.T) { + t.Parallel() + + for _, image := range []string{ + "example.test/unbounded/node:v1.33.1", + "example.test/unbounded/node@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "oci-layout:///var/lib/unbounded/layouts/node", + "", + } { + require.Equal(t, image, canonicalImageIdentity(image)) + } + + // An unparseable HTTPS reference is rejected later with a better message. + // Identity just has to stay deterministic rather than panic. + const malformed = "https://artifacts.example.test/\x7f" + require.Equal(t, malformed, canonicalImageIdentity(malformed)) +} + +// TestNodeStartPersistsAppliedConfig pins where the applied config is written. +// It has to happen in the stage that starts the node, and after kubelet has +// bootstrapped, so the record always describes the configuration the running +// node was built from. Moving it into the daemon stage would record a +// configuration the node never saw, which then reads as "no drift" and is +// never reconciled. +func TestNodeStartPersistsAppliedConfig(t *testing.T) { + t.Parallel() + + stages := newNodeStartStages() + + nodeStart := stages.nodeStartTask(false).Name() + require.Contains(t, nodeStart, "persist-applied-config") + require.Less(t, strings.Index(nodeStart, "wait-for-kubelet-bootstrap"), strings.Index(nodeStart, "persist-applied-config")) + + require.NotContains(t, stages.daemonInstallTask().Name(), "persist-applied-config") +} + +// TestNodeStartSkipsAppliedConfigWhenNodeAlreadyBuilt is the other half of the +// rule above, and the one a retry depends on. Every stage reapplies on every +// attempt, so the node stage runs again even when the node is already up. The +// attempt that finds a machine standing did not build it, and must not restate +// what it was built from. +// +// The case that makes this matter is a changed node label. Labels are outside +// the installation fingerprint, so a retry carrying a new one is admitted, but +// kubelet only takes --node-labels at registration and a restart under an +// existing node does not revise them. Writing the new label here would make the +// applied config match the desired config, which reads as no drift, which +// suppresses the repave that is the only thing that would deliver the label. +// The node would silently never get it. +func TestNodeStartSkipsAppliedConfigWhenNodeAlreadyBuilt(t *testing.T) { + t.Parallel() + + nodeStart := newNodeStartStages().nodeStartTask(true).Name() + + require.NotContains(t, nodeStart, "persist-applied-config") + require.Contains(t, nodeStart, "wait-for-kubelet-bootstrap") +} + +func newNodeStartStages() *agentStages { + return &agentStages{ + log: slog.New(slog.DiscardHandler), + cfg: &provision.UnboundedAgentConfig{}, + gs: &goalstates.MachineGoalState{ + NodeStart: &goalstates.NodeStart{MachineName: goalstates.NSpawnMachineKube1}, + }, + } +} + +func TestCompletedPreflightOutput(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + h := &preflightHandler{writer: &out, output: "json"} + require.NoError(t, h.writeReport(preflight.Report{})) + require.True(t, json.Valid(out.Bytes())) + + h.output = "unsupported" + require.Error(t, h.writeReport(preflight.Report{})) +} + +// TestClassifyNodeStartFailure pins the Machine condition reasons for a node +// that fails to come up. +// +// wait-for-kubelet-bootstrap has to map to KubeletBootstrapFailed alongside +// start-kubelet. It is its own task inside the node-start stage, and a failure +// there is the most common real one: a rejected or expired bootstrap token, an +// unreachable API server, a CA mismatch. Reporting it as a generic failure +// tells an operator nothing about where to look. +func TestClassifyNodeStartFailure(t *testing.T) { + t.Parallel() + + for _, tc := range []struct{ task, want string }{ + {"start-kubelet", "KubeletBootstrapFailed"}, + {"wait-for-kubelet-bootstrap", "KubeletBootstrapFailed"}, + {"start-nspawn-machine", "NSpawnFailed"}, + {"import-container-images", "Failed"}, + } { + t.Run(tc.task, func(t *testing.T) { + t.Parallel() + + err := fmt.Errorf("%s: %w", tc.task, errors.New("boom")) + require.Equal(t, tc.want, classifyNodeStartFailure(err)) + }) + } +} + +// TestBootstrapFingerprintTracksTheInstallationPrefix covers both halves of how +// the prefix enters installation identity, because the two pull in opposite +// directions. +// +// Configuring a prefix has to change the fingerprint. The agent's own binaries +// live under it, so a start with a different prefix is not a retry of the same +// installation: continuing would leave the first installation's files behind +// and build a second one beside them. Admission must refuse and ask for a +// reset, which is what a changed fingerprint does. +// +// Configuring nothing has to change nothing. Every host already in the field +// was fingerprinted without this input, and if the default hashed differently +// each of them would read as a different installation and demand an explicit +// reset on upgrade, for a field they never set. +func TestBootstrapFingerprintTracksTheInstallationPrefix(t *testing.T) { + load := func(t *testing.T) *provision.UnboundedAgentConfig { + t.Helper() + + cfg, err := loadConfigFromFile(filepath.Join("testdata", "bootstrap-v1", "input.json")) + require.NoError(t, err) + + return cfg + } + + baseline, err := bootstrapIdentity(load(t)) + require.NoError(t, err) + + // Whitespace is not a configuration choice, so it must not be one here + // either; otherwise a stray space rewrites the identity of a default host. + for _, blank := range []string{"", " ", "\t"} { + cfg := load(t) + cfg.HostPrefix = blank + + unset, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.Equal(t, baseline.ConfigFingerprint, unset.ConfigFingerprint, + "an unset prefix must hash as it did before the field existed, got %q", blank) + } + + // Naming the default explicitly puts the files in the same place as leaving + // it unset, so the two are the same installation. Hashing them differently + // would tell an operator who wrote down what was already true that they + // must reset the host. + explicit := load(t) + explicit.HostPrefix = goalstates.DefaultHostPrefix + + explicitID, err := bootstrapIdentity(explicit) + require.NoError(t, err) + require.Equal(t, baseline.ConfigFingerprint, explicitID.ConfigFingerprint, + "identity follows where the files land, not how the prefix was spelled") + + moved := load(t) + moved.HostPrefix = "/opt/unbounded" + + movedID, err := bootstrapIdentity(moved) + require.NoError(t, err) + require.NotEqual(t, baseline.ConfigFingerprint, movedID.ConfigFingerprint, + "moving the installation prefix must not read as a retry of the same installation") + require.Equal(t, "/opt/unbounded", movedID.HostPrefix) + + // The record needs a real directory, not an empty string standing for + // whatever the default was when it was written. + require.Equal(t, goalstates.DefaultHostPrefix, baseline.HostPrefix) +} diff --git a/cmd/agent/internal/cmd/cmd.go b/cmd/agent/internal/cmd/cmd.go index 40300728a..18f444fa1 100644 --- a/cmd/agent/internal/cmd/cmd.go +++ b/cmd/agent/internal/cmd/cmd.go @@ -4,10 +4,13 @@ package cmd import ( + "errors" "fmt" "os" "github.com/spf13/cobra" + + "github.com/Azure/unbounded/cmd/agent/internal/daemon" ) func Run() { @@ -36,6 +39,14 @@ func Run() { ) if err := root.Execute(); err != nil { + // The daemon standing down for an unfinished installation is an + // ordinary state, not a fault. It has already said so in the journal, + // and the unit treats this code as success so systemd leaves it alone + // rather than restarting it into its start limit. + if errors.Is(err, daemon.ErrDeferred) { + os.Exit(daemon.DeferredExitCode) + } + fmt.Printf("error: %v\n", err) os.Exit(1) } diff --git a/cmd/agent/internal/cmd/daemon.go b/cmd/agent/internal/cmd/daemon.go index 0f5f6679a..29ab555be 100644 --- a/cmd/agent/internal/cmd/daemon.go +++ b/cmd/agent/internal/cmd/daemon.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "os" "os/signal" @@ -18,15 +19,32 @@ func newCmdDaemon(cmdCtx *CommandContext) *cobra.Command { Short: "Long-running daemon for node lifecycle management", Long: "Long-running daemon that manages the nspawn machine lifecycle. " + "Runs as a systemd unit after initial provisioning.", + // systemd runs this with fixed arguments, so a failure here is never a + // usage problem and the flag listing is noise in the journal. + SilenceUsage: true, RunE: func(cmd *cobra.Command, _ []string) error { ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) defer cancel() cmdCtx.Setup() - return daemon.Run(ctx, cmdCtx.Logger) + return quietWhenDeferred(cmd, daemon.Run(ctx, cmdCtx.Logger)) }, } return cmd } + +// quietWhenDeferred stops cobra reporting a deferred daemon as an error. +// +// Standing down because an installation owns the host is an ordinary state +// that the daemon has already recorded as a warning. Letting cobra print +// "Error:" over the top of that puts a fault in the journal where there is +// none, which is exactly the confusion this whole path exists to remove. +func quietWhenDeferred(cmd *cobra.Command, err error) error { + if errors.Is(err, daemon.ErrDeferred) { + cmd.SilenceErrors = true + } + + return err +} diff --git a/cmd/agent/internal/cmd/daemon_test.go b/cmd/agent/internal/cmd/daemon_test.go new file mode 100644 index 000000000..d40791b87 --- /dev/null +++ b/cmd/agent/internal/cmd/daemon_test.go @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/cmd/agent/internal/daemon" +) + +// TestDaemonCommandKeepsDeferredOutOfTheJournalAsAnError covers what an +// operator actually sees when the daemon stands down for an unfinished +// installation. +// +// The daemon already reports it as a warning. Cobra would then print "Error:" +// over the top of that, and without SilenceUsage the whole flag listing too, so +// an ordinary state reads as both a fault and a misuse of the command. The +// journal is the one place this behavior is observed, so narrating it wrongly +// there undoes the fix where it counts. +func TestDaemonCommandKeepsDeferredOutOfTheJournalAsAnError(t *testing.T) { + t.Parallel() + + cmd := newCmdDaemon(&CommandContext{LogFormat: "text"}) + require.True(t, cmd.SilenceUsage, "systemd passes fixed arguments; a runtime failure is never a usage problem") + + // Run wraps the sentinel with context before it reaches here, which is how + // it appears in the journal, so recognition has to survive wrapping. + wrapped := fmt.Errorf("find active machine: %w", daemon.ErrDeferred) + + require.ErrorIs(t, quietWhenDeferred(cmd, wrapped), daemon.ErrDeferred) + require.True(t, cmd.SilenceErrors, "a deferred daemon must not be narrated as an error") +} + +// TestDaemonCommandStillReportsRealFailures keeps the suppression narrow. A +// daemon that cannot reach the API server has genuinely failed, and silencing +// that would hide the fault this path exists to distinguish from. +func TestDaemonCommandStillReportsRealFailures(t *testing.T) { + t.Parallel() + + cmd := &cobra.Command{} + + real := errors.New("kube client unreachable") + require.ErrorIs(t, quietWhenDeferred(cmd, real), real) + require.False(t, cmd.SilenceErrors, "genuine failures must still be reported") + + require.NoError(t, quietWhenDeferred(cmd, nil)) + require.False(t, cmd.SilenceErrors) +} diff --git a/cmd/agent/internal/cmd/preflight.go b/cmd/agent/internal/cmd/preflight.go index b5d1e6e20..33b8d174c 100644 --- a/cmd/agent/internal/cmd/preflight.go +++ b/cmd/agent/internal/cmd/preflight.go @@ -13,6 +13,7 @@ import ( "github.com/spf13/cobra" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/phases/host" @@ -77,6 +78,22 @@ func (h *preflightHandler) execute(ctx context.Context) error { return fmt.Errorf("validate agent config: %w", err) } + id, err := bootstrapIdentity(cfg) + if err != nil { + return err + } + + _, disposition, err := installstate.Admit(installstate.DefaultStore(), id.MachineName, id.ConfigFingerprint) + if err != nil { + return err + } + + if disposition == installstate.AlreadyComplete { + // Admission is non-mutating. start rechecks ownership under lock before + // verifying or repairing daemon assets, without the original artifacts. + return h.writeReport(preflight.Report{}) + } + downloads, _, err := provision.ResolveDownloadOverridesWithOfflineArtifacts(ctx, cfg) if err != nil { return fmt.Errorf("resolve download overrides: %w", err) @@ -98,12 +115,30 @@ func (h *preflightHandler) execute(ctx context.Context) error { rootfs.Preflight(logger, cfg.AgentConfig, goalState), ) + if disposition == installstate.Resume { + // A resumed installation owns the artifacts a clean host must not have. + // Bind addresses remain checked, and accept only owned listeners. + var filtered []preflight.Checker + + for _, check := range checks { + if check.Name() != host.CheckExistingDeploymentName { + filtered = append(filtered, check) + } + } + + checks = filtered + } + opts := preflight.Options{ IgnoreErrors: h.ignorePreflightErrors, FailOnWarnings: h.failOnWarnings, } report := preflight.Run(ctx, checks, opts) + return h.writeReport(report) +} + +func (h *preflightHandler) writeReport(report preflight.Report) error { switch strings.ToLower(h.output) { case "", "text": if err := writePreflightText(h.writer, report); err != nil { diff --git a/cmd/agent/internal/cmd/reset.go b/cmd/agent/internal/cmd/reset.go index 6398ddee9..6b6cebef7 100644 --- a/cmd/agent/internal/cmd/reset.go +++ b/cmd/agent/internal/cmd/reset.go @@ -45,12 +45,5 @@ Both possible nspawn machine names (kube1 and kube2) are stopped and removed.`, // resetAgent returns a task that resets the host by stopping the daemon and // removing the unbounded-agent and all associated resources. func resetAgent(log *slog.Logger) phases.Task { - return phases.Serial(log, - // CLI reset runs outside the daemon, so it can stop the daemon first to - // keep it from reconciling while files are removed. The daemon operation - // path stops the daemon last because stopping the unit terminates the - // reconciler before it can mark the MachineOperation complete. - daemon.StopDaemon(log), - daemon.ResetAgentResources(log), - ) + return daemon.ResetAgent(log) } diff --git a/cmd/agent/internal/cmd/start.go b/cmd/agent/internal/cmd/start.go index 61b9192af..e28ee8fcd 100644 --- a/cmd/agent/internal/cmd/start.go +++ b/cmd/agent/internal/cmd/start.go @@ -4,24 +4,18 @@ package cmd import ( - "context" "encoding/base64" - "log/slog" "os" "os/signal" "strings" "github.com/spf13/cobra" - "github.com/Azure/unbounded/cmd/agent/internal/attest" - "github.com/Azure/unbounded/cmd/agent/internal/daemon" + "github.com/Azure/unbounded/cmd/agent/internal/bootstrap" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/internal/version" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/phases" - "github.com/Azure/unbounded/pkg/agent/phases/host" - "github.com/Azure/unbounded/pkg/agent/phases/nodestart" - "github.com/Azure/unbounded/pkg/agent/phases/rootfs" ) func newCmdStart(cmdCtx *CommandContext) *cobra.Command { @@ -47,81 +41,29 @@ func newCmdStart(cmdCtx *CommandContext) *cobra.Command { log := cmdCtx.Logger - downloads, containerImageArchives, err := provision.ResolveDownloadOverridesWithOfflineArtifacts(ctx, cfg) - if err != nil { + if err := cfg.Validate(); err != nil { return err } - gs, err := goalstates.ResolveMachine(log, &cfg.AgentConfig, goalstates.NSpawnMachineKube1, downloads) + id, err := bootstrapIdentity(cfg) if err != nil { return err } - rootFSGoalState := gs.RootFS - nodeStartGoalState := gs.NodeStart - - if err := host.EnsureNoExistingDeployment(ctx, log); err != nil { - return err - } - - // Run host setup and attestation first. Metalman bootstrap tokens are - // only available after attestation, so Machine status reporting starts - // after this block. - preBootstrapTasks := []phases.Task{ - // Phase 1: host - host.InstallPackages(log), - phases.Parallel(log, - host.ConfigureOS(log), - host.ConfigureNFTables(log), - phases.Serial(log, host.DisableDocker(log), host.ConfigureDocker(log)), - host.DisableContainerd(log), - host.DisableKubelet(log), - host.DisableSwap(log), - host.HardenAPT(log), - ), - - // TPM Attestation (no-op when not configured). - attest.ApplyAttestation(log, cfg.Attest, cfg.MachineName, nodeStartGoalState), - - // Stage offline container image archives before status reporting starts. - rootfs.DownloadContainerImageArchives(log, containerImageArchives), - } - - if err := phases.Serial(log, preBootstrapTasks...).Do(ctx); err != nil { - return err - } - - syncAttestedKubeletConfig(&cfg.AgentConfig, nodeStartGoalState) - - reporter := daemon.NewBootstrapStatusReporter(ctx, log, &cfg.AgentConfig) - reporter.Running(ctx) - - if err := runBootstrapTask(ctx, log, reporter, "RootFSFailed", rootfs.Provision(log, rootFSGoalState)); err != nil { - return err - } - - if err := phases.ExecuteTask(ctx, log, nodestart.StartNode(log, nodeStartGoalState)); err != nil { - reporter.Failed(ctx, classifyNodeStartFailure(err), err) - return err - } + stages := &agentStages{log: log, cfg: cfg} - if err := runBootstrapTask(ctx, log, reporter, "KubeletBootstrapFailed", nodestart.WaitForKubeletBootstrap(log, nodeStartGoalState.MachineName)); err != nil { + outcome, err := bootstrap.New(log, installstate.DefaultStore(), stages, stages).Run(ctx, id) + if err != nil { return err } - if err := phases.Serial(log, - // Phase 4: Persist the applied config for drift detection. - daemon.PersistAppliedConfig(log, nodeStartGoalState.MachineName, &cfg.AgentConfig), - - // Phase 5: Enable and start the daemon that watches the - // Machine CR for drift detection and reconciliation. - daemon.EnableDaemon(log), - ).Do(ctx); err != nil { - reporter.Failed(ctx, "Failed", err) - return err + if outcome.AlreadyComplete { + log.Info("installation already complete") } - reporter.Succeeded(ctx) + // Safe when bootstrap never reached credential setup: the reporter + // reports through a nil-receiver check. + stages.reporter.Succeeded(ctx) return nil }, @@ -140,19 +82,19 @@ func syncAttestedKubeletConfig(cfg *provision.AgentConfig, nodeStart *goalstates } } -func runBootstrapTask(ctx context.Context, log *slog.Logger, reporter *daemon.BootstrapStatusReporter, reason string, task phases.Task) error { - if err := phases.ExecuteTask(ctx, log, task); err != nil { - reporter.Failed(ctx, reason, err) - return err - } - - return nil -} - +// classifyNodeStartFailure maps a node-start failure onto the Machine condition +// reason, by the name of the task that reported it. +// +// wait-for-kubelet-bootstrap is matched as well as start-kubelet because both +// mean the node did not join. That wait is its own task inside this stage, and +// a failure there is the most common real one: a rejected or expired token, an +// unreachable API server, a CA mismatch. Reporting it as a generic failure +// would tell an operator nothing. func classifyNodeStartFailure(err error) string { message := err.Error() switch { - case strings.Contains(message, "start-kubelet"): + case strings.Contains(message, "start-kubelet"), + strings.Contains(message, "wait-for-kubelet-bootstrap"): return "KubeletBootstrapFailed" case strings.Contains(message, "start-nspawn-machine"): return "NSpawnFailed" diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json new file mode 100644 index 000000000..0d3009fa3 --- /dev/null +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json @@ -0,0 +1,7 @@ +{ + "schemaVersion": 1, + "installID": "00112233445566778899aabbccddeeff", + "machineName": "bootstrap-fixture", + "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", + "phase": "complete" +} diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/input.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/input.json new file mode 100644 index 000000000..1089499e5 --- /dev/null +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/input.json @@ -0,0 +1,14 @@ +{ + "MachineName": "bootstrap-fixture", + "NodeName": "bootstrap-fixture", + "Cluster": { + "Version": "v1.34.0", + "CaCertBase64": "dGVzdC1jYQ==", + "ClusterDNS": "10.96.0.10" + }, + "Kubelet": { + "ApiServer": "https://api.example.test:6443", + "Auth": {"BootstrapToken": "abcdef.0123456789abcdef"} + }, + "OCIImage": "example.test/unbounded/node:fixture" +} diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/installing.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/installing.json new file mode 100644 index 000000000..29daac0b6 --- /dev/null +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/installing.json @@ -0,0 +1,7 @@ +{ + "schemaVersion": 1, + "installID": "00112233445566778899aabbccddeeff", + "machineName": "bootstrap-fixture", + "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", + "phase": "installing" +} diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json new file mode 100644 index 000000000..629816493 --- /dev/null +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json @@ -0,0 +1,7 @@ +{ + "schemaVersion": 1, + "installID": "00112233445566778899aabbccddeeff", + "machineName": "bootstrap-fixture", + "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", + "phase": "resetting" +} diff --git a/cmd/agent/internal/daemon/agentupgrade.go b/cmd/agent/internal/daemon/agentupgrade.go index eb0bb2dd2..979bb1206 100644 --- a/cmd/agent/internal/daemon/agentupgrade.go +++ b/cmd/agent/internal/daemon/agentupgrade.go @@ -61,7 +61,7 @@ func parseAgentUpgradeRequest(parameters map[string]string) (agentUpgradeRequest } func upgradeDaemonBinary(ctx context.Context, log *slog.Logger, request agentUpgradeRequest) error { - paths, err := goalstates.ResolvedAgentUpgradePaths() + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) if err != nil { return fmt.Errorf("resolve current daemon binary symlink: %w", err) } @@ -85,7 +85,7 @@ func upgradeDaemonBinary(ctx context.Context, log *slog.Logger, request agentUpg } func newAgentUpgradeSignalOperator() (agentUpgradeSignalOperator, error) { - paths, err := goalstates.ResolvedAgentUpgradePaths() + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) if err != nil { return nil, fmt.Errorf("resolve AgentUpgrade signal path: %w", err) } diff --git a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh index 314f2ca60..288dec7de 100644 --- a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh +++ b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh @@ -4,6 +4,9 @@ set -euo pipefail +# Last-resort recovery does not wait on lifecycle locks: the failing activation +# may still hold them. Reset stops this unit before removing daemon artifacts. + current="{{ .DaemonBinaryCurrentPath }}" last_good="$(readlink -f {{ .DaemonBinaryLastGoodPath }} || true)" upgrade_signal="{{ .DaemonAgentUpgradeSignalPath }}" diff --git a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon.service b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon.service index 6c9dc49e4..3a92ee30c 100644 --- a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon.service +++ b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon.service @@ -14,6 +14,12 @@ Type=simple ExecStart={{ .DaemonBinaryCurrentPath }} daemon Restart=always RestartSec=10 +# Standing down because an installation owns the host is not a failure. Without +# these the daemon would exit, restart, exhaust StartLimitBurst, and trip +# OnFailure into a binary rollback for a problem the binary does not have. +# Both must match daemon.DeferredExitCode. +SuccessExitStatus={{ .DaemonDeferredExitCode }} +RestartPreventExitStatus={{ .DaemonDeferredExitCode }} [Install] WantedBy=multi-user.target diff --git a/cmd/agent/internal/daemon/bootstrap_status_test.go b/cmd/agent/internal/daemon/bootstrap_status_test.go index 284125231..1e87fe777 100644 --- a/cmd/agent/internal/daemon/bootstrap_status_test.go +++ b/cmd/agent/internal/daemon/bootstrap_status_test.go @@ -109,3 +109,21 @@ func findMachineCondition(t *testing.T, conditions []metav1.Condition, condition return metav1.Condition{} } + +// TestBootstrapStatusReporter_NilIsSafe pins that every reporting entry point +// tolerates a nil receiver. +// +// Bootstrap builds the reporter only once attestation has supplied credentials, +// so stages that run before that, and a failure on one of them, report through a +// nil reporter. The callers rely on this instead of guarding each call, so it is +// a contract rather than an accident. +func TestBootstrapStatusReporter_NilIsSafe(t *testing.T) { + t.Parallel() + + var reporter *BootstrapStatusReporter + + assert.NotPanics(t, func() { reporter.Running(context.Background()) }) + assert.NotPanics(t, func() { reporter.Failed(context.Background(), "Failed", errors.New("boom")) }) + assert.NotPanics(t, func() { reporter.Failed(context.Background(), "Failed", nil) }) + assert.NotPanics(t, func() { reporter.Succeeded(context.Background()) }) +} diff --git a/cmd/agent/internal/daemon/controller.go b/cmd/agent/internal/daemon/controller.go index a0895f5ad..039b05fa3 100644 --- a/cmd/agent/internal/daemon/controller.go +++ b/cmd/agent/internal/daemon/controller.go @@ -22,11 +22,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" daemon "github.com/Azure/unbounded/pkg/agent/daemon" "github.com/Azure/unbounded/pkg/agent/goalstates" ) type repaveReconciler struct { + installation *installstate.Store client.Client log *slog.Logger machineName string @@ -41,6 +43,7 @@ func runController( machineName string, nodeName string, nodeOperator nodeOperator, + installation *installstate.Store, ) error { mgr, err := ctrl.NewManager(restCfg, manager.Options{ Scheme: newScheme(), @@ -69,6 +72,7 @@ func runController( c := mgr.GetClient() machineOperations := &machineOperationTarget{ + installation: installation, Client: c, log: log, machineName: machineName, @@ -92,6 +96,7 @@ func runController( } repaveReconciler := &repaveReconciler{ + installation: installation, Client: c, log: log, machineName: machineName, diff --git a/cmd/agent/internal/daemon/controller_machineoperation.go b/cmd/agent/internal/daemon/controller_machineoperation.go index 9b6990988..175ae332d 100644 --- a/cmd/agent/internal/daemon/controller_machineoperation.go +++ b/cmd/agent/internal/daemon/controller_machineoperation.go @@ -14,6 +14,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/pkg/agent/agentbinary" daemon "github.com/Azure/unbounded/pkg/agent/daemon" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -22,6 +23,7 @@ import ( const agentUpgradeLockRetryDelay = 2 * time.Second type machineOperationTarget struct { + installation *installstate.Store client.Client log *slog.Logger machineName string @@ -30,6 +32,17 @@ type machineOperationTarget struct { } func (t *machineOperationTarget) reconcileNodeReboot(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) { + lock, err := t.installation.AcquireMutationLock() + if errors.Is(err, installstate.ErrLockHeld) { + return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil + } + + if err != nil { + return ctrl.Result{}, err + } + + defer releaseInstallationLock(t.log, lock) + if err := store.MarkInProgress(ctx, op, "restarting active nspawn node"); err != nil { return ctrl.Result{}, err } @@ -57,6 +70,17 @@ func (t *machineOperationTarget) reconcileNodeReboot(ctx context.Context, store } func (t *machineOperationTarget) reconcileAgentUpgrade(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) { + installationLock, err := t.installation.AcquireMutationLock() + if errors.Is(err, installstate.ErrLockHeld) { + return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil + } + + if err != nil { + return ctrl.Result{}, err + } + + defer releaseInstallationLock(t.log, installationLock) + lockPath := t.agentUpgradeLockPath if lockPath == "" { lockPath = goalstates.DaemonAgentUpgradeLockPath @@ -119,6 +143,17 @@ func (t *machineOperationTarget) reconcileAgentUpgrade(ctx context.Context, stor } func (t *machineOperationTarget) reconcileAgentReset(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) { + lock, err := t.installation.AcquireLock() + if errors.Is(err, installstate.ErrLockHeld) { + return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil + } + + if err != nil { + return ctrl.Result{}, err + } + + defer releaseInstallationLock(t.log, lock) + if err := store.MarkInProgress(ctx, op, "resetting unbounded agent"); err != nil { return ctrl.Result{}, err } diff --git a/cmd/agent/internal/daemon/controller_node.go b/cmd/agent/internal/daemon/controller_node.go index a91fdf249..72d09a6ac 100644 --- a/cmd/agent/internal/daemon/controller_node.go +++ b/cmd/agent/internal/daemon/controller_node.go @@ -5,6 +5,7 @@ package daemon import ( "context" + "errors" "fmt" "strings" "time" @@ -17,11 +18,23 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/machineconfigs" "github.com/Azure/unbounded/internal/provision" ) func (r *repaveReconciler) ReconcileRepave(ctx context.Context, _ string) (reconcile.Result, error) { + lock, err := r.installation.AcquireMutationLock() + if errors.Is(err, installstate.ErrLockHeld) { + return reconcile.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil + } + + if err != nil { + return reconcile.Result{}, err + } + + defer releaseInstallationLock(r.log, lock) + active, err := r.nodeOperator.FindActiveMachine(r.log) if err != nil { return reconcile.Result{}, fmt.Errorf("find active machine: %w", err) diff --git a/cmd/agent/internal/daemon/controller_test.go b/cmd/agent/internal/daemon/controller_test.go index f70e11b87..702ccd1ad 100644 --- a/cmd/agent/internal/daemon/controller_test.go +++ b/cmd/agent/internal/daemon/controller_test.go @@ -21,6 +21,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/agentbinary" daemon "github.com/Azure/unbounded/pkg/agent/daemon" @@ -153,6 +154,7 @@ func newTestMachinaMachineOperationReconcilerWithLockPath( t.Helper() target := &machineOperationTarget{ + installation: installstate.NewStore(filepath.Join(t.TempDir(), "state"), filepath.Join(t.TempDir(), "install.lock")), Client: c, log: discardLogger(), machineName: "test-machine", @@ -557,6 +559,7 @@ func TestReconcileRepave_UsesDesiredMachineConfigurationVersion(t *testing.T) { op := &fakeNodeOperator{active: active} c := fakeStatusClient(machine, mcv) reconciler := &repaveReconciler{ + installation: installstate.NewStore(filepath.Join(t.TempDir(), "state"), filepath.Join(t.TempDir(), "install.lock")), Client: c, log: discardLogger(), machineName: "test-machine", @@ -612,6 +615,7 @@ func TestReconcileRepave_NoDriftMarksDesiredConfigurationApplied(t *testing.T) { op := &fakeNodeOperator{active: &ActiveMachine{Name: "kube1", Config: base}} c := fakeStatusClient(machine, mcv) reconciler := &repaveReconciler{ + installation: installstate.NewStore(filepath.Join(t.TempDir(), "state"), filepath.Join(t.TempDir(), "install.lock")), Client: c, log: discardLogger(), machineName: "test-machine", diff --git a/cmd/agent/internal/daemon/daemon.go b/cmd/agent/internal/daemon/daemon.go index 06a582e75..61f612fe1 100644 --- a/cmd/agent/internal/daemon/daemon.go +++ b/cmd/agent/internal/daemon/daemon.go @@ -6,6 +6,7 @@ package daemon import ( "context" "encoding/base64" + "errors" "fmt" "log/slog" "path/filepath" @@ -21,6 +22,7 @@ import ( v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/config" "github.com/Azure/unbounded/pkg/agent/daemoncred" @@ -31,14 +33,36 @@ const ( daemonControllerCertificateName = "unbounded-agent-daemon-controller" daemonControllerGroup = "unbounded-agent-daemons" daemonControllerCertWaitTimeout = 2 * time.Minute + + // installationLockWaitTimeout bounds how long daemon startup waits for the + // launching bootstrap or activation to release installation ownership. + installationLockWaitTimeout = 30 * time.Second + + // DeferredExitCode is returned when the daemon has no work because an + // installation owns the host. It is not a failure, and the daemon unit + // names it in both SuccessExitStatus and RestartPreventExitStatus so + // systemd leaves the unit inactive instead of restarting it, exhausting the + // start limit, and running OnFailure recovery. Those two directives and this + // constant have to agree; TestDaemonUnitDeclaresDeferredExitCode pins that. + // + // 69 is EX_UNAVAILABLE: the service is correct but cannot run yet. + DeferredExitCode = 69 ) +// ErrDeferred reports that the daemon cannot run because an installation owns +// the host, and that this is expected rather than a fault. The command layer +// turns it into DeferredExitCode without printing an error. +// +// Run wraps it with context, so callers must test with errors.Is. +var ErrDeferred = errors.New("daemon deferred until installation completes") + // kubeClientFunc constructs a controller-runtime client from a rest.Config. // The production implementation is client.NewWithWatch; tests can supply a fake. type kubeClientFunc func(cfg *rest.Config, opts client.Options) (client.WithWatch, error) // runOptions configures daemon runtime behavior. type runOptions struct { + installation *installstate.Store // DaemonCredentialDir stores the daemon-controller client certificate and key. // When empty, the default path under the agent config directory is used. DaemonCredentialDir string @@ -63,6 +87,10 @@ func (o *runOptions) validate() error { o.NodeOperator = nspawnNodeOperator{} } + if o.installation == nil { + o.installation = installstate.DefaultStore() + } + if o.DaemonCredentialDir == "" { o.DaemonCredentialDir = filepath.Join(goalstates.AgentConfigDir, "daemon-controller") } @@ -84,8 +112,9 @@ func run(ctx context.Context, log *slog.Logger, opts runOptions) error { return err } - // Find the active machine and its applied config. - active, err := runOpts.NodeOperator.FindActiveMachine(log) + // Discovery and migration share ownership. Read once after the launcher + // releases its lock rather than mutating a previously discovered snapshot. + active, err := discoverAndMigrate(ctx, log, runOpts.installation, runOpts.NodeOperator) if err != nil { return fmt.Errorf("find active machine: %w", err) } @@ -96,10 +125,6 @@ func run(ctx context.Context, log *slog.Logger, opts runOptions) error { "applied_version", active.Config.Cluster.Version, ) - if err := runOpts.NodeOperator.EnsureLifecycleMigration(ctx, log, active); err != nil { - return fmt.Errorf("ensure nspawn lifecycle migration: %w", err) - } - controllerCfg, stopControllerCreds, err := daemonControllerCredentials(ctx, log, active.Config, runOpts) if err != nil { return fmt.Errorf("build daemon controller credentials: %w", err) @@ -126,7 +151,56 @@ func run(ctx context.Context, log *slog.Logger, opts runOptions) error { log.Warn("failed to publish and clear AgentUpgrade daemon signals", "error", err) } - return runController(ctx, log, controllerCfg, active.Config.MachineName, active.Config.NodeName, runOpts.NodeOperator) + return runController(ctx, log, controllerCfg, active.Config.MachineName, active.Config.NodeName, runOpts.NodeOperator, runOpts.installation) +} + +func discoverAndMigrate(ctx context.Context, log *slog.Logger, store *installstate.Store, operator nodeOperator) (*ActiveMachine, error) { + waitCtx, cancel := context.WithTimeout(ctx, installationLockWaitTimeout) + defer cancel() + + for { + lock, err := store.AcquireMutationLock() + if err == nil { + defer releaseInstallationLock(log, lock) + + active, err := operator.FindActiveMachine(log) + if err != nil { + return nil, err + } + + if err := operator.EnsureLifecycleMigration(ctx, log, active); err != nil { + return nil, err + } + + return active, nil + } + + // An installation owns this host and has not finished. Nothing here can + // finish it: only a bootstrap run can, and that run starts the daemon + // when it succeeds. Stand down rather than fail. + if errors.Is(err, installstate.ErrInstallationInProgress) { + log.Warn("installation has not finished; daemon is standing down until bootstrap completes", "error", err) + + return nil, ErrDeferred + } + + if !errors.Is(err, installstate.ErrLockHeld) { + return nil, err + } + + select { + case <-waitCtx.Done(): + // A bootstrap is holding ownership for longer than a normal handoff. + // It starts the daemon again when it finishes, so waiting longer + // buys nothing and exiting as a failure would look like a crash. + log.Warn("bootstrap still holds installation ownership; daemon is standing down until it completes", + "waited", installationLockWaitTimeout, + ) + + return nil, ErrDeferred + case <-time.After(250 * time.Millisecond): + } + } } func daemonControllerCredentials( diff --git a/cmd/agent/internal/daemon/installation_test.go b/cmd/agent/internal/daemon/installation_test.go new file mode 100644 index 000000000..07677cc93 --- /dev/null +++ b/cmd/agent/internal/daemon/installation_test.go @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package daemon + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/Azure/unbounded/cmd/agent/internal/installstate" + shared "github.com/Azure/unbounded/pkg/agent/daemon" +) + +func TestInstallationContentionPreventsControllerWork(t *testing.T) { + t.Parallel() + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + lock, err := store.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + + target := &machineOperationTarget{installation: store, log: discardLogger()} + // Nil clients, operation stores and node operators ensure contention returns + // before discovery, status publication, binary staging or local mutation. + for name, handler := range map[string]func(context.Context, shared.MachineOperationStore[int64], shared.MachineOperation) (ctrl.Result, error){ + "reboot": target.reconcileNodeReboot, + "upgrade": target.reconcileAgentUpgrade, + "reset": target.reconcileAgentReset, + } { + t.Run(name, func(t *testing.T) { + result, err := handler(t.Context(), nil, shared.MachineOperation{}) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + }) + } + + repave := &repaveReconciler{installation: store, log: discardLogger()} + result, err := repave.ReconcileRepave(t.Context(), "node-delete") + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) +} diff --git a/cmd/agent/internal/daemon/lifecycle.go b/cmd/agent/internal/daemon/lifecycle.go index 6896281e3..8099521a0 100644 --- a/cmd/agent/internal/daemon/lifecycle.go +++ b/cmd/agent/internal/daemon/lifecycle.go @@ -7,12 +7,17 @@ import ( "bytes" "context" _ "embed" + "errors" "fmt" "log/slog" + "os" + "os/exec" "path/filepath" + "strings" "text/template" "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/phases" @@ -46,7 +51,7 @@ func EnableDaemon(log *slog.Logger) phases.Task { func (d *enableDaemon) Name() string { return "enable-daemon" } func (d *enableDaemon) Do(ctx context.Context) error { - paths, err := goalstates.ResolvedAgentUpgradePaths() + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) if err != nil { return fmt.Errorf("resolve current daemon binary symlink: %w", err) } @@ -86,27 +91,74 @@ func (d *enableDaemon) Do(ctx context.Context) error { return fmt.Errorf("writing %s: %w", goalstates.DaemonRecoveryScriptPath, err) } - sc := executil.Systemctl() + return activateDaemonUnit(ctx, d.log, executil.Systemctl()) +} - if err := executil.RunCmd(ctx, d.log, sc, "daemon-reload"); err != nil { +// activateDaemonUnit reloads, enables and starts the daemon unit. +// +// It is separate from writing the unit files so the command sequence can be +// exercised without a writable /etc, and because the order matters: see the +// reset-failed step below. +func activateDaemonUnit(ctx context.Context, log *slog.Logger, sc func(context.Context) *exec.Cmd) error { + if err := executil.RunCmd(ctx, log, sc, "daemon-reload"); err != nil { return fmt.Errorf("systemctl daemon-reload: %w", err) } - if err := executil.RunCmd(ctx, d.log, sc, "enable", goalstates.DaemonUnit); err != nil { + if err := executil.RunCmd(ctx, log, sc, "enable", goalstates.DaemonUnit); err != nil { return fmt.Errorf("systemctl enable %s: %w", goalstates.DaemonUnit, err) } - if err := executil.RunCmd(ctx, d.log, sc, "start", goalstates.DaemonUnit); err != nil { + // Clear any start-limit failure before starting. systemd refuses to start a + // unit that exhausted StartLimitBurst until the failure is reset, and that + // applies to manual starts too, so without this a retry cannot recover a + // host whose daemon was already rate-limited into failure. + // + // Tolerated when host policy denies it: SELinux can withhold this from the + // caller, and it unblocks a start rather than being required for one. + if err := executil.RunCmd(ctx, log, sc, "reset-failed", goalstates.DaemonUnit); err != nil { + log.Debug("could not reset daemon unit failure state", "unit", goalstates.DaemonUnit, "error", err) + } + + if err := executil.RunCmd(ctx, log, sc, "start", goalstates.DaemonUnit); err != nil { return fmt.Errorf("systemctl start %s: %w", goalstates.DaemonUnit, err) } - d.log.Info("daemon unit started", "unit", goalstates.DaemonUnit) + log.Info("daemon unit started", "unit", goalstates.DaemonUnit) return nil } +// InstallBootstrapBinary installs the staged bootstrap executable unless the +// host already has a usable daemon binary. The caller holds installation +// ownership; existing binary layouts are retained and upgrades use their normal +// activation path. +func InstallBootstrapBinary() error { + if usableDaemonBinary(goalstates.DaemonBinaryPath) { + return nil + } + + source, err := os.Executable() + if err != nil { + return err + } + + return fsutil.InstallFile(source, goalstates.DaemonBinaryPath, 0o755) +} + +// usableDaemonBinary resolves symlinks on purpose. The healthy layout reaches +// the active slot through a symlink chain, so only the target tells us whether +// the host can actually run the daemon. A dangling link, or one aimed at +// something that is not an executable file, is exactly the state that sends a +// completed install into repair, and repair cannot replace a bad link either. +// Treating the link's mere presence as a usable binary would strand the host. +func usableDaemonBinary(path string) bool { + info, err := os.Stat(path) + + return err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 +} + func renderDaemonAsset(name string, content []byte) ([]byte, error) { - paths, err := goalstates.ResolvedAgentUpgradePaths() + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) if err != nil { return nil, err } @@ -122,6 +174,7 @@ func renderDaemonAssetForPaths(name string, content []byte, paths goalstates.Age DaemonBinaryLastGoodPath string DaemonRecoveryScriptPath string DaemonAgentUpgradeSignalPath string + DaemonDeferredExitCode int }{ DaemonUnit: goalstates.DaemonUnit, DaemonRecoveryUnit: goalstates.DaemonRecoveryUnit, @@ -129,6 +182,7 @@ func renderDaemonAssetForPaths(name string, content []byte, paths goalstates.Age DaemonBinaryLastGoodPath: paths.LastGoodPath, DaemonRecoveryScriptPath: goalstates.DaemonRecoveryScriptPath, DaemonAgentUpgradeSignalPath: paths.SignalPath, + DaemonDeferredExitCode: DeferredExitCode, } tmpl, err := template.New(name).Parse(string(content)) @@ -153,8 +207,8 @@ type stopDaemon struct { } // StopDaemon returns a task that stops, disables, and removes the -// unbounded-agent-daemon systemd unit. Errors from stop and disable are -// logged but do not fail the task since the unit may not be present. +// unbounded-agent-daemon systemd unit. Only an absent unit permits a failed +// stop; substantive service errors must retain reset ownership. func StopDaemon(log *slog.Logger) phases.Task { return &stopDaemon{log: log} } @@ -163,7 +217,10 @@ func (t *stopDaemon) Name() string { return "stop-daemon" } func (t *stopDaemon) Do(ctx context.Context) error { if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "stop", goalstates.DaemonUnit); err != nil { - t.log.Warn("failed to stop daemon (may not be running)", "error", err) + state, inspectErr := executil.OutputCmd(ctx, t.log, "systemctl", "show", goalstates.DaemonUnit, "--property=LoadState", "--value") + if inspectErr != nil || strings.TrimSpace(state) != "not-found" { + return fmt.Errorf("stop daemon: %w", err) + } } return disableAndRemoveDaemonUnit(ctx, t.log) @@ -189,17 +246,79 @@ func (t *removeDaemonUnit) Do(ctx context.Context) error { return disableAndRemoveDaemonUnit(ctx, t.log) } +type removeFirstBootUnit struct { + log *slog.Logger +} + +// RemoveFirstBootBootstrapUnit returns a task that disables and removes the +// unit an Ignition config installs to bootstrap the agent. +func RemoveFirstBootBootstrapUnit(log *slog.Logger) phases.Task { + return &removeFirstBootUnit{log: log} +} + +func (t *removeFirstBootUnit) Name() string { return "remove-first-boot-unit" } + +func (t *removeFirstBootUnit) Do(ctx context.Context) error { + return removeFirstBootBootstrapUnit(ctx, t.log) +} + +// removeFirstBootBootstrapUnit disables and removes the unit an Ignition config +// installs to bootstrap the agent. +// +// Reset has to take this with it. The unit is installed into +// multi-user.target and carries no completion condition, so it runs on every +// boot and relies on the agent's ownership record to decide there is nothing to +// do. Reset removes that record, so a unit left behind would find a host with +// no installation and bootstrap it again, undoing the reset on the next boot. +// +// Absent on every host not provisioned through Ignition, which is the common +// case, so a missing unit is success rather than something to report. +func removeFirstBootBootstrapUnit(ctx context.Context, log *slog.Logger) error { + return removeFirstBootBootstrapUnitIn(ctx, log, goalstates.SystemdSystemDir) +} + +// removeFirstBootBootstrapUnitIn takes the unit directory so the sequence can +// be exercised without writing to /etc. +func removeFirstBootBootstrapUnitIn(ctx context.Context, log *slog.Logger, unitDir string) error { + unitPath := filepath.Join(unitDir, goalstates.FirstBootBootstrapUnit) + + if _, err := os.Lstat(unitPath); errors.Is(err, os.ErrNotExist) { + return nil + } + + log.Info("removing first-boot bootstrap unit", "unit", goalstates.FirstBootBootstrapUnit) + + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", goalstates.FirstBootBootstrapUnit); err != nil { + // Disable removes the enablement symlink. If it failed but the unit + // file is already gone, there is nothing left to start. + if _, statErr := os.Lstat(unitPath); !errors.Is(statErr, os.ErrNotExist) { + return fmt.Errorf("disable %s: %w", goalstates.FirstBootBootstrapUnit, err) + } + } + + return removeOwnedFile(unitPath) +} + func disableAndRemoveDaemonUnit(ctx context.Context, log *slog.Logger) error { if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", goalstates.DaemonUnit); err != nil { - log.Warn("failed to disable daemon (may already be absent or systemd unavailable)", "error", err) + if _, statErr := os.Lstat(filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonUnit)); !errors.Is(statErr, os.ErrNotExist) { + return err + } } unitPath := filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonUnit) - removeFileIfExists(log, unitPath) + if err := removeOwnedFile(unitPath); err != nil { + return err + } recoveryUnitPath := filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonRecoveryUnit) - removeFileIfExists(log, recoveryUnitPath) - removeFileIfExists(log, goalstates.DaemonRecoveryScriptPath) + if err := removeOwnedFile(recoveryUnitPath); err != nil { + return err + } + + if err := removeOwnedFile(goalstates.DaemonRecoveryScriptPath); err != nil { + return err + } return nil } @@ -235,7 +354,9 @@ func (t *removeAgentArtifacts) Do(_ context.Context) error { "/usr/local/bin/unbounded-agent-install.sh", "/usr/local/bin/unbounded-agent-uninstall.sh", } { - removeFileIfExists(t.log, path) + if err := removeOwnedFile(path); err != nil { + return err + } } // Remove directories. @@ -243,14 +364,90 @@ func (t *removeAgentArtifacts) Do(_ context.Context) error { "/etc/unbounded/agent", "/tmp/unbounded-agent", } { - removeAllIfExists(t.log, dir) + if err := os.RemoveAll(dir); err != nil { + return err + } } // Remove temp config files matching /tmp/unbounded-agent-config.*.json. matches, _ := filepath.Glob("/tmp/unbounded-agent-config.*.json") //nolint:errcheck // Pattern is valid; only errors on malformed globs. for _, m := range matches { - removeFileIfExists(t.log, m) + if err := removeOwnedFile(m); err != nil { + return err + } + } + + return nil +} + +func removeOwnedFile(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove owned artifact %s: %w", path, err) + } + + return nil +} + +// VerifyDaemonInstalled checks installed daemon assets and service state. An +// active daemon already proves it resolved an applied config at startup, so the +// applied-config check belongs to RepairDaemon rather than here. +func VerifyDaemonInstalled(ctx context.Context, log *slog.Logger) error { + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) + if err != nil { + return err + } + + for _, name := range []string{goalstates.DaemonUnit, goalstates.DaemonRecoveryUnit} { + if _, err := os.Stat(filepath.Join(goalstates.SystemdSystemDir, name)); err != nil { + return err + } + } + + for _, path := range []string{paths.CurrentPath, paths.LastGoodPath, paths.BinaryPath, goalstates.DaemonRecoveryScriptPath} { + info, err := os.Stat(path) + if err != nil { + return err + } + + if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return fmt.Errorf("daemon binary is not executable: %s", path) + } + } + + for _, check := range []string{"is-enabled", "is-active"} { + out, err := executil.OutputCmd(ctx, log, "systemctl", check, goalstates.DaemonUnit) + + want := "active" + if check == "is-enabled" { + want = "enabled" + } + + if err != nil { + return fmt.Errorf("daemon %s check: %w", check, err) + } + + if strings.TrimSpace(out) != want { + return fmt.Errorf("daemon %s check failed: %s", check, out) + } } return nil } + +// RepairDaemon requires the caller's installation lock. It uses current applied +// configuration, never the original bootstrap input that may name a retired slot. +func RepairDaemon(ctx context.Context, log *slog.Logger) error { + if _, err := (nspawnNodeOperator{}).FindActiveMachine(log); err != nil { + return err + } + + if err := InstallBootstrapBinary(); err != nil { + return err + } + + if err := EnableDaemon(log).Do(ctx); err != nil { + return err + } + + return fsutil.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) +} diff --git a/cmd/agent/internal/daemon/lifecycle_test.go b/cmd/agent/internal/daemon/lifecycle_test.go index db8d98079..cc6097705 100644 --- a/cmd/agent/internal/daemon/lifecycle_test.go +++ b/cmd/agent/internal/daemon/lifecycle_test.go @@ -4,11 +4,17 @@ package daemon import ( + "os" + "path/filepath" + "strconv" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -34,3 +40,193 @@ func TestRenderDaemonAsset(t *testing.T) { assert.Contains(t, renderedRecovery, goalstates.DaemonAgentUpgradeSignalPath) assert.Contains(t, renderedRecovery, "record-agent-upgrade-failure-signal") } + +func TestInstallBinaryStreamsAndReplacesAtomically(t *testing.T) { + t.Parallel() + dir := t.TempDir() + source, target := filepath.Join(dir, "source"), filepath.Join(dir, "bin", "target") + require.NoError(t, os.WriteFile(source, []byte("candidate"), 0o600)) + require.NoError(t, fsutil.InstallFile(source, target, 0o755)) + data, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "candidate", string(data)) + + info, err := os.Stat(target) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + require.Error(t, fsutil.InstallFile(filepath.Join(dir, "missing"), target, 0o755)) + data, err = os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "candidate", string(data)) +} + +// TestUsableDaemonBinaryRequiresAResolvableExecutable pins what counts as "the +// host already has a daemon binary". Only the resolved target matters: a broken +// or non-executable link is the state that sends a completed install into +// repair, and repair cannot replace the link, so it must not be mistaken for a +// working installation. +func TestUsableDaemonBinaryRequiresAResolvableExecutable(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + executable := filepath.Join(dir, "executable") + require.NoError(t, os.WriteFile(executable, []byte("binary"), 0o755)) + + plain := filepath.Join(dir, "plain") + require.NoError(t, os.WriteFile(plain, []byte("data"), 0o644)) + + // The production layout reaches the active slot through a symlink chain. + current := filepath.Join(dir, "current") + require.NoError(t, os.Symlink(executable, current)) + + chained := filepath.Join(dir, "chained") + require.NoError(t, os.Symlink(current, chained)) + + dangling := filepath.Join(dir, "dangling") + require.NoError(t, os.Symlink(filepath.Join(dir, "absent"), dangling)) + + toPlain := filepath.Join(dir, "to-plain") + require.NoError(t, os.Symlink(plain, toPlain)) + + directory := filepath.Join(dir, "directory") + require.NoError(t, os.Mkdir(directory, 0o755)) + + for path, want := range map[string]bool{ + executable: true, + current: true, + chained: true, + dangling: false, + toPlain: false, + plain: false, + directory: false, + filepath.Join(dir, "absent"): false, + } { + assert.Equal(t, want, usableDaemonBinary(path), "path %s", path) + } +} + +// TestDaemonUnitDeclaresDeferredExitCode pins the agreement between the exit +// code the daemon returns when it stands down and the two directives that tell +// systemd to accept it. +// +// If they ever disagree, the daemon still exits quietly but systemd treats the +// code as a crash: it restarts the unit, exhausts StartLimitBurst, and runs +// OnFailure, which is the last-resort binary rollback. Nothing else would fail, +// so the only thing standing between a silent regression and a host rolling its +// agent back for an unfinished install is this test. +func TestDaemonUnitDeclaresDeferredExitCode(t *testing.T) { + t.Parallel() + + rendered, err := renderDaemonAssetForPaths("daemon-service", daemonServiceContent, goalstates.AgentUpgradePaths{ + CurrentPath: "/usr/local/bin/unbounded-agent-current", + LastGoodPath: "/usr/local/bin/unbounded-agent-last-good", + BinaryPath: "/usr/local/bin/unbounded-agent", + SignalPath: "/var/lib/unbounded/agent/upgrade-signal", + }) + require.NoError(t, err) + + unit := string(rendered) + code := strconv.Itoa(DeferredExitCode) + + require.Contains(t, unit, "SuccessExitStatus="+code, + "systemd must not treat standing down as a failure, or OnFailure runs the binary rollback") + require.Contains(t, unit, "RestartPreventExitStatus="+code, + "systemd must not restart a deferred daemon, or repeated starts exhaust the start limit") + + // The safety net for genuine crashes has to survive the above. + require.Contains(t, unit, "Restart=always") + require.Contains(t, unit, "OnFailure="+goalstates.DaemonRecoveryUnit) +} + +// TestActivateDaemonUnitClearsFailureBeforeStarting pins the order that lets a +// retry recover a host this bug already broke. +// +// A daemon that exhausted its start limit sits in failed state, and systemd +// refuses to start it again until the failure is reset. That refusal applies to +// manual starts too, so a bootstrap retry that only ran enable and start would +// fail on exactly the hosts most in need of repair. +func TestActivateDaemonUnitClearsFailureBeforeStarting(t *testing.T) { + dir := t.TempDir() + calls := filepath.Join(dir, "calls") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), + []byte("#!/bin/sh\necho \"$@\" >> \""+calls+"\"\n"), 0o755)) + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + + require.NoError(t, activateDaemonUnit(t.Context(), discardLogger(), executil.Systemctl())) + + recorded, err := os.ReadFile(calls) + require.NoError(t, err) + + got := string(recorded) + resetAt := strings.Index(got, "reset-failed") + startAt := strings.Index(got, "start ") + + require.NotEqual(t, -1, resetAt, "reset-failed must run; without it a rate-limited unit cannot be started:\n%s", got) + require.NotEqual(t, -1, startAt, "start must run:\n%s", got) + require.Less(t, resetAt, startAt, "reset-failed must precede start, or it cannot unblock it:\n%s", got) +} + +// TestActivateDaemonUnitToleratesDeniedResetFailed covers hosts where policy +// withholds reset-failed. It unblocks a start rather than being required for +// one, so a denial must not fail the install. +func TestActivateDaemonUnitToleratesDeniedResetFailed(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), + []byte("#!/bin/sh\ncase \"$1\" in reset-failed) exit 1 ;; esac\nexit 0\n"), 0o755)) + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + + require.NoError(t, activateDaemonUnit(t.Context(), discardLogger(), executil.Systemctl())) +} + +// TestResetRemovesTheFirstBootBootstrapUnit covers the interaction between +// reset and an Ignition-provisioned host. +// +// The unit carries no completion condition and runs on every boot, deciding +// there is nothing to do from the agent's ownership record. Reset removes that +// record. A unit left behind would therefore find an uninstalled host on the +// next boot and bootstrap it, quietly undoing the reset. +func TestResetRemovesTheFirstBootBootstrapUnit(t *testing.T) { + dir := t.TempDir() + calls := filepath.Join(dir, "calls") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), + []byte("#!/bin/sh\necho \"$@\" >> \""+calls+"\"\n"), 0o755)) + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + + unitDir := t.TempDir() + unitPath := filepath.Join(unitDir, goalstates.FirstBootBootstrapUnit) + require.NoError(t, os.WriteFile(unitPath, []byte("[Unit]\n"), 0o644)) + + require.NoError(t, removeFirstBootBootstrapUnitIn(t.Context(), discardLogger(), unitDir)) + + require.NoFileExists(t, unitPath, "the unit file must be gone, or systemd can still start it") + + recorded, err := os.ReadFile(calls) + require.NoError(t, err) + require.Contains(t, string(recorded), "disable "+goalstates.FirstBootBootstrapUnit, + "removing the file alone leaves the enablement symlink in multi-user.target.wants") +} + +// TestFirstBootBootstrapUnitAbsentIsSuccess covers every host not provisioned +// through Ignition, which is the common case. There is nothing to remove and +// nothing to report. +func TestFirstBootBootstrapUnitAbsentIsSuccess(t *testing.T) { + t.Parallel() + + require.NoError(t, removeFirstBootBootstrapUnitIn(t.Context(), discardLogger(), t.TempDir())) +} + +// TestFirstBootBootstrapUnitNameIsShared pins that the command writing the unit +// and the reset removing it agree on its name. +// +// They live in packages that cannot import each other, so the name is held in +// goalstates. If it were duplicated and drifted, reset would leave an enabled +// unit on a host it had just torn down, and the host would re-bootstrap on the +// next boot with nothing reporting why. +func TestFirstBootBootstrapUnitNameIsShared(t *testing.T) { + t.Parallel() + + require.Equal(t, "unbounded-agent-bootstrap.service", goalstates.FirstBootBootstrapUnit) +} diff --git a/cmd/agent/internal/daemon/migration_test.go b/cmd/agent/internal/daemon/migration_test.go index 366b00e05..b39e00324 100644 --- a/cmd/agent/internal/daemon/migration_test.go +++ b/cmd/agent/internal/daemon/migration_test.go @@ -6,10 +6,13 @@ package daemon import ( "context" "errors" + "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" ) @@ -20,11 +23,54 @@ func TestDaemonStartupRunsLifecycleMigrationBeforeControllerSetup(t *testing.T) Name: "kube1", Config: &provision.AgentConfig{MachineName: "machine-1", NodeName: "node-1"}, }} - err := run(context.Background(), discardLogger(), runOptions{NodeOperator: op}) + err := run(context.Background(), discardLogger(), runOptions{NodeOperator: op, installation: installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock"))}) require.ErrorContains(t, err, "build daemon controller credentials") require.Equal(t, 1, op.lifecycleCalls) } +// TestStartupLockWaitStandsDownWithoutMigration covers a bootstrap that holds +// installation ownership for longer than the daemon is willing to wait. +// +// The daemon must not migrate anything, and it must not report a failure. A +// bootstrap that holds the lock this long is still working, and it starts the +// daemon again when it finishes. Exiting as a failure here is what previously +// drove the unit through its start limit and into OnFailure recovery. +func TestStartupLockWaitStandsDownWithoutMigration(t *testing.T) { + t.Parallel() + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + lock, err := store.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond) + defer cancel() + + op := &fakeNodeOperator{} + _, err = discoverAndMigrate(ctx, discardLogger(), store, op) + require.ErrorIs(t, err, ErrDeferred, "waiting out a live bootstrap must defer, not fail") + require.Zero(t, op.lifecycleCalls) +} + +// TestStartupStandsDownWhileInstallationUnfinished covers the other way the +// daemon can find itself with no work: the record says an installation is under +// way and nobody holds the lock, so no bootstrap is running to finish it. +// +// Only a bootstrap run can complete the install, and that run starts the daemon +// on success, so the daemon defers instead of failing. +func TestStartupStandsDownWhileInstallationUnfinished(t *testing.T) { + t.Parallel() + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + + record, err := installstate.NewRecord("machine-1", "fingerprint", "") + require.NoError(t, err) + require.NoError(t, store.Save(record)) + + op := &fakeNodeOperator{} + _, err = discoverAndMigrate(t.Context(), discardLogger(), store, op) + require.ErrorIs(t, err, ErrDeferred, "an unfinished installation must defer, not fail") + require.Zero(t, op.lifecycleCalls) +} + func TestDaemonStartupFailsLifecycleMigrationWithoutRetry(t *testing.T) { t.Parallel() @@ -36,7 +82,7 @@ func TestDaemonStartupFailsLifecycleMigrationWithoutRetry(t *testing.T) { }, lifecycleErrs: []error{resolveErr}, } - err := run(context.Background(), discardLogger(), runOptions{NodeOperator: op}) + err := run(context.Background(), discardLogger(), runOptions{NodeOperator: op, installation: installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock"))}) require.ErrorIs(t, err, resolveErr) require.Equal(t, 1, op.lifecycleCalls) } diff --git a/cmd/agent/internal/daemon/nodeoperator.go b/cmd/agent/internal/daemon/nodeoperator.go index 14c72253f..3bf98d21d 100644 --- a/cmd/agent/internal/daemon/nodeoperator.go +++ b/cmd/agent/internal/daemon/nodeoperator.go @@ -13,6 +13,7 @@ import ( "reflect" "strings" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -233,7 +234,8 @@ func (nspawnNodeOperator) RestartNode(ctx context.Context, log *slog.Logger, act } func (nspawnNodeOperator) ResetAgentResources(ctx context.Context, log *slog.Logger) error { - return ResetAgentResources(log).Do(ctx) + // The MachineOperation holds installation ownership through daemon stop. + return resetUnderLock(ctx, log, installstate.DefaultStore(), resetResources(log)) } func (nspawnNodeOperator) StopDaemon(ctx context.Context, log *slog.Logger) error { diff --git a/cmd/agent/internal/daemon/persist_config.go b/cmd/agent/internal/daemon/persist_config.go index 822243d5a..173bc2371 100644 --- a/cmd/agent/internal/daemon/persist_config.go +++ b/cmd/agent/internal/daemon/persist_config.go @@ -89,11 +89,3 @@ func removeFileIfExists(log *slog.Logger, path string) { log.Warn("failed to remove file", "path", path, "error", err) } } - -// removeAllIfExists removes a path and all children if it exists. Errors are -// logged at Warn so we have a trace but don't abort the flow. -func removeAllIfExists(log *slog.Logger, path string) { - if err := os.RemoveAll(path); err != nil { - log.Warn("failed to remove directory", "path", path, "error", err) - } -} diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index 11161ec7f..f35f704a2 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -4,16 +4,150 @@ package daemon import ( + "context" + "errors" + "fmt" "log/slog" + "os" + "path/filepath" + "strings" + "golang.org/x/sys/unix" + + "github.com/Azure/unbounded/cmd/agent/internal/installstate" + "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/phases" "github.com/Azure/unbounded/pkg/agent/phases/reset" ) -// ResetAgentResources returns a task that removes the unbounded-agent and all -// associated resources without stopping the daemon process. -func ResetAgentResources(log *slog.Logger) phases.Task { +// ResetAgent removes the unbounded-agent and all associated resources, stopping +// the daemon first. The daemon's own operation path stops it last instead, so +// that ordering stays with the caller. +func ResetAgent(log *slog.Logger) phases.Task { + return ownedReset(log, installstate.DefaultStore(), phases.Serial(log, StopDaemon(log), resetResources(log))) +} + +type lifecycleTask struct { + name string + run func(context.Context) error +} + +func (t lifecycleTask) Name() string { return t.name } +func (t lifecycleTask) Do(ctx context.Context) error { return t.run(ctx) } + +func ownedReset(log *slog.Logger, store *installstate.Store, inner phases.Task) phases.Task { + // The composed name keeps the underlying cleanup sequence visible to callers + // and to the reset ordering test. + return lifecycleTask{name: "owned-reset(" + inner.Name() + ")", run: func(ctx context.Context) error { + lock, err := store.AcquireLock() + if err != nil { + return err + } + defer func() { + if err := lock.Release(); err != nil { + log.Error("release reset lock", "error", err) + } + }() + + return resetUnderLock(ctx, log, store, inner) + }} +} + +// recordForTeardown returns the record reset should mark as resetting. +// +// Any record it cannot read is replaced rather than obeyed. Reset is about to +// delete it, so refusing to proceed protects nothing and costs everything: +// decide rejects the same unreadable record, so start is refused too, and the +// host is left with no way out through either path. The guide tells operators +// to keep this file intact, so it must not be the thing that strands them. +func recordForTeardown(log *slog.Logger, store *installstate.Store) (installstate.Record, error) { + r, err := store.Load() + if err == nil { + return r, nil + } + + if !errors.Is(err, installstate.ErrNotFound) { + log.Warn("installation record is unreadable; replacing it for teardown", "error", err) + } + + return installstate.NewRecord("legacy-reset", "legacy-reset", "") +} + +func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.Store, inner phases.Task) error { + r, err := recordForTeardown(log, store) + if err != nil { + return err + } + + r.Phase = installstate.Resetting + if err := store.Save(r); err != nil { + return err + } + // Cancel recovery waiting on ownership before removing its executable. + if err := stopRecoveryUnit(ctx, log); err != nil { + return err + } + + return durableReset(ctx, store, inner, []string{"/etc", "/var/lib/machines", "/usr/local", store.Root()}, unix.Syncfs) +} + +func stopRecoveryUnit(ctx context.Context, log *slog.Logger) error { + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "stop", goalstates.DaemonRecoveryUnit); err != nil { + out, inspectErr := executil.OutputCmd(ctx, log, "systemctl", "show", goalstates.DaemonRecoveryUnit, "--property=LoadState", "--value") + if inspectErr != nil || strings.TrimSpace(out) != "not-found" { + return fmt.Errorf("stop daemon recovery: %w", err) + } + } + + return nil +} + +func durableReset(ctx context.Context, store *installstate.Store, inner phases.Task, paths []string, syncfs func(int) error) error { + var handles []*os.File + defer func() { + for _, f := range handles { + _ = f.Close() //nolint:errcheck // Read-only directory descriptor; teardown sync errors are returned below. + } + }() + + for _, path := range paths { + for { + if _, err := os.Stat(path); err == nil { + break + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + + parent := filepath.Dir(path) + if parent == path { + return fmt.Errorf("no filesystem ancestor for %s", path) + } + + path = parent + } + + f, err := os.Open(path) + if err != nil { + return err + } + + handles = append(handles, f) + } + + if err := inner.Do(ctx); err != nil { + return err + } + + if err := fsutil.SyncOpenFilesystems(handles, syncfs); err != nil { + return err + } + + return store.Remove() +} + +func resetResources(log *slog.Logger) phases.Task { return phases.Serial(log, RemoveDaemonUnit(log), phases.Parallel(log, @@ -34,7 +168,17 @@ func ResetAgentResources(log *slog.Logger) phases.Task { reset.RemoveBPFFSMount(log, goalstates.NSpawnMachineKube2), ), reset.CleanupNetwork(log), + // Before the artifacts, so a failure here stops the reset while the + // host is still recognizably installed. A unit that survived a reset + // would bootstrap the host again on the next boot. + RemoveFirstBootBootstrapUnit(log), RemoveAgentArtifacts(log), reset.ReloadSystemd(log), ) } + +func releaseInstallationLock(log *slog.Logger, lock *installstate.Lock) { + if err := lock.Release(); err != nil { + log.Error("release installation lock", "error", err) + } +} diff --git a/cmd/agent/internal/daemon/reset_test.go b/cmd/agent/internal/daemon/reset_test.go index ddbfeadc0..4b231adba 100644 --- a/cmd/agent/internal/daemon/reset_test.go +++ b/cmd/agent/internal/daemon/reset_test.go @@ -4,19 +4,157 @@ package daemon import ( + "context" + "errors" "log/slog" + "os" + "path/filepath" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/cmd/agent/internal/installstate" ) -func TestResetAgentResourcesIncludesBPFFSMountCleanup(t *testing.T) { +func TestResetResourcesIncludesBPFFSMountCleanup(t *testing.T) { t.Parallel() - taskName := ResetAgentResources(slog.New(slog.DiscardHandler)).Name() + taskName := resetResources(slog.New(slog.DiscardHandler)).Name() assert.Contains(t, taskName, "parallel(remove-bpffs-mount, remove-bpffs-mount)") assert.Less(t, strings.Index(taskName, "parallel(remove-machine, remove-machine)"), strings.Index(taskName, "parallel(remove-bpffs-mount, remove-bpffs-mount)")) assert.Less(t, strings.Index(taskName, "parallel(remove-bpffs-mount, remove-bpffs-mount)"), strings.Index(taskName, "cleanup-routes")) } + +func TestResetRetainsOwnershipUntilTeardownAndSyncSucceed(t *testing.T) { + t.Parallel() + + for _, failure := range []string{"cleanup", "sync", ""} { + t.Run(failure, func(t *testing.T) { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + r, err := installstate.NewRecord("machine", "f", "") + require.NoError(t, err) + + r.Phase = installstate.Resetting + require.NoError(t, store.Save(r)) + + injected := errors.New("injected reset failure") + cleaned := false + task := lifecycleTask{name: "cleanup", run: func(context.Context) error { + if failure == "cleanup" { + return injected + } + + cleaned = true + + return nil + }} + synced := false + + err = durableReset(t.Context(), store, task, []string{store.Root()}, func(int) error { + require.True(t, cleaned) + + _, err := store.Load() + require.NoError(t, err, "ownership must remain during filesystem barrier") + + if failure == "sync" { + return injected + } + + synced = true + + return nil + }) + if failure != "" { + require.ErrorIs(t, err, injected) + loaded, err := store.Load() + require.NoError(t, err) + require.Equal(t, r, loaded) + } else { + require.NoError(t, err) + require.True(t, synced) + + _, err = store.Load() + require.ErrorIs(t, err, installstate.ErrNotFound) + } + }) + } +} + +// TestTeardownProceedsThroughAnUnreadableRecord covers the one file that can +// strand a host through both of its exits. +// +// decide rejects a record it cannot parse, so start is refused. If reset also +// refuses, nothing documented recovers the machine, and the guide tells +// operators to keep this file intact rather than delete it. Reset deletes it +// moments later regardless, so reading it is a courtesy, not a prerequisite. +// +// resetUnderLock itself syncs real host filesystems and needs root, so this +// covers the decision it delegates. Reintroducing a direct store.Load there +// would leave this helper uncalled, which staticcheck reports. +func TestTeardownProceedsThroughAnUnreadableRecord(t *testing.T) { + t.Parallel() + + for _, content := range []string{"{", "null", `{"schemaVersion":99}`, `{"schemaVersion":1,"phase":"bogus"}`} { + t.Run(content, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + + require.NoError(t, os.MkdirAll(store.Root(), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(store.Root(), "install-state.json"), []byte(content), 0o600)) + + // Confirm the premise: this is a record start would refuse. + _, loadErr := store.Load() + require.Error(t, loadErr, "fixture must be a record the store rejects") + + r, err := recordForTeardown(discardLogger(), store) + require.NoError(t, err, "an unreadable record must not block the thing that deletes it") + require.NoError(t, r.Validate(), "the replacement must be usable for teardown") + }) + } +} + +// TestTeardownKeepsAReadableRecord confirms the replacement above is a fallback +// and not the normal path: a record reset can read is the one it tears down, +// so the machine name and fingerprint stay accurate through reset. +func TestTeardownKeepsAReadableRecord(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + + saved, err := installstate.NewRecord("machine-1", "fingerprint-1", "") + require.NoError(t, err) + require.NoError(t, store.Save(saved)) + + r, err := recordForTeardown(discardLogger(), store) + require.NoError(t, err) + require.Equal(t, "machine-1", r.MachineName) + require.Equal(t, "fingerprint-1", r.ConfigFingerprint) +} + +// TestResetRemovesTheFirstBootUnitBeforeArtifacts pins that reset actually runs +// the removal, not merely that the removal works. +// +// The unit runs on every boot and decides there is nothing to do from the +// ownership record that reset is about to delete. Left behind, it would find an +// uninstalled host and bootstrap it again, undoing the reset with nothing +// reporting why. Ordering it before the artifacts means a failure stops the +// reset while the host is still recognizably installed. +func TestResetRemovesTheFirstBootUnitBeforeArtifacts(t *testing.T) { + t.Parallel() + + taskName := resetResources(slog.New(slog.DiscardHandler)).Name() + + assert.Contains(t, taskName, "remove-first-boot-unit", + "reset must remove the Ignition bootstrap unit or the host re-bootstraps on next boot") + assert.Less(t, + strings.Index(taskName, "remove-first-boot-unit"), + strings.Index(taskName, "remove-agent-artifacts"), + "a failure here must stop the reset while the host is still recognizably installed") +} diff --git a/cmd/agent/internal/installstate/doc.go b/cmd/agent/internal/installstate/doc.go new file mode 100644 index 000000000..1e9e1809d --- /dev/null +++ b/cmd/agent/internal/installstate/doc.go @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Package installstate owns the agent's installation ownership record and the +// lock that guards it. The record is internal bookkeeping, not a configuration +// input. +// +// It answers three questions nothing else on the host can answer: +// +// - Is there an installation here, and is it the one we are being asked to +// perform? The record carries the machine name and a fingerprint of the +// identity-defining configuration, so a retry can tell its own interrupted +// attempt from a different installation. Without it, bootstrap can only +// demand a pristine host and abort otherwise. +// - How far did the last attempt get? It does not say, deliberately. Every +// stage reapplies on every attempt and decides from the host what it still +// has to do, so there is nothing here that could disagree with the host. +// The phase records only whether an installation is under way, finished, or +// being torn down, which is a thing the host cannot be asked. +// - Is anything else mutating this host? The lock serializes bootstrap, reset, +// repave, NodeReboot, host agent activation and AgentUpgrade, which all +// touch the same files and services. +// +// It is separate from the bootstrap coordinator because the daemon's repave, +// NodeReboot, AgentUpgrade and AgentReset paths, reset, and host agent +// activation all need the record or the lock without running a bootstrap. +package installstate diff --git a/cmd/agent/internal/installstate/lock.go b/cmd/agent/internal/installstate/lock.go new file mode 100644 index 000000000..b9db8458a --- /dev/null +++ b/cmd/agent/internal/installstate/lock.go @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package installstate + +import ( + "errors" + "os" + "path/filepath" + + "github.com/gofrs/flock" +) + +var ErrLockHeld = errors.New("another host lifecycle operation holds the installation lock") + +type Lock struct{ flock *flock.Flock } + +// acquireLockAt is nonblocking. The kernel releases the lock on process exit; a +// leftover lock file does not imply a held lock and must not be deleted by reset. +func acquireLockAt(path string) (*Lock, error) { + // flock.New does not create the parent directory. + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, err + } + + l := flock.New(path) + + switch locked, err := l.TryLock(); { + case err != nil: + return nil, err + case !locked: + return nil, ErrLockHeld + } + + return &Lock{flock: l}, nil +} + +func (l *Lock) Release() error { + if l == nil || l.flock == nil { + return nil + } + + return l.flock.Unlock() +} diff --git a/cmd/agent/internal/installstate/mutation.go b/cmd/agent/internal/installstate/mutation.go new file mode 100644 index 000000000..64bdc40f8 --- /dev/null +++ b/cmd/agent/internal/installstate/mutation.go @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package installstate + +import ( + "errors" + "fmt" +) + +// ErrInstallationInProgress reports that an installation owns this host and has +// not finished. It is distinct from ErrLockHeld, which says another process is +// working right now: this one says the host is mid-installation whether or not +// anyone is currently acting on it. +// +// Callers that run because systemd started them, rather than because a person +// asked, must not treat this as a failure. An installation in progress is an +// ordinary state, and reporting it as a crash is how a service ends up being +// restarted, rate-limited, and recovered for a problem it does not have. +var ErrInstallationInProgress = errors.New("installation has not finished") + +// AcquireMutationLock serializes ordinary lifecycle work with bootstrap and +// reset. Legacy hosts without a record remain supported. When both locks are +// needed, acquire installation ownership before the binary activation lock. +func (s *Store) AcquireMutationLock() (*Lock, error) { + lock, err := s.AcquireLock() + if err != nil { + return nil, err + } + + r, loadErr := s.Load() + if errors.Is(loadErr, ErrNotFound) || loadErr == nil && r.Phase == Complete { + return lock, nil + } + + closeErr := lock.Release() + + if loadErr != nil { + return nil, errors.Join(loadErr, closeErr) + } + + return nil, errors.Join(fmt.Errorf("%w: installation is %s; finish bootstrap or reset before lifecycle operations", ErrInstallationInProgress, r.Phase), closeErr) +} diff --git a/cmd/agent/internal/installstate/store.go b/cmd/agent/internal/installstate/store.go new file mode 100644 index 000000000..ace82b371 --- /dev/null +++ b/cmd/agent/internal/installstate/store.go @@ -0,0 +1,262 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package installstate + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/Azure/unbounded/internal/fsutil" +) + +const ( + DefaultDirectory = "/var/lib/unbounded/agent" + defaultLockPath = "/run/unbounded-agent-install.lock" + schemaVersion = 1 +) + +// Phase is what an installation is currently doing, not how far it has got. +// +// It deliberately records no progress. A record that claimed a stage was +// finished would be a statement about the host that could stop being true +// without anyone noticing, and a retry that trusted it would skip work the host +// no longer has. The stages instead decide what to do by looking at the host, +// so the only thing worth persisting is which mode we are in. +type Phase string + +const ( + // Installing means an installation is under way. It says nothing about what + // has been done, so there is nothing in it that can go stale. + Installing Phase = "installing" + // Complete means the installation finished. A later start verifies and + // repairs from the applied config rather than reapplying bootstrap inputs, + // which after an ordinary repave describe a retired slot. + Complete Phase = "complete" + // Resetting means a teardown started and may not have finished. This is the + // one thing the host cannot be asked: a half-removed installation and a + // half-built one look the same, because direction of travel is not visible. + Resetting Phase = "resetting" +) + +type Record struct { + SchemaVersion int `json:"schemaVersion"` + InstallID string `json:"installID"` + MachineName string `json:"machineName"` + ConfigFingerprint string `json:"configFingerprint"` + Phase Phase `json:"phase"` + + // HostPrefix is the resolved installation prefix, recorded so teardown can + // find the agent's own files without being told where they are. + // + // It is written before the first host mutation, which makes it the only + // source that survives a bootstrap that failed before the node started. The + // applied config carries the same prefix but does not exist until then, so + // reset on a half-built host has nothing else to go on. + // + // Optional, and absent means the default. The schema version does not move + // for it: a record written by an agent that knows about the prefix stays + // readable by one that does not, because unknown fields are ignored, and a + // record written before it existed is read here as the default, which is + // what such a host actually has on disk. + HostPrefix string `json:"hostPrefix,omitempty"` +} + +func (r Record) Validate() error { + if r.SchemaVersion != schemaVersion || strings.TrimSpace(r.InstallID) == "" || strings.TrimSpace(r.MachineName) == "" || strings.TrimSpace(r.ConfigFingerprint) == "" { + return fmt.Errorf("invalid installation record identity or schema") + } + + switch r.Phase { + case Installing, Complete, Resetting: + return nil + default: + return fmt.Errorf("unknown installation phase %q", r.Phase) + } +} + +var ErrNotFound = errors.New("installation record not found") + +type Store struct { + root, lockPath string + // syncDir is a seam for exercising the window where the record is unlinked + // but the directory entry has not reached disk. + syncDir func(string) error +} + +func NewStore(root, lockPath string) *Store { + return &Store{root: root, lockPath: lockPath, syncDir: fsutil.SyncDir} +} + +func DefaultStore() *Store { return NewStore(DefaultDirectory, defaultLockPath) } +func (s *Store) Root() string { return s.root } +func (s *Store) statePath() string { return filepath.Join(s.root, "install-state.json") } +func (s *Store) AcquireLock() (*Lock, error) { return acquireLockAt(s.lockPath) } + +func (s *Store) Load() (Record, error) { + var r Record + + data, err := os.ReadFile(s.statePath()) + if errors.Is(err, os.ErrNotExist) { + return r, ErrNotFound + } + + if err != nil { + return r, err + } + + if err := json.Unmarshal(data, &r); err != nil { + return r, err + } + + return r, r.Validate() +} + +func (s *Store) Save(r Record) error { + if err := r.Validate(); err != nil { + return err + } + + data, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + + return fsutil.WriteFileDurable(s.statePath(), append(data, '\n'), 0o600) +} + +// MarkComplete commits completion. The durable record is the only completion +// signal; no separate marker file is maintained. +func (s *Store) MarkComplete(r Record) error { + r.Phase = Complete + return s.Save(r) +} + +// Remove is called only after teardown's filesystem barriers succeed. +// +// Dropping ownership is itself a durable step. If the unlink cannot be flushed, +// the record is put back, because a reset that reports failure must leave the +// host visibly owned. Otherwise the removal survives in page cache only, the +// error sends the operator away, and the next start is admitted as a fresh +// install onto a half-torn-down host. +func (s *Store) Remove() error { + if _, err := os.Stat(s.root); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + // Read what is about to be dropped so ownership can be restored below. A + // record that does not load is not restored: it granted no usable ownership + // and admission rejects it either way. + previous, loadErr := s.Load() + + if err := os.Remove(s.statePath()); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + + err := s.syncDir(s.root) + if err != nil && loadErr == nil { + if saveErr := s.Save(previous); saveErr != nil { + return errors.Join(err, saveErr) + } + } + + return err +} + +// NewRecord returns a record for a fresh installation. +// +// hostPrefix is a parameter rather than a field callers set afterwards because +// forgetting it is silent and only surfaces at teardown, on a host whose files +// are somewhere reset would not look. An empty prefix means the default. +// +// The value is stored as given and not validated here. This package deals in +// stdlib and durability only, and pulling in config validation to re-check a +// string this agent wrote from an already validated config would buy little. +func NewRecord(machine, fingerprint, hostPrefix string) (Record, error) { + id := make([]byte, 16) + if _, err := rand.Read(id); err != nil { + return Record{}, err + } + + return Record{ + SchemaVersion: schemaVersion, InstallID: hex.EncodeToString(id), MachineName: machine, + ConfigFingerprint: fingerprint, Phase: Installing, HostPrefix: hostPrefix, + }, nil +} + +// Fingerprint hashes canonical JSON supplied before ephemeral credentials are +// resolved. +// +// The caller decides what canonical means, and the hash is over exactly the +// bytes it is given. A release that adds a field to the fingerprinted struct +// changes the hash of every host that did not have it, and each of those reads +// as a different installation demanding an explicit reset. An optional field +// therefore has to carry omitempty and be absent at its default, so records +// written before it existed keep hashing the same way. +// +// TestBootstrapV1CompatibilityFixtures enforces this: its fixtures carry a +// literal fingerprint, so any change to what is hashed fails there rather than +// on upgraded hosts. +func Fingerprint(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) } + +type Disposition int + +const ( + Fresh Disposition = iota + Resume + AlreadyComplete +) + +func decide(r Record, loadErr error, machine, fingerprint string) (Disposition, error) { + if strings.TrimSpace(machine) == "" || strings.TrimSpace(fingerprint) == "" { + return Fresh, fmt.Errorf("bootstrap identity is required") + } + + if errors.Is(loadErr, ErrNotFound) { + return Fresh, nil + } + + if loadErr != nil { + return Fresh, fmt.Errorf("cannot read installation ownership: %w", loadErr) + } + + if err := r.Validate(); err != nil { + return Fresh, err + } + + if r.Phase == Resetting { + return Fresh, fmt.Errorf("reset is incomplete; run unbounded-agent reset again") + } + + if r.MachineName != machine || r.ConfigFingerprint != fingerprint { + return Fresh, fmt.Errorf("installation intent differs; explicit reset is required") + } + + if r.Phase == Complete { + return AlreadyComplete, nil + } + + return Resume, nil +} + +// Admit reads ownership and classifies a bootstrap attempt. Callers that mutate +// the host must hold the installation lock around this call. +func Admit(store *Store, machine, fingerprint string) (Record, Disposition, error) { + r, loadErr := store.Load() + + disposition, err := decide(r, loadErr, machine, fingerprint) + if err != nil { + return Record{}, Fresh, err + } + + return r, disposition, nil +} diff --git a/cmd/agent/internal/installstate/store_test.go b/cmd/agent/internal/installstate/store_test.go new file mode 100644 index 000000000..6877cf98a --- /dev/null +++ b/cmd/agent/internal/installstate/store_test.go @@ -0,0 +1,263 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package installstate + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func testStore(t *testing.T) *Store { + t.Helper() + dir := t.TempDir() + + return NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "install.lock")) +} + +func TestStoreLifecycle(t *testing.T) { + t.Parallel() + s := testStore(t) + require.NoError(t, s.Remove()) + _, err := s.Load() + require.ErrorIs(t, err, ErrNotFound) + r, err := NewRecord("machine", Fingerprint([]byte(`{"machineName":"machine"}`)), "") + require.NoError(t, err) + require.NoError(t, s.Save(r)) + loaded, err := s.Load() + require.NoError(t, err) + require.Equal(t, r, loaded) + + info, err := os.Stat(s.statePath()) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + require.NoError(t, s.MarkComplete(r)) + loaded, err = s.Load() + require.NoError(t, err) + require.Equal(t, Complete, loaded.Phase) + require.NoError(t, s.Remove()) + _, err = s.Load() + require.ErrorIs(t, err, ErrNotFound) +} + +func TestOwnershipAdmission(t *testing.T) { + t.Parallel() + + r, err := NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + + for _, phase := range []Phase{Installing, Complete, Resetting} { + t.Run(string(phase), func(t *testing.T) { + r := r + r.Phase = phase + + disposition, err := decide(r, nil, r.MachineName, r.ConfigFingerprint) + if phase == Resetting { + require.Error(t, err) + return + } + + require.NoError(t, err) + + want := Resume + if phase == Complete { + want = AlreadyComplete + } + + require.Equal(t, want, disposition) + + _, err = decide(r, nil, "other", r.ConfigFingerprint) + require.Error(t, err) + _, err = decide(r, nil, r.MachineName, "other") + require.Error(t, err) + }) + } + + _, err = decide(Record{}, ErrNotFound, "", "") + require.Error(t, err) +} + +func TestStoreRejectsCorruptAndOrphanedOwnership(t *testing.T) { + t.Parallel() + + for _, data := range []string{"{", "null", `{}`, `{"schemaVersion":2}`, `{"schemaVersion":1,"installID":"id","machineName":"machine","configFingerprint":"f"}`} { + t.Run(data, func(t *testing.T) { + s := testStore(t) + require.NoError(t, os.MkdirAll(s.Root(), 0o755)) + require.NoError(t, os.WriteFile(s.statePath(), []byte(data), 0o600)) + r, err := s.Load() + require.Error(t, err) + _, err = decide(r, err, "machine", "f") + require.Error(t, err) + }) + } +} + +func TestInstallationLockSurvivesStateRemoval(t *testing.T) { + t.Parallel() + s := testStore(t) + lock, err := s.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + + r, err := NewRecord("machine", "f", "") + require.NoError(t, err) + require.NoError(t, s.Save(r)) + require.NoError(t, s.Remove()) + _, err = s.AcquireLock() + require.ErrorIs(t, err, ErrLockHeld) + require.NoError(t, lock.Release()) + + second, err := s.AcquireLock() + require.NoError(t, err) + require.NoError(t, second.Release()) +} + +// TestRemoveRestoresOwnershipWhenUndurable covers the window where the record +// is unlinked but the directory entry never reaches disk. Reporting the error +// while leaving the removal in page cache would let the next start be admitted +// as a fresh install onto a host that was only partially torn down. +func TestRemoveRestoresOwnershipWhenUndurable(t *testing.T) { + t.Parallel() + + s := testStore(t) + r, err := NewRecord("machine", "f", "") + require.NoError(t, err) + + r.Phase = Resetting + require.NoError(t, s.Save(r)) + + failure := errors.New("sync failed") + s.syncDir = func(string) error { return failure } + + require.ErrorIs(t, s.Remove(), failure) + + restored, err := s.Load() + require.NoError(t, err) + require.Equal(t, r, restored) + + // Reset stays incomplete, so a retry is refused rather than admitted fresh. + _, _, err = Admit(s, r.MachineName, r.ConfigFingerprint) + require.Error(t, err) + + // Once the removal can be made durable, ownership is released. + s.syncDir = func(string) error { return nil } + require.NoError(t, s.Remove()) + _, err = s.Load() + require.ErrorIs(t, err, ErrNotFound) +} + +// A record that cannot be loaded grants no usable ownership, so a failed +// removal reports the error without resurrecting it. +func TestRemoveDoesNotRestoreUnusableOwnership(t *testing.T) { + t.Parallel() + + s := testStore(t) + require.NoError(t, os.MkdirAll(s.Root(), 0o755)) + require.NoError(t, os.WriteFile(s.statePath(), []byte("{"), 0o600)) + + failure := errors.New("sync failed") + s.syncDir = func(string) error { return failure } + + require.ErrorIs(t, s.Remove(), failure) + _, err := s.Load() + require.ErrorIs(t, err, ErrNotFound) +} + +func TestMutationAdmission(t *testing.T) { + t.Parallel() + + for _, phase := range []Phase{"", Installing, Complete, Resetting} { + t.Run(string(phase), func(t *testing.T) { + s := testStore(t) + + if phase != "" { + r, err := NewRecord("machine", "f", "") + require.NoError(t, err) + + r.Phase = phase + require.NoError(t, s.Save(r)) + } + + lock, err := s.AcquireMutationLock() + if phase == "" || phase == Complete { + require.NoError(t, err) + _, err = s.AcquireMutationLock() + require.ErrorIs(t, err, ErrLockHeld) + require.NoError(t, lock.Release()) + } else { + require.Error(t, err) + } + + lock, err = s.AcquireLock() + require.NoError(t, err, "failed admission must release the lock for reset") + require.NoError(t, lock.Release()) + }) + } +} + +// TestStoreIgnoresUnknownFields pins a property the record format depends on +// for cross-version upgrades, and which nothing else asserts. +// +// Fields are ignored rather than rejected, so a record written by a newer agent +// stays readable by an older one. A release that adds an optional field would +// otherwise brick every host that later ran an agent predating it: the record +// would fail to parse, start would refuse it, and reset would be the only way +// out. Adding DisallowUnknownFields would be the intuitive hardening and would +// take that guarantee away. +func TestStoreIgnoresUnknownFields(t *testing.T) { + t.Parallel() + + s := testStore(t) + require.NoError(t, os.MkdirAll(s.Root(), 0o755)) + require.NoError(t, os.WriteFile(s.statePath(), []byte( + `{"schemaVersion":1,"installID":"id","machineName":"machine","configFingerprint":"f","phase":"installing","fieldFromALaterRelease":"value"}`, + ), 0o600)) + + r, err := s.Load() + require.NoError(t, err, "an unknown field must not make a record unreadable") + require.Equal(t, Installing, r.Phase) + require.Equal(t, "machine", r.MachineName) + + disposition, err := decide(r, nil, "machine", "f") + require.NoError(t, err) + require.Equal(t, Resume, disposition, "the record must still be usable, not merely parseable") +} + +// TestRecordCarriesTheInstallationPrefix covers what the prefix is recorded +// for: teardown on a host where bootstrap failed before the node started. +// +// The applied config carries the same value but does not exist until the node +// runs, so on a half-built host this record is the only thing that knows where +// the agent put its files. Absent means the default, which is what a host +// installed before the prefix existed actually has on disk. +func TestRecordCarriesTheInstallationPrefix(t *testing.T) { + t.Parallel() + + s := testStore(t) + + prefixed, err := NewRecord("machine", "f", "/opt/unbounded") + require.NoError(t, err) + require.NoError(t, s.Save(prefixed)) + + loaded, err := s.Load() + require.NoError(t, err) + require.Equal(t, "/opt/unbounded", loaded.HostPrefix) + require.NoError(t, loaded.Validate()) + + // A default installation records nothing, so its record is byte-identical + // to one written before the field existed and stays readable by an agent + // that predates it. + def, err := NewRecord("machine", "f", "") + require.NoError(t, err) + + encoded, err := json.Marshal(def) + require.NoError(t, err) + require.NotContains(t, string(encoded), "hostPrefix", + "a default installation must not write the field, or older agents see a record they did not write") +} diff --git a/cmd/kubectl-unbounded/app/ignition.go b/cmd/kubectl-unbounded/app/ignition.go new file mode 100644 index 000000000..0e05a4038 --- /dev/null +++ b/cmd/kubectl-unbounded/app/ignition.go @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "encoding/base64" + "fmt" + "net/url" + "strings" +) + +// Ignition configuration types, covering only the subset this command emits. +// +// These are hand-written rather than taken from github.com/coreos/ignition to +// avoid a dependency carrying the whole specification for the handful of fields +// used here. The schema version is pinned and asserted by tests. +const ignitionSpecVersion = "3.4.0" + +// File modes are serialized as decimal integers in an Ignition config. +const ( + ignitionModeConfig = 0o600 + ignitionModeScript = 0o755 + ignitionModeData = 0o644 + ignitionModeDir = 0o755 +) + +type ignitionConfig struct { + Ignition ignitionVersion `json:"ignition"` + Storage *ignitionStorage `json:"storage,omitempty"` + Systemd *ignitionSystemd `json:"systemd,omitempty"` +} + +type ignitionVersion struct { + Version string `json:"version"` +} + +type ignitionStorage struct { + Directories []ignitionDirectory `json:"directories,omitempty"` + Files []ignitionFile `json:"files,omitempty"` +} + +// ignitionDirectory declares a directory Ignition creates before writing files +// into it. Ignition creates parents implicitly, so this exists to pin the mode +// of the agent's bin directory rather than to make the write succeed. +type ignitionDirectory struct { + Path string `json:"path"` + Mode int `json:"mode,omitempty"` +} + +type ignitionFile struct { + Path string `json:"path"` + Mode int `json:"mode,omitempty"` + Overwrite *bool `json:"overwrite,omitempty"` + Contents ignitionContents `json:"contents"` +} + +type ignitionContents struct { + Source string `json:"source"` + Verification *ignitionVerification `json:"verification,omitempty"` +} + +type ignitionVerification struct { + // Hash is "-", for example "sha256-abc123...". + Hash string `json:"hash"` +} + +type ignitionSystemd struct { + Units []ignitionUnit `json:"units,omitempty"` +} + +type ignitionUnit struct { + Name string `json:"name"` + Enabled *bool `json:"enabled,omitempty"` + Contents string `json:"contents,omitempty"` +} + +// ignitionDataURL encodes content as a data URL, which is how Ignition carries +// inline file contents. +func ignitionDataURL(content string) string { + return "data:;base64," + base64.StdEncoding.EncodeToString([]byte(content)) +} + +// ignitionRemoteFetchable reports whether Ignition can fetch a source itself. +// +// Ignition understands http, https, tftp, s3, arn, gs and data. It does not +// understand oci, which the agent resolves through its own artifact source. +// A source Ignition cannot fetch has to be left to the agent, which means the +// file lands after dbus has already started. +func ignitionRemoteFetchable(source string) bool { + parsed, err := url.Parse(strings.TrimSpace(source)) + if err != nil { + return false + } + + switch parsed.Scheme { + case "http", "https", "tftp", "s3", "arn", "gs": + return true + default: + return false + } +} + +// ignitionHashFromSHA256 converts a hex digest into the form Ignition expects. +func ignitionHashFromSHA256(hex string) (string, error) { + trimmed := strings.TrimSpace(hex) + + // Accept a plain digest or the first field of sha256sum output. + if fields := strings.Fields(trimmed); len(fields) > 0 { + trimmed = fields[0] + } + + if len(trimmed) != 64 { + return "", fmt.Errorf("sha256 digest must be 64 hex characters, got %d", len(trimmed)) + } + + for _, r := range trimmed { + isHex := (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') + if !isHex { + return "", fmt.Errorf("sha256 digest contains a non-hex character %q", r) + } + } + + return "sha256-" + strings.ToLower(trimmed), nil +} diff --git a/cmd/kubectl-unbounded/app/ignition_test.go b/cmd/kubectl-unbounded/app/ignition_test.go new file mode 100644 index 000000000..bae56575d --- /dev/null +++ b/cmd/kubectl-unbounded/app/ignition_test.go @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "encoding/base64" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestIgnitionSpecVersionIsPinned guards the one constant an operator cannot +// recover from being wrong. +// +// Ignition refuses a config whose version it does not implement, and it refuses +// it on first boot with no shell and no agent yet installed. There is nothing on +// the host to report the mismatch, so the failure presents as a machine that +// provisioned into nothing. +func TestIgnitionSpecVersionIsPinned(t *testing.T) { + t.Parallel() + + require.Equal(t, "3.4.0", ignitionSpecVersion) +} + +// TestIgnitionDataURLRoundTrips covers how inline file contents reach the host. +// Ignition reads them from a data URL, so anything lost in the encoding is lost +// silently: the file appears, with the wrong bytes in it. +func TestIgnitionDataURLRoundTrips(t *testing.T) { + t.Parallel() + + for _, content := range []string{ + "", + "plain", + "{\n \"MachineName\": \"kube1\"\n}\n", + "trailing newline\n", + "unicode: \u00e9\u00e8\u00ea and emoji bytes", + "null\x00byte", + } { + t.Run(strings.SplitN(content, "\n", 2)[0], func(t *testing.T) { + t.Parallel() + + url := ignitionDataURL(content) + require.True(t, strings.HasPrefix(url, "data:;base64,"), "got %q", url) + + decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(url, "data:;base64,")) + require.NoError(t, err) + require.Equal(t, content, string(decoded)) + }) + } +} + +// TestIgnitionRemoteFetchable pins which sources Ignition can retrieve itself. +// +// This decides where a file lands in the boot. A fetchable source is written +// before dbus starts; anything else has to wait for the agent, which is after. +// Reading it the wrong way round produces a config Ignition rejects, or a file +// that silently arrives too late to be useful. +func TestIgnitionRemoteFetchable(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + source string + want bool + }{ + {"https://example.test/unbounded-agent", true}, + {"http://example.test/unbounded-agent", true}, + {"tftp://example.test/unbounded-agent", true}, + {"s3://bucket/unbounded-agent", true}, + {"arn:aws:s3:::bucket/unbounded-agent", true}, + {"gs://bucket/unbounded-agent", true}, + {" https://example.test/spaced ", true}, + + // oci is the one that matters: it is the agent's own artifact scheme, + // and Ignition has no idea what to do with it. + {"oci://ghcr.io/azure/unbounded-agent:v1", false}, + {"file:///tmp/unbounded-agent", false}, + {"ftp://example.test/unbounded-agent", false}, + {"/usr/local/bin/unbounded-agent", false}, + {"", false}, + {"://not a url", false}, + } { + t.Run(tc.source, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.want, ignitionRemoteFetchable(tc.source)) + }) + } +} + +// TestIgnitionHashFromSHA256 covers the digest conversion, including the +// sha256sum shape an operator is most likely to paste in. +func TestIgnitionHashFromSHA256(t *testing.T) { + t.Parallel() + + const digest = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + t.Run("plain digest", func(t *testing.T) { + t.Parallel() + + got, err := ignitionHashFromSHA256(digest) + require.NoError(t, err) + require.Equal(t, "sha256-"+digest, got) + }) + + t.Run("sha256sum output", func(t *testing.T) { + t.Parallel() + + got, err := ignitionHashFromSHA256(digest + " unbounded-agent\n") + require.NoError(t, err) + require.Equal(t, "sha256-"+digest, got) + }) + + t.Run("uppercase is normalized", func(t *testing.T) { + t.Parallel() + + got, err := ignitionHashFromSHA256(strings.ToUpper(digest)) + require.NoError(t, err) + require.Equal(t, "sha256-"+digest, got, "Ignition compares the hash as written") + }) + + for _, tc := range []struct{ name, input string }{ + {"empty", ""}, + {"too short", digest[:63]}, + {"too long", digest + "0"}, + {"non-hex", strings.Replace(digest, "9", "z", 1)}, + } { + t.Run("rejects "+tc.name, func(t *testing.T) { + t.Parallel() + + _, err := ignitionHashFromSHA256(tc.input) + require.Error(t, err, "a malformed digest must fail here, not on the host at first boot") + }) + } +} + +// TestIgnitionConfigOmitsEmptySections pins that the emitted document contains +// only what was asked for. +// +// Ignition validates the whole config before acting on any of it, so an empty +// section serialized as null or [] can reject a config that is otherwise fine, +// again on a host with nothing available to say so. +func TestIgnitionConfigOmitsEmptySections(t *testing.T) { + t.Parallel() + + encoded, err := json.Marshal(ignitionConfig{Ignition: ignitionVersion{Version: ignitionSpecVersion}}) + require.NoError(t, err) + + require.JSONEq(t, `{"ignition":{"version":"3.4.0"}}`, string(encoded)) + require.NotContains(t, string(encoded), "storage") + require.NotContains(t, string(encoded), "systemd") +} + +// TestIgnitionFileModesSerializeAsDecimal covers a trap in the format: Ignition +// file modes are decimal integers, and Go's octal literals are easy to read as +// if they were being emitted verbatim. +// +// A mode written as 600 rather than 0o600 is 0o1130 on disk, which for the +// agent config means credentials readable by everyone. +func TestIgnitionFileModesSerializeAsDecimal(t *testing.T) { + t.Parallel() + + encoded, err := json.Marshal(ignitionFile{ + Path: "/etc/unbounded/agent/config.json", + Mode: ignitionModeConfig, + Contents: ignitionContents{Source: ignitionDataURL("{}")}, + }) + require.NoError(t, err) + + // 0o600 is 384 decimal. Asserting the number rather than the constant is + // the point: it is what a reader of the emitted config would see. + require.Contains(t, string(encoded), `"mode":384`) + + require.Equal(t, 0o600, ignitionModeConfig, "the agent config carries credentials") + require.Equal(t, 0o755, ignitionModeScript) + require.Equal(t, 0o644, ignitionModeData) + require.Equal(t, 0o755, ignitionModeDir) +} diff --git a/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go b/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go index 6f0c740ec..e2dfc1c17 100644 --- a/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go +++ b/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go @@ -30,6 +30,7 @@ import ( "github.com/Azure/unbounded/internal/kube" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/config" + "github.com/Azure/unbounded/pkg/agent/goalstates" ) //go:embed assets/node-bootstrap/script.sh @@ -47,6 +48,12 @@ const ( // variantCloudInit produces a cloud-init user-data document. variantCloudInit bootstrapVariant = "cloud-init" + + // variantIgnition produces an Ignition config. It is the only mechanism + // image-based hosts such as Azure Container Linux consume: they ship no + // cloud-init at all, so a cloud-init payload passed as user data is never + // acted on and nothing reports an error. + variantIgnition bootstrapVariant = "ignition" ) func parseBootstrapVariant(s string) (bootstrapVariant, error) { @@ -55,8 +62,10 @@ func parseBootstrapVariant(s string) (bootstrapVariant, error) { return variantScript, nil case variantCloudInit: return variantCloudInit, nil + case variantIgnition: + return variantIgnition, nil default: - return "", fmt.Errorf("unknown variant %q (valid: script, cloud-init)", s) + return "", fmt.Errorf("unknown variant %q (valid: script, cloud-init, ignition)", s) } } @@ -103,8 +112,22 @@ type manualBootstrapHandler struct { // agentURL is a fully qualified override for the unbounded-agent download // URL. When set it takes precedence over agentVersion and agentBaseURL. + // + // The ignition variant requires it, and requires it to name the bare agent + // binary rather than the release tarball: Ignition fetches files, it does + // not extract archives. agentURL string + // agentSHA256 is the expected digest of the agent binary. Required by the + // ignition variant, which fetches the binary without a script that could + // verify it afterwards. + agentSHA256 string + + // hostPrefix is the installation prefix for the agent's own host-side + // files. Required by the ignition variant, whose target hosts mount /usr + // read-only. + hostPrefix string + // agentBaseURL overrides the base URL used to construct the download URL // for the unbounded-agent. Useful for self-hosted release mirrors. Must // follow the same layout as GitHub releases @@ -186,6 +209,8 @@ func (h *manualBootstrapHandler) execute(ctx context.Context) error { switch bootstrapVariant(h.variant) { case variantCloudInit: output, err = h.renderCloudInit(cfg) + case variantIgnition: + output, err = h.renderIgnition(cfg) default: output, err = h.renderScript(cfg) } @@ -348,6 +373,14 @@ func (h *manualBootstrapHandler) validate() error { return errors.New("site name is required") } + // Rejected here rather than on the host. The prefix is interpolated into + // generated systemd units and into a shell script, neither of which quotes + // it, and a value that breaks those does so on a machine with no operator + // watching and no way to report it. + if err := config.ValidateHostPrefix(h.hostPrefix); err != nil { + return fmt.Errorf("invalid host prefix: %w", err) + } + // The machine name is optional. When omitted, the unbounded-agent resolves // it at startup from the AGENT_MACHINE_NAME environment variable or the host // hostname, which lets a single bootstrap payload be reused across many @@ -481,6 +514,12 @@ func (h *manualBootstrapHandler) buildAgentConfig(ctx context.Context) (*provisi }) cfg.Kubelet.NodeIP = strings.TrimSpace(h.nodeIP) + + // Carried in the config rather than only in the generated output, because + // the agent re-reads it long after bootstrap: the daemon and the nspawn + // lifecycle hooks are started by systemd and cannot inherit it from the + // environment that provisioned the host. + cfg.HostPrefix = strings.TrimSpace(h.hostPrefix) if source := strings.TrimSpace(h.offlineArtifactsSource); source != "" { cfg.OfflineArtifacts = &provision.AgentOfflineArtifacts{Source: source} } @@ -712,7 +751,9 @@ Examples: cmd.Flags().StringVar(&handler.kubernetesVersion, "kubernetes-version", "", "Override the Kubernetes version (default: auto-detected from API server)") cmd.Flags().StringVar(&handler.variant, "variant", "script", "Output format: script or cloud-init") cmd.Flags().StringVar(&handler.agentVersion, "agent-version", "", "Pin the unbounded-agent release tag to download on the host (default: latest GitHub release)") - cmd.Flags().StringVar(&handler.agentURL, "agent-url", "", "Fully qualified download URL for the unbounded-agent tarball (overrides --agent-version and --agent-base-url)") + cmd.Flags().StringVar(&handler.agentURL, "agent-url", "", "Fully qualified download URL for the unbounded-agent tarball (overrides --agent-version and --agent-base-url). With --variant ignition this must name the bare binary, not the tarball") + cmd.Flags().StringVar(&handler.agentSHA256, "agent-sha256", "", "SHA-256 digest of the agent binary, published in checksums.txt. Required with --variant ignition") + cmd.Flags().StringVar(&handler.hostPrefix, "host-prefix", "", "Installation prefix for the agent's own host-side files. Required with --variant ignition, whose target hosts mount /usr read-only") cmd.Flags().StringVar(&handler.agentBaseURL, "agent-base-url", "", "Base URL for unbounded-agent release downloads (default: https://github.com/Azure/unbounded/releases). Use this to self-host or mirror release assets") // Rootfs binary download overrides. See `kubectl unbounded machine register --help` @@ -803,3 +844,184 @@ func resolveBootstrapToken(ctx context.Context, logger *slog.Logger, kubeCli kub return nil, fmt.Errorf("no bootstrap token found for site %q and no tokens available in the cluster (run 'kubectl unbounded site init' first)", siteName) } + +// Paths the Ignition variant writes on the target host. The config path is the +// one `unbounded-agent start` reads from UNBOUNDED_AGENT_CONFIG_FILE. +const ( + ignitionAgentConfigPath = "/etc/unbounded/agent/config.json" + ignitionAgentBinaryName = "unbounded-agent" +) + +func boolPtr(v bool) *bool { return &v } + +// renderIgnition emits an Ignition config that provisions the host with no +// shell and no operator present. +// +// Ignition is declarative and runs from the initramfs, so everything it writes +// is in place before any service starts. That is what lets the agent config, +// the agent binary and the bootstrap unit all be present on the first boot +// rather than fetched by something running on the host. +func (h *manualBootstrapHandler) renderIgnition(cfg *provision.UnboundedAgentConfig) (string, error) { + configJSON, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return "", fmt.Errorf("marshaling agent config: %w", err) + } + + binaryFile, err := h.ignitionAgentBinaryFile(cfg) + if err != nil { + return "", err + } + + config := ignitionConfig{ + Ignition: ignitionVersion{Version: ignitionSpecVersion}, + Storage: &ignitionStorage{ + Directories: []ignitionDirectory{{ + Path: ignitionAgentBinDir(cfg), + Mode: ignitionModeDir, + }}, + Files: []ignitionFile{ + { + Path: ignitionAgentConfigPath, + Mode: ignitionModeConfig, + Overwrite: boolPtr(true), + Contents: ignitionContents{Source: ignitionDataURL(string(configJSON) + "\n")}, + }, + *binaryFile, + }, + }, + Systemd: &ignitionSystemd{Units: []ignitionUnit{{ + Name: goalstates.FirstBootBootstrapUnit, + Enabled: boolPtr(true), + Contents: h.ignitionBootstrapUnitContents(cfg), + }}}, + } + + rendered, err := json.MarshalIndent(config, "", " ") + if err != nil { + return "", fmt.Errorf("marshaling ignition config: %w", err) + } + + return string(rendered) + "\n", nil +} + +// ignitionAgentBinDir returns the directory the agent binary is placed in, +// derived from the configured host prefix so that a host with a read-only /usr +// puts it somewhere writable. +func ignitionAgentBinDir(cfg *provision.UnboundedAgentConfig) string { + prefix := "" + if cfg != nil { + prefix = cfg.HostPrefix + } + + return goalstates.ResolveHostPaths(prefix).BinDir +} + +// ignitionAgentBinaryFile fetches the agent binary straight to its final +// location, verified against a caller-supplied digest. +// +// Every input here is required rather than defaulted, because this variant has +// no shell to fall back on. Ignition declares state; it cannot resolve a +// version, detect an architecture, or extract an archive at boot, so the +// artifact has to be named exactly and the host has no way to report that it +// was not. +func (h *manualBootstrapHandler) ignitionAgentBinaryFile(cfg *provision.UnboundedAgentConfig) (*ignitionFile, error) { + source := strings.TrimSpace(h.agentURL) + digest := strings.TrimSpace(h.agentSHA256) + + // Ignition writes the binary itself, so an unset prefix would place it + // under the default /usr/local and fail at first boot on exactly the + // immutable hosts this variant exists to serve. Refuse at render time, + // where the message can say what to do. + if cfg == nil || strings.TrimSpace(cfg.HostPrefix) == "" { + return nil, fmt.Errorf("--host-prefix is required with --variant %s: Ignition places the agent binary itself, and the default prefix /usr/local is read-only on immutable hosts", variantIgnition) + } + + if source == "" { + return nil, fmt.Errorf("--agent-url is required with --variant %s, and must point at the bare agent binary rather than the release tarball, because Ignition cannot extract an archive", variantIgnition) + } + + if !ignitionRemoteFetchable(source) { + return nil, fmt.Errorf("--agent-url %q cannot be fetched by Ignition; use an http, https, tftp, s3, arn, or gs URL", source) + } + + if digest == "" { + return nil, fmt.Errorf("--agent-sha256 is required with --variant %s; the digest for each release binary is published in checksums.txt", variantIgnition) + } + + hash, err := ignitionHashFromSHA256(digest) + if err != nil { + return nil, fmt.Errorf("invalid --agent-sha256: %w", err) + } + + return &ignitionFile{ + Path: ignitionAgentBinDir(cfg) + "/" + ignitionAgentBinaryName, + Mode: ignitionModeScript, + Overwrite: boolPtr(true), + Contents: ignitionContents{ + Source: source, + Verification: &ignitionVerification{Hash: hash}, + }, + }, nil +} + +// ignitionBootstrapUnitContents renders the oneshot unit that bootstraps the +// agent on first boot. +// +// The unit runs the agent directly rather than a shell script. Ignition has +// already placed and verified the binary, so a script here would only +// re-implement that imperatively. +// +// It carries no completion condition, and so runs on every boot. That is +// deliberate. A condition needs a marker file, and a marker is a second record +// of completion that can disagree with the ownership record the agent already +// keeps; the agent's own admission answers the same question from the record, +// which is written before the first host mutation and therefore cannot be +// missing on a host that started installing. Both commands below return +// immediately once that record says the installation is complete: preflight +// reports an empty result and start verifies the daemon and repairs it if it +// is not running, neither resolving artifacts nor touching the network. The +// cost is two short-lived processes per boot, and the benefit is that a node +// whose daemon was stopped or damaged comes back on reboot. +func (h *manualBootstrapHandler) ignitionBootstrapUnitContents(cfg *provision.UnboundedAgentConfig) string { + binary := ignitionAgentBinDir(cfg) + "/" + ignitionAgentBinaryName + + var b strings.Builder + + b.WriteString("[Unit]\n") + b.WriteString("Description=Bootstrap the unbounded agent\n") + b.WriteString("Wants=network-online.target\n") + // The agent downloads the node rootfs and the Kubernetes, CRI and CNI + // binaries, so it needs the network even though Ignition already fetched + // the agent itself. Ordering after systemd-sysext keeps any extension + // merged before the agent runs. + b.WriteString("After=network-online.target nss-lookup.target systemd-sysext.service\n") + b.WriteString("ConditionPathExists=" + binary + "\n") + // Retry indefinitely rather than giving up after systemd's default start + // limit. Bootstrap has no later opportunity to run, so a burst of early + // failures must not permanently disable it. + b.WriteString("StartLimitIntervalSec=0\n\n") + + b.WriteString("[Service]\n") + b.WriteString("Type=oneshot\n") + b.WriteString("RemainAfterExit=yes\n") + // network-online.target only means a link is configured, not that DNS + // resolves. On a first boot the agent can start before systemd-resolved is + // answering and fail with an unresolved host, so retry rather than ordering + // against something that does not carry that guarantee. Verified on systemd + // 255 that Type=oneshot honors Restart=. + b.WriteString("Restart=on-failure\n") + b.WriteString("RestartSec=10s\n") + // `unbounded-agent start` has no --config flag and reads this variable. + b.WriteString("Environment=UNBOUNDED_AGENT_CONFIG_FILE=" + ignitionAgentConfigPath + "\n") + + // Preflight runs as ExecStartPre so a failure is reported against this unit + // before any host mutation, and shows up in its status rather than being + // buried in a script's output. + b.WriteString("ExecStartPre=" + binary + " preflight\n") + b.WriteString("ExecStart=" + binary + " start\n\n") + + b.WriteString("[Install]\n") + b.WriteString("WantedBy=multi-user.target\n") + + return b.String() +} diff --git a/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go b/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go index 2ced9a19d..6d8cf2d3d 100644 --- a/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go +++ b/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go @@ -20,6 +20,7 @@ import ( "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/config" + "github.com/Azure/unbounded/pkg/agent/goalstates" ) // --------------------------------------------------------------------------- @@ -1189,3 +1190,199 @@ func TestManualBootstrapHandler_BuildAgentConfig_AdditionalHostDevices(t *testin require.Equal(t, []string{"/dev/uinput", "char-input"}, cfg.AdditionalHostDevices) } + +// ignitionTestConfig returns an agent config shaped like one the command would +// build, with the prefix the ignition variant requires. +func ignitionTestConfig(prefix string) *provision.UnboundedAgentConfig { + return &provision.UnboundedAgentConfig{ + AgentConfig: provision.AgentConfig{ + MachineName: "test-node", + HostPrefix: prefix, + Cluster: provision.AgentClusterConfig{ + CaCertBase64: "dGVzdA==", + ClusterDNS: "10.0.0.10", + Version: "v1.30.0", + }, + Kubelet: provision.AgentKubeletConfig{ + ApiServer: "https://api-server:6443", + Auth: provision.KubeletAuthInfo{BootstrapToken: "abc123.0123456789abcdef"}, + }, + }, + } +} + +const ignitionTestDigest = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + +func ignitionTestHandler() *manualBootstrapHandler { + return &manualBootstrapHandler{ + logger: discardLogger(), + agentURL: "https://example.test/unbounded-agent-linux-amd64", + agentSHA256: ignitionTestDigest, + hostPrefix: "/opt/unbounded", + } +} + +// TestRenderIgnitionPlacesEverythingBeforeFirstBoot covers the property the +// whole variant exists for: on a host with no shell and no operator, every file +// the agent needs is already present when the unit starts. +func TestRenderIgnitionPlacesEverythingBeforeFirstBoot(t *testing.T) { + t.Parallel() + + out, err := ignitionTestHandler().renderIgnition(ignitionTestConfig("/opt/unbounded")) + require.NoError(t, err) + + var cfg ignitionConfig + require.NoError(t, json.Unmarshal([]byte(out), &cfg), "emitted document must be valid JSON") + require.Equal(t, ignitionSpecVersion, cfg.Ignition.Version) + + require.NotNil(t, cfg.Storage) + + paths := map[string]ignitionFile{} + for _, f := range cfg.Storage.Files { + paths[f.Path] = f + } + + agentConfig, ok := paths[ignitionAgentConfigPath] + require.True(t, ok, "the agent config must be written, got %v", paths) + require.Equal(t, ignitionModeConfig, agentConfig.Mode, "the agent config carries a bootstrap token") + + binary, ok := paths["/opt/unbounded/bin/unbounded-agent"] + require.True(t, ok, "the agent binary must land under the configured prefix, got %v", paths) + require.Equal(t, ignitionModeScript, binary.Mode) + require.Equal(t, "https://example.test/unbounded-agent-linux-amd64", binary.Contents.Source) + require.NotNil(t, binary.Contents.Verification, "an unattended host must not accept whatever the URL returns") + require.Equal(t, "sha256-"+ignitionTestDigest, binary.Contents.Verification.Hash) + + require.NotNil(t, cfg.Systemd) + require.Len(t, cfg.Systemd.Units, 1) + require.Equal(t, goalstates.FirstBootBootstrapUnit, cfg.Systemd.Units[0].Name) + require.NotNil(t, cfg.Systemd.Units[0].Enabled) + require.True(t, *cfg.Systemd.Units[0].Enabled, "an unenabled unit never runs and nothing reports it") +} + +// TestRenderIgnitionHonoursTheHostPrefix pins that every host-side path moves +// together. A binary under the prefix and a unit pointing at /usr/local would +// produce a host that provisions into a unit which cannot start. +func TestRenderIgnitionHonoursTheHostPrefix(t *testing.T) { + t.Parallel() + + h := ignitionTestHandler() + h.hostPrefix = "/var/lib/unbounded-agent" + + out, err := h.renderIgnition(ignitionTestConfig("/var/lib/unbounded-agent")) + require.NoError(t, err) + + require.Contains(t, out, "/var/lib/unbounded-agent/bin/unbounded-agent") + require.NotContains(t, out, "/usr/local/bin/unbounded-agent", + "nothing may resolve to the default prefix once one is configured") +} + +// TestRenderIgnitionRefusesRatherThanGuessing covers each input this variant +// cannot default. +// +// Ignition declares state: it cannot resolve a version, detect an architecture, +// or extract an archive at boot. Every one of these failures would otherwise +// land on a machine with no shell and no way to say what went wrong, so they +// are refused at render time where the message reaches a person. +func TestRenderIgnitionRefusesRatherThanGuessing(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mutate func(*manualBootstrapHandler) + prefix string + wantErr string + }{ + { + name: "no host prefix", + prefix: "", + wantErr: "--host-prefix is required", + }, + { + name: "no agent url", + mutate: func(h *manualBootstrapHandler) { h.agentURL = "" }, + wantErr: "--agent-url is required", + }, + { + name: "agent url Ignition cannot fetch", + mutate: func(h *manualBootstrapHandler) { h.agentURL = "oci://ghcr.io/azure/unbounded-agent:v1" }, + wantErr: "cannot be fetched by Ignition", + }, + { + name: "no digest", + mutate: func(h *manualBootstrapHandler) { h.agentSHA256 = "" }, + wantErr: "--agent-sha256 is required", + }, + { + name: "malformed digest", + mutate: func(h *manualBootstrapHandler) { h.agentSHA256 = "not-a-digest" }, + wantErr: "invalid --agent-sha256", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := ignitionTestHandler() + prefix := "/opt/unbounded" + + if tc.mutate != nil { + tc.mutate(h) + } else { + prefix = tc.prefix + } + + _, err := h.renderIgnition(ignitionTestConfig(prefix)) + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// TestIgnitionBootstrapUnitRunsOnEveryBoot pins the decision not to carry a +// completion condition. +// +// A condition needs a marker file, which is a second record of completion that +// can disagree with the ownership record the agent already keeps. Instead the +// unit runs every boot and the agent's own admission answers the question, +// returning immediately once the record says the installation is complete. The +// benefit is that a node whose daemon was stopped comes back on reboot. +func TestIgnitionBootstrapUnitRunsOnEveryBoot(t *testing.T) { + t.Parallel() + + unit := ignitionTestHandler().ignitionBootstrapUnitContents(ignitionTestConfig("/opt/unbounded")) + + require.NotContains(t, unit, "ConditionPathExists=!", + "a completion marker would be a second source of truth beside the ownership record") + + // The one condition that stays guards against running a binary Ignition + // failed to place, which would otherwise fail confusingly every boot. + require.Contains(t, unit, "ConditionPathExists=/opt/unbounded/bin/unbounded-agent") + + require.Contains(t, unit, "WantedBy=multi-user.target", "the unit has to be started on every boot for this to work") +} + +// TestIgnitionBootstrapUnitSurvivesEarlyBootRaces covers two failures seen on +// real hardware rather than reasoned about. +// +// network-online.target means a link is configured, not that DNS resolves: a +// unit can start in the same second the target is reached while +// systemd-resolved is still coming up, and fail on an unresolved host. And +// bootstrap has no later opportunity to run, so systemd's default start limit +// would turn a burst of early failures into a permanently disabled unit. +func TestIgnitionBootstrapUnitSurvivesEarlyBootRaces(t *testing.T) { + t.Parallel() + + unit := ignitionTestHandler().ignitionBootstrapUnitContents(ignitionTestConfig("/opt/unbounded")) + + require.Contains(t, unit, "Restart=on-failure", "DNS may not answer yet on the first attempt") + require.Contains(t, unit, "StartLimitIntervalSec=0", "bootstrap gets no second chance if systemd gives up on it") + require.Contains(t, unit, "Type=oneshot") + require.Contains(t, unit, "After=network-online.target nss-lookup.target systemd-sysext.service") + + // start reads the config path from the environment; it has no flag for it. + require.Contains(t, unit, "Environment=UNBOUNDED_AGENT_CONFIG_FILE="+ignitionAgentConfigPath) + + // Preflight runs before any host mutation and reports against this unit. + require.Contains(t, unit, "ExecStartPre=/opt/unbounded/bin/unbounded-agent preflight") + require.Contains(t, unit, "ExecStart=/opt/unbounded/bin/unbounded-agent start") +} diff --git a/docs/content/guides/agent.md b/docs/content/guides/agent.md index 3fdfe9c75..e8dc3d751 100644 --- a/docs/content/guides/agent.md +++ b/docs/content/guides/agent.md @@ -28,6 +28,81 @@ in sequence: containerd and kubelet inside it. Kubelet performs TLS bootstrapping against the API server using the configured bootstrap token. +## Retrying bootstrap and repairing the daemon + +Initial bootstrap records installation ownership before changing the host. If a +stage fails or the process is interrupted, rerun the same saved bootstrap script +or invoke `unbounded-agent start` with the same original configuration. + +Every stage runs again on every attempt, and each decides what it still has to +do by looking at the host. Packages already present are not reinstalled. A +rootfs a machine is already registered from is left in place rather than +rebuilt. A running nspawn machine is left running. Node service configuration is +rewritten, and the service is restarted only if that configuration actually +changed. The nftables ruleset a running node depends on is not flushed. + +This is why a retry is safe to run repeatedly, and why a host that was modified +between attempts is repaired rather than skipped past: nothing is assumed from +how far a previous attempt got. + +The ownership record is `/var/lib/unbounded/agent/install-state.json`. It is an +internal file, not a configuration input. Keep it intact when retrying. It +records which phase the installation is in, never how much of it has been done, +so it cannot fall out of step with the host. A different machine name, +Kubernetes version, rootfs image, or API server endpoint is rejected and +requires an explicit reset before a new initial installation. Credentials and +artifact locations can be refreshed for a retry. + +A retry that finds the node already running does not rewrite +`-applied-config.json`, because that file records what the running node +was built from and this attempt did not build it. Configuration that is allowed +to change between attempts but only takes effect at node registration, node +labels in particular, therefore reaches the node through the daemon's ordinary +drift repave rather than through the retry itself. + +After completion, the same `start` invocation checks required daemon files, +executable permissions, and enabled/active service state, and repairs the daemon +when they are missing or stopped. It does not compare unit contents or overwrite +working local unit customizations. This path uses the current applied +configuration, including after an ordinary repave has switched from kube1 to +kube2. It does not resolve the original node image or binary-download sources. +The bootstrap shell still needs its agent download; to repair without that +download, run an available agent executable directly with the original config: + +```bash +sudo env UNBOUNDED_AGENT_CONFIG_FILE=/path/to/original-agent-config.json \ + /path/to/unbounded-agent start +``` + +The agent daemon does not run while an installation is unfinished. If it starts +in that state, on a reboot or because an interrupted bootstrap had already +enabled it, it logs a warning and exits with status 69. The unit reports +`inactive`, not `failed`, and is not restarted; `systemctl status +unbounded-agent-daemon` showing `inactive (dead)` together with that warning +means the install still has to be completed, not that the daemon is broken. Run +the bootstrap again and the daemon starts with it. Do not reset the host to +clear this state. + +Reset also completes on a host where bootstrap never got far enough to install +the packages it inspects with, and on one whose ownership record cannot be +parsed. Both are cases where refusing would leave nothing able to clear the +record, so reset proceeds and removes it. + +Bootstrap, reset, node lifecycle operations, and binary activation share an +installation lock. Last-resort daemon rollback does not wait on these locks; +reset stops the recovery unit before teardown. A busy lock is retryable, and +MachineOperation handlers requeue instead of starting concurrent host mutations. +Reset retains a `resetting` record when cleanup fails; correct the reported error +and run reset again. Ownership is removed only after teardown and its filesystem +synchronization barriers succeed. The lock file in `/run` remains and must not be +deleted to force an operation through. + +Recovery applies to initial installations that carry an ownership record. +Existing installations without one retain ordinary daemon operations and reset +support; `start` requires a clean host before creating new ownership. Interrupted +ordinary repaves still use the existing repave model, without a persistent repave +recovery operation. + ## Configuration The agent reads a JSON config file whose path is set through the diff --git a/go.mod b/go.mod index 8f5ef57c3..e7fc9829f 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/fatih/color v1.19.0 github.com/fsnotify/fsnotify v1.10.1 github.com/go-logr/logr v1.4.4 + github.com/gofrs/flock v0.10.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/cel-go v0.29.2 github.com/google/go-github/v75 v75.0.0 @@ -175,7 +176,6 @@ require ( github.com/go-openapi/swag/yamlutils v0.27.1 // indirect github.com/gobuffalo/flect v1.0.3 // indirect github.com/goccy/go-json v0.10.6 // indirect - github.com/gofrs/flock v0.10.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect diff --git a/hack/agent/e2e-kind/e2e.py b/hack/agent/e2e-kind/e2e.py index ea414b291..e6bd4a641 100755 --- a/hack/agent/e2e-kind/e2e.py +++ b/hack/agent/e2e-kind/e2e.py @@ -65,6 +65,7 @@ import re import secrets import shutil +import shlex import subprocess import sys import textwrap @@ -836,11 +837,8 @@ def node_config_bootstrap_args(node_config: NodeConfig) -> list[str]: return args -def inject_kubelet_configuration(bootstrap_script: str, node_config: NodeConfig) -> str: - """Inject a scenario's kubelet configuration into generated agent config JSON.""" - if not node_config.kubelet_configuration: - return bootstrap_script - +def patch_agent_config(bootstrap_script: str, mutate: Callable[[dict], None]) -> str: + """Rewrite the agent config JSON embedded in a generated bootstrap script.""" start_marker = "cat > \"${UNBOUNDED_AGENT_CONFIG_FILE}\" <<'AGENT_CONFIG_EOF'\n" end_marker = "\nAGENT_CONFIG_EOF" prefix, separator, remainder = bootstrap_script.partition(start_marker) @@ -853,14 +851,22 @@ def inject_kubelet_configuration(bootstrap_script: str, node_config: NodeConfig) try: agent_config = json.loads(agent_config_json) - kubelet = agent_config["Kubelet"] + mutate(agent_config) except (KeyError, TypeError, json.JSONDecodeError) as exc: die(f"generated bootstrap script contains invalid agent config: {exc}") - kubelet["Configuration"] = node_config.kubelet_configuration - rendered_config = json.dumps(agent_config, indent=2) + return prefix + start_marker + json.dumps(agent_config, indent=2) + end_marker + suffix + + +def inject_kubelet_configuration(bootstrap_script: str, node_config: NodeConfig) -> str: + """Inject a scenario's kubelet configuration into generated agent config JSON.""" + if not node_config.kubelet_configuration: + return bootstrap_script + + def set_kubelet_configuration(agent_config: dict) -> None: + agent_config["Kubelet"]["Configuration"] = node_config.kubelet_configuration - return prefix + start_marker + rendered_config + end_marker + suffix + return patch_agent_config(bootstrap_script, set_kubelet_configuration) def log_active_node_config(node_config: NodeConfig) -> None: @@ -2059,6 +2065,19 @@ def run_agent(node_config: NodeConfig) -> None: log("Agent bootstrap completed") +def run_agent_recovery(node_config: NodeConfig) -> None: + """Inject a late bootstrap failure, then retry the identical generated input.""" + previous = os.environ.get("E2E_BOOTSTRAP_RECOVERY") + os.environ["E2E_BOOTSTRAP_RECOVERY"] = "1" + try: + run_agent(node_config) + finally: + if previous is None: + os.environ.pop("E2E_BOOTSTRAP_RECOVERY", None) + else: + os.environ["E2E_BOOTSTRAP_RECOVERY"] = previous + + def prepare_agent_artifacts() -> str: """Build agent artifacts and return the URL that serves the tarball.""" VM_DIR.mkdir(parents=True, exist_ok=True) @@ -2570,6 +2589,80 @@ def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: log("Running bootstrap script on VM...") log("This will download the agent, bootstrap the node, and join it to the Kind cluster.") env_prefix = f"AGENT_URL={agent_url} AGENT_DEBUG={AGENT_DEBUG}" + if os.environ.get("E2E_BOOTSTRAP_RECOVERY") == "1": + # Make a daemon asset unwritable after ownership admission. The node + # stage precedes the failing asset write, without a production failpoint. + inject = ( + "for i in $(seq 1 600); do " + "if test -f /var/lib/unbounded/agent/install-state.json; then " + "mkdir -p /usr/local/bin/unbounded-agent-daemon-recovery.sh; exit 0; fi; " + "sleep 1; done; exit 1" + ) + ssh_cmd("sudo systemd-run --unit=p6-bootstrap-injection --collect /bin/bash -c " + shlex.quote(inject)) + result = subprocess.run([ + "timeout", "1200", "ssh", *SSH_OPTS, SSH_TARGET, + f"sudo {env_prefix} /tmp/bootstrap.sh", + ], check=False) + if result.returncode == 0: + die("blocked daemon asset did not interrupt initial bootstrap") + snapshot = bounded_ssh( + "sudo cat /var/lib/unbounded/agent/install-state.json; " + "systemctl show systemd-nspawn@kube1.service --property=MainPID --value", + time.monotonic() + 30, check=True).stdout + state_text, pid = snapshot.rstrip().rsplit("\n", 1) + before = json.loads(state_text) + # The record says only that an installation is under way. It deliberately + # does not say how far it got, because that would be a claim about the + # host that could stop being true. What proves the failure landed late + # is the host itself: the node is up, so the retry has to converge + # around a running machine rather than rebuild underneath it. + if before["phase"] != "installing" or not pid.isdigit() or int(pid) <= 0: + die(f"bootstrap did not fail with a running node: {snapshot}") + + # The applied config records what actually configured the running node. + # Retry with a changed node label: admission still allows it, because + # labels are deliberately outside the installation fingerprint, but the + # node was started before the change and never saw it. Re-persisting it + # here would read as "no drift" forever after. The retry reapplies the + # node stage like every other, so what keeps the record still is that + # the node is already running with the old configuration and the label + # change is not one the stage acts on. + applied_config = "/etc/unbounded/agent/kube1-applied-config.json" + before_applied = bounded_ssh( + f"sudo sha256sum {applied_config}", time.monotonic() + 30, check=True).stdout.split()[0] + + def add_retry_label(agent_config: dict) -> None: + agent_config["Kubelet"].setdefault("Labels", {})["e2e.unbounded.test/retry"] = "changed" + + retry_script = patch_agent_config(bootstrap_script, add_retry_label) + if retry_script == bootstrap_script: + die("failed to change a node label for the bootstrap retry") + + retry_script_path = VM_DIR / "bootstrap-retry.sh" + retry_script_path.write_text(retry_script) + retry_script_path.chmod(0o600) + scp_cmd(str(retry_script_path), f"{SSH_TARGET}:/tmp/bootstrap-retry.sh") + ssh_cmd("chmod +x /tmp/bootstrap-retry.sh") + + ssh_cmd("sudo rmdir /usr/local/bin/unbounded-agent-daemon-recovery.sh") + run(["timeout", "1200", "ssh", *SSH_OPTS, SSH_TARGET, + f"sudo {env_prefix} /tmp/bootstrap-retry.sh"]) + after_text = bounded_ssh( + "sudo cat /var/lib/unbounded/agent/install-state.json; " + "systemctl show systemd-nspawn@kube1.service --property=MainPID --value", + time.monotonic() + 30, check=True).stdout + state_text, after_pid = after_text.rstrip().rsplit("\n", 1) + after = json.loads(state_text) + if after["installID"] != before["installID"] or after["phase"] != "complete" or after_pid != pid: + die("bootstrap retry changed ownership or restarted the running node") + + after_applied = bounded_ssh( + f"sudo sha256sum {applied_config}", time.monotonic() + 30, check=True).stdout.split()[0] + if after_applied != before_applied: + die("bootstrap retry overwrote the applied config with a label the running node never saw") + + log("Retry converged around the running node, preserving installation, nspawn PID and applied config") + return run([ "timeout", "1200", "ssh", *SSH_OPTS, "-o", "ServerAliveInterval=30", SSH_TARGET, @@ -4742,6 +4835,44 @@ def reinstall_agent(node_config: NodeConfig) -> None: die("same-disk reinstall changed host boot identity") +def validate_bootstrap_repair() -> None: + """Repair after ordinary repave using the original input and installed agent.""" + script = textwrap.dedent(r""" + set -eu + test ! -e /etc/unbounded/agent/kube1-applied-config.json + test -f /etc/unbounded/agent/kube2-applied-config.json + before=$(sha256sum /etc/unbounded/agent/kube2-applied-config.json) + node_pid=$(systemctl show systemd-nspawn@kube2.service --property=MainPID --value) + test "$node_pid" -gt 0 + cp /usr/local/bin/unbounded-agent-current /tmp/p6-repair-agent + chmod 0755 /tmp/p6-repair-agent + python3 - <<'PY' + from pathlib import Path + script = Path('/tmp/bootstrap.sh').read_text() + marker = "<<'AGENT_CONFIG_EOF'\n" + if marker not in script: + raise SystemExit('original bootstrap config not found') + config = script.split(marker, 1)[1].split('\nAGENT_CONFIG_EOF', 1)[0] + Path('/tmp/p6-original-config.json').write_text(config) + Path('/tmp/p6-original-config.json').chmod(0o600) + PY + systemctl stop unbounded-agent-daemon.service + rm /etc/systemd/system/unbounded-agent-daemon.service + systemctl daemon-reload + export UNBOUNDED_AGENT_CONFIG_FILE=/tmp/p6-original-config.json + /tmp/p6-repair-agent preflight --output json + /tmp/p6-repair-agent start + test "$before" = "$(sha256sum /etc/unbounded/agent/kube2-applied-config.json)" + test ! -e /etc/unbounded/agent/kube1-applied-config.json + test "$node_pid" = "$(systemctl show systemd-nspawn@kube2.service --property=MainPID --value)" + grep -q '"phase": "complete"' /var/lib/unbounded/agent/install-state.json + systemctl is-active unbounded-agent-daemon.service + """) + bounded_ssh("sudo bash -c " + shlex.quote(script), time.monotonic() + 180, check=True) + validate_workload() + log("Completed-install repair preserved the repaved slot and workload") + + SUITES: dict[str, list[str]] = { "setup": ["configure-kind-kube-proxy", "install-machine-crd", "deploy-unbounded-net-controller", "start-machina-controller", "validate-machina-controller", "validate-controllers-healthy"], @@ -4754,6 +4885,8 @@ def reinstall_agent(node_config: NodeConfig) -> None: "validate-node-reboot-operation", "validate-workload", "validate-node-repave-upgrade"], "configuration": ["validate-node-configs"], "fresh-bootstrap": ["run-agent", "wait-for-node", "validate-workload"], + "bootstrap-recovery": ["run-agent-recovery", "wait-for-node", "validate-workload", + "validate-node-repave-upgrade", "validate-bootstrap-repair"], } @@ -4791,6 +4924,8 @@ def command(_node_config: NodeConfig) -> None: COMMANDS: dict[str, Command] = { + "run-agent-recovery": run_agent_recovery, + "validate-bootstrap-repair": _without_node_config(validate_bootstrap_repair), "configure-kind-kube-proxy": _without_node_config(configure_kind_kube_proxy), "validate-host-reboot": _without_node_config(validate_host_reboot), "reinstall-agent": reinstall_agent, diff --git a/hack/agent/e2e-kind/test_reliability.py b/hack/agent/e2e-kind/test_reliability.py index 768f4655f..7e8ac798c 100644 --- a/hack/agent/e2e-kind/test_reliability.py +++ b/hack/agent/e2e-kind/test_reliability.py @@ -24,13 +24,60 @@ def test_suites_preserve_main_lifecycle_and_exclude_future_features(self): self.assertIn(name, steps) self.assertLess(steps.index("validate-host-reboot"), steps.index("reset-agent")) self.assertEqual(steps.count("run-agent"), 1) - self.assertFalse(any("recovery" in name or "ignition" in name for name in e2e.COMMANDS)) + self.assertFalse(any("repave-recovery" in name or "ignition" in name for name in e2e.COMMANDS)) + self.assertEqual(e2e.SUITES["bootstrap-recovery"], [ + "run-agent-recovery", "wait-for-node", "validate-workload", + "validate-node-repave-upgrade", "validate-bootstrap-repair", + ]) def test_reboot_disconnect_requires_new_identity(self): values = [(0, "old"), (255, ""), (255, ""), (0, "old"), (0, "new")] with patch.object(e2e, "bounded_ssh", side_effect=[subprocess.CompletedProcess([], code, out, "") for code, out in values]), patch.object(e2e.time, "sleep"): self.assertEqual(e2e.reboot_host_and_wait(), "new") + def test_repair_script_is_valid_shell_and_embedded_python(self): + with patch.object(e2e, "bounded_ssh") as ssh, patch.object(e2e, "validate_workload"): + e2e.validate_bootstrap_repair() + import shlex + script = shlex.split(ssh.call_args.args[0])[-1] + subprocess.run(["bash", "-n"], input=script, text=True, check=True) + python = script.split("python3 - <<'PY'\n", 1)[1].split("\nPY\n", 1)[0] + compile(python, "repair-fixture", "exec") + + def test_agent_config_patch_changes_only_the_embedded_config(self): + script = ( + "#!/bin/bash\nset -eu\n" + "cat > \"${UNBOUNDED_AGENT_CONFIG_FILE}\" <<'AGENT_CONFIG_EOF'\n" + + json.dumps({"Kubelet": {"ApiServer": "https://api.test", "Labels": {"keep": "yes"}}}, indent=2) + + "\nAGENT_CONFIG_EOF\n\"${AGENT_BIN}\" start\n" + ) + + def add_label(config): + config["Kubelet"].setdefault("Labels", {})["e2e.unbounded.test/retry"] = "changed" + + patched = e2e.patch_agent_config(script, add_label) + self.assertNotEqual(patched, script) + self.assertTrue(patched.startswith("#!/bin/bash\nset -eu\n")) + self.assertTrue(patched.endswith("\nAGENT_CONFIG_EOF\n\"${AGENT_BIN}\" start\n")) + + config = json.loads(patched.split("<<'AGENT_CONFIG_EOF'\n", 1)[1].split("\nAGENT_CONFIG_EOF", 1)[0]) + self.assertEqual(config["Kubelet"]["Labels"], {"keep": "yes", "e2e.unbounded.test/retry": "changed"}) + self.assertEqual(config["Kubelet"]["ApiServer"], "https://api.test") + + def test_agent_config_patch_fails_on_malformed_script(self): + for script in ("#!/bin/bash\ntrue\n", + "cat > \"${UNBOUNDED_AGENT_CONFIG_FILE}\" <<'AGENT_CONFIG_EOF'\n{}", + "cat > \"${UNBOUNDED_AGENT_CONFIG_FILE}\" <<'AGENT_CONFIG_EOF'\nnot-json\nAGENT_CONFIG_EOF\n"): + with self.assertRaises(SystemExit): + e2e.patch_agent_config(script, lambda config: None) + + def test_recovery_mode_is_scoped_to_attempt(self): + cfg = e2e.NodeConfig(name="test", node_labels={}, register_with_taints=[]) + with patch.dict(os.environ, {}, clear=True), patch.object(e2e, "run_agent", side_effect=RuntimeError("injected")): + with self.assertRaises(RuntimeError): + e2e.run_agent_recovery(cfg) + self.assertNotIn("E2E_BOOTSTRAP_RECOVERY", os.environ) + def test_reboot_permission_failure_fails(self): with patch.object(e2e, "bounded_ssh", side_effect=[subprocess.CompletedProcess([], 0, "old", ""), subprocess.CompletedProcess([], 1, "", "denied")]), self.assertRaises(SystemExit): e2e.reboot_host_and_wait() diff --git a/internal/fsutil/fsutil.go b/internal/fsutil/fsutil.go new file mode 100644 index 000000000..59cdd6702 --- /dev/null +++ b/internal/fsutil/fsutil.go @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Package fsutil provides durable filesystem helpers shared by the agent +// library and the agent commands. +package fsutil + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/google/renameio/v2" + "golang.org/x/sys/unix" +) + +// SyncDir persists a directory entry so newly written names survive a crash. +func SyncDir(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + + return errors.Join(f.Sync(), f.Close()) +} + +// WriteFileDurable also persists newly created parent directories, so the +// written file cannot outlive the directory entries needed to reach it. +func WriteFileDurable(path string, data []byte, mode os.FileMode) error { + var parents []string + + for dir := filepath.Dir(path); ; dir = filepath.Dir(dir) { + _, err := os.Stat(dir) + if err == nil { + parents = append(parents, dir) + break + } + + if !errors.Is(err, os.ErrNotExist) { + return err + } + + parents = append(parents, dir) + } + + if err := writeFile(path, data, mode); err != nil { + return err + } + + for _, dir := range parents { + if err := SyncDir(dir); err != nil { + return err + } + } + + return nil +} + +// writeFile writes content atomically, creating parent directories as needed. +// The temporary file shares the destination directory so it inherits the +// correct SELinux label instead of the temp-directory label. +func writeFile(path string, data []byte, mode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return err + } + + return renameio.WriteFile(path, data, mode, renameio.WithTempDir(filepath.Dir(path))) +} + +// InstallFile streams source onto target atomically. Large executables are +// copied rather than buffered in memory. +func InstallFile(source, target string, mode os.FileMode) (err error) { + f, err := os.Open(source) + if err != nil { + return err + } + + defer func() { err = errors.Join(err, f.Close()) }() + + if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { + return err + } + + pending, err := renameio.NewPendingFile(target, renameio.WithPermissions(mode), renameio.WithTempDir(filepath.Dir(target))) + if err != nil { + return err + } + + defer pending.Cleanup() //nolint:errcheck // Pending file cleanup after atomic replacement. + + if _, err := io.Copy(pending, f); err != nil { + return err + } + + return pending.CloseAtomicallyReplace() +} + +// SyncFilesystems persists every filesystem backing the given paths. +func SyncFilesystems(paths ...string) error { + var files []*os.File + + defer func() { + for _, f := range files { + _ = f.Close() //nolint:errcheck // Read-only handle; sync errors are returned. + } + }() + + for _, path := range paths { + f, err := os.Open(path) + if err != nil { + return err + } + + files = append(files, f) + } + + return SyncOpenFilesystems(files, unix.Syncfs) +} + +// SyncOpenFilesystems synchronizes each distinct filesystem once, using open +// handles that stay valid after teardown removes their paths. +func SyncOpenFilesystems(files []*os.File, syncfs func(int) error) error { + seen := map[uint64]bool{} + + for _, f := range files { + var stat unix.Stat_t + if err := unix.Fstat(int(f.Fd()), &stat); err != nil { + return err + } + + if seen[uint64(stat.Dev)] { + continue + } + + seen[uint64(stat.Dev)] = true + + if err := syncfs(int(f.Fd())); err != nil { + return fmt.Errorf("sync %s: %w", f.Name(), err) + } + } + + return nil +} diff --git a/internal/fsutil/fsutil_test.go b/internal/fsutil/fsutil_test.go new file mode 100644 index 000000000..9a527b596 --- /dev/null +++ b/internal/fsutil/fsutil_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package fsutil_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/internal/fsutil" +) + +func TestInstallFileStreamsAndReplacesAtomically(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + source, target := filepath.Join(dir, "source"), filepath.Join(dir, "bin", "target") + require.NoError(t, os.WriteFile(source, []byte("candidate"), 0o600)) + require.NoError(t, fsutil.InstallFile(source, target, 0o755)) + + data, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "candidate", string(data)) + + info, err := os.Stat(target) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + + // A failed install must leave the previously installed file intact. + require.Error(t, fsutil.InstallFile(filepath.Join(dir, "missing"), target, 0o755)) + data, err = os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "candidate", string(data)) +} + +func TestWriteFileDurableCreatesAndPersistsParents(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "a", "b", "state.json") + require.NoError(t, fsutil.WriteFileDurable(path, []byte("{}\n"), 0o600)) + + data, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, "{}\n", string(data)) + + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) +} + +func TestSyncOpenFilesystemsDeduplicatesByDevice(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + first, err := os.Open(dir) + require.NoError(t, err) + + t.Cleanup(func() { require.NoError(t, first.Close()) }) + + second, err := os.Open(dir) + require.NoError(t, err) + + t.Cleanup(func() { require.NoError(t, second.Close()) }) + + calls := 0 + + require.NoError(t, fsutil.SyncOpenFilesystems([]*os.File{first, second}, func(int) error { + calls++ + return nil + })) + require.Equal(t, 1, calls, "one barrier per filesystem, not per path") + + require.Error(t, fsutil.SyncOpenFilesystems([]*os.File{first}, func(int) error { return os.ErrPermission })) +} + +func TestSyncFilesystemsReportsMissingPath(t *testing.T) { + t.Parallel() + require.Error(t, fsutil.SyncFilesystems(filepath.Join(t.TempDir(), "absent"))) +} diff --git a/internal/provision/assets/unbounded-agent-install.sh b/internal/provision/assets/unbounded-agent-install.sh index 499aee1fc..0fb3d4f59 100644 --- a/internal/provision/assets/unbounded-agent-install.sh +++ b/internal/provision/assets/unbounded-agent-install.sh @@ -69,13 +69,40 @@ if [ -z "${AGENT_URL}" ]; then else _version_desc="${AGENT_VERSION:-custom}" fi -AGENT_BIN="/usr/local/bin/unbounded-agent" - echo "Downloading unbounded-agent ${_version_desc} for ${arch} from ${AGENT_URL}..." -tmp_dir="$(mktemp -d)" +# Staged under /var/lib rather than the default temporary directory because the +# staged binary is executed, not just copied: admission runs from it below. +# Hardened hosts commonly mount /tmp noexec, which would fail the run outright, +# and image-based hosts are the ones most likely to do so. +staging_root="/var/lib/unbounded" +mkdir -p "${staging_root}" +tmp_dir="$(mktemp -d "${staging_root}/install.XXXXXX")" trap 'rm -rf "${tmp_dir}"' EXIT curl -fsSL "${AGENT_URL}" | tar -xz -C "${tmp_dir}" unbounded-agent -install -m 0755 "${tmp_dir}/unbounded-agent" "${AGENT_BIN}" +# Run admission from the staged executable. Bootstrap installs the daemon binary +# only after acquiring installation ownership; retries cannot overwrite a live +# current/compatibility binary link before their intent has been accepted. +AGENT_BIN="${tmp_dir}/unbounded-agent" +chmod 0755 "${AGENT_BIN}" + +# Seed the daemon binary path when nothing usable is there yet. The agent +# version is selected independently of this script - by AGENT_VERSION, by +# AGENT_URL, or by the default of tracking the latest published release - so an +# installer that relied on the agent to install its own binary would silently +# break every agent released before that behavior existed. Such an agent never +# writes the binary, and bootstrap then fails at daemon setup with no indication +# that the installer and the agent disagree. +# +# The test follows symlinks on purpose. On a host this installation already owns +# the path resolves through the compatibility symlink to a live blue-green slot, +# so it is left untouched and admission still runs from the staged executable +# above. A dangling link resolves to nothing and is replaced, because install +# would otherwise write through it to a stale location. +AGENT_BIN_TARGET="/usr/local/bin/unbounded-agent" +if [ ! -x "${AGENT_BIN_TARGET}" ]; then + rm -f "${AGENT_BIN_TARGET}" + install -m 0755 "${AGENT_BIN}" "${AGENT_BIN_TARGET}" +fi _START_ARGS="" case "${AGENT_DEBUG}" in diff --git a/internal/provision/script_test.go b/internal/provision/script_test.go index b16568c03..a9cb86a55 100644 --- a/internal/provision/script_test.go +++ b/internal/provision/script_test.go @@ -44,6 +44,32 @@ func TestUnboundedAgentInstallScript(t *testing.T) { require.Contains(t, script, "Running unbounded-agent preflight") require.Contains(t, script, "preflight ${_START_ARGS}") require.Contains(t, script, "0|false|no|FALSE|NO|False|No") + + // The installer must place the agent binary itself. The agent version is + // selected independently of this script, including the default of tracking + // the latest published release, so an installer that relies on the agent to + // install its own binary breaks every agent released before that behavior + // existed. The uninstall script removes this same path. + require.Contains(t, script, `AGENT_BIN_TARGET="/usr/local/bin/unbounded-agent"`) + require.Contains(t, script, `install -m 0755 "${AGENT_BIN}" "${AGENT_BIN_TARGET}"`) + + // It must not clobber a live binary. The test follows symlinks so a host + // this installation already owns resolves through the compatibility symlink + // to a live slot and is skipped, which keeps admission running from the + // staged executable rather than one the retry just wrote. + require.Contains(t, script, `if [ ! -x "${AGENT_BIN_TARGET}" ]; then`) + require.Contains(t, script, `AGENT_BIN="${tmp_dir}/unbounded-agent"`) + + // The staged binary is executed, not just copied: admission runs from it. + // The default temporary directory is therefore the wrong place for it, + // because a host that mounts /tmp noexec cannot run it at all, and + // image-based hosts are the ones most likely to be hardened that way. + require.Contains(t, script, `mkdir -p "${staging_root}"`) + require.Contains(t, script, `tmp_dir="$(mktemp -d "${staging_root}/install.XXXXXX")"`) + require.NotContains(t, script, `tmp_dir="$(mktemp -d)"`) + + // Whatever is staged must still be cleaned up. + require.Contains(t, script, `trap 'rm -rf "${tmp_dir}"' EXIT`) } func TestUnboundedAgentUninstallScript(t *testing.T) { diff --git a/pkg/agent/agentbinary/agentbinary.go b/pkg/agent/agentbinary/agentbinary.go index ef9ad8e09..6d696f487 100644 --- a/pkg/agent/agentbinary/agentbinary.go +++ b/pkg/agent/agentbinary/agentbinary.go @@ -50,9 +50,14 @@ func EnsureDaemonBinaryLinks(ctx context.Context, log *slog.Logger, paths goalst } currentTarget := paths.CurrentTargetPath - if _, err := os.Lstat(paths.CurrentPath); err != nil { + // Resolved rather than stat'd, matching the last-good check below. Lstat + // succeeds on a symlink whose target is gone, so a dangling current link + // read as healthy and was left alone. VerifyDaemonInstalled resolves it and + // fails, which made the link the one fault that verify could report and + // repair could not fix: start returned the same stat error forever. + if _, err := filepath.EvalSymlinks(paths.CurrentPath); err != nil { if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("stat current daemon binary symlink: %w", err) + return fmt.Errorf("resolve current daemon binary symlink: %w", err) } target, targetErr := initialDaemonBinaryTarget(paths) diff --git a/pkg/agent/agentbinary/agentbinary_test.go b/pkg/agent/agentbinary/agentbinary_test.go index 3b3654350..fa25bb39f 100644 --- a/pkg/agent/agentbinary/agentbinary_test.go +++ b/pkg/agent/agentbinary/agentbinary_test.go @@ -233,3 +233,31 @@ func assertFileContent(t *testing.T, path, expected string) { require.NoError(t, err) assert.Equal(t, expected, string(data)) } + +// TestEnsureDaemonBinaryLinks_RepairsDanglingCurrent covers the one fault that +// verify could report and repair could not fix. +// +// VerifyDaemonInstalled resolves the current link and fails when its target is +// gone. Link initialization used to stat the link itself, which succeeds on a +// dangling symlink, so it saw a healthy link and left it. start on a completed +// installation then verified, repaired nothing, verified again, and returned +// the same stat error on every run forever. +func TestEnsureDaemonBinaryLinks_RepairsDanglingCurrent(t *testing.T) { + paths := setupDaemonBinaryTestPaths(t) + require.NoError(t, os.WriteFile(paths.BluePath, []byte("blue"), 0o755)) + + // A current link whose target no longer exists, as an interrupted + // activation or a removed slot leaves behind. + require.NoError(t, os.Symlink(filepath.Join(t.TempDir(), "removed-slot"), paths.CurrentPath)) + + _, err := filepath.EvalSymlinks(paths.CurrentPath) + require.Error(t, err, "fixture must be a link that cannot resolve") + + require.NoError(t, EnsureDaemonBinaryLinks(context.Background(), slog.Default(), paths)) + + assertSymlinkTarget(t, paths.CurrentPath, paths.BluePath) + + resolved, err := filepath.EvalSymlinks(paths.CurrentPath) + require.NoError(t, err, "the repaired link must resolve, or verify still fails") + require.Equal(t, paths.BluePath, resolved) +} diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index c8900082a..9f4602c8d 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -77,6 +77,18 @@ type AgentConfig struct { // Empty remains unobserved for legacy installations; it is not inferred from // the host distribution. The daemon reports explicit values in Machine status. ProvisioningFormat string `json:"ProvisioningFormat,omitempty"` + + // HostPrefix is the installation prefix for the agent's own host-side + // files: the daemon binaries under /bin and helper scripts + // under /libexec. It does not affect paths inside the nspawn + // machine, which are always relative to the machine directory. + // + // Empty means /usr/local, so hosts that do not set it are unaffected. Hosts + // with a read-only /usr must set it to a writable prefix; the agent refuses + // to bootstrap rather than guessing one, because where the agent may write + // is a property of the filesystem and not something that can be safely + // inferred from the distribution. + HostPrefix string `json:"HostPrefix,omitempty"` } const ( @@ -94,6 +106,65 @@ func ValidateProvisioningFormat(format string) error { } } +// hostPrefixAllowedRune reports whether r may appear in a host installation +// prefix. +// +// The prefix is interpolated into generated systemd units and into a shell +// script, neither of which quotes it. Rather than adding two kinds of escaping +// and having to keep them correct in every consumer, the accepted syntax is +// narrow enough that the value is inert in both contexts: no whitespace, no +// quoting or substitution characters, and no systemd "%" specifiers. +func hostPrefixAllowedRune(r rune) bool { + switch { + case r >= 'a' && r <= 'z': + return true + case r >= 'A' && r <= 'Z': + return true + case r >= '0' && r <= '9': + return true + case r == '/' || r == '.' || r == '_' || r == '-': + return true + default: + return false + } +} + +// ValidateHostPrefix checks that a configured host installation prefix is an +// absolute, normalized path that can hold a bin and libexec directory, and that +// it is safe to interpolate into the assets generated from it. An empty prefix +// is valid and selects the default. +func ValidateHostPrefix(prefix string) error { + trimmed := strings.TrimSpace(prefix) + if trimmed == "" { + return nil + } + + if !filepath.IsAbs(trimmed) { + return fmt.Errorf("HostPrefix must be an absolute path") + } + + if cleaned := filepath.Clean(trimmed); cleaned != trimmed { + return fmt.Errorf("HostPrefix must be a normalized path, for example %s", cleaned) + } + + if trimmed == "/" { + return fmt.Errorf("HostPrefix must not be the filesystem root") + } + + // Report the offending character rather than only the rule, because the + // caller cannot otherwise tell which byte of a long path was rejected. + for _, r := range trimmed { + if !hostPrefixAllowedRune(r) { + return fmt.Errorf( + "HostPrefix may only contain letters, digits, '/', '.', '_' and '-', but contains %q", + r, + ) + } + } + + return nil +} + // AgentOfflineArtifacts configures a complete offline source for binaries the // agent installs into the nspawn rootfs. type AgentOfflineArtifacts struct { @@ -250,6 +321,10 @@ func (a *AgentConfig) Validate() error { errs = append(errs, err) } + if err := ValidateHostPrefix(a.HostPrefix); err != nil { + errs = append(errs, err) + } + apiServer := strings.TrimSpace(a.Kubelet.ApiServer) if apiServer == "" { errs = append(errs, fmt.Errorf("Kubelet.ApiServer is required")) diff --git a/pkg/agent/config/config_test.go b/pkg/agent/config/config_test.go index e1709bbe1..efecca055 100644 --- a/pkg/agent/config/config_test.go +++ b/pkg/agent/config/config_test.go @@ -582,3 +582,72 @@ func TestAgentConfig_BackfillNodeName_UsesHostHostname(t *testing.T) { assert.Equal(t, want, cfg.NodeName) } + +// TestValidateHostPrefix pins what may be configured as an installation prefix. +// The value is interpolated into generated systemd units and into a shell +// script, neither of which quotes it, so the accepted syntax is deliberately +// narrow enough to be inert in both rather than requiring two kinds of +// escaping that every consumer would have to keep correct. +func TestValidateHostPrefix(t *testing.T) { + t.Parallel() + + for _, prefix := range []string{ + "", + "/usr/local", + "/opt/unbounded", + "/var/lib/unbounded-agent", + "/opt/Unbounded_1.0-rc.2", + } { + if err := ValidateHostPrefix(prefix); err != nil { + t.Errorf("ValidateHostPrefix(%q) = %v, want nil", prefix, err) + } + } + + for _, tc := range []struct{ prefix, reason string }{ + {"usr/local", "relative"}, + {"./opt", "relative"}, + {"/opt/", "trailing separator is not normalized"}, + {"/opt/../opt", "unnormalized"}, + {"/", "filesystem root"}, + {"/opt/un bounded", "whitespace"}, + {"/opt/$HOME", "shell substitution"}, + {"/opt/%i", "systemd specifier"}, + {"/opt/un;rm -rf /", "shell metacharacter"}, + {"/opt/\"quoted\"", "quoting"}, + {"/opt/un`cmd`", "command substitution"}, + } { + if err := ValidateHostPrefix(tc.prefix); err == nil { + t.Errorf("ValidateHostPrefix(%q) = nil, want an error (%s)", tc.prefix, tc.reason) + } + } +} + +// TestValidateRejectsBadHostPrefix checks the prefix is actually reached by +// whole-config validation, not merely validatable in isolation. +func TestValidateRejectsBadHostPrefix(t *testing.T) { + t.Parallel() + + cfg := validAgentConfigForHostPrefix() + if err := cfg.Validate(); err != nil { + t.Fatalf("baseline config should be valid: %v", err) + } + + cfg.HostPrefix = "/opt/$INJECTED" + if err := cfg.Validate(); err == nil { + t.Fatal("Validate() = nil, want an error for an unsafe HostPrefix") + } + + cfg.HostPrefix = "/opt/unbounded" + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() = %v, want nil for a valid HostPrefix", err) + } +} + +func validAgentConfigForHostPrefix() *AgentConfig { + return &AgentConfig{ + MachineName: "machine", + NodeName: "node", + Cluster: AgentClusterConfig{ClusterDNS: "10.96.0.10"}, + Kubelet: AgentKubeletConfig{ApiServer: "https://api.example.test"}, + } +} diff --git a/pkg/agent/goalstates/agentupgrade.go b/pkg/agent/goalstates/agentupgrade.go index 9fb64dc30..0e1df0755 100644 --- a/pkg/agent/goalstates/agentupgrade.go +++ b/pkg/agent/goalstates/agentupgrade.go @@ -24,13 +24,36 @@ type AgentUpgradePaths struct { // ResolvedAgentUpgradePaths returns the host-side agent binary paths after // applying environment overrides. +// +// Deprecated: use ResolvedAgentUpgradePathsFor, which resolves the binaries +// under a configured installation prefix. This entry point is equivalent to +// passing an empty prefix and is kept for callers outside this repository. func ResolvedAgentUpgradePaths() (AgentUpgradePaths, error) { + return ResolvedAgentUpgradePathsFor("") +} + +// ResolvedAgentUpgradePathsFor returns the host-side agent binary paths under an +// installation prefix, after applying environment overrides. +// +// An empty prefix selects DefaultHostPrefix, so a host that does not configure +// one resolves exactly the paths this package has always used. +// +// Environment overrides are absolute and win over the prefix. They name a +// specific file, which is more particular than a directory to look in, and the +// nspawn lifecycle hooks rely on that to pin a binary across an upgrade. +// +// The AgentUpgrade signal path is deliberately not prefixed. It lives under the +// agent config directory rather than the installation prefix, because it is +// state about an upgrade rather than part of the installed layout. +func ResolvedAgentUpgradePathsFor(prefix string) (AgentUpgradePaths, error) { + binDir := ResolveHostPaths(prefix).BinDir + paths := AgentUpgradePaths{ - BinaryPath: resolveDaemonBinaryPath(EnvDaemonBinary, DaemonBinaryPath), - BluePath: resolveDaemonBinaryPath(EnvDaemonBinaryBlue, DaemonBinaryBluePath), - GreenPath: resolveDaemonBinaryPath(EnvDaemonBinaryGreen, DaemonBinaryGreenPath), - CurrentPath: resolveDaemonBinaryPath(EnvDaemonBinaryCurrent, DaemonBinaryCurrentPath), - LastGoodPath: resolveDaemonBinaryPath(EnvDaemonBinaryLastGood, DaemonBinaryLastGoodPath), + BinaryPath: resolveDaemonBinaryPath(EnvDaemonBinary, filepath.Join(binDir, daemonBinaryName)), + BluePath: resolveDaemonBinaryPath(EnvDaemonBinaryBlue, filepath.Join(binDir, daemonBinaryBlueName)), + GreenPath: resolveDaemonBinaryPath(EnvDaemonBinaryGreen, filepath.Join(binDir, daemonBinaryGreenName)), + CurrentPath: resolveDaemonBinaryPath(EnvDaemonBinaryCurrent, filepath.Join(binDir, daemonBinaryCurrentName)), + LastGoodPath: resolveDaemonBinaryPath(EnvDaemonBinaryLastGood, filepath.Join(binDir, daemonBinaryLastGoodName)), SignalPath: resolveDaemonBinaryPath(EnvDaemonAgentUpgradeSignalPath, DaemonAgentUpgradeSignalPath), } diff --git a/pkg/agent/goalstates/agentupgrade_test.go b/pkg/agent/goalstates/agentupgrade_test.go index 0a50521ed..189a741e3 100644 --- a/pkg/agent/goalstates/agentupgrade_test.go +++ b/pkg/agent/goalstates/agentupgrade_test.go @@ -40,7 +40,7 @@ func TestResolvedAgentUpgradePaths(t *testing.T) { t.Setenv(EnvDaemonBinaryLastGood, lastGoodPath) t.Setenv(EnvDaemonAgentUpgradeSignalPath, signalPath) - paths, err := ResolvedAgentUpgradePaths() + paths, err := ResolvedAgentUpgradePathsFor("") require.NoError(t, err) assert.Equal(t, binaryPath, paths.BinaryPath) @@ -56,7 +56,7 @@ func TestResolvedAgentUpgradePaths_UsesDefaultsForBlankOverrides(t *testing.T) { t.Setenv(EnvDaemonBinary, "") t.Setenv(EnvDaemonBinaryBlue, " ") - paths, err := ResolvedAgentUpgradePaths() + paths, err := ResolvedAgentUpgradePathsFor("") require.NoError(t, err) assert.Equal(t, DaemonBinaryPath, paths.BinaryPath) @@ -87,7 +87,7 @@ func TestResolvedAgentUpgradePaths_ResolvesCurrentTarget(t *testing.T) { t.Setenv(EnvDaemonBinary, binaryPath) t.Setenv(EnvDaemonBinaryCurrent, currentPath) - paths, err := ResolvedAgentUpgradePaths() + paths, err := ResolvedAgentUpgradePathsFor("") require.NoError(t, err) assert.Equal(t, currentTargetPath, paths.CurrentTargetPath) @@ -97,8 +97,63 @@ func TestResolvedAgentUpgradePaths_CurrentTargetFallsBackToBinaryPath(t *testing t.Setenv(EnvDaemonBinary, "/agent") t.Setenv(EnvDaemonBinaryCurrent, filepath.Join(t.TempDir(), "missing-current")) - paths, err := ResolvedAgentUpgradePaths() + paths, err := ResolvedAgentUpgradePathsFor("") require.NoError(t, err) assert.Equal(t, "/agent", paths.CurrentTargetPath) } + +// TestResolvedAgentUpgradePathsForPrefix covers the reason the prefix-aware +// entry point exists: a host whose /usr is read-only cannot hold the agent's +// own binaries under /usr/local, so they move with the prefix. +// +// The signal path deliberately does not move. It is state about an upgrade +// rather than part of the installed layout, and it lives under the agent config +// directory, which is writable on such hosts. +func TestResolvedAgentUpgradePathsForPrefix(t *testing.T) { + paths, err := ResolvedAgentUpgradePathsFor("/opt/unbounded") + require.NoError(t, err) + + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent", paths.BinaryPath) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-blue", paths.BluePath) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-green", paths.GreenPath) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-current", paths.CurrentPath) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-last-good", paths.LastGoodPath) + assert.Equal(t, DaemonAgentUpgradeSignalPath, paths.SignalPath) +} + +// TestResolvedAgentUpgradePathsForDefaultMatchesLegacyConstants pins that a host +// which configures no prefix resolves exactly what this package resolved before +// the prefix existed. +// +// These paths are baked into generated systemd units and into the blue-green +// symlinks on every host already in the field. If the default drifted, an +// upgraded agent would look for its binaries somewhere the installed host does +// not have them, and the daemon would fail to start with nothing having changed +// on disk. +func TestResolvedAgentUpgradePathsForDefaultMatchesLegacyConstants(t *testing.T) { + paths, err := ResolvedAgentUpgradePathsFor("") + require.NoError(t, err) + + assert.Equal(t, DaemonBinaryPath, paths.BinaryPath) + assert.Equal(t, DaemonBinaryBluePath, paths.BluePath) + assert.Equal(t, DaemonBinaryGreenPath, paths.GreenPath) + assert.Equal(t, DaemonBinaryCurrentPath, paths.CurrentPath) + assert.Equal(t, DaemonBinaryLastGoodPath, paths.LastGoodPath) + assert.Equal(t, DaemonAgentUpgradeSignalPath, paths.SignalPath) +} + +// TestDeprecatedResolvedAgentUpgradePathsStillWorks keeps the compatibility +// promise honest. The entry point is deprecated rather than removed because it +// is published from pkg/, and callers outside this repository compose their own +// phases from it. +func TestDeprecatedResolvedAgentUpgradePathsStillWorks(t *testing.T) { + //nolint:staticcheck // Exercising the deprecated entry point is the point. + legacy, err := ResolvedAgentUpgradePaths() + require.NoError(t, err) + + current, err := ResolvedAgentUpgradePathsFor("") + require.NoError(t, err) + + assert.Equal(t, current, legacy, "the deprecated entry point must stay equivalent to an empty prefix") +} diff --git a/pkg/agent/goalstates/constants.go b/pkg/agent/goalstates/constants.go index fcfc18043..41413fd38 100644 --- a/pkg/agent/goalstates/constants.go +++ b/pkg/agent/goalstates/constants.go @@ -26,6 +26,15 @@ const ( // DaemonRecoveryUnit is the systemd recovery unit for the agent daemon. DaemonRecoveryUnit = "unbounded-agent-daemon-recovery.service" + // FirstBootBootstrapUnit is the unit an Ignition config installs to bootstrap + // the agent on boot. + // + // Named here rather than in the command that writes it because reset has to + // remove it, and the two live in packages that cannot import each other. A + // name that drifted between them would leave the unit enabled on a host that + // had been reset, which re-bootstraps it on the next boot. + FirstBootBootstrapUnit = "unbounded-agent-bootstrap.service" + DaemonBinaryPath = "/usr/local/bin/unbounded-agent" DaemonBinaryBluePath = "/usr/local/bin/unbounded-agent-blue" DaemonBinaryGreenPath = "/usr/local/bin/unbounded-agent-green" diff --git a/pkg/agent/goalstates/hostpaths.go b/pkg/agent/goalstates/hostpaths.go new file mode 100644 index 000000000..90d9fbdc7 --- /dev/null +++ b/pkg/agent/goalstates/hostpaths.go @@ -0,0 +1,173 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package goalstates + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/Azure/unbounded/pkg/agent/config" +) + +// DefaultHostPrefix is the installation prefix used when the agent config does +// not set one. +const DefaultHostPrefix = "/usr/local" + +// Base names of the agent's own host-side files. They are joined with the +// resolved prefix rather than being absolute constants so that hosts with a +// read-only /usr can place them somewhere writable. +const ( + daemonBinaryName = "unbounded-agent" + daemonBinaryBlueName = "unbounded-agent-blue" + daemonBinaryGreenName = "unbounded-agent-green" + daemonBinaryCurrentName = "unbounded-agent-current" + daemonBinaryLastGoodName = "unbounded-agent-last-good" + nspawnLifecycleName = "unbounded-agent-nspawn-lifecycle" + daemonRecoveryScriptName = "unbounded-agent-daemon-recovery.sh" + localDNSNetworkHelperName = "unbounded-localdns-network" +) + +// HostPaths is the resolved host-side layout of the agent's own files under an +// installation prefix. +// +// These are paths on the host. Files inside the nspawn machine are always +// resolved relative to the machine directory and are unaffected by the prefix. +type HostPaths struct { + // Prefix is the resolved installation prefix. + Prefix string + // BinDir is /bin. + BinDir string + // LibexecDir is /libexec. + LibexecDir string + + // NSpawnLifecycleBinary is the rollback-stable helper invoked by the + // generated nspawn hook units. + NSpawnLifecycleBinary string + // DaemonRecoveryScript is executed by the daemon recovery unit. + DaemonRecoveryScript string + // LocalDNSNetworkHelper backs unbounded-localdns-network.service. + LocalDNSNetworkHelper string +} + +// HostPrefixOrDefault returns the configured prefix, or DefaultHostPrefix when +// it is empty. +func HostPrefixOrDefault(prefix string) string { + if trimmed := strings.TrimSpace(prefix); trimmed != "" { + return trimmed + } + + return DefaultHostPrefix +} + +// ResolveHostPaths returns the host-side agent layout for an installation +// prefix. An empty prefix selects DefaultHostPrefix. +func ResolveHostPaths(prefix string) HostPaths { + resolved := HostPrefixOrDefault(prefix) + binDir := filepath.Join(resolved, "bin") + libexecDir := filepath.Join(resolved, "libexec") + + return HostPaths{ + Prefix: resolved, + BinDir: binDir, + LibexecDir: libexecDir, + NSpawnLifecycleBinary: filepath.Join(binDir, nspawnLifecycleName), + DaemonRecoveryScript: filepath.Join(binDir, daemonRecoveryScriptName), + LocalDNSNetworkHelper: filepath.Join(libexecDir, localDNSNetworkHelperName), + } +} + +// KnownHostPrefixes returns the prefixes that teardown and existing-deployment +// detection must consider. +// +// A host provisioned before the prefix was configurable, or by an agent using a +// different prefix, still has files under the default. Cleanup and +// already-provisioned checks therefore look at both, so that changing the +// prefix cannot orphan files or let a dirty host be silently reprovisioned. +func KnownHostPrefixes(prefix string) []string { + resolved := HostPrefixOrDefault(prefix) + if resolved == DefaultHostPrefix { + return []string{DefaultHostPrefix} + } + + return []string{resolved, DefaultHostPrefix} +} + +// MergeHostPrefixes returns every distinct prefix teardown must sweep, given +// candidates gathered from different sources. +// +// Teardown cannot rely on any single source. The installation record has the +// prefix from before the first mutation but may be absent on hosts provisioned +// by an older agent; the applied config has it only once the node started. An +// empty candidate contributes nothing but never suppresses the default. +func MergeHostPrefixes(candidates ...string) []string { + var ( + out []string + seen = map[string]struct{}{} + ) + + add := func(prefix string) { + if _, ok := seen[prefix]; ok { + return + } + + seen[prefix] = struct{}{} + + out = append(out, prefix) + } + + for _, candidate := range candidates { + if strings.TrimSpace(candidate) == "" { + continue + } + + for _, prefix := range KnownHostPrefixes(candidate) { + add(prefix) + } + } + + add(DefaultHostPrefix) + + return out +} + +// HostPrefixFromAppliedConfig returns the installation prefix recorded in the +// applied config of whichever machine is provisioned on this host. +// +// Processes started by systemd, such as the agent daemon and the nspawn +// lifecycle hooks, cannot inherit the prefix from the environment that +// bootstrapped the host. The applied config is the authoritative record: it is +// written once at bootstrap and re-read here so that later upgrades and +// teardown resolve the same paths the bootstrap used. +// +// An absent or unreadable config yields the default prefix, which is what a +// host provisioned before the prefix was configurable actually has on disk. +// +// Note that the applied config only exists once the node has started. Callers +// that must work after a *failed* bootstrap should prefer the installation +// record, which carries the same prefix and is written before the first host +// mutation. That package is internal to the agent binary, so it cannot be named +// from here. +func HostPrefixFromAppliedConfig() string { + for _, name := range []string{NSpawnMachineKube1, NSpawnMachineKube2} { + data, err := os.ReadFile(AppliedConfigPath(name)) + if err != nil { + continue + } + + // Only the prefix is needed here, so decode into the shared config type + // rather than a consumer-specific wrapper. Unknown fields are ignored. + var cfg config.AgentConfig + if err := json.Unmarshal(data, &cfg); err != nil { + continue + } + + if prefix := HostPrefixOrDefault(cfg.HostPrefix); prefix != DefaultHostPrefix { + return prefix + } + } + + return DefaultHostPrefix +} diff --git a/pkg/agent/goalstates/hostpaths_test.go b/pkg/agent/goalstates/hostpaths_test.go new file mode 100644 index 000000000..63b02256a --- /dev/null +++ b/pkg/agent/goalstates/hostpaths_test.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package goalstates + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/config" +) + +func TestHostPrefixOrDefault(t *testing.T) { + t.Parallel() + + assert.Equal(t, DefaultHostPrefix, HostPrefixOrDefault("")) + assert.Equal(t, DefaultHostPrefix, HostPrefixOrDefault(" ")) + assert.Equal(t, "/opt/unbounded", HostPrefixOrDefault("/opt/unbounded")) + assert.Equal(t, "/opt/unbounded", HostPrefixOrDefault(" /opt/unbounded ")) +} + +// TestResolveHostPathsDefaultsAreUnchanged pins the pre-existing absolute paths. +// Hosts that do not configure a prefix must keep exactly the layout they had +// before the prefix became configurable. +func TestResolveHostPathsDefaultsAreUnchanged(t *testing.T) { + t.Parallel() + + paths := ResolveHostPaths("") + + assert.Equal(t, "/usr/local", paths.Prefix) + assert.Equal(t, "/usr/local/bin", paths.BinDir) + assert.Equal(t, "/usr/local/libexec", paths.LibexecDir) + assert.Equal(t, "/usr/local/bin/unbounded-agent-nspawn-lifecycle", paths.NSpawnLifecycleBinary) + assert.Equal(t, "/usr/local/bin/unbounded-agent-daemon-recovery.sh", paths.DaemonRecoveryScript) + assert.Equal(t, "/usr/local/libexec/unbounded-localdns-network", paths.LocalDNSNetworkHelper) +} + +func TestResolveHostPathsWithPrefix(t *testing.T) { + t.Parallel() + + paths := ResolveHostPaths("/opt/unbounded") + + assert.Equal(t, "/opt/unbounded", paths.Prefix) + assert.Equal(t, "/opt/unbounded/bin", paths.BinDir) + assert.Equal(t, "/opt/unbounded/libexec", paths.LibexecDir) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle", paths.NSpawnLifecycleBinary) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-daemon-recovery.sh", paths.DaemonRecoveryScript) + assert.Equal(t, "/opt/unbounded/libexec/unbounded-localdns-network", paths.LocalDNSNetworkHelper) +} + +// TestResolvedAgentUpgradePathsDefaultsAreUnchanged is the equivalent regression +// guard for the blue-green daemon binary layout. +// TestResolvedAgentUpgradePathsEnvOverridesPrefix keeps the existing escape +// hatch working: an explicit environment override wins over the prefix. +func TestKnownHostPrefixes(t *testing.T) { + t.Parallel() + + assert.Equal(t, []string{DefaultHostPrefix}, KnownHostPrefixes("")) + assert.Equal(t, []string{DefaultHostPrefix}, KnownHostPrefixes(DefaultHostPrefix)) + + // A non-default prefix must still sweep the default, so that a host + // provisioned under the old layout is not left with orphaned files. + assert.Equal(t, []string{"/opt/unbounded", DefaultHostPrefix}, KnownHostPrefixes("/opt/unbounded")) +} + +func TestHostPrefixFromAppliedConfig(t *testing.T) { + // AppliedConfigPath is absolute, so redirect it by pointing AgentConfigDir's + // consumers at a temporary root is not possible; instead assert the + // fallback, which is the branch reachable without writing to /etc. + assert.Equal(t, DefaultHostPrefix, HostPrefixFromAppliedConfig()) +} + +// TestHostPrefixRoundTripsThroughAppliedConfig proves the persisted config +// carries the prefix, which is what lets systemd-started processes resolve the +// same paths the bootstrap used. +func TestHostPrefixRoundTripsThroughAppliedConfig(t *testing.T) { + t.Parallel() + + cfg := config.AgentConfig{MachineName: "m", HostPrefix: "/opt/unbounded"} + + data, err := json.Marshal(cfg) + require.NoError(t, err) + + path := filepath.Join(t.TempDir(), "applied-config.json") + require.NoError(t, os.WriteFile(path, data, 0o600)) + + raw, err := os.ReadFile(path) + require.NoError(t, err) + + var decoded config.AgentConfig + require.NoError(t, json.Unmarshal(raw, &decoded)) + + assert.Equal(t, "/opt/unbounded", decoded.HostPrefix) + assert.Equal(t, "/opt/unbounded/bin", ResolveHostPaths(decoded.HostPrefix).BinDir) +} + +// TestDefaultHostPathsMatchTheExistingConstants is the regression guard for +// every host that does not set a prefix. Those hosts must resolve to exactly +// the paths they had before the prefix existed, because the lifecycle helper +// path is baked as an absolute path into generated systemd units that are +// already on disk. +func TestDefaultHostPathsMatchTheExistingConstants(t *testing.T) { + t.Parallel() + + paths := ResolveHostPaths("") + + require.Equal(t, DefaultHostPrefix, paths.Prefix) + require.Equal(t, filepath.Dir(DaemonBinaryPath), paths.BinDir) + require.Equal(t, NSpawnLifecycleBinaryPath, paths.NSpawnLifecycleBinary) + require.Equal(t, DaemonRecoveryScriptPath, paths.DaemonRecoveryScript) +} diff --git a/pkg/agent/internal/utilio/io.go b/pkg/agent/internal/utilio/io.go index b32862b67..abc6341c1 100644 --- a/pkg/agent/internal/utilio/io.go +++ b/pkg/agent/internal/utilio/io.go @@ -4,6 +4,7 @@ package utilio import ( + "bytes" "errors" "fmt" "io" @@ -76,3 +77,28 @@ func WriteFile(filename string, content []byte, perm os.FileMode) error { return renameio.WriteFile(filename, content, perm, renameio.WithTempDir(filepath.Dir(filename))) } + +// WriteFileIfChanged writes content like WriteFile and reports whether the +// content differed beforehand. +// +// It exists so a caller that reapplies configuration can tell an actual change +// from a no-op. Writing unconditionally would be correct on disk but would make +// every reapply look like a change, so callers that act on one, such as +// restarting the service that reads the file, would act every time. +// +// Only content is compared. WriteFile preserves an existing file's permissions +// rather than resetting them to perm, so a drifted mode cannot be corrected +// here; reporting it as changed would make this return true on every call and +// restart the reader forever. perm therefore applies only when the file is +// being created. +func WriteFileIfChanged(filename string, content []byte, perm os.FileMode) (bool, error) { + if existing, err := os.ReadFile(filename); err == nil && bytes.Equal(existing, content) { + return false, nil + } + + if err := WriteFile(filename, content, perm); err != nil { + return false, err + } + + return true, nil +} diff --git a/pkg/agent/internal/utilio/io_test.go b/pkg/agent/internal/utilio/io_test.go index 21db44ca0..59e40feec 100644 --- a/pkg/agent/internal/utilio/io_test.go +++ b/pkg/agent/internal/utilio/io_test.go @@ -5,6 +5,8 @@ package utilio import ( "math" + "os" + "path/filepath" "strings" "testing" ) @@ -17,3 +19,44 @@ func TestInstallFileWithLimitedSizeRejectsOverflowingLimit(t *testing.T) { t.Fatal("InstallFileWithLimitedSize error = nil") } } + +// TestWriteFileIfChanged covers the signal callers act on. A reapply that +// changes nothing must report false, or every reapply would look like a change +// and anything keyed on it, such as restarting a service, would fire each time. +func TestWriteFileIfChanged(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "nested", "config") + + changed, err := WriteFileIfChanged(path, []byte("one"), 0o644) + if err != nil || !changed { + t.Fatalf("first write: changed=%v err=%v, want changed=true", changed, err) + } + + changed, err = WriteFileIfChanged(path, []byte("one"), 0o644) + if err != nil || changed { + t.Fatalf("identical rewrite: changed=%v err=%v, want changed=false", changed, err) + } + + changed, err = WriteFileIfChanged(path, []byte("two"), 0o644) + if err != nil || !changed { + t.Fatalf("content change: changed=%v err=%v, want changed=true", changed, err) + } + + data, err := os.ReadFile(path) + if err != nil || string(data) != "two" { + t.Fatalf("content = %q err=%v, want \"two\"", data, err) + } + + // A drifted mode is deliberately not reported as changed. WriteFile + // preserves existing permissions, so it could not be corrected here, and + // reporting it would restart the file's reader on every call forever. + if err := os.Chmod(path, 0o600); err != nil { + t.Fatalf("chmod: %v", err) + } + + changed, err = WriteFileIfChanged(path, []byte("two"), 0o644) + if err != nil || changed { + t.Fatalf("mode drift: changed=%v err=%v, want changed=false", changed, err) + } +} diff --git a/pkg/agent/phases/host/configure_nftables.go b/pkg/agent/phases/host/configure_nftables.go index f2a8231a6..f3cf603b9 100644 --- a/pkg/agent/phases/host/configure_nftables.go +++ b/pkg/agent/phases/host/configure_nftables.go @@ -16,6 +16,7 @@ import ( "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/internal/utilio" "github.com/Azure/unbounded/pkg/agent/phases" + "github.com/Azure/unbounded/pkg/agent/phases/reset" ) const ( @@ -35,6 +36,9 @@ var nftablesClearRules []byte type configureNFTables struct { log *slog.Logger + // machineRegistered reports whether any nspawn machine is registered, and + // is injectable so the replay behavior can be tested without a host. + machineRegistered func(context.Context, *slog.Logger) (bool, error) } // ConfigureNFTables returns a task that installs a oneshot systemd unit which @@ -42,7 +46,18 @@ type configureNFTables struct { // This ensures stale rules (e.g. left behind by Docker) do not interfere with // Kubernetes networking. func ConfigureNFTables(log *slog.Logger) phases.Task { - return &configureNFTables{log: log} + return &configureNFTables{log: log, machineRegistered: anyMachineRegistered} +} + +// anyMachineRegistered reports whether either node slot is registered. It fails +// closed: an uninspectable host is not reported as having no machine. +func anyMachineRegistered(ctx context.Context, log *slog.Logger) (bool, error) { + name, err := reset.FirstRegisteredMachine(ctx, log) + if err != nil { + return false, err + } + + return name != "", nil } func (c *configureNFTables) Name() string { return "configure-nftables" } @@ -94,9 +109,55 @@ func (c *configureNFTables) ensureNFTablesFlushUnit(ctx context.Context) error { return fmt.Errorf("systemctl enable %s: %w", nftablesFlushUnit, err) } + // Starting the unit applies `flush ruleset`, which erases every nftables + // rule on the host. See shouldStartFlush for why that is conditional. + start, err := c.shouldStartFlush(ctx) + if err != nil { + return err + } + + if !start { + return nil + } + if err := executil.RunCmd(ctx, c.log, systemctl, "start", nftablesFlushUnit); err != nil { return fmt.Errorf("systemctl start %s: %w", nftablesFlushUnit, err) } return nil } + +// shouldStartFlush reports whether the flush unit may be started now. +// +// The flush erases every nftables rule on the host. That is the point on a +// fresh host, and it is safe at boot because the unit is ordered before the +// nspawn machine and LocalDNS re-adds its table after it. +// +// Starting it here is a different thing entirely. The nspawn container shares +// the host network namespace, so a running node's kube-proxy and CNI rules live +// in the ruleset being erased, and LocalDNS's NOTRACK table goes with them. +// Nothing puts it back: systemd ordering only sequences units within a single +// transaction, so starting this unit alone does not pull in +// unbounded-localdns-network.service, and that unit otherwise runs only when +// the machine starts. kube-proxy resyncs on its own; LocalDNS does not. +// +// The flush exists to hand a clean slate to a node that has not started yet. +// Once one is registered it has already served that purpose, so the live +// ruleset is left alone. The unit stays installed and enabled either way, so +// the next boot still gets its clean slate in the correct order. +func (c *configureNFTables) shouldStartFlush(ctx context.Context) (bool, error) { + registered, err := c.machineRegistered(ctx, c.log) + if err != nil { + return false, fmt.Errorf("inspect registered machines before flushing nftables: %w", err) + } + + if registered { + c.log.Info("nspawn machine is registered; leaving the live nftables ruleset alone", + "unit", nftablesFlushUnit, + ) + + return false, nil + } + + return true, nil +} diff --git a/pkg/agent/phases/host/configure_nftables_test.go b/pkg/agent/phases/host/configure_nftables_test.go new file mode 100644 index 000000000..90fd9684e --- /dev/null +++ b/pkg/agent/phases/host/configure_nftables_test.go @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package host + +import ( + "context" + "errors" + "log/slog" + "testing" + + "github.com/stretchr/testify/require" +) + +func nftablesTask(registered bool, err error) *configureNFTables { + return &configureNFTables{ + log: slog.New(slog.DiscardHandler), + machineRegistered: func(context.Context, *slog.Logger) (bool, error) { + return registered, err + }, + } +} + +// TestFlushNotStartedWhileMachineRegistered is the replay guard. +// +// Starting the flush unit applies `flush ruleset`, and the nspawn container +// shares the host network namespace, so on a host with a running node that +// erases kube-proxy, CNI and LocalDNS rules at once. Only kube-proxy resyncs; +// LocalDNS's NOTRACK table is reinstated by a unit that runs when the machine +// starts, and systemd ordering does not pull that unit in when this one is +// started on its own. +func TestFlushNotStartedWhileMachineRegistered(t *testing.T) { + t.Parallel() + + start, err := nftablesTask(true, nil).shouldStartFlush(t.Context()) + require.NoError(t, err) + require.False(t, start, "must not flush a ruleset a running node depends on") +} + +// TestFlushStartedOnFreshHost keeps the original behavior where it is correct: +// no machine is registered, so no running node can lose rules. +func TestFlushStartedOnFreshHost(t *testing.T) { + t.Parallel() + + start, err := nftablesTask(false, nil).shouldStartFlush(t.Context()) + require.NoError(t, err) + require.True(t, start) +} + +// TestFlushFailsClosedOnUninspectableHost keeps an inspection failure from +// being read as "nothing registered", which would flush a ruleset that may +// belong to a running node. +func TestFlushFailsClosedOnUninspectableHost(t *testing.T) { + t.Parallel() + + injected := errors.New("machinectl unavailable") + + start, err := nftablesTask(false, injected).shouldStartFlush(t.Context()) + require.ErrorIs(t, err, injected) + require.False(t, start) +} diff --git a/pkg/agent/phases/host/preflight_existing_deployment.go b/pkg/agent/phases/host/preflight_existing_deployment.go index 4c22d61ac..7e34cf616 100644 --- a/pkg/agent/phases/host/preflight_existing_deployment.go +++ b/pkg/agent/phases/host/preflight_existing_deployment.go @@ -16,7 +16,9 @@ import ( "github.com/Azure/unbounded/pkg/agent/preflight" ) -const checkExistingDeploymentName = "existing-deployment" +// CheckExistingDeploymentName identifies the clean-host check, which a +// resumed installation intentionally skips. +const CheckExistingDeploymentName = "existing-deployment" // CheckExistingDeployment verifies the host does not already contain // node deployment artifacts. Bootstrap must start from a clean host; @@ -26,14 +28,14 @@ func CheckExistingDeployment(log *slog.Logger) preflight.Checker { } func checkExistingDeployment(log *slog.Logger, deps hostCheckDeps) preflight.Checker { - return simpleHostChecker{name: checkExistingDeploymentName, check: func(ctx context.Context) []preflight.Result { + return simpleHostChecker{name: CheckExistingDeploymentName, check: func(ctx context.Context) []preflight.Result { results := existingDeploymentResults(ctx, log, deps) if len(results) > 0 { return results } return preflight.ResultsOK( - checkExistingDeploymentName, + CheckExistingDeploymentName, "host deployment", "no existing node deployment was detected", ) @@ -151,7 +153,7 @@ func appendExistingDeploymentArtifactResult( return append(results, existingDeploymentResult(artifact.description, artifact.path, artifact.description+" "+artifact.path)) } else if !errors.Is(err, os.ErrNotExist) { return append(results, preflight.Error( - checkExistingDeploymentName, + CheckExistingDeploymentName, artifact.path, "existing deployment artifact cannot be inspected: %s; node reset is needed before running preflight or start again", artifact.path, @@ -163,7 +165,7 @@ func appendExistingDeploymentArtifactResult( func existingDeploymentResult(description, target, detail string) preflight.Result { return preflight.Error( - checkExistingDeploymentName, + CheckExistingDeploymentName, target, "existing node deployment artifact detected (%s): %s; node reset is needed before running preflight or start again", description, diff --git a/pkg/agent/phases/nodestart/change_tracker.go b/pkg/agent/phases/nodestart/change_tracker.go new file mode 100644 index 000000000..35f69e65c --- /dev/null +++ b/pkg/agent/phases/nodestart/change_tracker.go @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package nodestart + +import ( + "os" + + "github.com/Azure/unbounded/pkg/agent/internal/utilio" +) + +// changeTracker records whether any file a configuration task owns actually +// differed from what was already on disk. +// +// The node services read their configuration at start, so a reapply that alters +// one has to restart it and a reapply that alters nothing must not. Embedders +// write through this rather than calling utilio directly, so the answer covers +// every file the task owns rather than only the last one written. +type changeTracker struct { + changed bool +} + +// write applies content and records whether it differed from what was there. +func (t *changeTracker) write(path string, content []byte, perm os.FileMode) error { + changed, err := utilio.WriteFileIfChanged(path, content, perm) + if err != nil { + return err + } + + t.changed = t.changed || changed + + return nil +} diff --git a/pkg/agent/phases/nodestart/cri.go b/pkg/agent/phases/nodestart/cri.go index df72490a5..c8678cd58 100644 --- a/pkg/agent/phases/nodestart/cri.go +++ b/pkg/agent/phases/nodestart/cri.go @@ -37,11 +37,17 @@ const ( type configureContainerd struct { goalState *goalstates.NodeStart + + changeTracker } // ConfigureContainerd returns a task that writes the containerd configuration, systemd unit, // and optional GPU drop-in configs into the machine rootfs. It runs before the nspawn machine // is started, so all paths are relative to the machine directory on the host filesystem. +// +// The agent reaches this work through StartNode, which builds the task directly +// so it can tell whether the configuration it wrote differed. This entry point +// remains for callers outside the agent that compose phases themselves. func ConfigureContainerd(goalState *goalstates.NodeStart) phases.Task { return &configureContainerd{goalState: goalState} } @@ -88,7 +94,7 @@ func (c *configureContainerd) ensureContainerdConfig() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.ContainerdConfigPath) - return utilio.WriteFile(dest, buf.Bytes(), 0o644) + return c.write(dest, buf.Bytes(), 0o644) } func (c *configureContainerd) ensureGantryHostsConfig() error { @@ -109,7 +115,7 @@ func (c *configureContainerd) ensureGantryHostsConfig() error { return err } - return utilio.WriteFile(dest, []byte(gantryHostsConfig), 0o644) + return c.write(dest, []byte(gantryHostsConfig), 0o644) } func hasGantryHostsManagedMarker(content []byte) bool { @@ -136,7 +142,7 @@ func (c *configureContainerd) ensureContainerdServiceUnit() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.SystemdSystemDir, goalstates.SystemdUnitContainerd) - return utilio.WriteFile(dest, buf.Bytes(), 0o644) + return c.write(dest, buf.Bytes(), 0o644) } // ensureGPUDropInConfigs manages GPU-related containerd drop-in configs. diff --git a/pkg/agent/phases/nodestart/kubelet.go b/pkg/agent/phases/nodestart/kubelet.go index 238077b6d..93d92c3c3 100644 --- a/pkg/agent/phases/nodestart/kubelet.go +++ b/pkg/agent/phases/nodestart/kubelet.go @@ -20,17 +20,22 @@ import ( "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/internal/utilio" "github.com/Azure/unbounded/pkg/agent/phases" ) type configureKubelet struct { goalState *goalstates.NodeStart + + changeTracker } // ConfigureKubelet returns a task that writes the kubelet configuration into the machine rootfs. // It runs before the nspawn machine is started, so all paths are relative to // the machine directory on the host filesystem. +// +// The agent reaches this work through StartNode, which builds the task directly +// so it can tell whether the configuration it wrote differed. This entry point +// remains for callers outside the agent that compose phases themselves. func ConfigureKubelet(goalState *goalstates.NodeStart) phases.Task { return &configureKubelet{goalState: goalState} } @@ -103,7 +108,7 @@ func (s *startKubelet) Do(ctx context.Context) error { func (c *configureKubelet) ensureKubeletCACert() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletAPIServerCACertPath) - return utilio.WriteFile(dest, c.goalState.Kubelet.CACertData, 0o644) + return c.write(dest, c.goalState.Kubelet.CACertData, 0o644) } // ensureKubeletConfiguration writes a KubeletConfiguration assembled from the @@ -137,7 +142,7 @@ func (c *configureKubelet) ensureKubeletConfiguration() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletConfigurationPath) - return utilio.WriteFile(dest, data, 0o644) + return c.write(dest, data, 0o644) } func defaultKubeletConfiguration() map[string]any { @@ -208,7 +213,7 @@ func (c *configureKubelet) ensureKubeletServiceUnit() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.SystemdSystemDir, goalstates.SystemdUnitKubelet) - return utilio.WriteFile(dest, buf.Bytes(), 0o644) + return c.write(dest, buf.Bytes(), 0o644) } // ensureKubeletDropIns renders and writes all kubelet systemd drop-in files @@ -259,7 +264,7 @@ func (c *configureKubelet) ensureKubeletDropIns() error { } dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletServiceDropInDir, d.name) - if err := utilio.WriteFile(dest, buf.Bytes(), 0o644); err != nil { + if err := c.write(dest, buf.Bytes(), 0o644); err != nil { return fmt.Errorf("write %s: %w", dest, err) } } @@ -321,7 +326,7 @@ func (c *configureKubelet) ensureBootstrapKubeconfig() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletBootstrapKubeconfigPath) - return utilio.WriteFile(dest, data, 0o600) + return c.write(dest, data, 0o600) } // ensureExecKubeconfig writes a kubeconfig that uses an exec credential @@ -339,7 +344,7 @@ func (c *configureKubelet) ensureExecKubeconfig() error { // to the kubelet kubeconfig path (no TLS bootstrap needed). dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletKubeconfigPath) - return utilio.WriteFile(dest, data, 0o600) + return c.write(dest, data, 0o600) } // formatNodeLabels formats a map of node labels as a sorted, comma-separated diff --git a/pkg/agent/phases/nodestart/kubelet_test.go b/pkg/agent/phases/nodestart/kubelet_test.go index d9c6c2e99..81794c871 100644 --- a/pkg/agent/phases/nodestart/kubelet_test.go +++ b/pkg/agent/phases/nodestart/kubelet_test.go @@ -278,3 +278,40 @@ func TestConfigureKubeletOmitsEmptyNodeLabelsAndTaints(t *testing.T) { require.NotContains(t, string(data), "--image-credential-provider-config=") require.NotContains(t, string(data), "--image-credential-provider-bin-dir=") } + +// TestConfigureKubeletReportsOnlyRealChanges backs the restart decision. If an +// identical reapply reported a change, every retry would bounce kubelet on a +// running node; if a real change went unreported, the files and the running +// service would silently disagree. +func TestConfigureKubeletReportsOnlyRealChanges(t *testing.T) { + t.Parallel() + + machineDir := t.TempDir() + goalState := &goalstates.NodeStart{ + MachineDir: machineDir, + NodeName: "worker-1", + Kubelet: goalstates.Kubelet{ + APIServer: "https://api.example.com", + CACertData: []byte("ca"), + NodeIP: "10.0.0.15", + KubeletAuthInfo: config.KubeletAuthInfo{ + BootstrapToken: "token", + }, + ClusterDNS: "10.0.0.10", + }, + } + + first := &configureKubelet{goalState: goalState} + require.NoError(t, first.Do(context.Background())) + require.True(t, first.changed, "creating the configuration is a change") + + second := &configureKubelet{goalState: goalState} + require.NoError(t, second.Do(context.Background())) + require.False(t, second.changed, "an identical reapply is not a change") + + goalState.Kubelet.ClusterDNS = "10.0.0.11" + + third := &configureKubelet{goalState: goalState} + require.NoError(t, third.Do(context.Background())) + require.True(t, third.changed, "a different cluster DNS is a change") +} diff --git a/pkg/agent/phases/nodestart/nspawn.go b/pkg/agent/phases/nodestart/nspawn.go index 985a2d97f..83c2e0676 100644 --- a/pkg/agent/phases/nodestart/nspawn.go +++ b/pkg/agent/phases/nodestart/nspawn.go @@ -25,12 +25,22 @@ type machinectlRunner interface { Terminate(ctx context.Context, name string) error Exists(ctx context.Context, name string) bool ResetFailed(ctx context.Context, name string) error + Running(ctx context.Context, name string) (bool, error) } type defaultMachinectlRunner struct { log *slog.Logger } +func (r defaultMachinectlRunner) Running(ctx context.Context, name string) (bool, error) { + out, err := executil.OutputCmd(ctx, r.log, "systemctl", "show", "systemd-nspawn@"+name+".service", "--property=ActiveState", "--value") + if err != nil { + return false, err + } + + return strings.TrimSpace(out) == "active" || strings.TrimSpace(out) == "activating", nil +} + func (r defaultMachinectlRunner) Enable(ctx context.Context, name string) error { return executil.RunCmd(ctx, r.log, executil.Machinectl(), "enable", name) } @@ -58,11 +68,21 @@ type startNSpawnMachine struct { // runner is the machinectl/systemctl driver. Tests inject a fake. runner machinectlRunner + + // wasRunning records whether the machine was already up before this task + // touched it. A reapply against a live node has to restart any service + // whose configuration it changed, because those are read at start; on a + // machine this task boots, the services read the new files anyway. + wasRunning bool } // StartNSpawnMachine returns a task that starts the systemd-nspawn machine using machinectl and // waits until D-Bus is responsive inside the machine so that subsequent phases // can safely use executil.MachineRun(). +// +// The agent reaches this work through StartNode, which builds the task directly +// so it can tell whether the configuration it wrote differed. This entry point +// remains for callers outside the agent that compose phases themselves. func StartNSpawnMachine(log *slog.Logger, goalState *goalstates.NodeStart) phases.Task { return &startNSpawnMachine{ log: log, @@ -96,6 +116,17 @@ func (s *startNSpawnMachine) Do(ctx context.Context) error { // under us, leaving an orphaned registration with errno 17 / "File exists"), // terminates the stale registration and retries once. func (s *startNSpawnMachine) startWithRecovery(ctx context.Context, name string) error { + running, err := s.runner.Running(ctx, name) + if err != nil { + return fmt.Errorf("inspect nspawn service before replay: %w", err) + } + + s.wasRunning = running + + if running { + return nil + } + startErr := s.runner.Start(ctx, name) if startErr == nil { return nil diff --git a/pkg/agent/phases/nodestart/nspawn_test.go b/pkg/agent/phases/nodestart/nspawn_test.go index 4fe0da0fe..9cf30f301 100644 --- a/pkg/agent/phases/nodestart/nspawn_test.go +++ b/pkg/agent/phases/nodestart/nspawn_test.go @@ -22,7 +22,9 @@ func silentLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } // fakeRunner is a scriptable machinectlRunner for exercising the // startWithRecovery state machine without touching real binaries. type fakeRunner struct { - mu sync.Mutex + running bool + runningErr error + mu sync.Mutex // startResults are returned by successive calls to Start. // If the slice is exhausted, the test fails. @@ -97,6 +99,8 @@ func (f *fakeRunner) Exists(_ context.Context, _ string) bool { return f.existsAfterStart } +func (f *fakeRunner) Running(context.Context, string) (bool, error) { return f.running, f.runningErr } + func (f *fakeRunner) ResetFailed(_ context.Context, _ string) error { f.mu.Lock() defer f.mu.Unlock() @@ -122,6 +126,26 @@ func runStart(t *testing.T, runner *fakeRunner) error { } // TestStartWithRecovery_HappyPath: clean start, no recovery needed. +func TestStartWithRecoveryPreservesRunningMachine(t *testing.T) { + t.Parallel() + + r := &fakeRunner{running: true, existsAfterStart: true} + require.NoError(t, runStart(t, r)) + require.Zero(t, r.startCalls) + require.Zero(t, r.terminateCalls) + require.Zero(t, r.resetFailedCalls) +} + +func TestStartWithRecoveryRejectsInspectionFailure(t *testing.T) { + t.Parallel() + + injected := errors.New("inspection denied") + r := &fakeRunner{runningErr: injected} + require.ErrorIs(t, runStart(t, r), injected) + require.Zero(t, r.startCalls) + require.Zero(t, r.terminateCalls) +} + func TestStartWithRecovery_HappyPath(t *testing.T) { t.Parallel() @@ -247,3 +271,41 @@ func TestIsAlreadyExistsErr(t *testing.T) { }) } } + +// TestStartRecordsWhetherMachineWasAlreadyRunning pins the handoff to +// restartReconfigured. That task decides whether to restart a service whose +// configuration changed, and the only thing it has to go on is what this task +// observed before it touched the machine. +// +// Without this, the two halves can drift silently: restartReconfigured keeps +// reading the field correctly while nothing ever sets it, and a reapply against +// a live node stops restarting the services it just reconfigured. +func TestStartRecordsWhetherMachineWasAlreadyRunning(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + running bool + startResults []error + }{ + {name: "already running", running: true}, + {name: "booted by this run", running: false, startResults: []error{nil}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + s := &startNSpawnMachine{ + log: silentLogger(), + goalState: &goalstates.NodeStart{MachineName: "kube1"}, + runner: &fakeRunner{ + running: tc.running, + startResults: tc.startResults, + existsAfterStart: true, + }, + } + + require.NoError(t, s.startWithRecovery(context.Background(), "kube1")) + require.Equal(t, tc.running, s.wasRunning) + }) + } +} diff --git a/pkg/agent/phases/nodestart/preflight_api_server.go b/pkg/agent/phases/nodestart/preflight_api_server.go index fdf61d88e..9992a01fe 100644 --- a/pkg/agent/phases/nodestart/preflight_api_server.go +++ b/pkg/agent/phases/nodestart/preflight_api_server.go @@ -30,14 +30,25 @@ type apiServerReachableChecker struct { // Preflight returns the standard node-start checks that can run before the // nspawn machine starts. func Preflight(log *slog.Logger, cfg config.AgentConfig, goalState *goalstates.MachineGoalState) []preflight.Checker { + // A listener is acceptable only when its process root and executable match + // this installation's rootfs, which lets an interrupted bootstrap retry while + // its node keeps running. Foreign and uninspectable owners still fail, and on + // a clean host the rootfs is absent so the check behaves as a plain port test. + var root string + if goalState.RootFS != nil { + root = goalState.RootFS.MachineDir + } + return []preflight.Checker{ // TODO: Consider moving the kubelet bind address to the kubelet goal state. - CheckBindAddress(log, checkKubeletBindAddressName, kubeletBindAddress, "kubelet bind address"), - CheckBindAddress( + checkOwnedBindAddress(log, checkKubeletBindAddressName, kubeletBindAddress, "kubelet bind address", root, kubeletExecutablePath), + checkOwnedBindAddress( log, checkContainerdMetricsBindAddressName, goalState.NodeStart.Containerd.MetricsAddress, "containerd metrics bind address", + root, + containerdExecutablePath, ), CheckAPIServerReachable(log, cfg), } diff --git a/pkg/agent/phases/nodestart/preflight_bind_address.go b/pkg/agent/phases/nodestart/preflight_bind_address.go index aff7e0763..78d5c84f6 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address.go @@ -24,6 +24,11 @@ const ( checkKubeletBindAddressName = "kubelet-bind-address" checkContainerdMetricsBindAddressName = "containerd-metrics-bind-address" kubeletBindAddress = "0.0.0.0:10250" + + // Executable paths inside the nspawn machine rootfs, used to prove that a + // listener belongs to this installation. + kubeletExecutablePath = "usr/local/bin/kubelet" + containerdExecutablePath = "usr/local/bin/containerd" ) type bindAddressChecker struct { @@ -32,9 +37,18 @@ type bindAddressChecker struct { description string log *slog.Logger inspect func(address string) (string, bool, error) + owned func() bool } -// CheckBindAddress verifies no TCP listener currently occupies an address's port. +// CheckBindAddress verifies no TCP listener currently occupies an address's +// port. Any listener fails, including one belonging to this installation. +// +// Nothing in the agent calls this: Preflight is unconditionally ownership-aware +// and uses checkOwnedBindAddress, which accepts a listener it can prove this +// installation owns. It stays exported because it is part of this package's +// published surface, and callers outside the repository compose their own +// preflight sets from it. Removing it would break them at compile time, so it +// is kept deliberately rather than by oversight. func CheckBindAddress(log *slog.Logger, name, address, description string) preflight.Checker { return bindAddressChecker{ name: name, @@ -47,6 +61,73 @@ func CheckBindAddress(log *slog.Logger, name, address, description string) prefl } } +// checkOwnedBindAddress accepts only listeners with both the expected process +// root and executable inode. An uninspectable owner is not proof of ownership. +func checkOwnedBindAddress(log *slog.Logger, name, address, description, root, executable string) preflight.Checker { + c := bindAddressChecker{ + name: name, address: address, description: description, log: log, + inspect: func(address string) (string, bool, error) { return inspectTCPListener("/proc", address) }, + } + c.owned = func() bool { return listenerOwnedByRoot("/proc", address, root, executable) } + + return c +} + +func listenerOwnedByRoot(procRoot, address, root, executable string) bool { + _, portText, err := net.SplitHostPort(address) + if err != nil { + return false + } + + port, err := strconv.ParseUint(portText, 10, 16) + if err != nil { + return false + } + + wanted := map[string]struct{}{} + + for _, table := range []string{"tcp", "tcp6"} { + data, err := os.ReadFile(filepath.Join(procRoot, "net", table)) + if errors.Is(err, fs.ErrNotExist) && table == "tcp6" { + continue + } + + if err != nil { + return false + } + + for inode := range listenerSocketInodes(data, uint16(port)) { + wanted[inode] = struct{}{} + } + } + + rootInfo, err := os.Stat(root) + if err != nil { + return false + } + + exeInfo, err := os.Stat(filepath.Join(root, executable)) + if err != nil { + return false + } + + matched := map[string]bool{} + + for _, owner := range socketOwners(procRoot, wanted) { + base := filepath.Join(procRoot, strconv.Itoa(owner.pid)) + processRoot, rootErr := os.Stat(filepath.Join(base, "root")) + + processExe, exeErr := os.Stat(filepath.Join(base, "exe")) + if rootErr != nil || exeErr != nil || !os.SameFile(rootInfo, processRoot) || !os.SameFile(exeInfo, processExe) { + return false + } + + matched[owner.inode] = true + } + + return len(wanted) > 0 && len(matched) == len(wanted) +} + func (c bindAddressChecker) Name() string { return c.name } func (c bindAddressChecker) Check(context.Context) []preflight.Result { @@ -58,6 +139,10 @@ func (c bindAddressChecker) Check(context.Context) []preflight.Result { } if occupied { + if c.owned != nil && c.owned() { + return preflight.ResultsOK(c.name, c.address, c.description+" belongs to this installation") + } + if owner != "" { return preflight.ResultsError(c.name, c.address, "%s is already in use by process %s", c.description, owner) } @@ -128,11 +213,34 @@ func listenerSocketInodes(socketTable []byte, port uint16) map[string]struct{} { } func findSocketOwner(procRoot string, inodes map[string]struct{}) string { + owners := socketOwners(procRoot, inodes) + if len(owners) > 0 { + owner := owners[0] + + name, err := os.ReadFile(filepath.Join(procRoot, strconv.Itoa(owner.pid), "comm")) + if err == nil && strings.TrimSpace(string(name)) != "" { + return strconv.Quote(strings.TrimSpace(string(name))) + " (PID " + strconv.Itoa(owner.pid) + ")" + } + + return "PID " + strconv.Itoa(owner.pid) + } + + return "" +} + +type socketOwner struct { + pid int + inode string +} + +func socketOwners(procRoot string, inodes map[string]struct{}) []socketOwner { processes, err := os.ReadDir(procRoot) if err != nil { - return "" + return nil } + var owners []socketOwner + for _, process := range processes { pid, err := strconv.Atoi(process.Name()) if err != nil || !process.IsDir() { @@ -155,14 +263,9 @@ func findSocketOwner(procRoot string, inodes map[string]struct{}) string { continue } - name, err := os.ReadFile(filepath.Join(procRoot, process.Name(), "comm")) - if err == nil && strings.TrimSpace(string(name)) != "" { - return strconv.Quote(strings.TrimSpace(string(name))) + " (PID " + strconv.Itoa(pid) + ")" - } - - return "PID " + strconv.Itoa(pid) + owners = append(owners, socketOwner{pid: pid, inode: inode}) } } - return "" + return owners } diff --git a/pkg/agent/phases/nodestart/preflight_bind_address_test.go b/pkg/agent/phases/nodestart/preflight_bind_address_test.go index d9a4854aa..826e5d804 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address_test.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address_test.go @@ -7,6 +7,7 @@ import ( "context" "errors" "log/slog" + "net" "os" "path/filepath" "testing" @@ -22,9 +23,10 @@ import ( const procTCPHeader = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" func TestPreflightBindAddresses(t *testing.T) { - goalState := &goalstates.MachineGoalState{NodeStart: &goalstates.NodeStart{ - Containerd: goalstates.Containerd{MetricsAddress: "0.0.0.0:12345"}, - }} + goalState := &goalstates.MachineGoalState{ + NodeStart: &goalstates.NodeStart{Containerd: goalstates.Containerd{MetricsAddress: "0.0.0.0:12345"}}, + RootFS: &goalstates.RootFS{MachineDir: "/var/lib/machines/kube1"}, + } checks := Preflight(slog.New(slog.DiscardHandler), config.AgentConfig{}, goalState) @@ -32,9 +34,26 @@ func TestPreflightBindAddresses(t *testing.T) { assert.Equal(t, kubeletBindAddress, checks[0].(bindAddressChecker).address) assert.Equal(t, checkContainerdMetricsBindAddressName, checks[1].Name()) assert.Equal(t, "0.0.0.0:12345", checks[1].(bindAddressChecker).address) + + // Bind checks are always ownership-aware, so an interrupted bootstrap can + // retry while its own node keeps listening. + for _, check := range checks[:2] { + assert.NotNil(t, check.(bindAddressChecker).owned) + } +} + +// A partially resolved goal state must degrade to a plain port check instead of +// panicking, because callers build checks before the rootfs stage has run. +func TestPreflightBindAddressesWithoutResolvedRootFS(t *testing.T) { + goalState := &goalstates.MachineGoalState{NodeStart: &goalstates.NodeStart{ + Containerd: goalstates.Containerd{MetricsAddress: "0.0.0.0:12345"}, + }} + + checks := Preflight(slog.New(slog.DiscardHandler), config.AgentConfig{}, goalState) + assert.False(t, checks[0].(bindAddressChecker).owned()) } -func TestCheckBindAddressAvailable(t *testing.T) { +func TestBindAddressCheckerReportsAvailable(t *testing.T) { checker := testBindAddressChecker(func(string) (string, bool, error) { return "", false, nil }) results := checker.Check(context.Background()) @@ -43,7 +62,7 @@ func TestCheckBindAddressAvailable(t *testing.T) { assert.Equal(t, "kubelet bind address is available", results[0].Message) } -func TestCheckBindAddressInUseIncludesOwner(t *testing.T) { +func TestBindAddressCheckerReportsForeignOwner(t *testing.T) { checker := testBindAddressChecker(func(string) (string, bool, error) { return `"kubelet" (PID 123)`, true, nil }) @@ -54,7 +73,7 @@ func TestCheckBindAddressInUseIncludesOwner(t *testing.T) { assert.Equal(t, `kubelet bind address is already in use by process "kubelet" (PID 123)`, results[0].Message) } -func TestCheckBindAddressInspectionFailure(t *testing.T) { +func TestBindAddressCheckerReportsInspectionFailure(t *testing.T) { checker := testBindAddressChecker(func(string) (string, bool, error) { return "", false, errors.New("inspection failed") }) @@ -130,3 +149,75 @@ func createProcFixture(t *testing.T, tcp, tcp6 string) string { return procRoot } + +func TestListenerOwnershipRequiresRootAndExecutableForEverySocket(t *testing.T) { + t.Parallel() + + for _, mode := range []string{"owned", "foreign-root", "foreign-executable", "unknown-owner", "shared-with-foreign"} { + t.Run(mode, func(t *testing.T) { + proc := createProcFixture(t, procTCPHeader+"0: 00000000:280A 00000000:0000 0A 0 0 0 0 0 45678\n", procTCPHeader) + root := t.TempDir() + exe := filepath.Join(root, "kubelet") + require.NoError(t, os.WriteFile(exe, []byte("executable"), 0o755)) + + processRoot, processExe := root, exe + if mode == "foreign-root" { + processRoot = t.TempDir() + } + + if mode == "foreign-executable" { + processExe = filepath.Join(t.TempDir(), "kubelet") + require.NoError(t, os.WriteFile(processExe, []byte("executable"), 0o755)) + } + + if mode != "unknown-owner" { + require.NoError(t, os.MkdirAll(filepath.Join(proc, "123", "fd"), 0o755)) + require.NoError(t, os.Symlink("socket:[45678]", filepath.Join(proc, "123", "fd", "4"))) + require.NoError(t, os.Symlink(processRoot, filepath.Join(proc, "123", "root"))) + require.NoError(t, os.Symlink(processExe, filepath.Join(proc, "123", "exe"))) + } + + if mode == "shared-with-foreign" { + require.NoError(t, os.MkdirAll(filepath.Join(proc, "456", "fd"), 0o755)) + require.NoError(t, os.Symlink("socket:[45678]", filepath.Join(proc, "456", "fd", "4"))) + require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(proc, "456", "root"))) + require.NoError(t, os.Symlink(exe, filepath.Join(proc, "456", "exe"))) + } + + require.Equal(t, mode == "owned", listenerOwnedByRoot(proc, kubeletBindAddress, root, "kubelet")) + }) + } +} + +// TestCheckBindAddressRejectsAnyListener exercises the exported constructor +// rather than the checker type the other tests here build directly. +// +// It exists because nothing inside the agent calls CheckBindAddress: Preflight +// is ownership-aware and uses checkOwnedBindAddress. A sweep for symbols with no +// caller therefore reads it as dead and removes it, which breaks callers outside +// the repository that compose their own preflight sets. This test is the caller +// that keeps it honest, and it pins the behavior those callers rely on: the +// unowned constructor rejects any listener at all, where the owned variant +// accepts one it can prove belongs to this installation. +func TestCheckBindAddressRejectsAnyListener(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + t.Cleanup(func() { _ = listener.Close() }) + + address := listener.Addr().String() + log := slog.New(slog.DiscardHandler) + + occupied := CheckBindAddress(log, "test-bind-address", address, "test bind address") + require.Equal(t, "test-bind-address", occupied.Name()) + + results := occupied.Check(context.Background()) + require.NotEmpty(t, results) + assert.Equal(t, preflight.SeverityError, results[0].Severity, + "a listener this installation cannot claim must fail the unowned check") + + require.NoError(t, listener.Close()) + + free := CheckBindAddress(log, "test-bind-address", address, "test bind address") + assert.Equal(t, preflight.SeverityOK, free.Check(context.Background())[0].Severity) +} diff --git a/pkg/agent/phases/nodestart/restart_reconfigured.go b/pkg/agent/phases/nodestart/restart_reconfigured.go new file mode 100644 index 000000000..2c1b67c9e --- /dev/null +++ b/pkg/agent/phases/nodestart/restart_reconfigured.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package nodestart + +import ( + "context" + "fmt" + "log/slog" + + "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/phases" +) + +// restartReconfigured restarts containerd and kubelet when this invocation +// actually changed their configuration. +// +// Writing a configuration file does not affect a service that has already read +// it. That is harmless when this sequence boots the machine, because the +// services start afterwards and read the new files. It is not harmless when the +// sequence runs against a machine that is already up: without this, the files +// on disk and the running services would disagree, with nothing to reconcile +// them, which is a worse outcome than not reapplying at all. +// +// Only an actual change restarts anything. A reapply that writes identical +// content leaves the node alone, so the ordinary case of rerunning bootstrap +// after a failure costs nothing. +// +// It covers containerd and kubelet, and deliberately not everything the node +// stage writes. LocalDNS and the NVIDIA drop-in write through utilio directly +// and are not tracked, so a reapply that changes one updates the file without +// restarting its reader. Those inputs are all carried in the applied config, +// which a retry against a running node does not rewrite, so the daemon still +// sees drift and repaves. Extending tracking to them would make the reapply +// converge without a repave; until then the repave is what closes the gap. +type restartReconfigured struct { + log *slog.Logger + goalState *goalstates.NodeStart + + startMachine *startNSpawnMachine + containerd *configureContainerd + kubelet *configureKubelet +} + +func (r *restartReconfigured) Name() string { return "restart-reconfigured-services" } + +func (r *restartReconfigured) Do(ctx context.Context) error { + if !r.startMachine.wasRunning { + // This sequence started the machine, so its services have already read + // the configuration written above. + return nil + } + + // containerd first: kubelet talks to it, so restarting kubelet into a + // restarting runtime would only make it retry. + for _, unit := range []struct { + name string + changed bool + }{ + {goalstates.SystemdUnitContainerd, r.containerd.changed}, + {goalstates.SystemdUnitKubelet, r.kubelet.changed}, + } { + if !unit.changed { + continue + } + + r.log.Info("configuration changed on a running node, restarting service", + "unit", unit.name, + "machine", r.goalState.MachineName, + ) + + if _, err := executil.MachineRun(ctx, r.log, r.goalState.MachineName, + "systemctl", "restart", unit.name, + ); err != nil { + return fmt.Errorf("systemctl restart %s in %s: %w", unit.name, r.goalState.MachineName, err) + } + } + + return nil +} + +var _ phases.Task = (*restartReconfigured)(nil) diff --git a/pkg/agent/phases/nodestart/restart_reconfigured_test.go b/pkg/agent/phases/nodestart/restart_reconfigured_test.go new file mode 100644 index 000000000..eea1a5af8 --- /dev/null +++ b/pkg/agent/phases/nodestart/restart_reconfigured_test.go @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package nodestart + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/goalstates" +) + +// stubMachineRun puts a recording systemd-run on PATH, since MachineRun shells +// out to it, and returns the file each invocation appends its arguments to. +func stubMachineRun(t *testing.T) string { + t.Helper() + + dir := t.TempDir() + record := filepath.Join(dir, "calls") + script := "#!/bin/sh\necho \"$@\" >> " + record + "\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemd-run"), []byte(script), 0o755)) + t.Setenv("PATH", dir) + + return record +} + +func machineRunCalls(t *testing.T, record string) string { + t.Helper() + + data, err := os.ReadFile(record) + if os.IsNotExist(err) { + return "" + } + + require.NoError(t, err) + + return string(data) +} + +func reconfigureTask(t *testing.T, wasRunning, containerdChanged, kubeletChanged bool) (*restartReconfigured, string) { + t.Helper() + + record := stubMachineRun(t) + + return &restartReconfigured{ + log: slog.New(slog.DiscardHandler), + goalState: &goalstates.NodeStart{MachineName: goalstates.NSpawnMachineKube1}, + startMachine: &startNSpawnMachine{wasRunning: wasRunning}, + containerd: &configureContainerd{changeTracker: changeTracker{changed: containerdChanged}}, + kubelet: &configureKubelet{changeTracker: changeTracker{changed: kubeletChanged}}, + }, record +} + +// TestNoRestartWhenThisRunStartedTheMachine covers the fresh install. The +// services start after the configuration is written, so they already read it. +func TestNoRestartWhenThisRunStartedTheMachine(t *testing.T) { + task, record := reconfigureTask(t, false, true, true) + + require.NoError(t, task.Do(t.Context())) + require.Empty(t, machineRunCalls(t, record), "a machine this run started needs no restart") +} + +// TestNoRestartWhenNothingChanged is the ordinary retry: bootstrap is rerun +// after a failure, the configuration is identical, and the node is left alone. +func TestNoRestartWhenNothingChanged(t *testing.T) { + task, record := reconfigureTask(t, true, false, false) + + require.NoError(t, task.Do(t.Context())) + require.Empty(t, machineRunCalls(t, record), "an unchanged reapply must not disturb a running node") +} + +// TestRestartsOnlyTheServiceWhoseConfigChanged keeps a kubelet change from +// bouncing the container runtime underneath it. +func TestRestartsOnlyTheServiceWhoseConfigChanged(t *testing.T) { + task, record := reconfigureTask(t, true, false, true) + + require.NoError(t, task.Do(t.Context())) + + calls := machineRunCalls(t, record) + require.Contains(t, calls, "restart "+goalstates.SystemdUnitKubelet) + require.NotContains(t, calls, "restart "+goalstates.SystemdUnitContainerd) +} + +// TestRestartsContainerdBeforeKubelet pins the order. kubelet talks to +// containerd, so restarting kubelet into a restarting runtime would only make +// it retry. +func TestRestartsContainerdBeforeKubelet(t *testing.T) { + task, record := reconfigureTask(t, true, true, true) + + require.NoError(t, task.Do(t.Context())) + + calls := machineRunCalls(t, record) + require.Less(t, + indexOfUnit(calls, goalstates.SystemdUnitContainerd), + indexOfUnit(calls, goalstates.SystemdUnitKubelet), + "containerd must restart before kubelet", + ) +} + +func indexOfUnit(calls, unit string) int { + for i := 0; i+len(unit) <= len(calls); i++ { + if calls[i:i+len(unit)] == unit { + return i + } + } + + return -1 +} diff --git a/pkg/agent/phases/nodestart/start.go b/pkg/agent/phases/nodestart/start.go index 484a58028..e745b4f9e 100644 --- a/pkg/agent/phases/nodestart/start.go +++ b/pkg/agent/phases/nodestart/start.go @@ -20,17 +20,36 @@ import ( // and node update flows. Callers that need to persist the applied config for // drift detection should append that step separately. func StartNode(log *slog.Logger, gs *goalstates.NodeStart) phases.Task { + // These are built concretely rather than through the exported constructors + // so the sequence can tell whether the configuration it wrote differed, and + // whether the machine was already up. Both are needed to decide if a + // running service has to be restarted to read what changed. + containerd := &configureContainerd{goalState: gs} + kubelet := &configureKubelet{goalState: gs} + startMachine := &startNSpawnMachine{ + log: log, + goalState: gs, + runner: defaultMachinectlRunner{log: log}, + } + return phases.Serial(log, phases.Parallel(log, - ConfigureContainerd(gs), - ConfigureKubelet(gs), + containerd, + kubelet, ConfigureLocalDNS(gs), ), SetupLocalDNSNetwork(log, gs), - StartNSpawnMachine(log, gs), + startMachine, WaitForLocalDNS(log, gs), StartContainerd(log, gs), ImportContainerImages(log, gs), StartKubelet(log, gs), + &restartReconfigured{ + log: log, + goalState: gs, + startMachine: startMachine, + containerd: containerd, + kubelet: kubelet, + }, ) } diff --git a/pkg/agent/phases/reset/helpers.go b/pkg/agent/phases/reset/helpers.go index 80a62be49..548c402ad 100644 --- a/pkg/agent/phases/reset/helpers.go +++ b/pkg/agent/phases/reset/helpers.go @@ -5,22 +5,27 @@ package reset import ( "errors" + "fmt" "log/slog" "os" ) -// removeFileIfExists removes a file if it exists. Non-ENOENT errors are -// logged at Warn so we have a trace but don't abort the reset flow. -func removeFileIfExists(log *slog.Logger, path string) { +// removeFileIfExists ignores absence but propagates substantive removal errors. +func removeFileIfExists(log *slog.Logger, path string) error { if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { log.Warn("failed to remove file", "path", path, "error", err) + return fmt.Errorf("remove %s: %w", path, err) } + + return nil } -// removeAllIfExists removes a path and all children if it exists. Errors are -// logged at Warn so we have a trace but don't abort the reset flow. -func removeAllIfExists(log *slog.Logger, path string) { +// removeAllIfExists propagates removal failures so reset retains ownership. +func removeAllIfExists(log *slog.Logger, path string) error { if err := os.RemoveAll(path); err != nil { log.Warn("failed to remove directory", "path", path, "error", err) + return fmt.Errorf("remove %s: %w", path, err) } + + return nil } diff --git a/pkg/agent/phases/reset/machine.go b/pkg/agent/phases/reset/machine.go index b3f26d276..092441ce7 100644 --- a/pkg/agent/phases/reset/machine.go +++ b/pkg/agent/phases/reset/machine.go @@ -9,9 +9,11 @@ import ( "fmt" "log/slog" "os" + "strings" "time" "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/phases" ) @@ -29,22 +31,31 @@ func StopMachine(log *slog.Logger, machineName string) phases.Task { func (t *stopMachine) Name() string { return "stop-machine" } func (t *stopMachine) Do(ctx context.Context) error { + // Stop the systemd service that manages the nspawn container. This + // properly tears down mount namespaces and cgroups so that + // machinectl remove can succeed. + serviceName := fmt.Sprintf("systemd-nspawn@%s.service", t.machineName) + if err := executil.RunCmd(ctx, t.log, executil.Machinectl(), "disable", t.machineName); err != nil { - t.log.Warn("failed to disable machine (may not have been enabled)", "machine", t.machineName, "error", err) + if confirmErr := confirmNotEnabled(ctx, t.log, serviceName); confirmErr != nil { + return fmt.Errorf("disable machine %s: %w", t.machineName, errors.Join(err, confirmErr)) + } + + t.log.Warn("machine was not enabled; continuing with stop and removal", "machine", t.machineName, "error", err) + } + + exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName) + if err != nil { + return err } - if !machineExists(ctx, t.log, t.machineName) { + if !exists { t.log.Info("machine not running, nothing to stop", "machine", t.machineName) return nil } t.log.Info("stopping nspawn machine", "machine", t.machineName) - // Stop the systemd service that manages the nspawn container. This - // properly tears down mount namespaces and cgroups so that - // machinectl remove can succeed. - serviceName := fmt.Sprintf("systemd-nspawn@%s.service", t.machineName) - if !serviceIsActive(ctx, t.log, serviceName) { t.log.Info("nspawn service already inactive, skipping stop", "service", serviceName) } else if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "stop", serviceName); err != nil { @@ -52,12 +63,16 @@ func (t *stopMachine) Do(ctx context.Context) error { } // Wait up to 30 seconds for the machine to fully stop. - if t.waitForGone(ctx, 30*time.Second) { + if gone, err := t.waitForGone(ctx, 30*time.Second); err != nil { + return err + } else if gone { return nil } // Force terminate if still registered. - if machineExists(ctx, t.log, t.machineName) { + if exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName); err != nil { + return err + } else if exists { t.log.Warn("machine did not stop gracefully, terminating", "machine", t.machineName) if err := executil.RunCmd(ctx, t.log, executil.Machinectl(), "terminate", t.machineName); err != nil { @@ -65,29 +80,63 @@ func (t *stopMachine) Do(ctx context.Context) error { } // Wait up to 15 seconds for the terminate to take full effect. - t.waitForGone(ctx, 15*time.Second) + if gone, err := t.waitForGone(ctx, 15*time.Second); err != nil { + return err + } else if !gone { + return fmt.Errorf("machine %s remains registered after termination", t.machineName) + } } return ctx.Err() } +// confirmNotEnabled reports whether the nspawn unit is definitely not enabled. +// +// machinectl disable fails for benign reasons, most often a machine that was +// never enabled, so the failure alone does not justify aborting reset. It is +// not safe to simply ignore either: the enablement symlink outlives the config +// and rootfs that reset deletes, so a unit left enabled makes the host try to +// start a machine that no longer exists on the next boot, while reset reported +// success. Continue only when systemd positively reports a state that cannot +// start the unit at boot, and treat a failed inspection as unconfirmed. +func confirmNotEnabled(ctx context.Context, log *slog.Logger, service string) error { + // show exits zero whatever the state, unlike is-enabled, so the reported + // state is the signal rather than the exit status. + out, err := executil.OutputCmd(ctx, log, "systemctl", "show", service, "--property=UnitFileState", "--value") + if err != nil { + return fmt.Errorf("inspect %s enablement: %w", service, err) + } + + // An empty state means no unit file is installed, so nothing is enabled. + switch state := strings.TrimSpace(out); state { + case "", "disabled", "not-found", "masked", "masked-runtime", "static", "indirect": + return nil + default: + return fmt.Errorf("%s is %s", service, state) + } +} + // waitForGone polls machineExists until the machine disappears or the timeout // elapses. Returns true if the machine is gone. -func (t *stopMachine) waitForGone(ctx context.Context, timeout time.Duration) bool { +func (t *stopMachine) waitForGone(ctx context.Context, timeout time.Duration) (bool, error) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { - if !machineExists(ctx, t.log, t.machineName) { - return true + if exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName); err != nil { + return false, err + } else if !exists { + return true, nil } select { case <-ctx.Done(): - return false + return false, ctx.Err() case <-time.After(time.Second): } } - return !machineExists(ctx, t.log, t.machineName) + exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName) + + return !exists, err } type removeMachine struct { @@ -110,6 +159,8 @@ func (t *removeMachine) Do(ctx context.Context) error { if _, err := os.Stat(machineDir); errors.Is(err, os.ErrNotExist) { t.log.Info("machine rootfs not present, nothing to remove", "machine", t.machineName) return nil + } else if err != nil { + return err } t.log.Info("removing machine rootfs", "machine", t.machineName, "dir", machineDir) @@ -130,7 +181,9 @@ func (t *removeMachine) Do(ctx context.Context) error { return nil // machinectl removed both image metadata and directory } - if !machineExists(ctx, t.log, t.machineName) { + if exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName); err != nil { + return err + } else if !exists { // Once machined no longer knows the machine, the nspawn service is stopped // and the rootfs can be deleted directly. Some host configurations can still // make machinectl remove fail at this point. Fedora with SELinux enforcing, @@ -149,15 +202,72 @@ func (t *removeMachine) Do(ctx context.Context) error { // Fallback: force-remove the directory if machinectl keeps failing. t.log.Warn("machinectl remove did not succeed, force-removing directory", "dir", machineDir) - removeAllIfExists(t.log, machineDir) - return nil + if exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName); err != nil { + return err + } else if exists { + return fmt.Errorf("refusing to remove registered machine %s", t.machineName) + } + + return removeAllIfExists(t.log, machineDir) } -// machineExists checks whether the named nspawn machine is known to machinectl. -func machineExists(ctx context.Context, log *slog.Logger, name string) bool { - err := executil.RunCmd(ctx, log, executil.Machinectl(), "show", name) - return err == nil +// RegisteredMachine reports registration only after successful inventory. +// Bootstrap callers must not treat a missing inspection tool as a clean host. +func RegisteredMachine(ctx context.Context, log *slog.Logger, name string) (bool, error) { + names, err := registeredMachines(ctx, log) + if err != nil { + return false, err + } + + _, ok := names[name] + + return ok, nil +} + +// FirstRegisteredMachine returns the name of the first node slot that is +// registered, or the empty string if neither is. It inventories once rather +// than per slot, and like RegisteredMachine it reports an uninspectable host as +// an error rather than as a clean one. +// +// This is for questions about the host as a whole, where either slot being +// occupied is what matters: whether it is safe to flush a ruleset shared by +// every machine in the netns, for instance. A caller asking about the node it +// is building or tearing down wants RegisteredMachine with that slot's name, +// because a machine in the other slot answers a question it did not ask. +func FirstRegisteredMachine(ctx context.Context, log *slog.Logger) (string, error) { + names, err := registeredMachines(ctx, log) + if err != nil { + return "", err + } + + for _, name := range []string{goalstates.NSpawnMachineKube1, goalstates.NSpawnMachineKube2} { + if _, ok := names[name]; ok { + return name, nil + } + } + + return "", nil +} + +// registeredMachines inventories machinectl once and returns the registered +// names as a set. +func registeredMachines(ctx context.Context, log *slog.Logger) (map[string]struct{}, error) { + out, err := executil.OutputCmd(ctx, log, "machinectl", "list", "--no-legend", "--no-pager") + if err != nil { + return nil, fmt.Errorf("inspect registered machines: %w", err) + } + + names := make(map[string]struct{}) + + for _, line := range strings.Split(out, "\n") { + fields := strings.Fields(line) + if len(fields) > 0 { + names[fields[0]] = struct{}{} + } + } + + return names, nil } // serviceIsActive returns true if the named systemd service is currently active. diff --git a/pkg/agent/phases/reset/network.go b/pkg/agent/phases/reset/network.go index 6d50c875d..5044903de 100644 --- a/pkg/agent/phases/reset/network.go +++ b/pkg/agent/phases/reset/network.go @@ -66,10 +66,24 @@ func (t *cleanupLocalDNSRules) Name() string { return "cleanup-localdns-rules" } func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "disable", "--now", goalstates.LocalDNSNetworkUnit); err != nil { - t.log.Debug("LocalDNS network unit was not active", "error", err) + path := filepath.Join(goalstates.SystemdSystemDir, goalstates.LocalDNSNetworkUnit) + if _, statErr := os.Lstat(path); !errors.Is(statErr, os.ErrNotExist) { + return fmt.Errorf("disable LocalDNS unit: %w", err) + } + } + + tables, err := executil.OutputCmd(ctx, t.log, "nft", "list", "tables") + if missingTool(err) { + t.log.Warn("nft is not installed; no LocalDNS ruleset can exist") + + tables, err = "", nil } - if _, err := executil.OutputCmd(ctx, t.log, "nft", "list", "table", "ip", goalstates.LocalDNSNFTTable); err == nil { + if err != nil { + return fmt.Errorf("inspect LocalDNS tables: %w", err) + } + + if strings.Contains(tables, "table ip "+goalstates.LocalDNSNFTTable+"\n") { if err := executil.RunCmd(ctx, t.log, func(ctx context.Context) *exec.Cmd { return exec.CommandContext(ctx, "nft") }, "delete", "table", "ip", goalstates.LocalDNSNFTTable); err != nil { @@ -77,7 +91,22 @@ func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { } } - if output, err := executil.OutputCmd(ctx, t.log, "ip", "-d", "-o", "link", "show", "dev", goalstates.LocalDNSInterfaceName); err == nil { + links, err := executil.OutputCmd(ctx, t.log, "ip", "-d", "-o", "link", "show") + if missingTool(err) { + t.log.Warn("ip is not installed; no LocalDNS interface can exist") + + links, err = "", nil + } + + if err != nil { + return fmt.Errorf("inspect LocalDNS interface: %w", err) + } + + for _, output := range strings.Split(links, "\n") { + if !strings.Contains(output, ": "+goalstates.LocalDNSInterfaceName+":") { + continue + } + if !strings.Contains(" "+output+" ", " dummy ") { return fmt.Errorf("refusing to remove non-dummy interface %s", goalstates.LocalDNSInterfaceName) } @@ -91,7 +120,9 @@ func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { filepath.Join(goalstates.SystemdSystemDir, goalstates.LocalDNSNetworkUnit), "/usr/local/libexec/unbounded-localdns-network", } { - removeFileIfExists(t.log, path) + if err := removeFileIfExists(t.log, path); err != nil { + return err + } } return nil @@ -101,19 +132,27 @@ func (t *removeNetworkInterfaces) Do(ctx context.Context) error { // Remove WireGuard interfaces (wg51820, wg51821, ...). wgIfaces, err := listWireGuardInterfaces(ctx, t.log) if err != nil { - t.log.Warn("failed to list WireGuard interfaces", "error", err) + return err } for _, iface := range wgIfaces { t.log.Info("removing interface", "interface", iface) - deleteLink(ctx, t.log, iface) + + if err := deleteLink(ctx, t.log, iface); err != nil { + return err + } } // Remove tunnel and overlay interfaces. for _, iface := range knownOverlayInterfaces { - if linkExists(t.log, iface) { + if exists, err := linkExists(t.log, iface); err != nil { + return err + } else if exists { t.log.Info("removing interface", "interface", iface) - deleteLink(ctx, t.log, iface) + + if err := deleteLink(ctx, t.log, iface); err != nil { + return err + } } } @@ -139,7 +178,9 @@ func (t *removeWireGuardKeys) Do(_ context.Context) error { "/etc/wireguard/server.priv", "/etc/wireguard/server.pub", } { - removeFileIfExists(t.log, path) + if err := removeFileIfExists(t.log, path); err != nil { + return err + } } return nil @@ -149,6 +190,12 @@ func (t *removeWireGuardKeys) Do(_ context.Context) error { // interfaces visible on the host. func listWireGuardInterfaces(ctx context.Context, log *slog.Logger) ([]string, error) { out, err := executil.OutputCmd(ctx, log, "ip", "-o", "link", "show") + if missingTool(err) { + log.Warn("ip is not installed; no WireGuard interfaces can exist") + + return nil, nil + } + if err != nil { return nil, fmt.Errorf("ip link show: %w", err) } @@ -170,7 +217,7 @@ func listWireGuardInterfaces(ctx context.Context, log *slog.Logger) ([]string, e } } - return ifaces, nil + return ifaces, scanner.Err() } // isWireGuardInterface returns true if the interface name matches the @@ -196,25 +243,30 @@ func isWireGuardInterface(name string) bool { // linkExists checks whether a network interface exists by looking up its // entry in /sys/class/net. This avoids shelling out and cleanly distinguishes // "not found" from real errors. -func linkExists(log *slog.Logger, name string) bool { +func linkExists(log *slog.Logger, name string) (bool, error) { _, err := os.Stat(fmt.Sprintf("/sys/class/net/%s", name)) if err == nil { - return true + return true, nil } if errors.Is(err, os.ErrNotExist) { - return false + return false, nil } log.Warn("failed to check interface existence", "interface", name, "error", err) - return false + return false, err } -// deleteLink removes a network interface, logging a warning if the operation -// fails (e.g. the interface was already removed). -func deleteLink(ctx context.Context, log *slog.Logger, name string) { +// deleteLink ignores only verified absence after a failed deletion. +func deleteLink(ctx context.Context, log *slog.Logger, name string) error { if err := executil.RunCmd(ctx, log, executil.Ip(), "link", "delete", name); err != nil { - log.Warn("failed to delete interface (may already be gone)", "interface", name, "error", err) + if exists, inspectErr := linkExists(log, name); inspectErr == nil && !exists { + return nil + } + + return fmt.Errorf("delete interface %s: %w", name, err) } + + return nil } diff --git a/pkg/agent/phases/reset/nspawn.go b/pkg/agent/phases/reset/nspawn.go index c4637654d..4714c39ce 100644 --- a/pkg/agent/phases/reset/nspawn.go +++ b/pkg/agent/phases/reset/nspawn.go @@ -9,6 +9,7 @@ import ( "fmt" "log/slog" "os" + "os/exec" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -34,11 +35,7 @@ func (t *removeNSpawnConfig) Do(_ context.Context) error { configRegenerationUnit := fmt.Sprintf("%s/%s", goalstates.SystemdSystemDir, goalstates.ConfigRegenerationUnit(t.machineName)) t.log.Info("removing nspawn configuration", "nspawn_file", nspawnFile, "override_dir", overrideDir, "config_regeneration_unit", configRegenerationUnit) - removeFileIfExists(t.log, nspawnFile) - removeAllIfExists(t.log, overrideDir) - removeFileIfExists(t.log, configRegenerationUnit) - - return nil + return errors.Join(removeFileIfExists(t.log, nspawnFile), removeAllIfExists(t.log, overrideDir), removeFileIfExists(t.log, configRegenerationUnit)) } type removeBPFFSMount struct { @@ -63,18 +60,20 @@ func (t *removeBPFFSMount) Do(ctx context.Context) error { return fmt.Errorf("stat bpffs mount path %s: %w", mountPath, err) } - // mountpoint -q exits non-zero when an existing path is not a mount point. - // The agent runs as root and host preparation installs util-linux, so treat a - // non-zero exit here as "already unmounted" and remove the directory below. + // util-linux reserves exit 32 for a path that is not a mount point. + // Other failures do not authorize recursive removal. if err := executil.RunCmdAt(ctx, t.log, slog.LevelDebug, executil.Mountpoint(), "-q", mountPath); err == nil { if err := executil.RunCmd(ctx, t.log, executil.Umount(), mountPath); err != nil { return fmt.Errorf("unmount bpffs %s: %w", mountPath, err) } + } else { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 32 { + return fmt.Errorf("inspect bpffs mount %s: %w", mountPath, err) + } } - removeAllIfExists(t.log, mountPath) - - return nil + return removeAllIfExists(t.log, mountPath) } // CleanupMachine returns a composite task that removes all artifacts of an diff --git a/pkg/agent/phases/reset/routes.go b/pkg/agent/phases/reset/routes.go index 1f1732d29..1de32a0f7 100644 --- a/pkg/agent/phases/reset/routes.go +++ b/pkg/agent/phases/reset/routes.go @@ -5,8 +5,10 @@ package reset import ( "context" + "encoding/json" "fmt" "log/slog" + "strconv" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/pkg/agent/phases" @@ -20,7 +22,8 @@ const ( ) type cleanupRoutes struct { - log *slog.Logger + log *slog.Logger + output func(context.Context, ...string) (string, error) } // CleanupRoutes returns a task that removes policy routing rules and flushes @@ -34,21 +37,96 @@ func (t *cleanupRoutes) Name() string { return "cleanup-routes" } func (t *cleanupRoutes) Do(ctx context.Context) error { t.log.Info("cleaning up policy routing rules") - for table := wireguardTableStart; table <= wireguardTableEnd; table++ { - tableStr := fmt.Sprintf("%d", table) + output := t.output + if output == nil { + output = func(ctx context.Context, args ...string) (string, error) { + return executil.OutputCmd(ctx, t.log, "ip", args...) + } + } - // Remove all ip rules pointing to this table. - for { - if err := executil.RunCmd(ctx, t.log, executil.Ip(), "rule", "del", "table", tableStr); err != nil { - break // no more rules for this table + for _, family := range []string{"-4", "-6"} { + for _, kind := range []string{"rule", "route"} { + args := []string{family, "-N", "-j", kind, "show"} + if kind == "route" { + args = append(args, "table", "all") + } + + out, err := output(ctx, args...) + if missingTool(err) { + t.log.Warn("ip is not installed; no policy routing rules can exist") + + return nil + } + + if err != nil { + return fmt.Errorf("inspect %s %s: %w", family, kind, err) } - } - // Flush the routing table. - if err := executil.RunCmd(ctx, t.log, executil.Ip(), "route", "flush", "table", tableStr); err != nil { - t.log.Warn("failed to flush routing table (may be empty)", "table", tableStr, "error", err) + tables, err := ownedRoutingTables(out) + if err != nil { + return fmt.Errorf("decode %s %s: %w", family, kind, err) + } + + flushed := make(map[int]bool) + + for _, table := range tables { + action := "del" + + if kind == "route" { + if flushed[table] { + continue + } + + flushed[table] = true + action = "flush" + } + + if _, err := output(ctx, family, kind, action, "table", strconv.Itoa(table)); err != nil { + return fmt.Errorf("remove %s %s table %d: %w", family, kind, table, err) + } + } } } return nil } + +// Enumerate numeric table IDs before mutation so an inspection failure cannot +// be mistaken for absence. Preserve duplicate rules but flush each table once. +func ownedRoutingTables(output string) ([]int, error) { + var entries []struct { + Table json.RawMessage `json:"table"` + } + if err := json.Unmarshal([]byte(output), &entries); err != nil { + return nil, err + } + + var tables []int + + for _, entry := range entries { + if len(entry.Table) == 0 { + continue + } + + var table int + if err := json.Unmarshal(entry.Table, &table); err != nil { + var name string + if err := json.Unmarshal(entry.Table, &name); err != nil { + return nil, err + } + + var parseErr error + + table, parseErr = strconv.Atoi(name) + if parseErr != nil { + continue + } + } + + if table >= wireguardTableStart && table <= wireguardTableEnd { + tables = append(tables, table) + } + } + + return tables, nil +} diff --git a/pkg/agent/phases/reset/strict_test.go b/pkg/agent/phases/reset/strict_test.go new file mode 100644 index 000000000..9894c0760 --- /dev/null +++ b/pkg/agent/phases/reset/strict_test.go @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package reset + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMachineInspectionFailsClosed(t *testing.T) { + for _, tc := range []struct { + name, script string + exists, failed bool + }{ + {"registered", "printf 'kube1 container systemd-nspawn - - -\\n'", true, false}, + {"different", "printf 'kube10 container systemd-nspawn - - -\\n'", false, false}, + {"absent", "exit 0", false, false}, + {"denied", "exit 1", false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "machinectl"), []byte("#!/bin/sh\n"+tc.script+"\n"), 0o755)) + t.Setenv("PATH", dir) + exists, err := RegisteredMachine(t.Context(), slog.New(slog.DiscardHandler), "kube1") + require.Equal(t, tc.exists, exists) + + if tc.failed { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +// TestConfirmNotEnabledOnlyAcceptsUnstartableStates pins which reported unit +// file states let reset continue after machinectl disable fails. Anything that +// could still start the unit at boot, or an inspection that did not answer, has +// to stop reset rather than leave an enablement symlink behind for a machine +// whose config and rootfs are about to be deleted. +func TestConfirmNotEnabledOnlyAcceptsUnstartableStates(t *testing.T) { + for _, tc := range []struct { + name, script string + confirmed bool + }{ + {"never-enabled", "printf 'disabled\\n'", true}, + {"no-unit-file", "printf '\\n'", true}, + {"not-found", "printf 'not-found\\n'", true}, + {"masked", "printf 'masked\\n'", true}, + {"static", "printf 'static\\n'", true}, + {"indirect", "printf 'indirect\\n'", true}, + {"enabled", "printf 'enabled\\n'", false}, + {"enabled-runtime", "printf 'enabled-runtime\\n'", false}, + {"linked", "printf 'linked\\n'", false}, + {"generated", "printf 'generated\\n'", false}, + {"inspection-failed", "exit 1", false}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), []byte("#!/bin/sh\n"+tc.script+"\n"), 0o755)) + t.Setenv("PATH", dir) + + err := confirmNotEnabled(t.Context(), slog.New(slog.DiscardHandler), "systemd-nspawn@kube1.service") + if tc.confirmed { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} + +// TestStopMachineFailsWhenDisableLeavesUnitEnabled covers the whole task: a +// failed disable is tolerated only once systemd confirms the unit cannot start +// at boot. +func TestStopMachineFailsWhenDisableLeavesUnitEnabled(t *testing.T) { + for _, tc := range []struct { + name, disable, unitFileState string + wantErr bool + }{ + {"disable-succeeds", "exit 0", "enabled", false}, + {"disable-fails-but-not-enabled", "exit 1", "disabled", false}, + {"disable-fails-and-still-enabled", "exit 1", "enabled", true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + // list reports no registered machines, so a tolerated failure falls + // through to "nothing to stop" and the task succeeds. + machinectl := "#!/bin/sh\ncase \"$1\" in\ndisable) " + tc.disable + " ;;\nlist) exit 0 ;;\nesac\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "machinectl"), []byte(machinectl), 0o755)) + + systemctl := "#!/bin/sh\nif [ \"$1\" = show ]; then printf '" + tc.unitFileState + "\\n'; fi\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), []byte(systemctl), 0o755)) + t.Setenv("PATH", dir) + + err := StopMachine(slog.New(slog.DiscardHandler), "kube1").Do(t.Context()) + if tc.wantErr { + require.ErrorContains(t, err, "systemd-nspawn@kube1.service is enabled") + } else { + require.NoError(t, err) + } + }) + } +} + +func TestCleanupRoutesPropagatesInspectionAndMutationFailures(t *testing.T) { + t.Parallel() + + for _, failure := range []string{"-4 -N -j rule show", "-4 rule del table 51820", "-4 -N -j route show table all", "-4 route flush table 51820", "-6 -N -j rule show", "-6 rule del table 51820", "-6 -N -j route show table all", "-6 route flush table 51820"} { + t.Run(failure, func(t *testing.T) { + injected := errors.New("injected network failure") + task := &cleanupRoutes{log: slog.New(slog.DiscardHandler), output: func(_ context.Context, args ...string) (string, error) { + if strings.Join(args, " ") == failure { + return "", injected + } + + return `[{"table":51820}]`, nil + }} + require.ErrorIs(t, task.Do(t.Context()), injected) + }) + } +} + +func TestCleanupRoutesPreservesUnrelatedTables(t *testing.T) { + t.Parallel() + + var mutations []string + + task := &cleanupRoutes{log: slog.New(slog.DiscardHandler), output: func(_ context.Context, args ...string) (string, error) { + if args[1] == "-N" { + return `[{"table":"main"},{"table":51819},{"table":51820},{"table":"51820"},{"table":51899},{"table":51900}]`, nil + } + + mutations = append(mutations, strings.Join(args, " ")) + + return "", nil + }} + require.NoError(t, task.Do(t.Context())) + require.Equal(t, []string{ + "-4 rule del table 51820", "-4 rule del table 51820", "-4 rule del table 51899", + "-4 route flush table 51820", "-4 route flush table 51899", + "-6 rule del table 51820", "-6 rule del table 51820", "-6 rule del table 51899", + "-6 route flush table 51820", "-6 route flush table 51899", + }, mutations) +} + +func TestCleanupRoutesAbsenceAndMalformedInventory(t *testing.T) { + t.Parallel() + + for _, output := range []string{"[]", "invalid", `[{"table":"unexpected-name"}]`, `[{"table":{}}]`} { + t.Run(output, func(t *testing.T) { + task := &cleanupRoutes{log: slog.New(slog.DiscardHandler), output: func(_ context.Context, args ...string) (string, error) { + require.Contains(t, args, "show", "invalid inventory must not authorize mutation") + return output, nil + }} + + err := task.Do(t.Context()) + if output == "[]" || output == `[{"table":"unexpected-name"}]` { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} + +func TestFileCleanupPropagatesSubstantiveFailure(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "child"), []byte("data"), 0o600)) + + log := slog.New(slog.DiscardHandler) + require.Error(t, removeFileIfExists(log, dir)) + require.NoError(t, removeAllIfExists(log, dir)) + require.NoError(t, removeFileIfExists(log, dir)) +} + +// TestFirstRegisteredMachineInventoriesOnce covers the scan callers use to ask +// whether any node slot is occupied, without knowing which. +// +// The counter pins the reason it exists: machinectl is run once for the whole +// question rather than once per slot. The kube10 case pins that a slot name +// which merely starts with another slot's name is not a match, the same trap +// TestMachineInspectionFailsClosed covers for the single-name lookup. +func TestFirstRegisteredMachineInventoriesOnce(t *testing.T) { + for _, tc := range []struct { + name, listed string + want string + wantErr bool + }{ + {name: "neither", listed: "", want: ""}, + {name: "kube1", listed: "kube1 container systemd-nspawn - - -", want: "kube1"}, + {name: "kube2", listed: "kube2 container systemd-nspawn - - -", want: "kube2"}, + {name: "both prefers kube1", listed: "kube2 container systemd-nspawn - - -\\nkube1 container systemd-nspawn - - -", want: "kube1"}, + {name: "unrelated machine", listed: "someother container systemd-nspawn - - -", want: ""}, + {name: "longer name is not a match", listed: "kube10 container systemd-nspawn - - -", want: ""}, + {name: "uninspectable fails closed", listed: "", want: "", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + counter := filepath.Join(dir, "calls") + + script := "printf '" + tc.listed + "\\n'" + if tc.wantErr { + script = "exit 1" + } + + require.NoError(t, os.WriteFile(filepath.Join(dir, "machinectl"), + []byte("#!/bin/sh\necho x >> "+counter+"\n"+script+"\n"), 0o755)) + t.Setenv("PATH", dir) + + got, err := FirstRegisteredMachine(t.Context(), slog.New(slog.DiscardHandler)) + + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + require.Equal(t, tc.want, got) + + calls, err := os.ReadFile(counter) + require.NoError(t, err) + require.Len(t, strings.Split(strings.TrimSpace(string(calls)), "\n"), 1, + "machinectl must be inventoried once for the whole scan, not once per slot") + }) + } +} diff --git a/pkg/agent/phases/reset/tooling.go b/pkg/agent/phases/reset/tooling.go new file mode 100644 index 000000000..550f0dcb6 --- /dev/null +++ b/pkg/agent/phases/reset/tooling.go @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package reset + +import ( + "context" + "errors" + "log/slog" + "os/exec" +) + +// missingTool reports whether err is a host inspection tool that is not +// installed, as opposed to a tool that ran and failed. +// +// Reset and bootstrap ask the host the same questions for opposite reasons, so +// they need opposite answers here. +// +// Bootstrap must fail closed. A host it cannot inspect is not a host it can +// prove is clean, and assuming otherwise risks building over a running node. +// +// Reset is the other way round. The tools it inspects with, machinectl from +// systemd-container and nft from nftables, are the ones bootstrap installs. A +// bootstrap that died inside host preparation therefore leaves a host where the +// question cannot be asked and the answer is known anyway: nothing of ours is +// running, because the tools needed to start it were never there. Treating that +// as an error leaves an installation record nothing can clear, which is the one +// outcome reset exists to prevent. +func missingTool(err error) bool { return errors.Is(err, exec.ErrNotFound) } + +// registeredMachineForCleanup answers RegisteredMachine's question with the +// tolerance described on missingTool. Cleanup uses this; admission must not. +func registeredMachineForCleanup(ctx context.Context, log *slog.Logger, name string) (bool, error) { + exists, err := RegisteredMachine(ctx, log, name) + if missingTool(err) { + log.Warn("machine inventory tool is not installed; nothing of ours can be running", "machine", name) + + return false, nil + } + + return exists, err +} diff --git a/pkg/agent/phases/reset/tooling_test.go b/pkg/agent/phases/reset/tooling_test.go new file mode 100644 index 000000000..f7ea329fc --- /dev/null +++ b/pkg/agent/phases/reset/tooling_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package reset + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// hostMissingOurPackages builds a PATH resembling a host where bootstrap died +// inside host preparation: systemd and iproute2 are present because the distro +// ships them, but machinectl and nft are not, because installing +// systemd-container and nftables is the step that did not finish. +// +// Removing everything from PATH would be a easier fixture and a wrong one. +// systemctl always exists on a systemd host, so a test that also hides it +// proves nothing about the hosts this tolerance is for. +func hostMissingOurPackages(t *testing.T) { + t.Helper() + + dir := t.TempDir() + + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), []byte( + "#!/bin/sh\n"+ + "case \"$1\" in show) echo \"\" ;; esac\n"+ + "exit 0\n"), 0o755)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, "ip"), []byte( + "#!/bin/sh\n"+ + "for a in \"$@\"; do if test \"$a\" = -j; then echo \"[]\"; exit 0; fi; done\n"+ + "exit 0\n"), 0o755)) + + t.Setenv("PATH", dir) +} + +// TestResetCompletesWithoutOurPackages is the case this tolerance exists for, +// and it runs through the reset tasks rather than the helper they call. +// +// A bootstrap that died inside host preparation leaves an installation record +// on a host without machinectl or nft. Reset has to finish there. If it cannot, +// the record stays forever and every later start is refused against an +// installation that nothing can clear. +func TestResetCompletesWithoutOurPackages(t *testing.T) { + hostMissingOurPackages(t) + + log := slog.New(slog.DiscardHandler) + + require.NoError(t, (&stopMachine{log: log, machineName: "kube1"}).Do(t.Context()), + "a machine cannot be running if machinectl was never installed") + + require.NoError(t, (&cleanupLocalDNSRules{log: log}).Do(t.Context()), + "a LocalDNS ruleset cannot exist if nft was never installed") + + require.NoError(t, (&cleanupRoutes{log: log}).Do(t.Context()), + "policy routing rules cannot exist without our packages") +} + +// TestAdmissionStillFailsClosedWithoutOurPackages is the other half, and the +// reason the tolerance is a separate function rather than a change to +// RegisteredMachine. +// +// Bootstrap asks the same question for the opposite reason: it must prove the +// host is clean before building on it. A host it cannot inspect is not a host +// it can prove anything about, so an absent tool stays an error here. +func TestAdmissionStillFailsClosedWithoutOurPackages(t *testing.T) { + hostMissingOurPackages(t) + + log := slog.New(slog.DiscardHandler) + + _, err := RegisteredMachine(t.Context(), log, "kube1") + require.Error(t, err, "bootstrap must not read an uninspectable host as a clean one") + + _, err = FirstRegisteredMachine(t.Context(), log) + require.Error(t, err, "bootstrap must not read an uninspectable host as a clean one") +} + +// TestCleanupStillFailsOnRealInspectionErrors keeps the tolerance narrow. A +// tool that is present and fails is a genuine problem: it may be reporting a +// machine that is actually there, and reset must not delete around it. +func TestCleanupStillFailsOnRealInspectionErrors(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "machinectl"), + []byte("#!/bin/sh\necho denied >&2\nexit 1\n"), 0o755)) + t.Setenv("PATH", dir) + + _, err := registeredMachineForCleanup(t.Context(), slog.New(slog.DiscardHandler), "kube1") + require.Error(t, err, "a tool that ran and failed is not proof of absence") +} diff --git a/pkg/agent/phases/rootfs/nspawn.go b/pkg/agent/phases/rootfs/nspawn.go index 0f67bd578..30a5cbe52 100644 --- a/pkg/agent/phases/rootfs/nspawn.go +++ b/pkg/agent/phases/rootfs/nspawn.go @@ -29,8 +29,9 @@ var nspawnTemplates = template.Must( ) type ensureNSpawnWorkspace struct { - log *slog.Logger - goalState *goalstates.RootFS + log *slog.Logger + goalState *goalstates.RootFS + ownedReplay bool } type NSpawnBind struct { @@ -101,6 +102,10 @@ func (e *ensureNSpawnWorkspace) Do(ctx context.Context) error { func (e *ensureNSpawnWorkspace) bootstrapWorkspace(ctx context.Context) error { bootstrapTask := oci.DownloadRootFS(e.log, e.goalState.MachineDir, e.goalState.HostArch, e.goalState.OCIImage) + if e.ownedReplay { + bootstrapTask = oci.DownloadOwnedRootFS(e.log, e.goalState.MachineDir, e.goalState.HostArch, e.goalState.OCIImage) + } + return phases.ExecuteTask(ctx, e.log, bootstrapTask) } diff --git a/pkg/agent/phases/rootfs/oci/task.go b/pkg/agent/phases/rootfs/oci/task.go index be4949301..71deb4c3f 100644 --- a/pkg/agent/phases/rootfs/oci/task.go +++ b/pkg/agent/phases/rootfs/oci/task.go @@ -5,20 +5,24 @@ package oci import ( "context" + "errors" "fmt" "log/slog" "os" + "path/filepath" + "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/artifactsource/ocilayout" "github.com/Azure/unbounded/pkg/agent/internal/utilio" "github.com/Azure/unbounded/pkg/agent/phases" ) type downloadRootFS struct { - log *slog.Logger - machineDir string - ociImage string - hostArch string + log *slog.Logger + machineDir string + ociImage string + hostArch string + ownedReplay bool } // DownloadRootFS downloads an OCI image and unpacks it into the machine @@ -39,6 +43,14 @@ func DownloadRootFS( func (d *downloadRootFS) Name() string { return "oci-download-rootfs" } +// DownloadOwnedRootFS is for owned initial installation only. The caller must +// hold installation ownership and prove this slot has never started a node. +// The original DownloadRootFS entry point keeps its existing nonempty-rootfs +// behavior for callers managing legacy installations. +func DownloadOwnedRootFS(log *slog.Logger, machineDir, hostArch, image string) phases.Task { + return &downloadRootFS{log: log, machineDir: machineDir, hostArch: hostArch, ociImage: image, ownedReplay: true} +} + func (d *downloadRootFS) Do(ctx context.Context) error { empty, err := utilio.IsDirEmpty(d.machineDir) if err != nil { @@ -46,8 +58,20 @@ func (d *downloadRootFS) Do(ctx context.Context) error { } if !empty { - d.log.Warn("machine directory is not empty, skipping rootfs bootstrap", slog.String("dir", d.machineDir)) - return nil + if !d.ownedReplay { + d.log.Warn("machine directory is not empty, skipping rootfs bootstrap", slog.String("dir", d.machineDir)) + return nil + } + + if _, err := os.Stat(filepath.Join(d.machineDir, ".unbounded-rootfs-complete")); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + + if err := utilio.CleanDir(d.machineDir); err != nil { + return err + } } d.log.Info("acquiring OCI image", @@ -68,6 +92,16 @@ func (d *downloadRootFS) Do(ctx context.Context) error { return fmt.Errorf("unpack OCI image: %w", err) } + if d.ownedReplay { + if err := fsutil.SyncFilesystems(d.machineDir); err != nil { + return err + } + + if err := fsutil.WriteFileDurable(filepath.Join(d.machineDir, ".unbounded-rootfs-complete"), []byte("complete\n"), 0o600); err != nil { + return err + } + } + d.log.Info("OCI image extraction complete", slog.String("dest", d.machineDir)) return nil diff --git a/pkg/agent/phases/rootfs/oci/task_test.go b/pkg/agent/phases/rootfs/oci/task_test.go new file mode 100644 index 000000000..8e785f054 --- /dev/null +++ b/pkg/agent/phases/rootfs/oci/task_test.go @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package oci + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOwnedReplayDiscardsIncompleteTreeBeforeRetry(t *testing.T) { + t.Parallel() + dir := t.TempDir() + partial := filepath.Join(dir, "partial-layer") + require.NoError(t, os.WriteFile(partial, []byte("incomplete"), 0o600)) + task := DownloadOwnedRootFS(slog.New(slog.DiscardHandler), dir, "amd64", "oci-layout://"+filepath.Join(t.TempDir(), "missing")) + require.Error(t, task.Do(t.Context())) + + _, err := os.Stat(partial) + require.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(filepath.Join(dir, ".unbounded-rootfs-complete")) + require.ErrorIs(t, err, os.ErrNotExist, "failed extraction must remain replayable") +} + +func TestRootFSReplayPreservesCompletedOrLegacyTree(t *testing.T) { + t.Parallel() + + for _, owned := range []bool{false, true} { + dir := t.TempDir() + payload := filepath.Join(dir, "payload") + require.NoError(t, os.WriteFile(payload, []byte("preserve"), 0o600)) + + if owned { + require.NoError(t, os.WriteFile(filepath.Join(dir, ".unbounded-rootfs-complete"), []byte("complete\n"), 0o600)) + } + + task := DownloadRootFS(slog.New(slog.DiscardHandler), dir, "amd64", "unavailable") + if owned { + task = DownloadOwnedRootFS(slog.New(slog.DiscardHandler), dir, "amd64", "unavailable") + } + + require.NoError(t, task.Do(t.Context())) + + data, err := os.ReadFile(payload) + require.NoError(t, err) + require.Equal(t, "preserve", string(data)) + } +} diff --git a/pkg/agent/phases/rootfs/provision.go b/pkg/agent/phases/rootfs/provision.go index 7259b9670..93a02140b 100644 --- a/pkg/agent/phases/rootfs/provision.go +++ b/pkg/agent/phases/rootfs/provision.go @@ -17,8 +17,18 @@ import ( // This is the shared rootfs provisioning sequence used by both the initial // agent start and node update flows. func Provision(log *slog.Logger, gs *goalstates.RootFS) phases.Task { + return provisionWithWorkspace(log, gs, EnsureNSpawnWorkspace(log, gs)) +} + +// ProvisionOwned replays an initial bootstrap rootfs under established host +// ownership. Never use it for a slot that may have started a node. +func ProvisionOwned(log *slog.Logger, gs *goalstates.RootFS) phases.Task { + return provisionWithWorkspace(log, gs, &ensureNSpawnWorkspace{log: log, goalState: gs, ownedReplay: true}) +} + +func provisionWithWorkspace(log *slog.Logger, gs *goalstates.RootFS, workspace phases.Task) phases.Task { return phases.Serial(log, - EnsureNSpawnWorkspace(log, gs), + workspace, phases.Parallel(log, DownloadKubeBinaries(log, gs), DownloadCRIBinaries(log, gs),