From bb191a93d3dd5acc9277f10fb169a959d69af8cb Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:18:11 -0400 Subject: [PATCH 01/39] agent: make initial bootstrap resumable with strict reset --- .github/workflows/agent-e2e-kind.yaml | 34 +++ cmd/agent/internal/cmd/agentupgrade.go | 18 ++ cmd/agent/internal/cmd/agentupgrade_test.go | 18 ++ cmd/agent/internal/cmd/bootstrap.go | 183 ++++++++++++++ cmd/agent/internal/cmd/bootstrap_test.go | 88 +++++++ cmd/agent/internal/cmd/preflight.go | 51 ++++ cmd/agent/internal/cmd/reset.go | 9 +- cmd/agent/internal/cmd/start.go | 89 +------ .../cmd/testdata/bootstrap-v1/README.md | 16 ++ .../cmd/testdata/bootstrap-v1/complete.json | 8 + .../cmd/testdata/bootstrap-v1/input.json | 14 ++ .../bootstrap-v1/preparing-rootfs.json | 8 + .../cmd/testdata/bootstrap-v1/resetting.json | 8 + .../assets/unbounded-agent-daemon-recovery.sh | 7 + cmd/agent/internal/daemon/controller.go | 2 + .../daemon/controller_machineoperation.go | 35 +++ cmd/agent/internal/daemon/controller_node.go | 13 + cmd/agent/internal/daemon/controller_test.go | 4 + cmd/agent/internal/daemon/installation.go | 24 ++ .../internal/daemon/installation_test.go | 45 ++++ cmd/agent/internal/daemon/lifecycle.go | 164 ++++++++++++- cmd/agent/internal/daemon/nodeoperator.go | 36 ++- cmd/agent/internal/daemon/persist_config.go | 8 - .../internal/daemon/recovery_script_test.go | 2 + cmd/agent/internal/daemon/reset.go | 125 ++++++++++ cmd/agent/internal/daemon/reset_test.go | 62 +++++ docs/content/guides/agent.md | 42 ++++ hack/agent/e2e-kind/e2e.py | 94 ++++++++ hack/agent/e2e-kind/test_reliability.py | 22 +- .../assets/unbounded-agent-install.sh | 8 +- pkg/agent/bootstrap/coordinator.go | 171 +++++++++++++ pkg/agent/bootstrap/coordinator_test.go | 173 ++++++++++++++ pkg/agent/installstate/lock.go | 54 +++++ pkg/agent/installstate/mutation.go | 32 +++ pkg/agent/installstate/store.go | 224 ++++++++++++++++++ pkg/agent/installstate/store_test.go | 163 +++++++++++++ pkg/agent/internal/utilio/sync.go | 63 +++++ pkg/agent/phases/nodestart/nspawn.go | 19 ++ pkg/agent/phases/nodestart/nspawn_test.go | 26 +- .../nodestart/preflight_bind_address.go | 99 ++++++++ .../nodestart/preflight_bind_address_test.go | 39 +++ pkg/agent/phases/reset/helpers.go | 17 +- pkg/agent/phases/reset/machine.go | 79 ++++-- pkg/agent/phases/reset/network.go | 70 ++++-- pkg/agent/phases/reset/nspawn.go | 21 +- pkg/agent/phases/reset/routes.go | 99 +++++++- pkg/agent/phases/reset/strict_test.go | 114 +++++++++ pkg/agent/phases/rootfs/nspawn.go | 9 +- pkg/agent/phases/rootfs/oci/task.go | 45 +++- pkg/agent/phases/rootfs/oci/task_test.go | 52 ++++ pkg/agent/phases/rootfs/provision.go | 12 +- 51 files changed, 2637 insertions(+), 181 deletions(-) create mode 100644 cmd/agent/internal/cmd/bootstrap.go create mode 100644 cmd/agent/internal/cmd/bootstrap_test.go create mode 100644 cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md create mode 100644 cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json create mode 100644 cmd/agent/internal/cmd/testdata/bootstrap-v1/input.json create mode 100644 cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json create mode 100644 cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json create mode 100644 cmd/agent/internal/daemon/installation.go create mode 100644 cmd/agent/internal/daemon/installation_test.go create mode 100644 pkg/agent/bootstrap/coordinator.go create mode 100644 pkg/agent/bootstrap/coordinator_test.go create mode 100644 pkg/agent/installstate/lock.go create mode 100644 pkg/agent/installstate/mutation.go create mode 100644 pkg/agent/installstate/store.go create mode 100644 pkg/agent/installstate/store_test.go create mode 100644 pkg/agent/internal/utilio/sync.go create mode 100644 pkg/agent/phases/reset/strict_test.go create mode 100644 pkg/agent/phases/rootfs/oci/task_test.go diff --git a/.github/workflows/agent-e2e-kind.yaml b/.github/workflows/agent-e2e-kind.yaml index bdebf97c5..19db28bd4 100644 --- a/.github/workflows/agent-e2e-kind.yaml +++ b/.github/workflows/agent-e2e-kind.yaml @@ -121,6 +121,40 @@ jobs: if: always() run: python3 ./hack/agent/e2e-kind/e2e.py --verbose cleanup + agent-bootstrap-recovery: + name: agent bootstrap recovery (Ubuntu) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + env: + KIND_CLUSTER_NAME: agent-bootstrap-recovery + VM_NAME: agent-bootstrap-recovery + VM_SUBNET: "192.168.100" + VM_IP: "192.168.100.10" + AGENT_MACHINE_NAME: agent-bootstrap-recovery + HOST_BASE_OS: ubuntu2404 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up test control plane + uses: ./.github/actions/agent-e2e-kind-control-plane + with: + cluster-name: ${{ env.KIND_CLUSTER_NAME }} + vm-subnet: ${{ env.VM_SUBNET }} + - name: Set up machina resources + uses: ./.github/actions/agent-e2e-machina-setup + - name: Retry late bootstrap and repair after ordinary repave + run: python3 ./hack/agent/e2e-kind/e2e.py --verbose run-suite --suite bootstrap-recovery + - name: Verify strict reset + run: python3 ./hack/agent/e2e-kind/e2e.py --verbose reset-agent + - name: Collect logs + if: always() + uses: ./.github/actions/agent-e2e-kind-logs + with: + artifact-name: agent-bootstrap-recovery-logs + - name: Cleanup + if: always() + run: python3 ./hack/agent/e2e-kind/e2e.py --verbose cleanup + agent-config-e2e: name: agent config e2e runs-on: ubuntu-24.04 diff --git a/cmd/agent/internal/cmd/agentupgrade.go b/cmd/agent/internal/cmd/agentupgrade.go index 02ad6e766..8abc4d64d 100644 --- a/cmd/agent/internal/cmd/agentupgrade.go +++ b/cmd/agent/internal/cmd/agentupgrade.go @@ -17,6 +17,7 @@ import ( "github.com/Azure/unbounded/cmd/agent/internal/daemon" "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" ) const hostAgentBinaryMode = 0o755 @@ -36,6 +37,7 @@ type hostAgentUpgradeHandler struct { resolvedPath func() (goalstates.AgentUpgradePaths, error) newService func(goalstates.AgentUpgradePaths) agentbinary.DaemonService geteuid func() int + installation *installstate.Store } func newCmdHostAgentUpgrade(cmdCtx *CommandContext) *cobra.Command { @@ -115,6 +117,22 @@ func (h *hostAgentUpgradeHandler) execute(ctx context.Context) error { return fmt.Errorf("host agent upgrade requires root privileges") } + store := h.installation + if store == nil { + store = installstate.DefaultStore() + } + + lock, err := store.AcquireMutationLock() + if err != nil { + return err + } + + defer func() { + if err := lock.Release(); err != nil { + h.cmdCtx.Logger.Error("release installation lock", "error", err) + } + }() + result, err := agentbinary.ActivateHostDaemon(ctx, h.cmdCtx.Logger, options, service) if err != nil { return err diff --git a/cmd/agent/internal/cmd/agentupgrade_test.go b/cmd/agent/internal/cmd/agentupgrade_test.go index 702804c1b..bde59cae4 100644 --- a/cmd/agent/internal/cmd/agentupgrade_test.go +++ b/cmd/agent/internal/cmd/agentupgrade_test.go @@ -15,6 +15,7 @@ import ( "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" ) type preflightOnlyDaemonService struct{} @@ -82,6 +83,23 @@ func TestWriteHostAgentUpgradePlanOmitsUnchangedLastGood(t *testing.T) { assert.NotContains(t, output.String(), "Last-good link:") } +func TestHostAgentUpgradeTakesInstallationLockBeforeActivation(t *testing.T) { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + lock, err := store.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + + handler := &hostAgentUpgradeHandler{ + cmdCtx: &CommandContext{LogFormat: "text"}, installation: store, + executable: func() (string, error) { return filepath.Join(dir, "candidate"), nil }, + resolvedPath: func() (goalstates.AgentUpgradePaths, error) { return goalstates.AgentUpgradePaths{}, nil }, + newService: func(goalstates.AgentUpgradePaths) agentbinary.DaemonService { return preflightOnlyDaemonService{} }, + geteuid: func() int { return 0 }, + } + require.ErrorIs(t, handler.execute(t.Context()), installstate.ErrLockHeld) +} + func TestRecordAgentUpgradeFailureSignalCommand(t *testing.T) { dir := t.TempDir() signalPath := filepath.Join(dir, "agent-upgrade-signal") diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go new file mode 100644 index 000000000..9f0bd926d --- /dev/null +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "strings" + + "github.com/Azure/unbounded/cmd/agent/internal/attest" + "github.com/Azure/unbounded/cmd/agent/internal/daemon" + "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/bootstrap" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" + "github.com/Azure/unbounded/pkg/agent/phases" + "github.com/Azure/unbounded/pkg/agent/phases/host" + "github.com/Azure/unbounded/pkg/agent/phases/nodestart" + "github.com/Azure/unbounded/pkg/agent/phases/rootfs" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +type agentStages struct { + log *slog.Logger + cfg *provision.UnboundedAgentConfig + gs *goalstates.MachineGoalState + archives *goalstates.ContainerImageArchiveStaging + reporter *daemon.BootstrapStatusReporter + credentialsReady bool +} + +func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity, error) { + data, err := json.Marshal(cfg) + if err != nil { + return bootstrap.Identity{}, err + } + + return bootstrap.Identity{MachineName: cfg.MachineName, ConfigFingerprint: installstate.Fingerprint(data)}, nil +} + +func (s *agentStages) EnsureHostClean(ctx context.Context) error { + return host.EnsureNoExistingDeployment(ctx, s.log) +} + +func (s *agentStages) ResolveInputs(ctx context.Context) error { + downloads, archives, err := provision.ResolveDownloadOverridesWithOfflineArtifacts(ctx, s.cfg) + if err != nil { + return err + } + + s.archives = archives + + s.gs, err = goalstates.ResolveMachine(s.log, &s.cfg.AgentConfig, goalstates.NSpawnMachineKube1, downloads) + if err != nil { + return err + } + + return nil +} + +func (s *agentStages) PrepareHost(ctx context.Context) error { + if err := phases.Serial(s.log, host.InstallPackages(s.log), phases.Parallel(s.log, + host.ConfigureOS(s.log), host.ConfigureNFTables(s.log), phases.Serial(s.log, host.DisableDocker(s.log), host.ConfigureDocker(s.log)), + host.DisableContainerd(s.log), host.DisableKubelet(s.log), host.DisableSwap(s.log), host.HardenAPT(s.log))).Do(ctx); err != nil { + return err + } + + return bootstrap.SyncFilesystems("/etc", "/usr/local", installstate.DefaultDirectory) +} + +// Credentials must be resolved on every unfinished attempt, but TPM prerequisites +// must first be installed on a fresh host. This stage always precedes node work. +func (s *agentStages) prepareCredentials(ctx context.Context) error { + if s.credentialsReady { + return nil + } + + if err := attest.ApplyAttestation(s.log, s.cfg.Attest, s.cfg.MachineName, s.gs.NodeStart).Do(ctx); err != nil { + return err + } + + syncAttestedKubeletConfig(&s.cfg.AgentConfig, s.gs.NodeStart) + + if s.reporter == nil { + s.reporter = daemon.NewBootstrapStatusReporter(ctx, s.log, &s.cfg.AgentConfig) + s.reporter.Running(ctx) + } + + s.credentialsReady = true + + return nil +} + +func (s *agentStages) PrepareRootFS(ctx context.Context, _ bool) error { + // A pre-node checkpoint never authorizes deleting a registered machine, + // including one started independently after ownership was first recorded. + machines, err := executil.OutputCmd(ctx, s.log, "machinectl", "list", "--no-legend", "--no-pager") + if err != nil { + return fmt.Errorf("inspect machines before rootfs replay: %w", err) + } + + for _, line := range strings.Split(machines, "\n") { + fields := strings.Fields(line) + if len(fields) > 0 && (fields[0] == "kube1" || fields[0] == "kube2") { + return fmt.Errorf("refusing rootfs replay while %s is registered", fields[0]) + } + } + + if err := s.prepareCredentials(ctx); err != nil { + return err + } + + if err := phases.Serial(s.log, rootfs.DownloadContainerImageArchives(s.log, s.archives), rootfs.ProvisionOwned(s.log, s.gs.RootFS)).Do(ctx); err != nil { + return err + } + + return bootstrap.SyncFilesystems(s.gs.RootFS.MachineDir, "/usr/local", goalstates.SystemdSystemDir, goalstates.SystemdNSpawnDir) +} + +func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { + if err := s.prepareCredentials(ctx); err != nil { + return err + } + + checks := []preflight.Checker{ + nodestart.CheckOwnedBindAddress(s.log, "kubelet-bind-address", "0.0.0.0:10250", "kubelet bind address", s.gs.RootFS.MachineDir, "usr/local/bin/kubelet"), + nodestart.CheckOwnedBindAddress(s.log, "containerd-metrics-bind-address", s.gs.NodeStart.Containerd.MetricsAddress, "containerd metrics bind address", s.gs.RootFS.MachineDir, "usr/local/bin/containerd"), + } + if err := preflight.Run(ctx, checks, preflight.Options{}).Err(false); err != nil { + return err + } + + if err := phases.Serial(s.log, nodestart.StartNode(s.log, s.gs.NodeStart), nodestart.WaitForKubeletBootstrap(s.log, "kube1")).Do(ctx); err != nil { + return err + } + + return bootstrap.SyncFilesystems(s.gs.RootFS.MachineDir, goalstates.SystemdSystemDir) +} + +func (s *agentStages) EnsureDaemonInstalled(ctx context.Context) error { + if err := s.prepareCredentials(ctx); err != nil { + return err + } + + if err := daemon.InstallBootstrapBinary(); err != nil { + return err + } + + if err := phases.Serial(s.log, daemon.PersistAppliedConfig(s.log, "kube1", &s.cfg.AgentConfig), daemon.EnableDaemon(s.log)).Do(ctx); err != nil { + return err + } + + return bootstrap.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) +} + +func (s *agentStages) VerifyInstalled(ctx context.Context) error { + return daemon.VerifyDaemonInstalled(ctx, s.log) +} + +func (s *agentStages) RepairDaemon(ctx context.Context) error { return daemon.RepairDaemon(ctx, s.log) } + +func (s *agentStages) StageStarted(_ context.Context, stage installstate.Checkpoint) { + s.log.Info("bootstrap stage", "checkpoint", stage) +} + +func (s *agentStages) StageFailed(ctx context.Context, stage installstate.Checkpoint, err error) { + if s.reporter != nil { + reason := "Failed" + if stage == installstate.PreparingRootFS { + reason = "RootFSFailed" + } + + if stage == installstate.StartingNode { + reason = classifyNodeStartFailure(err) + } + + s.reporter.Failed(ctx, reason, err) + } +} diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go new file mode 100644 index 000000000..81389ad26 --- /dev/null +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "bytes" + "encoding/json" + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/installstate" + "github.com/Azure/unbounded/pkg/agent/preflight" +) + +// These fixtures are produced by P6's actual loader, normalizer, fingerprint, +// and Store.Save. Later releases must consume them with the original input. +func TestBootstrapV1CompatibilityFixtures(t *testing.T) { + dir := filepath.Join("testdata", "bootstrap-v1") + cfg, err := loadConfigFromFile(filepath.Join(dir, "input.json")) + require.NoError(t, err) + require.NoError(t, normalizeConfig(slog.New(slog.DiscardHandler), cfg)) + require.NoError(t, cfg.Validate()) + id, err := bootstrapIdentity(cfg) + require.NoError(t, err) + + for _, checkpoint := range []installstate.Checkpoint{installstate.PreparingRootFS, installstate.Complete, installstate.Resetting} { + fixture := filepath.Join(dir, string(checkpoint)+".json") + if os.Getenv("UPDATE_BOOTSTRAP_V1_FIXTURES") == "1" { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + record, err := installstate.NewRecord(id.MachineName, id.ConfigFingerprint) + require.NoError(t, err) + + record.InstallID = "00112233445566778899aabbccddeeff" + record.Checkpoint = checkpoint + require.NoError(t, store.Save(record)) + data, err := os.ReadFile(store.StatePath()) + require.NoError(t, err) + require.NoError(t, os.WriteFile(fixture, data, 0o644)) + } + + data, err := os.ReadFile(fixture) + require.NoError(t, err) + + var record installstate.Record + require.NoError(t, json.Unmarshal(data, &record)) + require.NoError(t, record.Validate()) + require.Equal(t, id.MachineName, record.MachineName) + require.Equal(t, id.ConfigFingerprint, record.ConfigFingerprint) + require.Equal(t, "/usr/local", record.HostPrefix) + + disposition, err := installstate.Decide(record, nil, id.MachineName, id.ConfigFingerprint) + if checkpoint == installstate.Resetting { + require.Error(t, err) + } else { + require.NoError(t, err) + + want := installstate.Resume + if checkpoint == installstate.Complete { + want = installstate.AlreadyComplete + } + + require.Equal(t, want, disposition) + } + } + + cfg.OCIImage = "example.test/unbounded/node:changed" + changed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.NotEqual(t, id.ConfigFingerprint, changed.ConfigFingerprint) +} + +func TestCompletedPreflightOutput(t *testing.T) { + t.Parallel() + + var out bytes.Buffer + + h := &preflightHandler{writer: &out, output: "json"} + require.NoError(t, h.writeReport(preflight.Report{})) + require.True(t, json.Valid(out.Bytes())) + + h.output = "unsupported" + require.Error(t, h.writeReport(preflight.Report{})) +} diff --git a/cmd/agent/internal/cmd/preflight.go b/cmd/agent/internal/cmd/preflight.go index b5d1e6e20..5f0b8ff3e 100644 --- a/cmd/agent/internal/cmd/preflight.go +++ b/cmd/agent/internal/cmd/preflight.go @@ -15,6 +15,7 @@ import ( "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases/host" "github.com/Azure/unbounded/pkg/agent/phases/nodestart" "github.com/Azure/unbounded/pkg/agent/phases/rootfs" @@ -77,6 +78,31 @@ func (h *preflightHandler) execute(ctx context.Context) error { return fmt.Errorf("validate agent config: %w", err) } + id, err := bootstrapIdentity(cfg) + if err != nil { + return err + } + + store := installstate.DefaultStore() + record, loadErr := store.Load() + + disposition, err := installstate.Decide(record, loadErr, id.MachineName, id.ConfigFingerprint) + if err != nil { + return err + } + + if disposition != installstate.Fresh { + if _, err := store.CheckMarker(record); err != nil { + return err + } + } + + if disposition == installstate.AlreadyComplete || record.Checkpoint == installstate.RepairingDaemon { + // Admission is non-mutating. start rechecks ownership under lock before + // verifying or repairing daemon assets, without the original artifacts. + return h.writeReport(preflight.Report{}) + } + downloads, _, err := provision.ResolveDownloadOverridesWithOfflineArtifacts(ctx, cfg) if err != nil { return fmt.Errorf("resolve download overrides: %w", err) @@ -97,6 +123,27 @@ func (h *preflightHandler) execute(ctx context.Context) error { nodestart.Preflight(logger, cfg.AgentConfig, goalState), rootfs.Preflight(logger, cfg.AgentConfig, goalState), ) + if disposition == installstate.Resume { + filtered := checks[:0] + for _, check := range checks { + if check.Name() == "existing-deployment" { + continue + } + + if record.Checkpoint.NodeMayBeRunning() { + switch check.Name() { + case "kubelet-bind-address": + check = nodestart.CheckOwnedBindAddress(logger, check.Name(), "0.0.0.0:10250", "kubelet bind address", goalState.RootFS.MachineDir, "usr/local/bin/kubelet") + case "containerd-metrics-bind-address": + check = nodestart.CheckOwnedBindAddress(logger, check.Name(), goalState.NodeStart.Containerd.MetricsAddress, "containerd metrics bind address", goalState.RootFS.MachineDir, "usr/local/bin/containerd") + } + } + + filtered = append(filtered, check) + } + + checks = filtered + } opts := preflight.Options{ IgnoreErrors: h.ignorePreflightErrors, @@ -104,6 +151,10 @@ func (h *preflightHandler) execute(ctx context.Context) error { } report := preflight.Run(ctx, checks, opts) + return h.writeReport(report) +} + +func (h *preflightHandler) writeReport(report preflight.Report) error { switch strings.ToLower(h.output) { case "", "text": if err := writePreflightText(h.writer, report); err != nil { diff --git a/cmd/agent/internal/cmd/reset.go b/cmd/agent/internal/cmd/reset.go index 6398ddee9..6b6cebef7 100644 --- a/cmd/agent/internal/cmd/reset.go +++ b/cmd/agent/internal/cmd/reset.go @@ -45,12 +45,5 @@ Both possible nspawn machine names (kube1 and kube2) are stopped and removed.`, // resetAgent returns a task that resets the host by stopping the daemon and // removing the unbounded-agent and all associated resources. func resetAgent(log *slog.Logger) phases.Task { - return phases.Serial(log, - // CLI reset runs outside the daemon, so it can stop the daemon first to - // keep it from reconciling while files are removed. The daemon operation - // path stops the daemon last because stopping the unit terminates the - // reconciler before it can mark the MachineOperation complete. - daemon.StopDaemon(log), - daemon.ResetAgentResources(log), - ) + return daemon.ResetAgent(log) } diff --git a/cmd/agent/internal/cmd/start.go b/cmd/agent/internal/cmd/start.go index 61b9192af..55bd753af 100644 --- a/cmd/agent/internal/cmd/start.go +++ b/cmd/agent/internal/cmd/start.go @@ -4,24 +4,18 @@ package cmd import ( - "context" "encoding/base64" - "log/slog" "os" "os/signal" "strings" "github.com/spf13/cobra" - "github.com/Azure/unbounded/cmd/agent/internal/attest" - "github.com/Azure/unbounded/cmd/agent/internal/daemon" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/internal/version" + "github.com/Azure/unbounded/pkg/agent/bootstrap" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/phases" - "github.com/Azure/unbounded/pkg/agent/phases/host" - "github.com/Azure/unbounded/pkg/agent/phases/nodestart" - "github.com/Azure/unbounded/pkg/agent/phases/rootfs" + "github.com/Azure/unbounded/pkg/agent/installstate" ) func newCmdStart(cmdCtx *CommandContext) *cobra.Command { @@ -47,82 +41,30 @@ func newCmdStart(cmdCtx *CommandContext) *cobra.Command { log := cmdCtx.Logger - downloads, containerImageArchives, err := provision.ResolveDownloadOverridesWithOfflineArtifacts(ctx, cfg) - if err != nil { + if err := cfg.Validate(); err != nil { return err } - gs, err := goalstates.ResolveMachine(log, &cfg.AgentConfig, goalstates.NSpawnMachineKube1, downloads) + id, err := bootstrapIdentity(cfg) if err != nil { return err } - rootFSGoalState := gs.RootFS - nodeStartGoalState := gs.NodeStart - - if err := host.EnsureNoExistingDeployment(ctx, log); err != nil { - return err - } - - // Run host setup and attestation first. Metalman bootstrap tokens are - // only available after attestation, so Machine status reporting starts - // after this block. - preBootstrapTasks := []phases.Task{ - // Phase 1: host - host.InstallPackages(log), - phases.Parallel(log, - host.ConfigureOS(log), - host.ConfigureNFTables(log), - phases.Serial(log, host.DisableDocker(log), host.ConfigureDocker(log)), - host.DisableContainerd(log), - host.DisableKubelet(log), - host.DisableSwap(log), - host.HardenAPT(log), - ), - - // TPM Attestation (no-op when not configured). - attest.ApplyAttestation(log, cfg.Attest, cfg.MachineName, nodeStartGoalState), - - // Stage offline container image archives before status reporting starts. - rootfs.DownloadContainerImageArchives(log, containerImageArchives), - } - - if err := phases.Serial(log, preBootstrapTasks...).Do(ctx); err != nil { - return err - } - - syncAttestedKubeletConfig(&cfg.AgentConfig, nodeStartGoalState) - - reporter := daemon.NewBootstrapStatusReporter(ctx, log, &cfg.AgentConfig) - reporter.Running(ctx) + stages := &agentStages{log: log, cfg: cfg} - if err := runBootstrapTask(ctx, log, reporter, "RootFSFailed", rootfs.Provision(log, rootFSGoalState)); err != nil { - return err - } - - if err := phases.ExecuteTask(ctx, log, nodestart.StartNode(log, nodeStartGoalState)); err != nil { - reporter.Failed(ctx, classifyNodeStartFailure(err), err) + outcome, err := bootstrap.New(log, installstate.DefaultStore(), stages, stages).Run(ctx, id) + if err != nil { return err } - if err := runBootstrapTask(ctx, log, reporter, "KubeletBootstrapFailed", nodestart.WaitForKubeletBootstrap(log, nodeStartGoalState.MachineName)); err != nil { - return err + if outcome.AlreadyComplete { + log.Info("installation already complete") } - if err := phases.Serial(log, - // Phase 4: Persist the applied config for drift detection. - daemon.PersistAppliedConfig(log, nodeStartGoalState.MachineName, &cfg.AgentConfig), - - // Phase 5: Enable and start the daemon that watches the - // Machine CR for drift detection and reconciliation. - daemon.EnableDaemon(log), - ).Do(ctx); err != nil { - reporter.Failed(ctx, "Failed", err) - return err + if stages.reporter != nil { + stages.reporter.Succeeded(ctx) } - reporter.Succeeded(ctx) - return nil }, } @@ -140,15 +82,6 @@ func syncAttestedKubeletConfig(cfg *provision.AgentConfig, nodeStart *goalstates } } -func runBootstrapTask(ctx context.Context, log *slog.Logger, reporter *daemon.BootstrapStatusReporter, reason string, task phases.Task) error { - if err := phases.ExecuteTask(ctx, log, task); err != nil { - reporter.Failed(ctx, reason, err) - return err - } - - return nil -} - func classifyNodeStartFailure(err error) string { message := err.Error() switch { diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md b/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md new file mode 100644 index 000000000..6482c2ef1 --- /dev/null +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md @@ -0,0 +1,16 @@ +# Bootstrap ownership v1 fixtures + +Produced by the P6 candidate on base `50286b2c`, using +`TestBootstrapV1CompatibilityFixtures`. The input uses synthetic credentials. +The producer runs the actual JSON loader, normalization, fingerprint, and +`installstate.Store.Save`; only the random installation ID is fixed. + +These files freeze the default-path contract for later releases. Consume the +original input when checking compatibility. Do not regenerate these fixtures to +make a changed serializer pass. Adding a new format requires new fixtures. + +Initial production command: + +```sh +UPDATE_BOOTSTRAP_V1_FIXTURES=1 go test ./cmd/agent/internal/cmd -run TestBootstrapV1CompatibilityFixtures -count=1 +``` diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json new file mode 100644 index 000000000..bd6f717a1 --- /dev/null +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json @@ -0,0 +1,8 @@ +{ + "schemaVersion": 1, + "installID": "00112233445566778899aabbccddeeff", + "machineName": "bootstrap-fixture", + "hostPrefix": "/usr/local", + "configFingerprint": "d6d6b8ead0b3ef26872432d1d3f82eacd16a3d2165bd81fcc3b7c48cace90d37", + "checkpoint": "complete" +} diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/input.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/input.json new file mode 100644 index 000000000..1089499e5 --- /dev/null +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/input.json @@ -0,0 +1,14 @@ +{ + "MachineName": "bootstrap-fixture", + "NodeName": "bootstrap-fixture", + "Cluster": { + "Version": "v1.34.0", + "CaCertBase64": "dGVzdC1jYQ==", + "ClusterDNS": "10.96.0.10" + }, + "Kubelet": { + "ApiServer": "https://api.example.test:6443", + "Auth": {"BootstrapToken": "abcdef.0123456789abcdef"} + }, + "OCIImage": "example.test/unbounded/node:fixture" +} diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json new file mode 100644 index 000000000..029067c44 --- /dev/null +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json @@ -0,0 +1,8 @@ +{ + "schemaVersion": 1, + "installID": "00112233445566778899aabbccddeeff", + "machineName": "bootstrap-fixture", + "hostPrefix": "/usr/local", + "configFingerprint": "d6d6b8ead0b3ef26872432d1d3f82eacd16a3d2165bd81fcc3b7c48cace90d37", + "checkpoint": "preparing-rootfs" +} diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json new file mode 100644 index 000000000..b2f479aa2 --- /dev/null +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json @@ -0,0 +1,8 @@ +{ + "schemaVersion": 1, + "installID": "00112233445566778899aabbccddeeff", + "machineName": "bootstrap-fixture", + "hostPrefix": "/usr/local", + "configFingerprint": "d6d6b8ead0b3ef26872432d1d3f82eacd16a3d2165bd81fcc3b7c48cace90d37", + "checkpoint": "resetting" +} diff --git a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh index 314f2ca60..537c1091f 100644 --- a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh +++ b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh @@ -4,6 +4,13 @@ set -euo pipefail +# Follow the Go lifecycle lock order before reading or switching binary links. +# The restarted Type=simple daemon can wait for this lock during migration. +exec 8>"{{ .InstallationLockPath }}" +flock 8 +exec 9>"{{ .ActivationLockPath }}" +flock 9 + current="{{ .DaemonBinaryCurrentPath }}" last_good="$(readlink -f {{ .DaemonBinaryLastGoodPath }} || true)" upgrade_signal="{{ .DaemonAgentUpgradeSignalPath }}" diff --git a/cmd/agent/internal/daemon/controller.go b/cmd/agent/internal/daemon/controller.go index a0895f5ad..9c8dec8d3 100644 --- a/cmd/agent/internal/daemon/controller.go +++ b/cmd/agent/internal/daemon/controller.go @@ -24,9 +24,11 @@ import ( v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" daemon "github.com/Azure/unbounded/pkg/agent/daemon" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" ) type repaveReconciler struct { + installation *installstate.Store client.Client log *slog.Logger machineName string diff --git a/cmd/agent/internal/daemon/controller_machineoperation.go b/cmd/agent/internal/daemon/controller_machineoperation.go index 9b6990988..e0345f2e7 100644 --- a/cmd/agent/internal/daemon/controller_machineoperation.go +++ b/cmd/agent/internal/daemon/controller_machineoperation.go @@ -17,11 +17,13 @@ import ( "github.com/Azure/unbounded/pkg/agent/agentbinary" daemon "github.com/Azure/unbounded/pkg/agent/daemon" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" ) const agentUpgradeLockRetryDelay = 2 * time.Second type machineOperationTarget struct { + installation *installstate.Store client.Client log *slog.Logger machineName string @@ -30,6 +32,17 @@ type machineOperationTarget struct { } func (t *machineOperationTarget) reconcileNodeReboot(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) { + lock, err := installationStore(t.installation).AcquireMutationLock() + if errors.Is(err, installstate.ErrLockHeld) { + return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil + } + + if err != nil { + return ctrl.Result{}, err + } + + defer releaseInstallationLock(t.log, lock) + if err := store.MarkInProgress(ctx, op, "restarting active nspawn node"); err != nil { return ctrl.Result{}, err } @@ -57,6 +70,17 @@ func (t *machineOperationTarget) reconcileNodeReboot(ctx context.Context, store } func (t *machineOperationTarget) reconcileAgentUpgrade(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) { + installationLock, err := installationStore(t.installation).AcquireMutationLock() + if errors.Is(err, installstate.ErrLockHeld) { + return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil + } + + if err != nil { + return ctrl.Result{}, err + } + + defer releaseInstallationLock(t.log, installationLock) + lockPath := t.agentUpgradeLockPath if lockPath == "" { lockPath = goalstates.DaemonAgentUpgradeLockPath @@ -119,6 +143,17 @@ func (t *machineOperationTarget) reconcileAgentUpgrade(ctx context.Context, stor } func (t *machineOperationTarget) reconcileAgentReset(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) { + lock, err := installationStore(t.installation).AcquireLock() + if errors.Is(err, installstate.ErrLockHeld) { + return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil + } + + if err != nil { + return ctrl.Result{}, err + } + + defer releaseInstallationLock(t.log, lock) + if err := store.MarkInProgress(ctx, op, "resetting unbounded agent"); err != nil { return ctrl.Result{}, err } diff --git a/cmd/agent/internal/daemon/controller_node.go b/cmd/agent/internal/daemon/controller_node.go index a91fdf249..717767726 100644 --- a/cmd/agent/internal/daemon/controller_node.go +++ b/cmd/agent/internal/daemon/controller_node.go @@ -5,6 +5,7 @@ package daemon import ( "context" + "errors" "fmt" "strings" "time" @@ -19,9 +20,21 @@ import ( v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" "github.com/Azure/unbounded/internal/machineconfigs" "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/installstate" ) func (r *repaveReconciler) ReconcileRepave(ctx context.Context, _ string) (reconcile.Result, error) { + lock, err := installationStore(r.installation).AcquireMutationLock() + if errors.Is(err, installstate.ErrLockHeld) { + return reconcile.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil + } + + if err != nil { + return reconcile.Result{}, err + } + + defer releaseInstallationLock(r.log, lock) + active, err := r.nodeOperator.FindActiveMachine(r.log) if err != nil { return reconcile.Result{}, fmt.Errorf("find active machine: %w", err) diff --git a/cmd/agent/internal/daemon/controller_test.go b/cmd/agent/internal/daemon/controller_test.go index f70e11b87..ee3f9862d 100644 --- a/cmd/agent/internal/daemon/controller_test.go +++ b/cmd/agent/internal/daemon/controller_test.go @@ -25,6 +25,7 @@ import ( "github.com/Azure/unbounded/pkg/agent/agentbinary" daemon "github.com/Azure/unbounded/pkg/agent/daemon" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" ) const testAgentUpgradeSHA256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" @@ -153,6 +154,7 @@ func newTestMachinaMachineOperationReconcilerWithLockPath( t.Helper() target := &machineOperationTarget{ + installation: installstate.NewStore(filepath.Join(t.TempDir(), "state"), filepath.Join(t.TempDir(), "install.lock")), Client: c, log: discardLogger(), machineName: "test-machine", @@ -557,6 +559,7 @@ func TestReconcileRepave_UsesDesiredMachineConfigurationVersion(t *testing.T) { op := &fakeNodeOperator{active: active} c := fakeStatusClient(machine, mcv) reconciler := &repaveReconciler{ + installation: installstate.NewStore(filepath.Join(t.TempDir(), "state"), filepath.Join(t.TempDir(), "install.lock")), Client: c, log: discardLogger(), machineName: "test-machine", @@ -612,6 +615,7 @@ func TestReconcileRepave_NoDriftMarksDesiredConfigurationApplied(t *testing.T) { op := &fakeNodeOperator{active: &ActiveMachine{Name: "kube1", Config: base}} c := fakeStatusClient(machine, mcv) reconciler := &repaveReconciler{ + installation: installstate.NewStore(filepath.Join(t.TempDir(), "state"), filepath.Join(t.TempDir(), "install.lock")), Client: c, log: discardLogger(), machineName: "test-machine", diff --git a/cmd/agent/internal/daemon/installation.go b/cmd/agent/internal/daemon/installation.go new file mode 100644 index 000000000..9dc568734 --- /dev/null +++ b/cmd/agent/internal/daemon/installation.go @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package daemon + +import ( + "log/slog" + + "github.com/Azure/unbounded/pkg/agent/installstate" +) + +func installationStore(store *installstate.Store) *installstate.Store { + if store != nil { + return store + } + + return installstate.DefaultStore() +} + +func releaseInstallationLock(log *slog.Logger, lock *installstate.Lock) { + if err := lock.Release(); err != nil { + log.Error("release installation lock", "error", err) + } +} diff --git a/cmd/agent/internal/daemon/installation_test.go b/cmd/agent/internal/daemon/installation_test.go new file mode 100644 index 000000000..b26d6a5d8 --- /dev/null +++ b/cmd/agent/internal/daemon/installation_test.go @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package daemon + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + ctrl "sigs.k8s.io/controller-runtime" + + shared "github.com/Azure/unbounded/pkg/agent/daemon" + "github.com/Azure/unbounded/pkg/agent/installstate" +) + +func TestInstallationContentionPreventsControllerWork(t *testing.T) { + t.Parallel() + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + lock, err := store.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + + target := &machineOperationTarget{installation: store, log: discardLogger()} + // Nil clients, operation stores and node operators ensure contention returns + // before discovery, status publication, binary staging or local mutation. + for name, handler := range map[string]func(context.Context, shared.MachineOperationStore[int64], shared.MachineOperation) (ctrl.Result, error){ + "reboot": target.reconcileNodeReboot, + "upgrade": target.reconcileAgentUpgrade, + "reset": target.reconcileAgentReset, + } { + t.Run(name, func(t *testing.T) { + result, err := handler(t.Context(), nil, shared.MachineOperation{}) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + }) + } + + repave := &repaveReconciler{installation: store, log: discardLogger()} + result, err := repave.ReconcileRepave(t.Context(), "node-delete") + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) +} diff --git a/cmd/agent/internal/daemon/lifecycle.go b/cmd/agent/internal/daemon/lifecycle.go index 6896281e3..1f1f0d30c 100644 --- a/cmd/agent/internal/daemon/lifecycle.go +++ b/cmd/agent/internal/daemon/lifecycle.go @@ -7,14 +7,19 @@ import ( "bytes" "context" _ "embed" + "errors" "fmt" "log/slog" + "os" "path/filepath" + "strings" "text/template" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/pkg/agent/agentbinary" + "github.com/Azure/unbounded/pkg/agent/bootstrap" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases" ) @@ -105,6 +110,29 @@ func (d *enableDaemon) Do(ctx context.Context) error { return nil } +// InstallBootstrapBinary installs the staged bootstrap executable if the host +// has no daemon binary yet. The caller holds installation ownership; existing +// binary layouts are retained and upgrades use their normal activation path. +func InstallBootstrapBinary() error { + if _, err := os.Lstat(goalstates.DaemonBinaryPath); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + + source, err := os.Executable() + if err != nil { + return err + } + + data, err := os.ReadFile(source) + if err != nil { + return err + } + + return writeFile(goalstates.DaemonBinaryPath, data, 0o755) +} + func renderDaemonAsset(name string, content []byte) ([]byte, error) { paths, err := goalstates.ResolvedAgentUpgradePaths() if err != nil { @@ -122,6 +150,8 @@ func renderDaemonAssetForPaths(name string, content []byte, paths goalstates.Age DaemonBinaryLastGoodPath string DaemonRecoveryScriptPath string DaemonAgentUpgradeSignalPath string + InstallationLockPath string + ActivationLockPath string }{ DaemonUnit: goalstates.DaemonUnit, DaemonRecoveryUnit: goalstates.DaemonRecoveryUnit, @@ -129,6 +159,8 @@ func renderDaemonAssetForPaths(name string, content []byte, paths goalstates.Age DaemonBinaryLastGoodPath: paths.LastGoodPath, DaemonRecoveryScriptPath: goalstates.DaemonRecoveryScriptPath, DaemonAgentUpgradeSignalPath: paths.SignalPath, + InstallationLockPath: installstate.DefaultLockPath, + ActivationLockPath: goalstates.DaemonAgentUpgradeLockPath, } tmpl, err := template.New(name).Parse(string(content)) @@ -153,8 +185,8 @@ type stopDaemon struct { } // StopDaemon returns a task that stops, disables, and removes the -// unbounded-agent-daemon systemd unit. Errors from stop and disable are -// logged but do not fail the task since the unit may not be present. +// unbounded-agent-daemon systemd unit. Only verified absence permits a failed +// stop; substantive service errors must retain reset ownership. func StopDaemon(log *slog.Logger) phases.Task { return &stopDaemon{log: log} } @@ -163,7 +195,10 @@ func (t *stopDaemon) Name() string { return "stop-daemon" } func (t *stopDaemon) Do(ctx context.Context) error { if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "stop", goalstates.DaemonUnit); err != nil { - t.log.Warn("failed to stop daemon (may not be running)", "error", err) + state, inspectErr := executil.OutputCmd(ctx, t.log, "systemctl", "show", goalstates.DaemonUnit, "--property=LoadState", "--value") + if inspectErr != nil || strings.TrimSpace(state) != "not-found" { + return fmt.Errorf("stop daemon: %w", err) + } } return disableAndRemoveDaemonUnit(ctx, t.log) @@ -191,15 +226,24 @@ func (t *removeDaemonUnit) Do(ctx context.Context) error { func disableAndRemoveDaemonUnit(ctx context.Context, log *slog.Logger) error { if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", goalstates.DaemonUnit); err != nil { - log.Warn("failed to disable daemon (may already be absent or systemd unavailable)", "error", err) + if _, statErr := os.Lstat(filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonUnit)); !errors.Is(statErr, os.ErrNotExist) { + return err + } } unitPath := filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonUnit) - removeFileIfExists(log, unitPath) + if err := removeOwnedFile(unitPath); err != nil { + return err + } recoveryUnitPath := filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonRecoveryUnit) - removeFileIfExists(log, recoveryUnitPath) - removeFileIfExists(log, goalstates.DaemonRecoveryScriptPath) + if err := removeOwnedFile(recoveryUnitPath); err != nil { + return err + } + + if err := removeOwnedFile(goalstates.DaemonRecoveryScriptPath); err != nil { + return err + } return nil } @@ -235,7 +279,9 @@ func (t *removeAgentArtifacts) Do(_ context.Context) error { "/usr/local/bin/unbounded-agent-install.sh", "/usr/local/bin/unbounded-agent-uninstall.sh", } { - removeFileIfExists(t.log, path) + if err := removeOwnedFile(path); err != nil { + return err + } } // Remove directories. @@ -243,14 +289,112 @@ func (t *removeAgentArtifacts) Do(_ context.Context) error { "/etc/unbounded/agent", "/tmp/unbounded-agent", } { - removeAllIfExists(t.log, dir) + if err := os.RemoveAll(dir); err != nil { + return err + } } // Remove temp config files matching /tmp/unbounded-agent-config.*.json. matches, _ := filepath.Glob("/tmp/unbounded-agent-config.*.json") //nolint:errcheck // Pattern is valid; only errors on malformed globs. for _, m := range matches { - removeFileIfExists(t.log, m) + if err := removeOwnedFile(m); err != nil { + return err + } + } + + return nil +} + +func removeOwnedFile(path string) error { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("remove owned artifact %s: %w", path, err) } return nil } + +func VerifyDaemonInstalled(ctx context.Context, log *slog.Logger) error { + if _, err := (nspawnNodeOperator{}).FindActiveMachine(log); err != nil { + return err + } + + paths, err := goalstates.ResolvedAgentUpgradePaths() + if err != nil { + return err + } + + service := NewHostDaemonActivationService(log, paths) + + assets, err := service.desiredAssets(paths.CurrentPath) + if err != nil { + return err + } + + for path, asset := range assets { + data, err := os.ReadFile(path) + if err != nil { + return err + } + + if !bytes.Equal(data, asset.content) { + return fmt.Errorf("daemon asset differs: %s", path) + } + + info, err := os.Stat(path) + if err != nil { + return err + } + + if info.Mode().Perm() != asset.mode { + return fmt.Errorf("daemon asset mode differs: %s", path) + } + } + + for _, path := range []string{paths.CurrentPath, paths.LastGoodPath, paths.BinaryPath} { + info, err := os.Stat(path) + if err != nil { + return err + } + + if !info.Mode().IsRegular() || info.Mode().Perm()&0o111 == 0 { + return fmt.Errorf("daemon binary is not executable: %s", path) + } + } + + for _, check := range []string{"is-enabled", "is-active"} { + out, err := executil.OutputCmd(ctx, log, "systemctl", check, goalstates.DaemonUnit) + + want := "active" + if check == "is-enabled" { + want = "enabled" + } + + if err != nil { + return fmt.Errorf("daemon %s check: %w", check, err) + } + + if strings.TrimSpace(out) != want { + return fmt.Errorf("daemon %s check failed: %s", check, out) + } + } + + return service.WaitHealthy(ctx, paths.CurrentPath) +} + +// RepairDaemon requires the caller's installation lock. It uses current applied +// configuration, never the original bootstrap input that may name a retired slot. +func RepairDaemon(ctx context.Context, log *slog.Logger) error { + if _, err := (nspawnNodeOperator{}).FindActiveMachine(log); err != nil { + return err + } + + if err := InstallBootstrapBinary(); err != nil { + return err + } + + if err := EnableDaemon(log).Do(ctx); err != nil { + return err + } + + return bootstrap.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) +} diff --git a/cmd/agent/internal/daemon/nodeoperator.go b/cmd/agent/internal/daemon/nodeoperator.go index 14c72253f..c713e71f3 100644 --- a/cmd/agent/internal/daemon/nodeoperator.go +++ b/cmd/agent/internal/daemon/nodeoperator.go @@ -12,10 +12,12 @@ import ( "os" "reflect" "strings" + "time" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases" "github.com/Azure/unbounded/pkg/agent/phases/nodestart" "github.com/Azure/unbounded/pkg/agent/phases/nodestop" @@ -187,6 +189,37 @@ func gantryDisabled(cfg *provision.AgentConfig) bool { } func (nspawnNodeOperator) EnsureLifecycleMigration(ctx context.Context, log *slog.Logger, active *ActiveMachine) error { + // Type=simple lets the launcher verify the running daemon while this startup + // migration waits for bootstrap or host activation to release ownership. + var lock *installstate.Lock + for { + var err error + + lock, err = installstate.DefaultStore().AcquireMutationLock() + if err == nil { + break + } + + if !errors.Is(err, installstate.ErrLockHeld) { + return err + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(250 * time.Millisecond): + } + } + + defer releaseInstallationLock(log, lock) + + current, err := (nspawnNodeOperator{}).FindActiveMachine(log) + if err != nil { + return err + } + + *active = *current + rootFS, err := goalstates.ResolveNSpawnConfig(active.Config, active.Name) if err != nil { return fmt.Errorf("resolve existing machine lifecycle: %w", err) @@ -233,7 +266,8 @@ func (nspawnNodeOperator) RestartNode(ctx context.Context, log *slog.Logger, act } func (nspawnNodeOperator) ResetAgentResources(ctx context.Context, log *slog.Logger) error { - return ResetAgentResources(log).Do(ctx) + // The MachineOperation holds installation ownership through daemon stop. + return resetUnderLock(ctx, log, installstate.DefaultStore(), resetResources(log)) } func (nspawnNodeOperator) StopDaemon(ctx context.Context, log *slog.Logger) error { diff --git a/cmd/agent/internal/daemon/persist_config.go b/cmd/agent/internal/daemon/persist_config.go index 822243d5a..173bc2371 100644 --- a/cmd/agent/internal/daemon/persist_config.go +++ b/cmd/agent/internal/daemon/persist_config.go @@ -89,11 +89,3 @@ func removeFileIfExists(log *slog.Logger, path string) { log.Warn("failed to remove file", "path", path, "error", err) } } - -// removeAllIfExists removes a path and all children if it exists. Errors are -// logged at Warn so we have a trace but don't abort the flow. -func removeAllIfExists(log *slog.Logger, path string) { - if err := os.RemoveAll(path); err != nil { - log.Warn("failed to remove directory", "path", path, "error", err) - } -} diff --git a/cmd/agent/internal/daemon/recovery_script_test.go b/cmd/agent/internal/daemon/recovery_script_test.go index 80ec39019..9bce45ea7 100644 --- a/cmd/agent/internal/daemon/recovery_script_test.go +++ b/cmd/agent/internal/daemon/recovery_script_test.go @@ -73,6 +73,8 @@ func TestRecoveryScript(t *testing.T) { "DaemonBinaryLastGoodPath": lastGood, "DaemonAgentUpgradeSignalPath": filepath.Join(dir, "signal"), "DaemonUnit": "test-agent.service", + "InstallationLockPath": filepath.Join(dir, "install.lock"), + "ActivationLockPath": filepath.Join(dir, "activation.lock"), })) stub := `#!/bin/bash diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index 11161ec7f..121d80015 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -4,9 +4,19 @@ package daemon import ( + "context" + "errors" + "fmt" "log/slog" + "os" + "path/filepath" + "strings" + "golang.org/x/sys/unix" + + "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases" "github.com/Azure/unbounded/pkg/agent/phases/reset" ) @@ -14,6 +24,121 @@ import ( // ResetAgentResources returns a task that removes the unbounded-agent and all // associated resources without stopping the daemon process. func ResetAgentResources(log *slog.Logger) phases.Task { + return resetWithOwnership(log, installstate.DefaultStore(), false) +} + +func ResetAgent(log *slog.Logger) phases.Task { + return resetWithOwnership(log, installstate.DefaultStore(), true) +} + +type lifecycleTask struct { + name string + run func(context.Context) error +} + +func (t lifecycleTask) Name() string { return t.name } +func (t lifecycleTask) Do(ctx context.Context) error { return t.run(ctx) } + +func resetWithOwnership(log *slog.Logger, store *installstate.Store, stop bool) phases.Task { + inner := resetResources(log) + if stop { + inner = phases.Serial(log, StopDaemon(log), inner) + } + + return lifecycleTask{name: "owned-reset(" + inner.Name() + ")", run: func(ctx context.Context) error { + lock, err := store.AcquireLock() + if err != nil { + return err + } + defer func() { + if err := lock.Release(); err != nil { + log.Error("release reset lock", "error", err) + } + }() + + return resetUnderLock(ctx, log, store, inner) + }} +} + +func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.Store, inner phases.Task) error { + r, err := store.Load() + if errors.Is(err, installstate.ErrNotFound) { + r, err = installstate.NewRecord("legacy-reset", "legacy-reset") + } + + if err != nil { + return err + } + + r.Checkpoint = installstate.Resetting + if err := store.Save(r); err != nil { + return err + } + // Cancel recovery waiting on ownership before removing its executable. + if err := stopRecoveryUnit(ctx, log); err != nil { + return err + } + + return durableReset(ctx, store, inner, []string{"/etc", "/var/lib/machines", "/usr/local", store.Root()}, unix.Syncfs) +} + +func stopRecoveryUnit(ctx context.Context, log *slog.Logger) error { + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "stop", goalstates.DaemonRecoveryUnit); err != nil { + out, inspectErr := executil.OutputCmd(ctx, log, "systemctl", "show", goalstates.DaemonRecoveryUnit, "--property=LoadState", "--value") + if inspectErr != nil || strings.TrimSpace(out) != "not-found" { + return fmt.Errorf("stop daemon recovery: %w", err) + } + } + + return nil +} + +func durableReset(ctx context.Context, store *installstate.Store, inner phases.Task, paths []string, syncfs func(int) error) error { + var handles []*os.File + defer func() { + for _, f := range handles { + _ = f.Close() //nolint:errcheck // Read-only directory descriptor; teardown sync errors are returned below. + } + }() + + for _, path := range paths { + for { + if _, err := os.Stat(path); err == nil { + break + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + + parent := filepath.Dir(path) + if parent == path { + return fmt.Errorf("no filesystem ancestor for %s", path) + } + + path = parent + } + + f, err := os.Open(path) + if err != nil { + return err + } + + handles = append(handles, f) + } + + if err := inner.Do(ctx); err != nil { + return err + } + + for _, f := range handles { + if err := syncfs(int(f.Fd())); err != nil { + return fmt.Errorf("sync teardown %s: %w", f.Name(), err) + } + } + + return store.Remove() +} + +func resetResources(log *slog.Logger) phases.Task { return phases.Serial(log, RemoveDaemonUnit(log), phases.Parallel(log, diff --git a/cmd/agent/internal/daemon/reset_test.go b/cmd/agent/internal/daemon/reset_test.go index ddbfeadc0..4d65c9b07 100644 --- a/cmd/agent/internal/daemon/reset_test.go +++ b/cmd/agent/internal/daemon/reset_test.go @@ -4,11 +4,17 @@ package daemon import ( + "context" + "errors" "log/slog" + "path/filepath" "strings" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/installstate" ) func TestResetAgentResourcesIncludesBPFFSMountCleanup(t *testing.T) { @@ -20,3 +26,59 @@ func TestResetAgentResourcesIncludesBPFFSMountCleanup(t *testing.T) { assert.Less(t, strings.Index(taskName, "parallel(remove-machine, remove-machine)"), strings.Index(taskName, "parallel(remove-bpffs-mount, remove-bpffs-mount)")) assert.Less(t, strings.Index(taskName, "parallel(remove-bpffs-mount, remove-bpffs-mount)"), strings.Index(taskName, "cleanup-routes")) } + +func TestResetRetainsOwnershipUntilTeardownAndSyncSucceed(t *testing.T) { + t.Parallel() + + for _, failure := range []string{"cleanup", "sync", ""} { + t.Run(failure, func(t *testing.T) { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + r, err := installstate.NewRecord("machine", "f") + require.NoError(t, err) + + r.Checkpoint = installstate.Resetting + require.NoError(t, store.Save(r)) + + injected := errors.New("injected reset failure") + cleaned := false + task := lifecycleTask{name: "cleanup", run: func(context.Context) error { + if failure == "cleanup" { + return injected + } + + cleaned = true + + return nil + }} + synced := false + + err = durableReset(t.Context(), store, task, []string{store.Root()}, func(int) error { + require.True(t, cleaned) + + _, err := store.Load() + require.NoError(t, err, "ownership must remain during filesystem barrier") + + if failure == "sync" { + return injected + } + + synced = true + + return nil + }) + if failure != "" { + require.ErrorIs(t, err, injected) + loaded, err := store.Load() + require.NoError(t, err) + require.Equal(t, r, loaded) + } else { + require.NoError(t, err) + require.True(t, synced) + + _, err = store.Load() + require.ErrorIs(t, err, installstate.ErrNotFound) + } + }) + } +} diff --git a/docs/content/guides/agent.md b/docs/content/guides/agent.md index 3fdfe9c75..bf2f12970 100644 --- a/docs/content/guides/agent.md +++ b/docs/content/guides/agent.md @@ -28,6 +28,48 @@ in sequence: containerd and kubelet inside it. Kubelet performs TLS bootstrapping against the API server using the configured bootstrap token. +## Retrying bootstrap and repairing the daemon + +Initial bootstrap records installation ownership before changing the host. If a +stage fails or the process is interrupted, rerun the same saved bootstrap script +or invoke `unbounded-agent start` with the same original configuration. Completed +stages are skipped; unfinished stages are replayed. A running nspawn machine is +preserved during node-start replay. + +The ownership record is `/var/lib/unbounded/agent/install-state.json`. The +completion marker is `/var/lib/unbounded/agent/bootstrap-complete`. These are +internal files, not configuration inputs. Keep them intact when retrying. A +different machine identity or configuration is rejected and requires an explicit +reset before a new initial installation. Regenerating a bootstrap script may +change its token and therefore its configuration identity. + +After completion, the same `start` invocation verifies daemon assets and local +process health, restores a missing completion marker, or repairs the daemon. +This path uses the current applied configuration, including after an ordinary +repave has switched from kube1 to kube2. It does not resolve the original node +image or binary-download sources. The bootstrap shell still needs its agent +download; to repair without that download, run an available agent executable +directly with the original config: + +```bash +sudo env UNBOUNDED_AGENT_CONFIG_FILE=/path/to/original-agent-config.json \ + /path/to/unbounded-agent start +``` + +Bootstrap, reset, node lifecycle operations, and binary activation share an +installation lock. A busy lock is retryable. MachineOperation handlers requeue +instead of starting concurrent host mutations. Reset retains a `resetting` +record when cleanup fails; correct the reported error and run reset again. +Ownership is removed only after teardown and its filesystem synchronization +barriers succeed. The lock file in `/run` remains and must not be deleted to +force an operation through. + +Recovery applies to checkpointed initial installations. Existing installations +without an ownership record retain ordinary daemon operations and reset support; +`start` requires a clean host before creating new ownership. Interrupted ordinary +repaves still use the existing repave model, without a persistent repave recovery +operation. + ## Configuration The agent reads a JSON config file whose path is set through the diff --git a/hack/agent/e2e-kind/e2e.py b/hack/agent/e2e-kind/e2e.py index ea414b291..1b5e5c1a0 100755 --- a/hack/agent/e2e-kind/e2e.py +++ b/hack/agent/e2e-kind/e2e.py @@ -65,6 +65,7 @@ import re import secrets import shutil +import shlex import subprocess import sys import textwrap @@ -2059,6 +2060,19 @@ def run_agent(node_config: NodeConfig) -> None: log("Agent bootstrap completed") +def run_agent_recovery(node_config: NodeConfig) -> None: + """Inject a late bootstrap failure, then retry the identical generated input.""" + previous = os.environ.get("E2E_BOOTSTRAP_RECOVERY") + os.environ["E2E_BOOTSTRAP_RECOVERY"] = "1" + try: + run_agent(node_config) + finally: + if previous is None: + os.environ.pop("E2E_BOOTSTRAP_RECOVERY", None) + else: + os.environ["E2E_BOOTSTRAP_RECOVERY"] = previous + + def prepare_agent_artifacts() -> str: """Build agent artifacts and return the URL that serves the tarball.""" VM_DIR.mkdir(parents=True, exist_ok=True) @@ -2570,6 +2584,43 @@ def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: log("Running bootstrap script on VM...") log("This will download the agent, bootstrap the node, and join it to the Kind cluster.") env_prefix = f"AGENT_URL={agent_url} AGENT_DEBUG={AGENT_DEBUG}" + if os.environ.get("E2E_BOOTSTRAP_RECOVERY") == "1": + # Make a daemon asset unwritable after ownership admission. The node + # stage precedes the failing asset write, without a production failpoint. + inject = ( + "for i in $(seq 1 600); do " + "if test -f /var/lib/unbounded/agent/install-state.json; then " + "mkdir -p /usr/local/bin/unbounded-agent-daemon-recovery.sh; exit 0; fi; " + "sleep 1; done; exit 1" + ) + ssh_cmd("sudo systemd-run --unit=p6-bootstrap-injection --collect /bin/bash -c " + shlex.quote(inject)) + result = subprocess.run([ + "timeout", "1200", "ssh", *SSH_OPTS, SSH_TARGET, + f"sudo {env_prefix} /tmp/bootstrap.sh", + ], check=False) + if result.returncode == 0: + die("blocked daemon asset did not interrupt initial bootstrap") + snapshot = bounded_ssh( + "sudo cat /var/lib/unbounded/agent/install-state.json; " + "systemctl show systemd-nspawn@kube1.service --property=MainPID --value", + time.monotonic() + 30, check=True).stdout + state_text, pid = snapshot.rstrip().rsplit("\n", 1) + before = json.loads(state_text) + if before["checkpoint"] != "installing-daemon" or not pid.isdigit() or int(pid) <= 0: + die(f"failure did not reach the late bootstrap checkpoint: {snapshot}") + ssh_cmd("sudo rmdir /usr/local/bin/unbounded-agent-daemon-recovery.sh") + run(["timeout", "1200", "ssh", *SSH_OPTS, SSH_TARGET, + f"sudo {env_prefix} /tmp/bootstrap.sh"]) + after_text = bounded_ssh( + "sudo cat /var/lib/unbounded/agent/install-state.json; " + "systemctl show systemd-nspawn@kube1.service --property=MainPID --value", + time.monotonic() + 30, check=True).stdout + state_text, after_pid = after_text.rstrip().rsplit("\n", 1) + after = json.loads(state_text) + if after["installID"] != before["installID"] or after["checkpoint"] != "complete" or after_pid != pid: + die("bootstrap retry changed ownership or restarted the running node") + log("Late bootstrap retry preserved installation and nspawn PID") + return run([ "timeout", "1200", "ssh", *SSH_OPTS, "-o", "ServerAliveInterval=30", SSH_TARGET, @@ -4742,6 +4793,45 @@ def reinstall_agent(node_config: NodeConfig) -> None: die("same-disk reinstall changed host boot identity") +def validate_bootstrap_repair() -> None: + """Repair after ordinary repave using the original input and installed agent.""" + script = textwrap.dedent(r""" + set -eu + test ! -e /etc/unbounded/agent/kube1-applied-config.json + test -f /etc/unbounded/agent/kube2-applied-config.json + before=$(sha256sum /etc/unbounded/agent/kube2-applied-config.json) + node_pid=$(systemctl show systemd-nspawn@kube2.service --property=MainPID --value) + test "$node_pid" -gt 0 + cp /usr/local/bin/unbounded-agent-current /tmp/p6-repair-agent + chmod 0755 /tmp/p6-repair-agent + python3 - <<'PY' + from pathlib import Path + script = Path('/tmp/bootstrap.sh').read_text() + marker = "<<'AGENT_CONFIG_EOF'\n" + if marker not in script: + raise SystemExit('original bootstrap config not found') + config = script.split(marker, 1)[1].split('\nAGENT_CONFIG_EOF', 1)[0] + Path('/tmp/p6-original-config.json').write_text(config) + Path('/tmp/p6-original-config.json').chmod(0o600) + PY + systemctl stop unbounded-agent-daemon.service + rm /etc/systemd/system/unbounded-agent-daemon.service + rm /var/lib/unbounded/agent/bootstrap-complete + systemctl daemon-reload + export UNBOUNDED_AGENT_CONFIG_FILE=/tmp/p6-original-config.json + /tmp/p6-repair-agent preflight --output json + /tmp/p6-repair-agent start + test "$before" = "$(sha256sum /etc/unbounded/agent/kube2-applied-config.json)" + test ! -e /etc/unbounded/agent/kube1-applied-config.json + test "$node_pid" = "$(systemctl show systemd-nspawn@kube2.service --property=MainPID --value)" + test -s /var/lib/unbounded/agent/bootstrap-complete + systemctl is-active unbounded-agent-daemon.service + """) + bounded_ssh("sudo bash -c " + shlex.quote(script), time.monotonic() + 180, check=True) + validate_workload() + log("Completed-install repair preserved the repaved slot and workload") + + SUITES: dict[str, list[str]] = { "setup": ["configure-kind-kube-proxy", "install-machine-crd", "deploy-unbounded-net-controller", "start-machina-controller", "validate-machina-controller", "validate-controllers-healthy"], @@ -4754,6 +4844,8 @@ def reinstall_agent(node_config: NodeConfig) -> None: "validate-node-reboot-operation", "validate-workload", "validate-node-repave-upgrade"], "configuration": ["validate-node-configs"], "fresh-bootstrap": ["run-agent", "wait-for-node", "validate-workload"], + "bootstrap-recovery": ["run-agent-recovery", "wait-for-node", "validate-workload", + "validate-node-repave-upgrade", "validate-bootstrap-repair"], } @@ -4791,6 +4883,8 @@ def command(_node_config: NodeConfig) -> None: COMMANDS: dict[str, Command] = { + "run-agent-recovery": run_agent_recovery, + "validate-bootstrap-repair": _without_node_config(validate_bootstrap_repair), "configure-kind-kube-proxy": _without_node_config(configure_kind_kube_proxy), "validate-host-reboot": _without_node_config(validate_host_reboot), "reinstall-agent": reinstall_agent, diff --git a/hack/agent/e2e-kind/test_reliability.py b/hack/agent/e2e-kind/test_reliability.py index 768f4655f..828be8fab 100644 --- a/hack/agent/e2e-kind/test_reliability.py +++ b/hack/agent/e2e-kind/test_reliability.py @@ -24,13 +24,33 @@ def test_suites_preserve_main_lifecycle_and_exclude_future_features(self): self.assertIn(name, steps) self.assertLess(steps.index("validate-host-reboot"), steps.index("reset-agent")) self.assertEqual(steps.count("run-agent"), 1) - self.assertFalse(any("recovery" in name or "ignition" in name for name in e2e.COMMANDS)) + self.assertFalse(any("repave-recovery" in name or "ignition" in name for name in e2e.COMMANDS)) + self.assertEqual(e2e.SUITES["bootstrap-recovery"], [ + "run-agent-recovery", "wait-for-node", "validate-workload", + "validate-node-repave-upgrade", "validate-bootstrap-repair", + ]) def test_reboot_disconnect_requires_new_identity(self): values = [(0, "old"), (255, ""), (255, ""), (0, "old"), (0, "new")] with patch.object(e2e, "bounded_ssh", side_effect=[subprocess.CompletedProcess([], code, out, "") for code, out in values]), patch.object(e2e.time, "sleep"): self.assertEqual(e2e.reboot_host_and_wait(), "new") + def test_repair_script_is_valid_shell_and_embedded_python(self): + with patch.object(e2e, "bounded_ssh") as ssh, patch.object(e2e, "validate_workload"): + e2e.validate_bootstrap_repair() + import shlex + script = shlex.split(ssh.call_args.args[0])[-1] + subprocess.run(["bash", "-n"], input=script, text=True, check=True) + python = script.split("python3 - <<'PY'\n", 1)[1].split("\nPY\n", 1)[0] + compile(python, "repair-fixture", "exec") + + def test_recovery_mode_is_scoped_to_attempt(self): + cfg = e2e.NodeConfig(name="test", node_labels={}, register_with_taints=[]) + with patch.dict(os.environ, {}, clear=True), patch.object(e2e, "run_agent", side_effect=RuntimeError("injected")): + with self.assertRaises(RuntimeError): + e2e.run_agent_recovery(cfg) + self.assertNotIn("E2E_BOOTSTRAP_RECOVERY", os.environ) + def test_reboot_permission_failure_fails(self): with patch.object(e2e, "bounded_ssh", side_effect=[subprocess.CompletedProcess([], 0, "old", ""), subprocess.CompletedProcess([], 1, "", "denied")]), self.assertRaises(SystemExit): e2e.reboot_host_and_wait() diff --git a/internal/provision/assets/unbounded-agent-install.sh b/internal/provision/assets/unbounded-agent-install.sh index 499aee1fc..c3ba638e3 100644 --- a/internal/provision/assets/unbounded-agent-install.sh +++ b/internal/provision/assets/unbounded-agent-install.sh @@ -69,13 +69,15 @@ if [ -z "${AGENT_URL}" ]; then else _version_desc="${AGENT_VERSION:-custom}" fi -AGENT_BIN="/usr/local/bin/unbounded-agent" - echo "Downloading unbounded-agent ${_version_desc} for ${arch} from ${AGENT_URL}..." tmp_dir="$(mktemp -d)" trap 'rm -rf "${tmp_dir}"' EXIT curl -fsSL "${AGENT_URL}" | tar -xz -C "${tmp_dir}" unbounded-agent -install -m 0755 "${tmp_dir}/unbounded-agent" "${AGENT_BIN}" +# Run admission from the staged executable. Bootstrap installs the daemon binary +# only after acquiring installation ownership; retries cannot overwrite a live +# current/compatibility binary link before their intent has been accepted. +AGENT_BIN="${tmp_dir}/unbounded-agent" +chmod 0755 "${AGENT_BIN}" _START_ARGS="" case "${AGENT_DEBUG}" in diff --git a/pkg/agent/bootstrap/coordinator.go b/pkg/agent/bootstrap/coordinator.go new file mode 100644 index 000000000..2d1d02d05 --- /dev/null +++ b/pkg/agent/bootstrap/coordinator.go @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Package bootstrap coordinates replay of owned initial installation stages. +package bootstrap + +import ( + "context" + "fmt" + "log/slog" + + "github.com/Azure/unbounded/pkg/agent/installstate" + "github.com/Azure/unbounded/pkg/agent/internal/utilio" +) + +type Identity struct{ MachineName, ConfigFingerprint string } + +type Stages interface { + EnsureHostClean(context.Context) error + ResolveInputs(context.Context) error + PrepareHost(context.Context) error + PrepareRootFS(context.Context, bool) error + EnsureNodeStarted(context.Context) error + EnsureDaemonInstalled(context.Context) error + RepairDaemon(context.Context) error + VerifyInstalled(context.Context) error +} + +type Reporter interface { + StageStarted(context.Context, installstate.Checkpoint) + StageFailed(context.Context, installstate.Checkpoint, error) +} + +type Coordinator struct { + log *slog.Logger + store *installstate.Store + stages Stages + reporter Reporter +} + +func New(log *slog.Logger, store *installstate.Store, stages Stages, reporter Reporter) *Coordinator { + return &Coordinator{log: log, store: store, stages: stages, reporter: reporter} +} + +type Outcome struct{ AlreadyComplete, Resumed bool } + +func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { + lock, err := c.store.AcquireLock() + if err != nil { + return Outcome{}, err + } + defer func() { + if err := lock.Release(); err != nil { + c.log.Error("release installation lock", "error", err) + } + }() + + r, loadErr := c.store.Load() + + disposition, err := installstate.Decide(r, loadErr, id.MachineName, id.ConfigFingerprint) + if err != nil { + return Outcome{}, err + } + + resumed := disposition != installstate.Fresh + if disposition == installstate.Fresh { + if err := c.stages.EnsureHostClean(ctx); err != nil { + return Outcome{}, err + } + + r, err = installstate.NewRecord(id.MachineName, id.ConfigFingerprint) + if err != nil { + return Outcome{}, err + } + + if err := c.store.Save(r); err != nil { + return Outcome{}, err + } + } else { + if _, err := c.store.CheckMarker(r); err != nil { + return Outcome{}, err + } + } + + if disposition == installstate.AlreadyComplete { + if err := c.stages.VerifyInstalled(ctx); err == nil { + if err := c.store.MarkComplete(r); err != nil { + return Outcome{}, err + } + + return Outcome{AlreadyComplete: true}, nil + } + + r.Checkpoint = installstate.RepairingDaemon + if err := c.store.Save(r); err != nil { + return Outcome{}, err + } + } + + if r.Checkpoint != installstate.RepairingDaemon { + if err := c.stages.ResolveInputs(ctx); err != nil { + return Outcome{}, fmt.Errorf("resolve bootstrap inputs: %w", err) + } + } + + for r.Checkpoint != installstate.Complete { + if err := ctx.Err(); err != nil { + return Outcome{}, err + } + + current := r.Checkpoint + if c.reporter != nil { + c.reporter.StageStarted(ctx, current) + } + + next, err := c.runStage(ctx, current, resumed) + if err != nil { + if c.reporter != nil { + c.reporter.StageFailed(ctx, current, err) + } + + return Outcome{}, fmt.Errorf("%s: %w", current, err) + } + + r.Checkpoint = next + if next == installstate.Complete { + if err := c.store.MarkComplete(r); err != nil { + return Outcome{}, err + } + } else if err := c.store.Save(r); err != nil { + return Outcome{}, err + } + } + + return Outcome{Resumed: resumed}, nil +} + +func (c *Coordinator) runStage(ctx context.Context, stage installstate.Checkpoint, resumed bool) (installstate.Checkpoint, error) { + switch stage { + case installstate.PreparingHost: + return installstate.PreparingRootFS, c.stages.PrepareHost(ctx) + case installstate.PreparingRootFS: + return installstate.StartingNode, c.stages.PrepareRootFS(ctx, resumed) + case installstate.StartingNode: + return installstate.InstallingDaemon, c.stages.EnsureNodeStarted(ctx) + case installstate.InstallingDaemon: + if err := c.stages.EnsureDaemonInstalled(ctx); err != nil { + return "", err + } + + return installstate.Complete, c.stages.VerifyInstalled(ctx) + case installstate.RepairingDaemon: + if err := c.stages.RepairDaemon(ctx); err != nil { + return "", err + } + + return installstate.Complete, c.stages.VerifyInstalled(ctx) + default: + return "", fmt.Errorf("unsupported checkpoint %s", stage) + } +} + +func SyncFilesystems(paths ...string) error { + for _, path := range paths { + if err := utilio.SyncFilesystem(path); err != nil { + return fmt.Errorf("sync %s: %w", path, err) + } + } + + return nil +} diff --git a/pkg/agent/bootstrap/coordinator_test.go b/pkg/agent/bootstrap/coordinator_test.go new file mode 100644 index 000000000..fe2e520b1 --- /dev/null +++ b/pkg/agent/bootstrap/coordinator_test.go @@ -0,0 +1,173 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package bootstrap + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/installstate" +) + +type fakeStages struct { + store *installstate.Store + calls []string + fail string + verifyErr error +} + +var errInjected = errors.New("injected stage failure") + +func (f *fakeStages) run(name string) error { + f.calls = append(f.calls, name) + if name != "clean" { + if _, err := f.store.Load(); err != nil { + return err + } + } + + if name == f.fail { + return errInjected + } + + return nil +} +func (f *fakeStages) EnsureHostClean(context.Context) error { return f.run("clean") } +func (f *fakeStages) ResolveInputs(context.Context) error { return f.run("resolve") } +func (f *fakeStages) PrepareHost(context.Context) error { return f.run("host") } +func (f *fakeStages) PrepareRootFS(context.Context, bool) error { return f.run("rootfs") } +func (f *fakeStages) EnsureNodeStarted(context.Context) error { return f.run("node") } +func (f *fakeStages) EnsureDaemonInstalled(context.Context) error { return f.run("daemon") } +func (f *fakeStages) RepairDaemon(context.Context) error { f.verifyErr = nil; return f.run("repair") } + +func (f *fakeStages) VerifyInstalled(context.Context) error { + if err := f.run("verify"); err != nil { + return err + } + + return f.verifyErr +} + +func TestInterruptedStagesResumeWithoutReplayingEarlierStages(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + fail string + checkpoint installstate.Checkpoint + want []string + }{ + {"host", installstate.PreparingHost, []string{"resolve", "host", "rootfs", "node", "daemon", "verify"}}, + {"rootfs", installstate.PreparingRootFS, []string{"resolve", "rootfs", "node", "daemon", "verify"}}, + {"node", installstate.StartingNode, []string{"resolve", "node", "daemon", "verify"}}, + {"daemon", installstate.InstallingDaemon, []string{"resolve", "daemon", "verify"}}, + } { + t.Run(tc.fail, func(t *testing.T) { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + stages := &fakeStages{store: store, fail: tc.fail} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + id := Identity{MachineName: "machine", ConfigFingerprint: "fingerprint"} + _, err := c.Run(t.Context(), id) + require.ErrorIs(t, err, errInjected) + record, err := store.Load() + require.NoError(t, err) + require.Equal(t, tc.checkpoint, record.Checkpoint) + + _, err = os.Stat(store.CompletePath()) + require.ErrorIs(t, err, os.ErrNotExist) + + stages.calls = nil + stages.fail = "" + outcome, err := c.Run(t.Context(), id) + require.NoError(t, err) + require.True(t, outcome.Resumed) + require.Equal(t, tc.want, stages.calls) + + complete, err := store.Load() + require.NoError(t, err) + require.Equal(t, record.InstallID, complete.InstallID) + require.Equal(t, installstate.Complete, complete.Checkpoint) + }) + } +} + +func TestCompletedRecoveryDoesNotResolveRetiredBootstrapInputs(t *testing.T) { + t.Parallel() + + for _, repair := range []bool{false, true} { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + r, err := installstate.NewRecord("machine", "fingerprint") + require.NoError(t, err) + + r.Checkpoint = installstate.Complete + require.NoError(t, store.Save(r)) + + stages := &fakeStages{store: store, fail: "resolve"} + if repair { + stages.verifyErr = errInjected + } + + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + outcome, err := c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.NoError(t, err) + require.Equal(t, !repair, outcome.AlreadyComplete) + + want := []string{"verify"} + if repair { + want = append(want, "repair", "verify") + } + + require.Equal(t, want, stages.calls) + + marker, err := store.CheckMarker(r) + require.NoError(t, err) + require.True(t, marker) + } +} + +func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { + t.Parallel() + + for _, mode := range []string{"different-intent", "resetting", "marker-conflict", "locked"} { + t.Run(mode, func(t *testing.T) { + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + r, err := installstate.NewRecord("machine", "fingerprint") + require.NoError(t, err) + + if mode == "resetting" { + r.Checkpoint = installstate.Resetting + } + + require.NoError(t, store.Save(r)) + + id := Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint} + if mode == "different-intent" { + id.ConfigFingerprint = "different" + } + + if mode == "marker-conflict" { + require.NoError(t, os.WriteFile(store.CompletePath(), []byte("other"), 0o644)) + } + + if mode == "locked" { + lock, err := store.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + } + + stages := &fakeStages{store: store} + _, err = New(slog.New(slog.DiscardHandler), store, stages, nil).Run(t.Context(), id) + require.Error(t, err) + require.Empty(t, stages.calls) + }) + } +} diff --git a/pkg/agent/installstate/lock.go b/pkg/agent/installstate/lock.go new file mode 100644 index 000000000..9be55b8e3 --- /dev/null +++ b/pkg/agent/installstate/lock.go @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package installstate + +import ( + "errors" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +var ErrLockHeld = errors.New("another host lifecycle operation holds the installation lock") + +type Lock struct{ file *os.File } + +func AcquireLock() (*Lock, error) { return AcquireLockAt(DefaultLockPath) } + +// AcquireLockAt is nonblocking. The kernel releases flock on process exit; a +// leftover lock file does not imply a held lock and must not be deleted by reset. +func AcquireLockAt(path string) (*Lock, error) { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return nil, err + } + + f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, err + } + + if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + closeErr := f.Close() + + if errors.Is(err, unix.EWOULDBLOCK) { + return nil, errors.Join(ErrLockHeld, closeErr) + } + + return nil, errors.Join(err, closeErr) + } + + return &Lock{file: f}, nil +} + +func (l *Lock) Release() error { + if l == nil || l.file == nil { + return nil + } + + err := l.file.Close() + l.file = nil + + return err +} diff --git a/pkg/agent/installstate/mutation.go b/pkg/agent/installstate/mutation.go new file mode 100644 index 000000000..1b5158ee8 --- /dev/null +++ b/pkg/agent/installstate/mutation.go @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package installstate + +import ( + "errors" + "fmt" +) + +// AcquireMutationLock serializes ordinary lifecycle work with bootstrap and +// reset. Legacy hosts without a record remain supported. When both locks are +// needed, acquire installation ownership before the binary activation lock. +func (s *Store) AcquireMutationLock() (*Lock, error) { + lock, err := s.AcquireLock() + if err != nil { + return nil, err + } + + r, loadErr := s.Load() + if errors.Is(loadErr, ErrNotFound) || loadErr == nil && r.Checkpoint == Complete { + return lock, nil + } + + closeErr := lock.Release() + + if loadErr != nil { + return nil, errors.Join(loadErr, closeErr) + } + + return nil, errors.Join(fmt.Errorf("installation is %s; finish bootstrap or reset before lifecycle operations", r.Checkpoint), closeErr) +} diff --git a/pkg/agent/installstate/store.go b/pkg/agent/installstate/store.go new file mode 100644 index 000000000..58b576905 --- /dev/null +++ b/pkg/agent/installstate/store.go @@ -0,0 +1,224 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Package installstate records ownership before initial bootstrap mutates a host. +package installstate + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/Azure/unbounded/pkg/agent/internal/utilio" +) + +const ( + DefaultDirectory = "/var/lib/unbounded/agent" + DefaultLockPath = "/run/unbounded-agent-install.lock" + DefaultHostPrefix = "/usr/local" + SchemaVersion = 1 +) + +type Checkpoint string + +const ( + PreparingHost Checkpoint = "preparing-host" + PreparingRootFS Checkpoint = "preparing-rootfs" + StartingNode Checkpoint = "starting-node" + InstallingDaemon Checkpoint = "installing-daemon" + RepairingDaemon Checkpoint = "repairing-daemon" + Complete Checkpoint = "complete" + Resetting Checkpoint = "resetting" +) + +func (c Checkpoint) NodeMayBeRunning() bool { + return c == StartingNode || c == InstallingDaemon || c == RepairingDaemon || c == Complete +} + +type Record struct { + SchemaVersion int `json:"schemaVersion"` + InstallID string `json:"installID"` + MachineName string `json:"machineName"` + // Store the resolved default now, so later configurable-prefix support can + // consume records from this release without changing their meaning. + HostPrefix string `json:"hostPrefix"` + ConfigFingerprint string `json:"configFingerprint"` + Checkpoint Checkpoint `json:"checkpoint"` +} + +func (r Record) Validate() error { + if r.SchemaVersion != SchemaVersion || strings.TrimSpace(r.InstallID) == "" || strings.TrimSpace(r.MachineName) == "" || strings.TrimSpace(r.ConfigFingerprint) == "" { + return fmt.Errorf("invalid installation record identity or schema") + } + // This release installs only at the default prefix. Unknown ownership must + // not authorize cleanup at a guessed location. + if r.HostPrefix != DefaultHostPrefix { + return fmt.Errorf("unsupported recorded host prefix %q", r.HostPrefix) + } + + switch r.Checkpoint { + case PreparingHost, PreparingRootFS, StartingNode, InstallingDaemon, RepairingDaemon, Complete, Resetting: + return nil + default: + return fmt.Errorf("unknown installation checkpoint %q", r.Checkpoint) + } +} + +var ErrNotFound = errors.New("installation record not found") + +type Store struct{ root, lockPath string } + +func NewStore(root, lockPath string) *Store { return &Store{root: root, lockPath: lockPath} } +func DefaultStore() *Store { return NewStore(DefaultDirectory, DefaultLockPath) } +func (s *Store) Root() string { return s.root } +func (s *Store) StatePath() string { return filepath.Join(s.root, "install-state.json") } +func (s *Store) CompletePath() string { return filepath.Join(s.root, "bootstrap-complete") } +func (s *Store) AcquireLock() (*Lock, error) { return AcquireLockAt(s.lockPath) } + +func (s *Store) Load() (Record, error) { + var r Record + + data, err := os.ReadFile(s.StatePath()) + if errors.Is(err, os.ErrNotExist) { + if _, markerErr := os.Lstat(s.CompletePath()); markerErr == nil { + return r, fmt.Errorf("completion marker exists without installation ownership") + } else if !errors.Is(markerErr, os.ErrNotExist) { + return r, markerErr + } + + return r, ErrNotFound + } + + if err != nil { + return r, err + } + + if err := json.Unmarshal(data, &r); err != nil { + return r, err + } + + return r, r.Validate() +} + +func (s *Store) Save(r Record) error { + if err := r.Validate(); err != nil { + return err + } + + data, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + + return utilio.WriteFileDurable(s.StatePath(), append(data, '\n'), 0o600) +} + +func (s *Store) CheckMarker(r Record) (bool, error) { + data, err := os.ReadFile(s.CompletePath()) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + + if err != nil { + return false, err + } + + if strings.TrimSpace(string(data)) != r.InstallID { + return false, fmt.Errorf("completion marker conflicts with installation identity") + } + + return true, nil +} + +func (s *Store) MarkComplete(r Record) error { + r.Checkpoint = Complete + if err := s.Save(r); err != nil { + return err + } + + return utilio.WriteFileDurable(s.CompletePath(), []byte(r.InstallID+"\n"), 0o644) +} + +// Remove is called only after teardown's filesystem barriers succeed. +func (s *Store) Remove() error { + if _, err := os.Stat(s.root); errors.Is(err, os.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + for _, path := range []string{s.CompletePath(), s.StatePath()} { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + // Persist marker removal before deleting ownership. After interruption, + // reset can resume from the record rather than encounter an orphan marker. + if err := utilio.SyncDir(s.root); err != nil { + return err + } + } + + return nil +} + +func NewRecord(machine, fingerprint string) (Record, error) { + id := make([]byte, 16) + if _, err := rand.Read(id); err != nil { + return Record{}, err + } + + return Record{ + SchemaVersion: SchemaVersion, InstallID: hex.EncodeToString(id), MachineName: machine, + HostPrefix: DefaultHostPrefix, ConfigFingerprint: fingerprint, Checkpoint: PreparingHost, + }, nil +} + +// Fingerprint hashes canonical JSON supplied before ephemeral credentials are +// resolved. Omitted optional fields stay omitted across compatible releases. +func Fingerprint(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) } + +type Disposition int + +const ( + Fresh Disposition = iota + Resume + AlreadyComplete +) + +func Decide(r Record, loadErr error, machine, fingerprint string) (Disposition, error) { + if strings.TrimSpace(machine) == "" || strings.TrimSpace(fingerprint) == "" { + return Fresh, fmt.Errorf("bootstrap identity is required") + } + + if errors.Is(loadErr, ErrNotFound) { + return Fresh, nil + } + + if loadErr != nil { + return Fresh, fmt.Errorf("cannot read installation ownership: %w", loadErr) + } + + if err := r.Validate(); err != nil { + return Fresh, err + } + + if r.Checkpoint == Resetting { + return Fresh, fmt.Errorf("reset is incomplete; run unbounded-agent reset again") + } + + if r.MachineName != machine || r.ConfigFingerprint != fingerprint { + return Fresh, fmt.Errorf("installation intent differs; explicit reset is required") + } + + if r.Checkpoint == Complete { + return AlreadyComplete, nil + } + + return Resume, nil +} diff --git a/pkg/agent/installstate/store_test.go b/pkg/agent/installstate/store_test.go new file mode 100644 index 000000000..48fff1c96 --- /dev/null +++ b/pkg/agent/installstate/store_test.go @@ -0,0 +1,163 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package installstate + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func testStore(t *testing.T) *Store { + t.Helper() + dir := t.TempDir() + + return NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "install.lock")) +} + +func TestStoreLifecycle(t *testing.T) { + t.Parallel() + s := testStore(t) + require.NoError(t, s.Remove()) + _, err := s.Load() + require.ErrorIs(t, err, ErrNotFound) + r, err := NewRecord("machine", Fingerprint([]byte(`{"machineName":"machine"}`))) + require.NoError(t, err) + require.NoError(t, s.Save(r)) + loaded, err := s.Load() + require.NoError(t, err) + require.Equal(t, r, loaded) + + info, err := os.Stat(s.StatePath()) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) + require.NoError(t, s.MarkComplete(r)) + loaded, err = s.Load() + require.NoError(t, err) + require.Equal(t, Complete, loaded.Checkpoint) + marker, err := s.CheckMarker(loaded) + require.NoError(t, err) + require.True(t, marker) + require.NoError(t, s.Remove()) + _, err = s.Load() + require.ErrorIs(t, err, ErrNotFound) +} + +func TestOwnershipAdmission(t *testing.T) { + t.Parallel() + + r, err := NewRecord("machine", "fingerprint") + require.NoError(t, err) + + for _, checkpoint := range []Checkpoint{PreparingHost, PreparingRootFS, StartingNode, InstallingDaemon, RepairingDaemon, Complete, Resetting} { + t.Run(string(checkpoint), func(t *testing.T) { + r := r + r.Checkpoint = checkpoint + + disposition, err := Decide(r, nil, r.MachineName, r.ConfigFingerprint) + if checkpoint == Resetting { + require.Error(t, err) + return + } + + require.NoError(t, err) + + want := Resume + if checkpoint == Complete { + want = AlreadyComplete + } + + require.Equal(t, want, disposition) + + _, err = Decide(r, nil, "other", r.ConfigFingerprint) + require.Error(t, err) + _, err = Decide(r, nil, r.MachineName, "other") + require.Error(t, err) + }) + } + + _, err = Decide(Record{}, ErrNotFound, "", "") + require.Error(t, err) +} + +func TestStoreRejectsCorruptAndOrphanedOwnership(t *testing.T) { + t.Parallel() + + for _, data := range []string{"{", "null", `{}`, `{"schemaVersion":2}`, `{"schemaVersion":1,"installID":"id","machineName":"machine","configFingerprint":"f","hostPrefix":"/opt/unbounded","checkpoint":"complete"}`} { + t.Run(data, func(t *testing.T) { + s := testStore(t) + require.NoError(t, os.MkdirAll(s.Root(), 0o755)) + require.NoError(t, os.WriteFile(s.StatePath(), []byte(data), 0o600)) + r, err := s.Load() + require.Error(t, err) + _, err = Decide(r, err, "machine", "f") + require.Error(t, err) + }) + } + + s := testStore(t) + require.NoError(t, os.MkdirAll(s.Root(), 0o755)) + require.NoError(t, os.WriteFile(s.CompletePath(), []byte("orphan"), 0o644)) + _, err := s.Load() + require.Error(t, err) + require.NotErrorIs(t, err, ErrNotFound) + r, err := NewRecord("machine", "f") + require.NoError(t, err) + _, err = s.CheckMarker(r) + require.Error(t, err) +} + +func TestInstallationLockSurvivesStateRemoval(t *testing.T) { + t.Parallel() + s := testStore(t) + lock, err := s.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + + r, err := NewRecord("machine", "f") + require.NoError(t, err) + require.NoError(t, s.Save(r)) + require.NoError(t, s.Remove()) + _, err = s.AcquireLock() + require.ErrorIs(t, err, ErrLockHeld) + require.NoError(t, lock.Release()) + + second, err := s.AcquireLock() + require.NoError(t, err) + require.NoError(t, second.Release()) +} + +func TestMutationAdmission(t *testing.T) { + t.Parallel() + + for _, checkpoint := range []Checkpoint{"", PreparingHost, StartingNode, Complete, RepairingDaemon, Resetting} { + t.Run(string(checkpoint), func(t *testing.T) { + s := testStore(t) + + if checkpoint != "" { + r, err := NewRecord("machine", "f") + require.NoError(t, err) + + r.Checkpoint = checkpoint + require.NoError(t, s.Save(r)) + } + + lock, err := s.AcquireMutationLock() + if checkpoint == "" || checkpoint == Complete { + require.NoError(t, err) + _, err = s.AcquireMutationLock() + require.ErrorIs(t, err, ErrLockHeld) + require.NoError(t, lock.Release()) + } else { + require.Error(t, err) + } + + lock, err = s.AcquireLock() + require.NoError(t, err, "failed admission must release the lock for reset") + require.NoError(t, lock.Release()) + }) + } +} diff --git a/pkg/agent/internal/utilio/sync.go b/pkg/agent/internal/utilio/sync.go new file mode 100644 index 000000000..0cdb8e2a9 --- /dev/null +++ b/pkg/agent/internal/utilio/sync.go @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package utilio + +import ( + "errors" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +func SyncDir(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + + return errors.Join(f.Sync(), f.Close()) +} + +// WriteFileDurable also persists newly created parent directories, so the state +// file cannot outlive the directory entries needed to reach it. +func WriteFileDurable(path string, data []byte, mode os.FileMode) error { + var parents []string + + for dir := filepath.Dir(path); ; dir = filepath.Dir(dir) { + _, err := os.Stat(dir) + if err == nil { + parents = append(parents, dir) + break + } + + if !errors.Is(err, os.ErrNotExist) { + return err + } + + parents = append(parents, dir) + } + + if err := WriteFile(path, data, mode); err != nil { + return err + } + + for _, dir := range parents { + if err := SyncDir(dir); err != nil { + return err + } + } + + return nil +} + +// SyncFilesystem persists all output on the filesystem before a checkpoint. +func SyncFilesystem(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + + return errors.Join(unix.Syncfs(int(f.Fd())), f.Close()) +} diff --git a/pkg/agent/phases/nodestart/nspawn.go b/pkg/agent/phases/nodestart/nspawn.go index 985a2d97f..3551dd8b0 100644 --- a/pkg/agent/phases/nodestart/nspawn.go +++ b/pkg/agent/phases/nodestart/nspawn.go @@ -25,12 +25,22 @@ type machinectlRunner interface { Terminate(ctx context.Context, name string) error Exists(ctx context.Context, name string) bool ResetFailed(ctx context.Context, name string) error + Running(ctx context.Context, name string) (bool, error) } type defaultMachinectlRunner struct { log *slog.Logger } +func (r defaultMachinectlRunner) Running(ctx context.Context, name string) (bool, error) { + out, err := executil.OutputCmd(ctx, r.log, "systemctl", "show", "systemd-nspawn@"+name+".service", "--property=ActiveState", "--value") + if err != nil { + return false, err + } + + return strings.TrimSpace(out) == "active" || strings.TrimSpace(out) == "activating", nil +} + func (r defaultMachinectlRunner) Enable(ctx context.Context, name string) error { return executil.RunCmd(ctx, r.log, executil.Machinectl(), "enable", name) } @@ -96,6 +106,15 @@ func (s *startNSpawnMachine) Do(ctx context.Context) error { // under us, leaving an orphaned registration with errno 17 / "File exists"), // terminates the stale registration and retries once. func (s *startNSpawnMachine) startWithRecovery(ctx context.Context, name string) error { + running, err := s.runner.Running(ctx, name) + if err != nil { + return fmt.Errorf("inspect nspawn service before replay: %w", err) + } + + if running { + return nil + } + startErr := s.runner.Start(ctx, name) if startErr == nil { return nil diff --git a/pkg/agent/phases/nodestart/nspawn_test.go b/pkg/agent/phases/nodestart/nspawn_test.go index 4fe0da0fe..d7fe40c3c 100644 --- a/pkg/agent/phases/nodestart/nspawn_test.go +++ b/pkg/agent/phases/nodestart/nspawn_test.go @@ -22,7 +22,9 @@ func silentLogger() *slog.Logger { return slog.New(slog.DiscardHandler) } // fakeRunner is a scriptable machinectlRunner for exercising the // startWithRecovery state machine without touching real binaries. type fakeRunner struct { - mu sync.Mutex + running bool + runningErr error + mu sync.Mutex // startResults are returned by successive calls to Start. // If the slice is exhausted, the test fails. @@ -97,6 +99,8 @@ func (f *fakeRunner) Exists(_ context.Context, _ string) bool { return f.existsAfterStart } +func (f *fakeRunner) Running(context.Context, string) (bool, error) { return f.running, f.runningErr } + func (f *fakeRunner) ResetFailed(_ context.Context, _ string) error { f.mu.Lock() defer f.mu.Unlock() @@ -122,6 +126,26 @@ func runStart(t *testing.T, runner *fakeRunner) error { } // TestStartWithRecovery_HappyPath: clean start, no recovery needed. +func TestStartWithRecoveryPreservesRunningMachine(t *testing.T) { + t.Parallel() + + r := &fakeRunner{running: true, existsAfterStart: true} + require.NoError(t, runStart(t, r)) + require.Zero(t, r.startCalls) + require.Zero(t, r.terminateCalls) + require.Zero(t, r.resetFailedCalls) +} + +func TestStartWithRecoveryRejectsInspectionFailure(t *testing.T) { + t.Parallel() + + injected := errors.New("inspection denied") + r := &fakeRunner{runningErr: injected} + require.ErrorIs(t, runStart(t, r), injected) + require.Zero(t, r.startCalls) + require.Zero(t, r.terminateCalls) +} + func TestStartWithRecovery_HappyPath(t *testing.T) { t.Parallel() diff --git a/pkg/agent/phases/nodestart/preflight_bind_address.go b/pkg/agent/phases/nodestart/preflight_bind_address.go index aff7e0763..785d7cfa2 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address.go @@ -32,6 +32,101 @@ type bindAddressChecker struct { description string log *slog.Logger inspect func(address string) (string, bool, error) + owned func() bool +} + +// CheckOwnedBindAddress accepts only listeners with both the expected process +// root and executable inode. An uninspectable owner is not proof of ownership. +func CheckOwnedBindAddress(log *slog.Logger, name, address, description, root, executable string) preflight.Checker { + c := bindAddressChecker{ + name: name, address: address, description: description, log: log, + inspect: func(address string) (string, bool, error) { return inspectTCPListener("/proc", address) }, + } + c.owned = func() bool { return listenerOwnedByRoot("/proc", address, root, executable) } + + return c +} + +func listenerOwnedByRoot(procRoot, address, root, executable string) bool { + _, portText, err := net.SplitHostPort(address) + if err != nil { + return false + } + + port, err := strconv.ParseUint(portText, 10, 16) + if err != nil { + return false + } + + wanted := map[string]struct{}{} + + for _, table := range []string{"tcp", "tcp6"} { + data, err := os.ReadFile(filepath.Join(procRoot, "net", table)) + if errors.Is(err, fs.ErrNotExist) && table == "tcp6" { + continue + } + + if err != nil { + return false + } + + for inode := range listenerSocketInodes(data, uint16(port)) { + wanted[inode] = struct{}{} + } + } + + rootInfo, err := os.Stat(root) + if err != nil { + return false + } + + exeInfo, err := os.Stat(filepath.Join(root, executable)) + if err != nil { + return false + } + + processes, err := os.ReadDir(procRoot) + if err != nil { + return false + } + + matched := map[string]bool{} + + for _, process := range processes { + if _, err := strconv.Atoi(process.Name()); err != nil { + continue + } + + base := filepath.Join(procRoot, process.Name()) + + fds, err := os.ReadDir(filepath.Join(base, "fd")) + if err != nil { + continue + } + + for _, fd := range fds { + target, err := os.Readlink(filepath.Join(base, "fd", fd.Name())) + if err != nil { + continue + } + + inode := strings.TrimSuffix(strings.TrimPrefix(target, "socket:["), "]") + if _, found := wanted[inode]; !found { + continue + } + + processRoot, rootErr := os.Stat(filepath.Join(base, "root")) + + processExe, exeErr := os.Stat(filepath.Join(base, "exe")) + if rootErr != nil || exeErr != nil || !os.SameFile(rootInfo, processRoot) || !os.SameFile(exeInfo, processExe) { + return false + } + + matched[inode] = true + } + } + + return len(wanted) > 0 && len(matched) == len(wanted) } // CheckBindAddress verifies no TCP listener currently occupies an address's port. @@ -58,6 +153,10 @@ func (c bindAddressChecker) Check(context.Context) []preflight.Result { } if occupied { + if c.owned != nil && c.owned() { + return preflight.ResultsOK(c.name, c.address, c.description+" belongs to this installation") + } + if owner != "" { return preflight.ResultsError(c.name, c.address, "%s is already in use by process %s", c.description, owner) } diff --git a/pkg/agent/phases/nodestart/preflight_bind_address_test.go b/pkg/agent/phases/nodestart/preflight_bind_address_test.go index d9a4854aa..3389d89d1 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address_test.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address_test.go @@ -130,3 +130,42 @@ func createProcFixture(t *testing.T, tcp, tcp6 string) string { return procRoot } + +func TestListenerOwnershipRequiresRootAndExecutableForEverySocket(t *testing.T) { + t.Parallel() + + for _, mode := range []string{"owned", "foreign-root", "foreign-executable", "unknown-owner", "shared-with-foreign"} { + t.Run(mode, func(t *testing.T) { + proc := createProcFixture(t, procTCPHeader+"0: 00000000:280A 00000000:0000 0A 0 0 0 0 0 45678\n", procTCPHeader) + root := t.TempDir() + exe := filepath.Join(root, "kubelet") + require.NoError(t, os.WriteFile(exe, []byte("executable"), 0o755)) + + processRoot, processExe := root, exe + if mode == "foreign-root" { + processRoot = t.TempDir() + } + + if mode == "foreign-executable" { + processExe = filepath.Join(t.TempDir(), "kubelet") + require.NoError(t, os.WriteFile(processExe, []byte("executable"), 0o755)) + } + + if mode != "unknown-owner" { + require.NoError(t, os.MkdirAll(filepath.Join(proc, "123", "fd"), 0o755)) + require.NoError(t, os.Symlink("socket:[45678]", filepath.Join(proc, "123", "fd", "4"))) + require.NoError(t, os.Symlink(processRoot, filepath.Join(proc, "123", "root"))) + require.NoError(t, os.Symlink(processExe, filepath.Join(proc, "123", "exe"))) + } + + if mode == "shared-with-foreign" { + require.NoError(t, os.MkdirAll(filepath.Join(proc, "456", "fd"), 0o755)) + require.NoError(t, os.Symlink("socket:[45678]", filepath.Join(proc, "456", "fd", "4"))) + require.NoError(t, os.Symlink(t.TempDir(), filepath.Join(proc, "456", "root"))) + require.NoError(t, os.Symlink(exe, filepath.Join(proc, "456", "exe"))) + } + + require.Equal(t, mode == "owned", listenerOwnedByRoot(proc, kubeletBindAddress, root, "kubelet")) + }) + } +} diff --git a/pkg/agent/phases/reset/helpers.go b/pkg/agent/phases/reset/helpers.go index 80a62be49..548c402ad 100644 --- a/pkg/agent/phases/reset/helpers.go +++ b/pkg/agent/phases/reset/helpers.go @@ -5,22 +5,27 @@ package reset import ( "errors" + "fmt" "log/slog" "os" ) -// removeFileIfExists removes a file if it exists. Non-ENOENT errors are -// logged at Warn so we have a trace but don't abort the reset flow. -func removeFileIfExists(log *slog.Logger, path string) { +// removeFileIfExists ignores absence but propagates substantive removal errors. +func removeFileIfExists(log *slog.Logger, path string) error { if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { log.Warn("failed to remove file", "path", path, "error", err) + return fmt.Errorf("remove %s: %w", path, err) } + + return nil } -// removeAllIfExists removes a path and all children if it exists. Errors are -// logged at Warn so we have a trace but don't abort the reset flow. -func removeAllIfExists(log *slog.Logger, path string) { +// removeAllIfExists propagates removal failures so reset retains ownership. +func removeAllIfExists(log *slog.Logger, path string) error { if err := os.RemoveAll(path); err != nil { log.Warn("failed to remove directory", "path", path, "error", err) + return fmt.Errorf("remove %s: %w", path, err) } + + return nil } diff --git a/pkg/agent/phases/reset/machine.go b/pkg/agent/phases/reset/machine.go index b3f26d276..4de156ce5 100644 --- a/pkg/agent/phases/reset/machine.go +++ b/pkg/agent/phases/reset/machine.go @@ -9,6 +9,7 @@ import ( "fmt" "log/slog" "os" + "strings" "time" "github.com/Azure/unbounded/internal/executil" @@ -30,10 +31,24 @@ func (t *stopMachine) Name() string { return "stop-machine" } func (t *stopMachine) Do(ctx context.Context) error { if err := executil.RunCmd(ctx, t.log, executil.Machinectl(), "disable", t.machineName); err != nil { - t.log.Warn("failed to disable machine (may not have been enabled)", "machine", t.machineName, "error", err) + out, inspectErr := executil.OutputCmd(ctx, t.log, "systemctl", "show", "systemd-nspawn@"+t.machineName+".service", "--property=UnitFileState", "--value") + if inspectErr != nil { + return fmt.Errorf("inspect nspawn enablement: %w", inspectErr) + } + + switch strings.TrimSpace(out) { + case "disabled", "static", "masked": + default: + return fmt.Errorf("disable nspawn machine %s: %w", t.machineName, err) + } + } + + exists, err := machineExists(ctx, t.log, t.machineName) + if err != nil { + return err } - if !machineExists(ctx, t.log, t.machineName) { + if !exists { t.log.Info("machine not running, nothing to stop", "machine", t.machineName) return nil } @@ -52,12 +67,16 @@ func (t *stopMachine) Do(ctx context.Context) error { } // Wait up to 30 seconds for the machine to fully stop. - if t.waitForGone(ctx, 30*time.Second) { + if gone, err := t.waitForGone(ctx, 30*time.Second); err != nil { + return err + } else if gone { return nil } // Force terminate if still registered. - if machineExists(ctx, t.log, t.machineName) { + if exists, err := machineExists(ctx, t.log, t.machineName); err != nil { + return err + } else if exists { t.log.Warn("machine did not stop gracefully, terminating", "machine", t.machineName) if err := executil.RunCmd(ctx, t.log, executil.Machinectl(), "terminate", t.machineName); err != nil { @@ -65,7 +84,11 @@ func (t *stopMachine) Do(ctx context.Context) error { } // Wait up to 15 seconds for the terminate to take full effect. - t.waitForGone(ctx, 15*time.Second) + if gone, err := t.waitForGone(ctx, 15*time.Second); err != nil { + return err + } else if !gone { + return fmt.Errorf("machine %s remains registered after termination", t.machineName) + } } return ctx.Err() @@ -73,21 +96,25 @@ func (t *stopMachine) Do(ctx context.Context) error { // waitForGone polls machineExists until the machine disappears or the timeout // elapses. Returns true if the machine is gone. -func (t *stopMachine) waitForGone(ctx context.Context, timeout time.Duration) bool { +func (t *stopMachine) waitForGone(ctx context.Context, timeout time.Duration) (bool, error) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { - if !machineExists(ctx, t.log, t.machineName) { - return true + if exists, err := machineExists(ctx, t.log, t.machineName); err != nil { + return false, err + } else if !exists { + return true, nil } select { case <-ctx.Done(): - return false + return false, ctx.Err() case <-time.After(time.Second): } } - return !machineExists(ctx, t.log, t.machineName) + exists, err := machineExists(ctx, t.log, t.machineName) + + return !exists, err } type removeMachine struct { @@ -110,6 +137,8 @@ func (t *removeMachine) Do(ctx context.Context) error { if _, err := os.Stat(machineDir); errors.Is(err, os.ErrNotExist) { t.log.Info("machine rootfs not present, nothing to remove", "machine", t.machineName) return nil + } else if err != nil { + return err } t.log.Info("removing machine rootfs", "machine", t.machineName, "dir", machineDir) @@ -130,7 +159,9 @@ func (t *removeMachine) Do(ctx context.Context) error { return nil // machinectl removed both image metadata and directory } - if !machineExists(ctx, t.log, t.machineName) { + if exists, err := machineExists(ctx, t.log, t.machineName); err != nil { + return err + } else if !exists { // Once machined no longer knows the machine, the nspawn service is stopped // and the rootfs can be deleted directly. Some host configurations can still // make machinectl remove fail at this point. Fedora with SELinux enforcing, @@ -149,15 +180,31 @@ func (t *removeMachine) Do(ctx context.Context) error { // Fallback: force-remove the directory if machinectl keeps failing. t.log.Warn("machinectl remove did not succeed, force-removing directory", "dir", machineDir) - removeAllIfExists(t.log, machineDir) - return nil + if exists, err := machineExists(ctx, t.log, t.machineName); err != nil { + return err + } else if exists { + return fmt.Errorf("refusing to remove registered machine %s", t.machineName) + } + + return removeAllIfExists(t.log, machineDir) } // machineExists checks whether the named nspawn machine is known to machinectl. -func machineExists(ctx context.Context, log *slog.Logger, name string) bool { - err := executil.RunCmd(ctx, log, executil.Machinectl(), "show", name) - return err == nil +func machineExists(ctx context.Context, log *slog.Logger, name string) (bool, error) { + out, err := executil.OutputCmd(ctx, log, "machinectl", "list", "--no-legend", "--no-pager") + if err != nil { + return false, fmt.Errorf("inspect registered machines: %w", err) + } + + for _, line := range strings.Split(out, "\n") { + fields := strings.Fields(line) + if len(fields) > 0 && fields[0] == name { + return true, nil + } + } + + return false, nil } // serviceIsActive returns true if the named systemd service is currently active. diff --git a/pkg/agent/phases/reset/network.go b/pkg/agent/phases/reset/network.go index 6d50c875d..f4ca0eaec 100644 --- a/pkg/agent/phases/reset/network.go +++ b/pkg/agent/phases/reset/network.go @@ -66,10 +66,18 @@ func (t *cleanupLocalDNSRules) Name() string { return "cleanup-localdns-rules" } func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "disable", "--now", goalstates.LocalDNSNetworkUnit); err != nil { - t.log.Debug("LocalDNS network unit was not active", "error", err) + path := filepath.Join(goalstates.SystemdSystemDir, goalstates.LocalDNSNetworkUnit) + if _, statErr := os.Lstat(path); !errors.Is(statErr, os.ErrNotExist) { + return fmt.Errorf("disable LocalDNS unit: %w", err) + } } - if _, err := executil.OutputCmd(ctx, t.log, "nft", "list", "table", "ip", goalstates.LocalDNSNFTTable); err == nil { + tables, err := executil.OutputCmd(ctx, t.log, "nft", "list", "tables") + if err != nil { + return fmt.Errorf("inspect LocalDNS tables: %w", err) + } + + if strings.Contains(tables, "table ip "+goalstates.LocalDNSNFTTable+"\n") { if err := executil.RunCmd(ctx, t.log, func(ctx context.Context) *exec.Cmd { return exec.CommandContext(ctx, "nft") }, "delete", "table", "ip", goalstates.LocalDNSNFTTable); err != nil { @@ -77,7 +85,16 @@ func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { } } - if output, err := executil.OutputCmd(ctx, t.log, "ip", "-d", "-o", "link", "show", "dev", goalstates.LocalDNSInterfaceName); err == nil { + links, err := executil.OutputCmd(ctx, t.log, "ip", "-d", "-o", "link", "show") + if err != nil { + return fmt.Errorf("inspect LocalDNS interface: %w", err) + } + + for _, output := range strings.Split(links, "\n") { + if !strings.Contains(output, ": "+goalstates.LocalDNSInterfaceName+":") { + continue + } + if !strings.Contains(" "+output+" ", " dummy ") { return fmt.Errorf("refusing to remove non-dummy interface %s", goalstates.LocalDNSInterfaceName) } @@ -91,7 +108,9 @@ func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { filepath.Join(goalstates.SystemdSystemDir, goalstates.LocalDNSNetworkUnit), "/usr/local/libexec/unbounded-localdns-network", } { - removeFileIfExists(t.log, path) + if err := removeFileIfExists(t.log, path); err != nil { + return err + } } return nil @@ -101,19 +120,27 @@ func (t *removeNetworkInterfaces) Do(ctx context.Context) error { // Remove WireGuard interfaces (wg51820, wg51821, ...). wgIfaces, err := listWireGuardInterfaces(ctx, t.log) if err != nil { - t.log.Warn("failed to list WireGuard interfaces", "error", err) + return err } for _, iface := range wgIfaces { t.log.Info("removing interface", "interface", iface) - deleteLink(ctx, t.log, iface) + + if err := deleteLink(ctx, t.log, iface); err != nil { + return err + } } // Remove tunnel and overlay interfaces. for _, iface := range knownOverlayInterfaces { - if linkExists(t.log, iface) { + if exists, err := linkExists(t.log, iface); err != nil { + return err + } else if exists { t.log.Info("removing interface", "interface", iface) - deleteLink(ctx, t.log, iface) + + if err := deleteLink(ctx, t.log, iface); err != nil { + return err + } } } @@ -139,7 +166,9 @@ func (t *removeWireGuardKeys) Do(_ context.Context) error { "/etc/wireguard/server.priv", "/etc/wireguard/server.pub", } { - removeFileIfExists(t.log, path) + if err := removeFileIfExists(t.log, path); err != nil { + return err + } } return nil @@ -170,7 +199,7 @@ func listWireGuardInterfaces(ctx context.Context, log *slog.Logger) ([]string, e } } - return ifaces, nil + return ifaces, scanner.Err() } // isWireGuardInterface returns true if the interface name matches the @@ -196,25 +225,30 @@ func isWireGuardInterface(name string) bool { // linkExists checks whether a network interface exists by looking up its // entry in /sys/class/net. This avoids shelling out and cleanly distinguishes // "not found" from real errors. -func linkExists(log *slog.Logger, name string) bool { +func linkExists(log *slog.Logger, name string) (bool, error) { _, err := os.Stat(fmt.Sprintf("/sys/class/net/%s", name)) if err == nil { - return true + return true, nil } if errors.Is(err, os.ErrNotExist) { - return false + return false, nil } log.Warn("failed to check interface existence", "interface", name, "error", err) - return false + return false, err } -// deleteLink removes a network interface, logging a warning if the operation -// fails (e.g. the interface was already removed). -func deleteLink(ctx context.Context, log *slog.Logger, name string) { +// deleteLink ignores only verified absence after a failed deletion. +func deleteLink(ctx context.Context, log *slog.Logger, name string) error { if err := executil.RunCmd(ctx, log, executil.Ip(), "link", "delete", name); err != nil { - log.Warn("failed to delete interface (may already be gone)", "interface", name, "error", err) + if exists, inspectErr := linkExists(log, name); inspectErr == nil && !exists { + return nil + } + + return fmt.Errorf("delete interface %s: %w", name, err) } + + return nil } diff --git a/pkg/agent/phases/reset/nspawn.go b/pkg/agent/phases/reset/nspawn.go index c4637654d..4714c39ce 100644 --- a/pkg/agent/phases/reset/nspawn.go +++ b/pkg/agent/phases/reset/nspawn.go @@ -9,6 +9,7 @@ import ( "fmt" "log/slog" "os" + "os/exec" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -34,11 +35,7 @@ func (t *removeNSpawnConfig) Do(_ context.Context) error { configRegenerationUnit := fmt.Sprintf("%s/%s", goalstates.SystemdSystemDir, goalstates.ConfigRegenerationUnit(t.machineName)) t.log.Info("removing nspawn configuration", "nspawn_file", nspawnFile, "override_dir", overrideDir, "config_regeneration_unit", configRegenerationUnit) - removeFileIfExists(t.log, nspawnFile) - removeAllIfExists(t.log, overrideDir) - removeFileIfExists(t.log, configRegenerationUnit) - - return nil + return errors.Join(removeFileIfExists(t.log, nspawnFile), removeAllIfExists(t.log, overrideDir), removeFileIfExists(t.log, configRegenerationUnit)) } type removeBPFFSMount struct { @@ -63,18 +60,20 @@ func (t *removeBPFFSMount) Do(ctx context.Context) error { return fmt.Errorf("stat bpffs mount path %s: %w", mountPath, err) } - // mountpoint -q exits non-zero when an existing path is not a mount point. - // The agent runs as root and host preparation installs util-linux, so treat a - // non-zero exit here as "already unmounted" and remove the directory below. + // util-linux reserves exit 32 for a path that is not a mount point. + // Other failures do not authorize recursive removal. if err := executil.RunCmdAt(ctx, t.log, slog.LevelDebug, executil.Mountpoint(), "-q", mountPath); err == nil { if err := executil.RunCmd(ctx, t.log, executil.Umount(), mountPath); err != nil { return fmt.Errorf("unmount bpffs %s: %w", mountPath, err) } + } else { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 32 { + return fmt.Errorf("inspect bpffs mount %s: %w", mountPath, err) + } } - removeAllIfExists(t.log, mountPath) - - return nil + return removeAllIfExists(t.log, mountPath) } // CleanupMachine returns a composite task that removes all artifacts of an diff --git a/pkg/agent/phases/reset/routes.go b/pkg/agent/phases/reset/routes.go index 1f1732d29..b877d386a 100644 --- a/pkg/agent/phases/reset/routes.go +++ b/pkg/agent/phases/reset/routes.go @@ -5,8 +5,10 @@ package reset import ( "context" + "encoding/json" "fmt" "log/slog" + "strconv" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/pkg/agent/phases" @@ -20,7 +22,8 @@ const ( ) type cleanupRoutes struct { - log *slog.Logger + log *slog.Logger + output func(context.Context, ...string) (string, error) } // CleanupRoutes returns a task that removes policy routing rules and flushes @@ -34,21 +37,95 @@ func (t *cleanupRoutes) Name() string { return "cleanup-routes" } func (t *cleanupRoutes) Do(ctx context.Context) error { t.log.Info("cleaning up policy routing rules") - for table := wireguardTableStart; table <= wireguardTableEnd; table++ { - tableStr := fmt.Sprintf("%d", table) + output := t.output + if output == nil { + output = func(ctx context.Context, args ...string) (string, error) { + return executil.OutputCmd(ctx, t.log, "ip", args...) + } + } - // Remove all ip rules pointing to this table. - for { - if err := executil.RunCmd(ctx, t.log, executil.Ip(), "rule", "del", "table", tableStr); err != nil { - break // no more rules for this table + for _, family := range []string{"-4", "-6"} { + for _, kind := range []string{"rule", "route"} { + args := []string{family, "-N", "-j", kind, "show"} + if kind == "route" { + args = append(args, "table", "all") } - } - // Flush the routing table. - if err := executil.RunCmd(ctx, t.log, executil.Ip(), "route", "flush", "table", tableStr); err != nil { - t.log.Warn("failed to flush routing table (may be empty)", "table", tableStr, "error", err) + out, err := output(ctx, args...) + if err != nil { + return fmt.Errorf("inspect %s %s: %w", family, kind, err) + } + + tables, err := ownedRoutingTables(out) + if err != nil { + return fmt.Errorf("decode %s %s: %w", family, kind, err) + } + + flushed := make(map[int]bool) + + for _, table := range tables { + action := "del" + + if kind == "route" { + if flushed[table] { + continue + } + + flushed[table] = true + action = "flush" + } + + if _, err := output(ctx, family, kind, action, "table", strconv.Itoa(table)); err != nil { + return fmt.Errorf("remove %s %s table %d: %w", family, kind, table, err) + } + } } } return nil } + +// Enumerate numeric table IDs before mutation so an inspection failure cannot +// be mistaken for absence. Preserve duplicate rules but flush each table once. +func ownedRoutingTables(output string) ([]int, error) { + var entries []struct { + Table json.RawMessage `json:"table"` + } + if err := json.Unmarshal([]byte(output), &entries); err != nil { + return nil, err + } + + var tables []int + + for _, entry := range entries { + if len(entry.Table) == 0 { + continue + } + + var table int + if err := json.Unmarshal(entry.Table, &table); err != nil { + var name string + if err := json.Unmarshal(entry.Table, &name); err != nil { + return nil, err + } + + var parseErr error + + table, parseErr = strconv.Atoi(name) + if parseErr != nil { + switch name { + case "main", "local", "default", "unspec": + continue + } + + return nil, fmt.Errorf("non-numeric routing table %q", name) + } + } + + if table >= wireguardTableStart && table <= wireguardTableEnd { + tables = append(tables, table) + } + } + + return tables, nil +} diff --git a/pkg/agent/phases/reset/strict_test.go b/pkg/agent/phases/reset/strict_test.go new file mode 100644 index 000000000..dc6f93279 --- /dev/null +++ b/pkg/agent/phases/reset/strict_test.go @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package reset + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMachineInspectionFailsClosed(t *testing.T) { + for _, tc := range []struct { + name, script string + exists, failed bool + }{ + {"registered", "printf 'kube1 container systemd-nspawn - - -\\n'", true, false}, + {"different", "printf 'kube10 container systemd-nspawn - - -\\n'", false, false}, + {"absent", "exit 0", false, false}, + {"denied", "exit 1", false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "machinectl"), []byte("#!/bin/sh\n"+tc.script+"\n"), 0o755)) + t.Setenv("PATH", dir) + exists, err := machineExists(t.Context(), slog.New(slog.DiscardHandler), "kube1") + require.Equal(t, tc.exists, exists) + + if tc.failed { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestCleanupRoutesPropagatesInspectionAndMutationFailures(t *testing.T) { + t.Parallel() + + for _, failure := range []string{"-4 -N -j rule show", "-4 rule del table 51820", "-4 -N -j route show table all", "-4 route flush table 51820", "-6 -N -j rule show", "-6 rule del table 51820", "-6 -N -j route show table all", "-6 route flush table 51820"} { + t.Run(failure, func(t *testing.T) { + injected := errors.New("injected network failure") + task := &cleanupRoutes{log: slog.New(slog.DiscardHandler), output: func(_ context.Context, args ...string) (string, error) { + if strings.Join(args, " ") == failure { + return "", injected + } + + return `[{"table":51820}]`, nil + }} + require.ErrorIs(t, task.Do(t.Context()), injected) + }) + } +} + +func TestCleanupRoutesPreservesUnrelatedTables(t *testing.T) { + t.Parallel() + + var mutations []string + + task := &cleanupRoutes{log: slog.New(slog.DiscardHandler), output: func(_ context.Context, args ...string) (string, error) { + if args[1] == "-N" { + return `[{"table":"main"},{"table":51819},{"table":51820},{"table":"51820"},{"table":51899},{"table":51900}]`, nil + } + + mutations = append(mutations, strings.Join(args, " ")) + + return "", nil + }} + require.NoError(t, task.Do(t.Context())) + require.Equal(t, []string{ + "-4 rule del table 51820", "-4 rule del table 51820", "-4 rule del table 51899", + "-4 route flush table 51820", "-4 route flush table 51899", + "-6 rule del table 51820", "-6 rule del table 51820", "-6 rule del table 51899", + "-6 route flush table 51820", "-6 route flush table 51899", + }, mutations) +} + +func TestCleanupRoutesAbsenceAndMalformedInventory(t *testing.T) { + t.Parallel() + + for _, output := range []string{"[]", "invalid", `[{"table":"unexpected-name"}]`, `[{"table":{}}]`} { + t.Run(output, func(t *testing.T) { + task := &cleanupRoutes{log: slog.New(slog.DiscardHandler), output: func(_ context.Context, args ...string) (string, error) { + require.Contains(t, args, "show", "invalid inventory must not authorize mutation") + return output, nil + }} + + err := task.Do(t.Context()) + if output == "[]" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} + +func TestFileCleanupPropagatesSubstantiveFailure(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "child"), []byte("data"), 0o600)) + + log := slog.New(slog.DiscardHandler) + require.Error(t, removeFileIfExists(log, dir)) + require.NoError(t, removeAllIfExists(log, dir)) + require.NoError(t, removeFileIfExists(log, dir)) +} diff --git a/pkg/agent/phases/rootfs/nspawn.go b/pkg/agent/phases/rootfs/nspawn.go index 0f67bd578..30a5cbe52 100644 --- a/pkg/agent/phases/rootfs/nspawn.go +++ b/pkg/agent/phases/rootfs/nspawn.go @@ -29,8 +29,9 @@ var nspawnTemplates = template.Must( ) type ensureNSpawnWorkspace struct { - log *slog.Logger - goalState *goalstates.RootFS + log *slog.Logger + goalState *goalstates.RootFS + ownedReplay bool } type NSpawnBind struct { @@ -101,6 +102,10 @@ func (e *ensureNSpawnWorkspace) Do(ctx context.Context) error { func (e *ensureNSpawnWorkspace) bootstrapWorkspace(ctx context.Context) error { bootstrapTask := oci.DownloadRootFS(e.log, e.goalState.MachineDir, e.goalState.HostArch, e.goalState.OCIImage) + if e.ownedReplay { + bootstrapTask = oci.DownloadOwnedRootFS(e.log, e.goalState.MachineDir, e.goalState.HostArch, e.goalState.OCIImage) + } + return phases.ExecuteTask(ctx, e.log, bootstrapTask) } diff --git a/pkg/agent/phases/rootfs/oci/task.go b/pkg/agent/phases/rootfs/oci/task.go index be4949301..0b68866d2 100644 --- a/pkg/agent/phases/rootfs/oci/task.go +++ b/pkg/agent/phases/rootfs/oci/task.go @@ -5,9 +5,11 @@ package oci import ( "context" + "errors" "fmt" "log/slog" "os" + "path/filepath" "github.com/Azure/unbounded/pkg/agent/artifactsource/ocilayout" "github.com/Azure/unbounded/pkg/agent/internal/utilio" @@ -15,10 +17,11 @@ import ( ) type downloadRootFS struct { - log *slog.Logger - machineDir string - ociImage string - hostArch string + log *slog.Logger + machineDir string + ociImage string + hostArch string + ownedReplay bool } // DownloadRootFS downloads an OCI image and unpacks it into the machine @@ -39,6 +42,14 @@ func DownloadRootFS( func (d *downloadRootFS) Name() string { return "oci-download-rootfs" } +// DownloadOwnedRootFS is for checkpointed initial installation only. The caller +// must hold installation ownership and prove this slot has never started a node. +// The original DownloadRootFS entry point keeps its existing nonempty-rootfs +// behavior for callers managing legacy installations. +func DownloadOwnedRootFS(log *slog.Logger, machineDir, hostArch, image string) phases.Task { + return &downloadRootFS{log: log, machineDir: machineDir, hostArch: hostArch, ociImage: image, ownedReplay: true} +} + func (d *downloadRootFS) Do(ctx context.Context) error { empty, err := utilio.IsDirEmpty(d.machineDir) if err != nil { @@ -46,8 +57,20 @@ func (d *downloadRootFS) Do(ctx context.Context) error { } if !empty { - d.log.Warn("machine directory is not empty, skipping rootfs bootstrap", slog.String("dir", d.machineDir)) - return nil + if !d.ownedReplay { + d.log.Warn("machine directory is not empty, skipping rootfs bootstrap", slog.String("dir", d.machineDir)) + return nil + } + + if _, err := os.Stat(filepath.Join(d.machineDir, ".unbounded-rootfs-complete")); err == nil { + return nil + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + + if err := utilio.CleanDir(d.machineDir); err != nil { + return err + } } d.log.Info("acquiring OCI image", @@ -68,6 +91,16 @@ func (d *downloadRootFS) Do(ctx context.Context) error { return fmt.Errorf("unpack OCI image: %w", err) } + if d.ownedReplay { + if err := utilio.SyncFilesystem(d.machineDir); err != nil { + return err + } + + if err := utilio.WriteFileDurable(filepath.Join(d.machineDir, ".unbounded-rootfs-complete"), []byte("complete\n"), 0o600); err != nil { + return err + } + } + d.log.Info("OCI image extraction complete", slog.String("dest", d.machineDir)) return nil diff --git a/pkg/agent/phases/rootfs/oci/task_test.go b/pkg/agent/phases/rootfs/oci/task_test.go new file mode 100644 index 000000000..8e785f054 --- /dev/null +++ b/pkg/agent/phases/rootfs/oci/task_test.go @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package oci + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestOwnedReplayDiscardsIncompleteTreeBeforeRetry(t *testing.T) { + t.Parallel() + dir := t.TempDir() + partial := filepath.Join(dir, "partial-layer") + require.NoError(t, os.WriteFile(partial, []byte("incomplete"), 0o600)) + task := DownloadOwnedRootFS(slog.New(slog.DiscardHandler), dir, "amd64", "oci-layout://"+filepath.Join(t.TempDir(), "missing")) + require.Error(t, task.Do(t.Context())) + + _, err := os.Stat(partial) + require.ErrorIs(t, err, os.ErrNotExist) + _, err = os.Stat(filepath.Join(dir, ".unbounded-rootfs-complete")) + require.ErrorIs(t, err, os.ErrNotExist, "failed extraction must remain replayable") +} + +func TestRootFSReplayPreservesCompletedOrLegacyTree(t *testing.T) { + t.Parallel() + + for _, owned := range []bool{false, true} { + dir := t.TempDir() + payload := filepath.Join(dir, "payload") + require.NoError(t, os.WriteFile(payload, []byte("preserve"), 0o600)) + + if owned { + require.NoError(t, os.WriteFile(filepath.Join(dir, ".unbounded-rootfs-complete"), []byte("complete\n"), 0o600)) + } + + task := DownloadRootFS(slog.New(slog.DiscardHandler), dir, "amd64", "unavailable") + if owned { + task = DownloadOwnedRootFS(slog.New(slog.DiscardHandler), dir, "amd64", "unavailable") + } + + require.NoError(t, task.Do(t.Context())) + + data, err := os.ReadFile(payload) + require.NoError(t, err) + require.Equal(t, "preserve", string(data)) + } +} diff --git a/pkg/agent/phases/rootfs/provision.go b/pkg/agent/phases/rootfs/provision.go index 7259b9670..93a02140b 100644 --- a/pkg/agent/phases/rootfs/provision.go +++ b/pkg/agent/phases/rootfs/provision.go @@ -17,8 +17,18 @@ import ( // This is the shared rootfs provisioning sequence used by both the initial // agent start and node update flows. func Provision(log *slog.Logger, gs *goalstates.RootFS) phases.Task { + return provisionWithWorkspace(log, gs, EnsureNSpawnWorkspace(log, gs)) +} + +// ProvisionOwned replays an initial bootstrap rootfs under established host +// ownership. Never use it for a slot that may have started a node. +func ProvisionOwned(log *slog.Logger, gs *goalstates.RootFS) phases.Task { + return provisionWithWorkspace(log, gs, &ensureNSpawnWorkspace{log: log, goalState: gs, ownedReplay: true}) +} + +func provisionWithWorkspace(log *slog.Logger, gs *goalstates.RootFS, workspace phases.Task) phases.Task { return phases.Serial(log, - EnsureNSpawnWorkspace(log, gs), + workspace, phases.Parallel(log, DownloadKubeBinaries(log, gs), DownloadCRIBinaries(log, gs), From ba6e67e91bbe936a850b1405982d2b8fc48e25c1 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Wed, 16 Sep 2026 13:29:32 -0400 Subject: [PATCH 02/39] agent: simplify bootstrap recovery and retry admission --- cmd/agent/internal/cmd/bootstrap.go | 44 +++++----- cmd/agent/internal/cmd/bootstrap_test.go | 31 +++++++ cmd/agent/internal/cmd/preflight.go | 2 +- .../cmd/testdata/bootstrap-v1/README.md | 6 +- .../cmd/testdata/bootstrap-v1/complete.json | 2 +- .../bootstrap-v1/preparing-rootfs.json | 2 +- .../cmd/testdata/bootstrap-v1/resetting.json | 2 +- .../assets/unbounded-agent-daemon-recovery.sh | 8 +- cmd/agent/internal/daemon/daemon.go | 45 ++++++++-- cmd/agent/internal/daemon/lifecycle.go | 50 +++-------- cmd/agent/internal/daemon/lifecycle_test.go | 21 +++++ cmd/agent/internal/daemon/migration_test.go | 23 ++++- cmd/agent/internal/daemon/nodeoperator.go | 32 ------- .../internal/daemon/recovery_script_test.go | 2 - cmd/agent/internal/daemon/reset.go | 11 ++- cmd/agent/internal/daemon/sysutil.go | 26 ++++++ docs/content/guides/agent.md | 21 +++-- pkg/agent/bootstrap/coordinator.go | 87 ++++++++++++------- pkg/agent/bootstrap/coordinator_test.go | 53 +++++++++-- pkg/agent/installstate/lock.go | 2 - pkg/agent/installstate/store.go | 5 +- pkg/agent/installstate/store_test.go | 4 +- .../nodestart/preflight_bind_address.go | 75 +++++++--------- pkg/agent/phases/reset/helpers.go | 19 ++++ pkg/agent/phases/reset/machine.go | 36 ++++---- pkg/agent/phases/reset/network.go | 53 ++++++----- pkg/agent/phases/reset/routes.go | 11 ++- pkg/agent/phases/reset/strict_test.go | 19 +++- pkg/agent/phases/reset/systemd.go | 4 + 29 files changed, 432 insertions(+), 264 deletions(-) diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index 9f0bd926d..0472866ae 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -12,7 +12,6 @@ import ( "github.com/Azure/unbounded/cmd/agent/internal/attest" "github.com/Azure/unbounded/cmd/agent/internal/daemon" - "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/bootstrap" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -20,8 +19,8 @@ import ( "github.com/Azure/unbounded/pkg/agent/phases" "github.com/Azure/unbounded/pkg/agent/phases/host" "github.com/Azure/unbounded/pkg/agent/phases/nodestart" + "github.com/Azure/unbounded/pkg/agent/phases/reset" "github.com/Azure/unbounded/pkg/agent/phases/rootfs" - "github.com/Azure/unbounded/pkg/agent/preflight" ) type agentStages struct { @@ -34,7 +33,13 @@ type agentStages struct { } func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity, error) { - data, err := json.Marshal(cfg) + // Keep identity tied to the cluster and installed rootfs, while allowing + // credentials and artifact locations to be refreshed for a retry. + data, err := json.Marshal(struct { + KubernetesVersion string + OCIImage string + APIServer string + }{strings.TrimPrefix(cfg.Cluster.Version, "v"), cfg.OCIImage, cfg.Kubelet.ApiServer}) if err != nil { return bootstrap.Identity{}, err } @@ -63,6 +68,10 @@ func (s *agentStages) ResolveInputs(ctx context.Context) error { } func (s *agentStages) PrepareHost(ctx context.Context) error { + if err := daemon.InstallBootstrapBinary(); err != nil { + return err + } + if err := phases.Serial(s.log, host.InstallPackages(s.log), phases.Parallel(s.log, host.ConfigureOS(s.log), host.ConfigureNFTables(s.log), phases.Serial(s.log, host.DisableDocker(s.log), host.ConfigureDocker(s.log)), host.DisableContainerd(s.log), host.DisableKubelet(s.log), host.DisableSwap(s.log), host.HardenAPT(s.log))).Do(ctx); err != nil { @@ -95,18 +104,17 @@ func (s *agentStages) prepareCredentials(ctx context.Context) error { return nil } -func (s *agentStages) PrepareRootFS(ctx context.Context, _ bool) error { +func (s *agentStages) PrepareRootFS(ctx context.Context) error { // A pre-node checkpoint never authorizes deleting a registered machine, // including one started independently after ownership was first recorded. - machines, err := executil.OutputCmd(ctx, s.log, "machinectl", "list", "--no-legend", "--no-pager") - if err != nil { - return fmt.Errorf("inspect machines before rootfs replay: %w", err) - } + for _, name := range []string{goalstates.NSpawnMachineKube1, goalstates.NSpawnMachineKube2} { + registered, err := reset.RegisteredMachine(ctx, s.log, name) + if err != nil { + return err + } - for _, line := range strings.Split(machines, "\n") { - fields := strings.Fields(line) - if len(fields) > 0 && (fields[0] == "kube1" || fields[0] == "kube2") { - return fmt.Errorf("refusing rootfs replay while %s is registered", fields[0]) + if registered { + return fmt.Errorf("refusing rootfs replay while %s is registered", name) } } @@ -126,14 +134,6 @@ func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { return err } - checks := []preflight.Checker{ - nodestart.CheckOwnedBindAddress(s.log, "kubelet-bind-address", "0.0.0.0:10250", "kubelet bind address", s.gs.RootFS.MachineDir, "usr/local/bin/kubelet"), - nodestart.CheckOwnedBindAddress(s.log, "containerd-metrics-bind-address", s.gs.NodeStart.Containerd.MetricsAddress, "containerd metrics bind address", s.gs.RootFS.MachineDir, "usr/local/bin/containerd"), - } - if err := preflight.Run(ctx, checks, preflight.Options{}).Err(false); err != nil { - return err - } - if err := phases.Serial(s.log, nodestart.StartNode(s.log, s.gs.NodeStart), nodestart.WaitForKubeletBootstrap(s.log, "kube1")).Do(ctx); err != nil { return err } @@ -146,10 +146,6 @@ func (s *agentStages) EnsureDaemonInstalled(ctx context.Context) error { return err } - if err := daemon.InstallBootstrapBinary(); err != nil { - return err - } - if err := phases.Serial(s.log, daemon.PersistAppliedConfig(s.log, "kube1", &s.cfg.AgentConfig), daemon.EnableDaemon(s.log)).Do(ctx); err != nil { return err } diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index 81389ad26..e535d1e64 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/preflight" ) @@ -74,6 +75,36 @@ func TestBootstrapV1CompatibilityFixtures(t *testing.T) { require.NotEqual(t, id.ConfigFingerprint, changed.ConfigFingerprint) } +func TestBootstrapFingerprintAllowsCredentialAndDownloadRefresh(t *testing.T) { + cfg, err := loadConfigFromFile(filepath.Join("testdata", "bootstrap-v1", "input.json")) + require.NoError(t, err) + original, err := bootstrapIdentity(cfg) + require.NoError(t, err) + + cfg.Kubelet.Auth.BootstrapToken = "rotated-token" + cfg.Cluster.CaCertBase64 = "rotated-ca" + cfg.Downloads = &provision.AgentDownloads{Kubernetes: &provision.AgentDownloadSource{BaseURL: "https://mirror.example.test"}} + refreshed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.Equal(t, original, refreshed) + + for _, change := range []func(){ + func() { cfg.Cluster.Version = "1.35.0" }, + func() { cfg.OCIImage = "other-image" }, + func() { cfg.Kubelet.ApiServer = "https://other-cluster" }, + } { + version, image, endpoint := cfg.Cluster.Version, cfg.OCIImage, cfg.Kubelet.ApiServer + + change() + + changed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.NotEqual(t, original.ConfigFingerprint, changed.ConfigFingerprint) + + cfg.Cluster.Version, cfg.OCIImage, cfg.Kubelet.ApiServer = version, image, endpoint + } +} + func TestCompletedPreflightOutput(t *testing.T) { t.Parallel() diff --git a/cmd/agent/internal/cmd/preflight.go b/cmd/agent/internal/cmd/preflight.go index 5f0b8ff3e..18344e7de 100644 --- a/cmd/agent/internal/cmd/preflight.go +++ b/cmd/agent/internal/cmd/preflight.go @@ -97,7 +97,7 @@ func (h *preflightHandler) execute(ctx context.Context) error { } } - if disposition == installstate.AlreadyComplete || record.Checkpoint == installstate.RepairingDaemon { + if disposition == installstate.AlreadyComplete { // Admission is non-mutating. start rechecks ownership under lock before // verifying or repairing daemon assets, without the original artifacts. return h.writeReport(preflight.Report{}) diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md b/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md index 6482c2ef1..94a149a21 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md @@ -1,6 +1,6 @@ # Bootstrap ownership v1 fixtures -Produced by the P6 candidate on base `50286b2c`, using +Produced by the simplified P6 candidate on base `50286b2c`, using `TestBootstrapV1CompatibilityFixtures`. The input uses synthetic credentials. The producer runs the actual JSON loader, normalization, fingerprint, and `installstate.Store.Save`; only the random installation ID is fixed. @@ -9,6 +9,10 @@ These files freeze the default-path contract for later releases. Consume the original input when checking compatibility. Do not regenerate these fixtures to make a changed serializer pass. Adding a new format requires new fixtures. +During pre-merge PR review the fingerprint was narrowed to Kubernetes version, +rootfs image and API server endpoint, with machine name checked separately. +These fixtures supersede the unreleased full-config fingerprint from `bb191a93`. + Initial production command: ```sh diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json index bd6f717a1..93abe4ca7 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json @@ -3,6 +3,6 @@ "installID": "00112233445566778899aabbccddeeff", "machineName": "bootstrap-fixture", "hostPrefix": "/usr/local", - "configFingerprint": "d6d6b8ead0b3ef26872432d1d3f82eacd16a3d2165bd81fcc3b7c48cace90d37", + "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", "checkpoint": "complete" } diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json index 029067c44..0893eedea 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json @@ -3,6 +3,6 @@ "installID": "00112233445566778899aabbccddeeff", "machineName": "bootstrap-fixture", "hostPrefix": "/usr/local", - "configFingerprint": "d6d6b8ead0b3ef26872432d1d3f82eacd16a3d2165bd81fcc3b7c48cace90d37", + "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", "checkpoint": "preparing-rootfs" } diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json index b2f479aa2..36abab04d 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json @@ -3,6 +3,6 @@ "installID": "00112233445566778899aabbccddeeff", "machineName": "bootstrap-fixture", "hostPrefix": "/usr/local", - "configFingerprint": "d6d6b8ead0b3ef26872432d1d3f82eacd16a3d2165bd81fcc3b7c48cace90d37", + "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", "checkpoint": "resetting" } diff --git a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh index 537c1091f..288dec7de 100644 --- a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh +++ b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon-recovery.sh @@ -4,12 +4,8 @@ set -euo pipefail -# Follow the Go lifecycle lock order before reading or switching binary links. -# The restarted Type=simple daemon can wait for this lock during migration. -exec 8>"{{ .InstallationLockPath }}" -flock 8 -exec 9>"{{ .ActivationLockPath }}" -flock 9 +# Last-resort recovery does not wait on lifecycle locks: the failing activation +# may still hold them. Reset stops this unit before removing daemon artifacts. current="{{ .DaemonBinaryCurrentPath }}" last_good="$(readlink -f {{ .DaemonBinaryLastGoodPath }} || true)" diff --git a/cmd/agent/internal/daemon/daemon.go b/cmd/agent/internal/daemon/daemon.go index 06a582e75..3d095a3b8 100644 --- a/cmd/agent/internal/daemon/daemon.go +++ b/cmd/agent/internal/daemon/daemon.go @@ -6,6 +6,7 @@ package daemon import ( "context" "encoding/base64" + "errors" "fmt" "log/slog" "path/filepath" @@ -25,6 +26,7 @@ import ( "github.com/Azure/unbounded/pkg/agent/config" "github.com/Azure/unbounded/pkg/agent/daemoncred" "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/installstate" ) const ( @@ -39,6 +41,7 @@ type kubeClientFunc func(cfg *rest.Config, opts client.Options) (client.WithWatc // runOptions configures daemon runtime behavior. type runOptions struct { + installation *installstate.Store // DaemonCredentialDir stores the daemon-controller client certificate and key. // When empty, the default path under the agent config directory is used. DaemonCredentialDir string @@ -84,8 +87,9 @@ func run(ctx context.Context, log *slog.Logger, opts runOptions) error { return err } - // Find the active machine and its applied config. - active, err := runOpts.NodeOperator.FindActiveMachine(log) + // Discovery and migration share ownership. Read once after the launcher + // releases its lock rather than mutating a previously discovered snapshot. + active, err := discoverAndMigrate(ctx, log, installationStore(runOpts.installation), runOpts.NodeOperator) if err != nil { return fmt.Errorf("find active machine: %w", err) } @@ -96,10 +100,6 @@ func run(ctx context.Context, log *slog.Logger, opts runOptions) error { "applied_version", active.Config.Cluster.Version, ) - if err := runOpts.NodeOperator.EnsureLifecycleMigration(ctx, log, active); err != nil { - return fmt.Errorf("ensure nspawn lifecycle migration: %w", err) - } - controllerCfg, stopControllerCreds, err := daemonControllerCredentials(ctx, log, active.Config, runOpts) if err != nil { return fmt.Errorf("build daemon controller credentials: %w", err) @@ -129,6 +129,39 @@ func run(ctx context.Context, log *slog.Logger, opts runOptions) error { return runController(ctx, log, controllerCfg, active.Config.MachineName, active.Config.NodeName, runOpts.NodeOperator) } +func discoverAndMigrate(ctx context.Context, log *slog.Logger, store *installstate.Store, operator nodeOperator) (*ActiveMachine, error) { + waitCtx, cancel := context.WithTimeout(ctx, hostDaemonHealthTimeout) + defer cancel() + + for { + lock, err := store.AcquireMutationLock() + if err == nil { + defer releaseInstallationLock(log, lock) + + active, err := operator.FindActiveMachine(log) + if err != nil { + return nil, err + } + + if err := operator.EnsureLifecycleMigration(ctx, log, active); err != nil { + return nil, err + } + + return active, nil + } + + if !errors.Is(err, installstate.ErrLockHeld) { + return nil, err + } + + select { + case <-waitCtx.Done(): + return nil, fmt.Errorf("wait for installation ownership at daemon startup: %w", waitCtx.Err()) + case <-time.After(250 * time.Millisecond): + } + } +} + func daemonControllerCredentials( ctx context.Context, log *slog.Logger, diff --git a/cmd/agent/internal/daemon/lifecycle.go b/cmd/agent/internal/daemon/lifecycle.go index 1f1f0d30c..9b1a414bd 100644 --- a/cmd/agent/internal/daemon/lifecycle.go +++ b/cmd/agent/internal/daemon/lifecycle.go @@ -19,8 +19,8 @@ import ( "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/bootstrap" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases" + "github.com/Azure/unbounded/pkg/agent/phases/reset" ) // --------------------------------------------------------------------------- @@ -125,12 +125,7 @@ func InstallBootstrapBinary() error { return err } - data, err := os.ReadFile(source) - if err != nil { - return err - } - - return writeFile(goalstates.DaemonBinaryPath, data, 0o755) + return installBinary(source, goalstates.DaemonBinaryPath) } func renderDaemonAsset(name string, content []byte) ([]byte, error) { @@ -150,8 +145,6 @@ func renderDaemonAssetForPaths(name string, content []byte, paths goalstates.Age DaemonBinaryLastGoodPath string DaemonRecoveryScriptPath string DaemonAgentUpgradeSignalPath string - InstallationLockPath string - ActivationLockPath string }{ DaemonUnit: goalstates.DaemonUnit, DaemonRecoveryUnit: goalstates.DaemonRecoveryUnit, @@ -159,8 +152,6 @@ func renderDaemonAssetForPaths(name string, content []byte, paths goalstates.Age DaemonBinaryLastGoodPath: paths.LastGoodPath, DaemonRecoveryScriptPath: goalstates.DaemonRecoveryScriptPath, DaemonAgentUpgradeSignalPath: paths.SignalPath, - InstallationLockPath: installstate.DefaultLockPath, - ActivationLockPath: goalstates.DaemonAgentUpgradeLockPath, } tmpl, err := template.New(name).Parse(string(content)) @@ -185,8 +176,8 @@ type stopDaemon struct { } // StopDaemon returns a task that stops, disables, and removes the -// unbounded-agent-daemon systemd unit. Only verified absence permits a failed -// stop; substantive service errors must retain reset ownership. +// unbounded-agent-daemon systemd unit. Offline hosts and absent units permit +// cleanup; substantive service errors on a running systemd remain failures. func StopDaemon(log *slog.Logger) phases.Task { return &stopDaemon{log: log} } @@ -194,7 +185,7 @@ func StopDaemon(log *slog.Logger) phases.Task { func (t *stopDaemon) Name() string { return "stop-daemon" } func (t *stopDaemon) Do(ctx context.Context) error { - if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "stop", goalstates.DaemonUnit); err != nil { + if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "stop", goalstates.DaemonUnit); err != nil && !reset.SystemdUnavailable() { state, inspectErr := executil.OutputCmd(ctx, t.log, "systemctl", "show", goalstates.DaemonUnit, "--property=LoadState", "--value") if inspectErr != nil || strings.TrimSpace(state) != "not-found" { return fmt.Errorf("stop daemon: %w", err) @@ -225,7 +216,7 @@ func (t *removeDaemonUnit) Do(ctx context.Context) error { } func disableAndRemoveDaemonUnit(ctx context.Context, log *slog.Logger) error { - if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", goalstates.DaemonUnit); err != nil { + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", goalstates.DaemonUnit); err != nil && !reset.SystemdUnavailable() { if _, statErr := os.Lstat(filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonUnit)); !errors.Is(statErr, os.ErrNotExist) { return err } @@ -323,34 +314,13 @@ func VerifyDaemonInstalled(ctx context.Context, log *slog.Logger) error { return err } - service := NewHostDaemonActivationService(log, paths) - - assets, err := service.desiredAssets(paths.CurrentPath) - if err != nil { - return err - } - - for path, asset := range assets { - data, err := os.ReadFile(path) - if err != nil { - return err - } - - if !bytes.Equal(data, asset.content) { - return fmt.Errorf("daemon asset differs: %s", path) - } - - info, err := os.Stat(path) - if err != nil { + for _, name := range []string{goalstates.DaemonUnit, goalstates.DaemonRecoveryUnit} { + if _, err := os.Stat(filepath.Join(goalstates.SystemdSystemDir, name)); err != nil { return err } - - if info.Mode().Perm() != asset.mode { - return fmt.Errorf("daemon asset mode differs: %s", path) - } } - for _, path := range []string{paths.CurrentPath, paths.LastGoodPath, paths.BinaryPath} { + for _, path := range []string{paths.CurrentPath, paths.LastGoodPath, paths.BinaryPath, goalstates.DaemonRecoveryScriptPath} { info, err := os.Stat(path) if err != nil { return err @@ -378,7 +348,7 @@ func VerifyDaemonInstalled(ctx context.Context, log *slog.Logger) error { } } - return service.WaitHealthy(ctx, paths.CurrentPath) + return nil } // RepairDaemon requires the caller's installation lock. It uses current applied diff --git a/cmd/agent/internal/daemon/lifecycle_test.go b/cmd/agent/internal/daemon/lifecycle_test.go index db8d98079..4783f9403 100644 --- a/cmd/agent/internal/daemon/lifecycle_test.go +++ b/cmd/agent/internal/daemon/lifecycle_test.go @@ -4,6 +4,8 @@ package daemon import ( + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -34,3 +36,22 @@ func TestRenderDaemonAsset(t *testing.T) { assert.Contains(t, renderedRecovery, goalstates.DaemonAgentUpgradeSignalPath) assert.Contains(t, renderedRecovery, "record-agent-upgrade-failure-signal") } + +func TestInstallBinaryStreamsAndReplacesAtomically(t *testing.T) { + t.Parallel() + dir := t.TempDir() + source, target := filepath.Join(dir, "source"), filepath.Join(dir, "bin", "target") + require.NoError(t, os.WriteFile(source, []byte("candidate"), 0o600)) + require.NoError(t, installBinary(source, target)) + data, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "candidate", string(data)) + + info, err := os.Stat(target) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + require.Error(t, installBinary(filepath.Join(dir, "missing"), target)) + data, err = os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "candidate", string(data)) +} diff --git a/cmd/agent/internal/daemon/migration_test.go b/cmd/agent/internal/daemon/migration_test.go index 366b00e05..42f215204 100644 --- a/cmd/agent/internal/daemon/migration_test.go +++ b/cmd/agent/internal/daemon/migration_test.go @@ -6,11 +6,14 @@ package daemon import ( "context" "errors" + "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/installstate" ) func TestDaemonStartupRunsLifecycleMigrationBeforeControllerSetup(t *testing.T) { @@ -20,11 +23,27 @@ func TestDaemonStartupRunsLifecycleMigrationBeforeControllerSetup(t *testing.T) Name: "kube1", Config: &provision.AgentConfig{MachineName: "machine-1", NodeName: "node-1"}, }} - err := run(context.Background(), discardLogger(), runOptions{NodeOperator: op}) + err := run(context.Background(), discardLogger(), runOptions{NodeOperator: op, installation: installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock"))}) require.ErrorContains(t, err, "build daemon controller credentials") require.Equal(t, 1, op.lifecycleCalls) } +func TestStartupLockWaitHonorsDeadlineWithoutMigration(t *testing.T) { + t.Parallel() + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + lock, err := store.AcquireLock() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lock.Release()) }) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond) + defer cancel() + + op := &fakeNodeOperator{} + _, err = discoverAndMigrate(ctx, discardLogger(), store, op) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.Zero(t, op.lifecycleCalls) +} + func TestDaemonStartupFailsLifecycleMigrationWithoutRetry(t *testing.T) { t.Parallel() @@ -36,7 +55,7 @@ func TestDaemonStartupFailsLifecycleMigrationWithoutRetry(t *testing.T) { }, lifecycleErrs: []error{resolveErr}, } - err := run(context.Background(), discardLogger(), runOptions{NodeOperator: op}) + err := run(context.Background(), discardLogger(), runOptions{NodeOperator: op, installation: installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock"))}) require.ErrorIs(t, err, resolveErr) require.Equal(t, 1, op.lifecycleCalls) } diff --git a/cmd/agent/internal/daemon/nodeoperator.go b/cmd/agent/internal/daemon/nodeoperator.go index c713e71f3..61c7234b2 100644 --- a/cmd/agent/internal/daemon/nodeoperator.go +++ b/cmd/agent/internal/daemon/nodeoperator.go @@ -12,7 +12,6 @@ import ( "os" "reflect" "strings" - "time" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/internal/provision" @@ -189,37 +188,6 @@ func gantryDisabled(cfg *provision.AgentConfig) bool { } func (nspawnNodeOperator) EnsureLifecycleMigration(ctx context.Context, log *slog.Logger, active *ActiveMachine) error { - // Type=simple lets the launcher verify the running daemon while this startup - // migration waits for bootstrap or host activation to release ownership. - var lock *installstate.Lock - for { - var err error - - lock, err = installstate.DefaultStore().AcquireMutationLock() - if err == nil { - break - } - - if !errors.Is(err, installstate.ErrLockHeld) { - return err - } - - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(250 * time.Millisecond): - } - } - - defer releaseInstallationLock(log, lock) - - current, err := (nspawnNodeOperator{}).FindActiveMachine(log) - if err != nil { - return err - } - - *active = *current - rootFS, err := goalstates.ResolveNSpawnConfig(active.Config, active.Name) if err != nil { return fmt.Errorf("resolve existing machine lifecycle: %w", err) diff --git a/cmd/agent/internal/daemon/recovery_script_test.go b/cmd/agent/internal/daemon/recovery_script_test.go index 9bce45ea7..80ec39019 100644 --- a/cmd/agent/internal/daemon/recovery_script_test.go +++ b/cmd/agent/internal/daemon/recovery_script_test.go @@ -73,8 +73,6 @@ func TestRecoveryScript(t *testing.T) { "DaemonBinaryLastGoodPath": lastGood, "DaemonAgentUpgradeSignalPath": filepath.Join(dir, "signal"), "DaemonUnit": "test-agent.service", - "InstallationLockPath": filepath.Join(dir, "install.lock"), - "ActivationLockPath": filepath.Join(dir, "activation.lock"), })) stub := `#!/bin/bash diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index 121d80015..7646868dd 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -15,6 +15,7 @@ import ( "golang.org/x/sys/unix" "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/pkg/agent/bootstrap" "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases" @@ -83,6 +84,10 @@ func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.S } func stopRecoveryUnit(ctx context.Context, log *slog.Logger) error { + if reset.SystemdUnavailable() { + return nil + } + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "stop", goalstates.DaemonRecoveryUnit); err != nil { out, inspectErr := executil.OutputCmd(ctx, log, "systemctl", "show", goalstates.DaemonRecoveryUnit, "--property=LoadState", "--value") if inspectErr != nil || strings.TrimSpace(out) != "not-found" { @@ -129,10 +134,8 @@ func durableReset(ctx context.Context, store *installstate.Store, inner phases.T return err } - for _, f := range handles { - if err := syncfs(int(f.Fd())); err != nil { - return fmt.Errorf("sync teardown %s: %w", f.Name(), err) - } + if err := bootstrap.SyncOpenFilesystems(handles, syncfs); err != nil { + return err } return store.Remove() diff --git a/cmd/agent/internal/daemon/sysutil.go b/cmd/agent/internal/daemon/sysutil.go index 385fd4014..ba5d954b1 100644 --- a/cmd/agent/internal/daemon/sysutil.go +++ b/cmd/agent/internal/daemon/sysutil.go @@ -4,6 +4,8 @@ package daemon import ( + "errors" + "io" "os" "path/filepath" @@ -20,3 +22,27 @@ func writeFile(filename string, content []byte, perm os.FileMode) error { return renameio.WriteFile(filename, content, perm, renameio.WithTempDir(filepath.Dir(filename))) } + +func installBinary(source, target string) (err error) { + f, err := os.Open(source) + if err != nil { + return err + } + defer func() { err = errors.Join(err, f.Close()) }() + + if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { + return err + } + + pending, err := renameio.NewPendingFile(target, renameio.WithPermissions(0o755), renameio.WithTempDir(filepath.Dir(target))) + if err != nil { + return err + } + defer pending.Cleanup() //nolint:errcheck // Pending file cleanup after atomic replacement. + + if _, err := io.Copy(pending, f); err != nil { + return err + } + + return pending.CloseAtomicallyReplace() +} diff --git a/docs/content/guides/agent.md b/docs/content/guides/agent.md index bf2f12970..b90cab082 100644 --- a/docs/content/guides/agent.md +++ b/docs/content/guides/agent.md @@ -39,12 +39,15 @@ preserved during node-start replay. The ownership record is `/var/lib/unbounded/agent/install-state.json`. The completion marker is `/var/lib/unbounded/agent/bootstrap-complete`. These are internal files, not configuration inputs. Keep them intact when retrying. A -different machine identity or configuration is rejected and requires an explicit -reset before a new initial installation. Regenerating a bootstrap script may -change its token and therefore its configuration identity. - -After completion, the same `start` invocation verifies daemon assets and local -process health, restores a missing completion marker, or repairs the daemon. +different machine name, Kubernetes version, rootfs image, or API server endpoint +is rejected and requires an explicit reset before a new initial installation. +Credentials and artifact locations can be refreshed for a retry. Other fields +do not participate in admission; a retry does not reapply stages already completed. + +After completion, the same `start` invocation checks required daemon files, +executable permissions, and enabled/active service state, restores a missing +completion marker, or repairs the daemon. It does not compare unit contents or +overwrite working local unit customizations. This path uses the current applied configuration, including after an ordinary repave has switched from kube1 to kube2. It does not resolve the original node image or binary-download sources. The bootstrap shell still needs its agent @@ -57,11 +60,13 @@ sudo env UNBOUNDED_AGENT_CONFIG_FILE=/path/to/original-agent-config.json \ ``` Bootstrap, reset, node lifecycle operations, and binary activation share an -installation lock. A busy lock is retryable. MachineOperation handlers requeue +installation lock. Last-resort daemon rollback does not wait on these locks; +reset stops the recovery unit before teardown. A busy lock is retryable. MachineOperation handlers requeue instead of starting concurrent host mutations. Reset retains a `resetting` record when cleanup fails; correct the reported error and run reset again. Ownership is removed only after teardown and its filesystem synchronization -barriers succeed. The lock file in `/run` remains and must not be deleted to +barriers succeed. Missing cleanup tools on a partially prepared host are skipped; +failures from installed tools remain errors. The lock file in `/run` remains and must not be deleted to force an operation through. Recovery applies to checkpointed initial installations. Existing installations diff --git a/pkg/agent/bootstrap/coordinator.go b/pkg/agent/bootstrap/coordinator.go index 2d1d02d05..1c1e8be6f 100644 --- a/pkg/agent/bootstrap/coordinator.go +++ b/pkg/agent/bootstrap/coordinator.go @@ -8,9 +8,11 @@ import ( "context" "fmt" "log/slog" + "os" + + "golang.org/x/sys/unix" "github.com/Azure/unbounded/pkg/agent/installstate" - "github.com/Azure/unbounded/pkg/agent/internal/utilio" ) type Identity struct{ MachineName, ConfigFingerprint string } @@ -19,7 +21,7 @@ type Stages interface { EnsureHostClean(context.Context) error ResolveInputs(context.Context) error PrepareHost(context.Context) error - PrepareRootFS(context.Context, bool) error + PrepareRootFS(context.Context) error EnsureNodeStarted(context.Context) error EnsureDaemonInstalled(context.Context) error RepairDaemon(context.Context) error @@ -42,7 +44,7 @@ func New(log *slog.Logger, store *installstate.Store, stages Stages, reporter Re return &Coordinator{log: log, store: store, stages: stages, reporter: reporter} } -type Outcome struct{ AlreadyComplete, Resumed bool } +type Outcome struct{ AlreadyComplete bool } func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { lock, err := c.store.AcquireLock() @@ -62,7 +64,6 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { return Outcome{}, err } - resumed := disposition != installstate.Fresh if disposition == installstate.Fresh { if err := c.stages.EnsureHostClean(ctx); err != nil { return Outcome{}, err @@ -83,24 +84,25 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { } if disposition == installstate.AlreadyComplete { - if err := c.stages.VerifyInstalled(ctx); err == nil { - if err := c.store.MarkComplete(r); err != nil { + if err := c.stages.VerifyInstalled(ctx); err != nil { + if err := c.stages.RepairDaemon(ctx); err != nil { return Outcome{}, err } - return Outcome{AlreadyComplete: true}, nil + if err := c.stages.VerifyInstalled(ctx); err != nil { + return Outcome{}, err + } } - r.Checkpoint = installstate.RepairingDaemon - if err := c.store.Save(r); err != nil { + if err := c.store.MarkComplete(r); err != nil { return Outcome{}, err } + + return Outcome{AlreadyComplete: true}, nil } - if r.Checkpoint != installstate.RepairingDaemon { - if err := c.stages.ResolveInputs(ctx); err != nil { - return Outcome{}, fmt.Errorf("resolve bootstrap inputs: %w", err) - } + if err := c.stages.ResolveInputs(ctx); err != nil { + return Outcome{}, fmt.Errorf("resolve bootstrap inputs: %w", err) } for r.Checkpoint != installstate.Complete { @@ -113,7 +115,7 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { c.reporter.StageStarted(ctx, current) } - next, err := c.runStage(ctx, current, resumed) + next, err := c.runStage(ctx, current) if err != nil { if c.reporter != nil { c.reporter.StageFailed(ctx, current, err) @@ -132,38 +134,63 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { } } - return Outcome{Resumed: resumed}, nil + return Outcome{}, nil } -func (c *Coordinator) runStage(ctx context.Context, stage installstate.Checkpoint, resumed bool) (installstate.Checkpoint, error) { +func (c *Coordinator) runStage(ctx context.Context, stage installstate.Checkpoint) (installstate.Checkpoint, error) { switch stage { case installstate.PreparingHost: return installstate.PreparingRootFS, c.stages.PrepareHost(ctx) case installstate.PreparingRootFS: - return installstate.StartingNode, c.stages.PrepareRootFS(ctx, resumed) + return installstate.StartingNode, c.stages.PrepareRootFS(ctx) case installstate.StartingNode: return installstate.InstallingDaemon, c.stages.EnsureNodeStarted(ctx) case installstate.InstallingDaemon: - if err := c.stages.EnsureDaemonInstalled(ctx); err != nil { - return "", err - } - - return installstate.Complete, c.stages.VerifyInstalled(ctx) - case installstate.RepairingDaemon: - if err := c.stages.RepairDaemon(ctx); err != nil { - return "", err - } - - return installstate.Complete, c.stages.VerifyInstalled(ctx) + return installstate.Complete, c.stages.EnsureDaemonInstalled(ctx) default: return "", fmt.Errorf("unsupported checkpoint %s", stage) } } func SyncFilesystems(paths ...string) error { + var files []*os.File + defer func() { + for _, f := range files { + _ = f.Close() //nolint:errcheck // Read-only handle; sync errors are returned. + } + }() //nolint:errcheck // Read-only handles; sync errors are returned. + for _, path := range paths { - if err := utilio.SyncFilesystem(path); err != nil { - return fmt.Errorf("sync %s: %w", path, err) + f, err := os.Open(path) + if err != nil { + return err + } + + files = append(files, f) + } + + return SyncOpenFilesystems(files, unix.Syncfs) +} + +// SyncOpenFilesystems synchronizes each filesystem once per barrier, using +// open handles that remain valid after teardown removes their paths. +func SyncOpenFilesystems(files []*os.File, syncfs func(int) error) error { + seen := map[uint64]bool{} + + for _, f := range files { + var stat unix.Stat_t + if err := unix.Fstat(int(f.Fd()), &stat); err != nil { + return err + } + + if seen[uint64(stat.Dev)] { + continue + } + + seen[uint64(stat.Dev)] = true + + if err := syncfs(int(f.Fd())); err != nil { + return fmt.Errorf("sync %s: %w", f.Name(), err) } } diff --git a/pkg/agent/bootstrap/coordinator_test.go b/pkg/agent/bootstrap/coordinator_test.go index fe2e520b1..54932e413 100644 --- a/pkg/agent/bootstrap/coordinator_test.go +++ b/pkg/agent/bootstrap/coordinator_test.go @@ -42,7 +42,7 @@ func (f *fakeStages) run(name string) error { func (f *fakeStages) EnsureHostClean(context.Context) error { return f.run("clean") } func (f *fakeStages) ResolveInputs(context.Context) error { return f.run("resolve") } func (f *fakeStages) PrepareHost(context.Context) error { return f.run("host") } -func (f *fakeStages) PrepareRootFS(context.Context, bool) error { return f.run("rootfs") } +func (f *fakeStages) PrepareRootFS(context.Context) error { return f.run("rootfs") } func (f *fakeStages) EnsureNodeStarted(context.Context) error { return f.run("node") } func (f *fakeStages) EnsureDaemonInstalled(context.Context) error { return f.run("daemon") } func (f *fakeStages) RepairDaemon(context.Context) error { f.verifyErr = nil; return f.run("repair") } @@ -63,10 +63,10 @@ func TestInterruptedStagesResumeWithoutReplayingEarlierStages(t *testing.T) { checkpoint installstate.Checkpoint want []string }{ - {"host", installstate.PreparingHost, []string{"resolve", "host", "rootfs", "node", "daemon", "verify"}}, - {"rootfs", installstate.PreparingRootFS, []string{"resolve", "rootfs", "node", "daemon", "verify"}}, - {"node", installstate.StartingNode, []string{"resolve", "node", "daemon", "verify"}}, - {"daemon", installstate.InstallingDaemon, []string{"resolve", "daemon", "verify"}}, + {"host", installstate.PreparingHost, []string{"resolve", "host", "rootfs", "node", "daemon"}}, + {"rootfs", installstate.PreparingRootFS, []string{"resolve", "rootfs", "node", "daemon"}}, + {"node", installstate.StartingNode, []string{"resolve", "node", "daemon"}}, + {"daemon", installstate.InstallingDaemon, []string{"resolve", "daemon"}}, } { t.Run(tc.fail, func(t *testing.T) { dir := t.TempDir() @@ -87,7 +87,7 @@ func TestInterruptedStagesResumeWithoutReplayingEarlierStages(t *testing.T) { stages.fail = "" outcome, err := c.Run(t.Context(), id) require.NoError(t, err) - require.True(t, outcome.Resumed) + require.False(t, outcome.AlreadyComplete) require.Equal(t, tc.want, stages.calls) complete, err := store.Load() @@ -118,7 +118,7 @@ func TestCompletedRecoveryDoesNotResolveRetiredBootstrapInputs(t *testing.T) { c := New(slog.New(slog.DiscardHandler), store, stages, nil) outcome, err := c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) require.NoError(t, err) - require.Equal(t, !repair, outcome.AlreadyComplete) + require.True(t, outcome.AlreadyComplete) want := []string{"verify"} if repair { @@ -171,3 +171,42 @@ func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { }) } } + +func TestSyncBarrierDeduplicatesFilesystem(t *testing.T) { + dir := t.TempDir() + a, err := os.Open(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, a.Close()) }) + + b, err := os.Open(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, b.Close()) }) + + calls := 0 + + require.NoError(t, SyncOpenFilesystems([]*os.File{a, b}, func(int) error { calls++; return nil })) + require.Equal(t, 1, calls) + require.ErrorIs(t, SyncOpenFilesystems([]*os.File{a, b}, func(int) error { return errInjected }), errInjected) +} + +func TestInterruptedRepairRemainsCompleteAndRetries(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + stages := &fakeStages{store: store, fail: "repair", verifyErr: errInjected} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + id := Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint} + _, err = c.Run(t.Context(), id) + require.ErrorIs(t, err, errInjected) + loaded, err := store.Load() + require.NoError(t, err) + require.Equal(t, installstate.Complete, loaded.Checkpoint) + + stages.fail = "" + stages.verifyErr = errInjected + stages.calls = nil + _, err = c.Run(t.Context(), id) + require.NoError(t, err) + require.Equal(t, []string{"verify", "repair", "verify"}, stages.calls) +} diff --git a/pkg/agent/installstate/lock.go b/pkg/agent/installstate/lock.go index 9be55b8e3..36f7bab27 100644 --- a/pkg/agent/installstate/lock.go +++ b/pkg/agent/installstate/lock.go @@ -15,8 +15,6 @@ var ErrLockHeld = errors.New("another host lifecycle operation holds the install type Lock struct{ file *os.File } -func AcquireLock() (*Lock, error) { return AcquireLockAt(DefaultLockPath) } - // AcquireLockAt is nonblocking. The kernel releases flock on process exit; a // leftover lock file does not imply a held lock and must not be deleted by reset. func AcquireLockAt(path string) (*Lock, error) { diff --git a/pkg/agent/installstate/store.go b/pkg/agent/installstate/store.go index 58b576905..aa466d29e 100644 --- a/pkg/agent/installstate/store.go +++ b/pkg/agent/installstate/store.go @@ -32,13 +32,12 @@ const ( PreparingRootFS Checkpoint = "preparing-rootfs" StartingNode Checkpoint = "starting-node" InstallingDaemon Checkpoint = "installing-daemon" - RepairingDaemon Checkpoint = "repairing-daemon" Complete Checkpoint = "complete" Resetting Checkpoint = "resetting" ) func (c Checkpoint) NodeMayBeRunning() bool { - return c == StartingNode || c == InstallingDaemon || c == RepairingDaemon || c == Complete + return c == StartingNode || c == InstallingDaemon || c == Complete } type Record struct { @@ -63,7 +62,7 @@ func (r Record) Validate() error { } switch r.Checkpoint { - case PreparingHost, PreparingRootFS, StartingNode, InstallingDaemon, RepairingDaemon, Complete, Resetting: + case PreparingHost, PreparingRootFS, StartingNode, InstallingDaemon, Complete, Resetting: return nil default: return fmt.Errorf("unknown installation checkpoint %q", r.Checkpoint) diff --git a/pkg/agent/installstate/store_test.go b/pkg/agent/installstate/store_test.go index 48fff1c96..6a74a670c 100644 --- a/pkg/agent/installstate/store_test.go +++ b/pkg/agent/installstate/store_test.go @@ -52,7 +52,7 @@ func TestOwnershipAdmission(t *testing.T) { r, err := NewRecord("machine", "fingerprint") require.NoError(t, err) - for _, checkpoint := range []Checkpoint{PreparingHost, PreparingRootFS, StartingNode, InstallingDaemon, RepairingDaemon, Complete, Resetting} { + for _, checkpoint := range []Checkpoint{PreparingHost, PreparingRootFS, StartingNode, InstallingDaemon, Complete, Resetting} { t.Run(string(checkpoint), func(t *testing.T) { r := r r.Checkpoint = checkpoint @@ -133,7 +133,7 @@ func TestInstallationLockSurvivesStateRemoval(t *testing.T) { func TestMutationAdmission(t *testing.T) { t.Parallel() - for _, checkpoint := range []Checkpoint{"", PreparingHost, StartingNode, Complete, RepairingDaemon, Resetting} { + for _, checkpoint := range []Checkpoint{"", PreparingHost, StartingNode, Complete, Resetting} { t.Run(string(checkpoint), func(t *testing.T) { s := testStore(t) diff --git a/pkg/agent/phases/nodestart/preflight_bind_address.go b/pkg/agent/phases/nodestart/preflight_bind_address.go index 785d7cfa2..397baed94 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address.go @@ -85,45 +85,18 @@ func listenerOwnedByRoot(procRoot, address, root, executable string) bool { return false } - processes, err := os.ReadDir(procRoot) - if err != nil { - return false - } - matched := map[string]bool{} - for _, process := range processes { - if _, err := strconv.Atoi(process.Name()); err != nil { - continue - } + for _, owner := range socketOwners(procRoot, wanted) { + base := filepath.Join(procRoot, strconv.Itoa(owner.pid)) + processRoot, rootErr := os.Stat(filepath.Join(base, "root")) - base := filepath.Join(procRoot, process.Name()) - - fds, err := os.ReadDir(filepath.Join(base, "fd")) - if err != nil { - continue + processExe, exeErr := os.Stat(filepath.Join(base, "exe")) + if rootErr != nil || exeErr != nil || !os.SameFile(rootInfo, processRoot) || !os.SameFile(exeInfo, processExe) { + return false } - for _, fd := range fds { - target, err := os.Readlink(filepath.Join(base, "fd", fd.Name())) - if err != nil { - continue - } - - inode := strings.TrimSuffix(strings.TrimPrefix(target, "socket:["), "]") - if _, found := wanted[inode]; !found { - continue - } - - processRoot, rootErr := os.Stat(filepath.Join(base, "root")) - - processExe, exeErr := os.Stat(filepath.Join(base, "exe")) - if rootErr != nil || exeErr != nil || !os.SameFile(rootInfo, processRoot) || !os.SameFile(exeInfo, processExe) { - return false - } - - matched[inode] = true - } + matched[owner.inode] = true } return len(wanted) > 0 && len(matched) == len(wanted) @@ -227,11 +200,34 @@ func listenerSocketInodes(socketTable []byte, port uint16) map[string]struct{} { } func findSocketOwner(procRoot string, inodes map[string]struct{}) string { + owners := socketOwners(procRoot, inodes) + if len(owners) > 0 { + owner := owners[0] + + name, err := os.ReadFile(filepath.Join(procRoot, strconv.Itoa(owner.pid), "comm")) + if err == nil && strings.TrimSpace(string(name)) != "" { + return strconv.Quote(strings.TrimSpace(string(name))) + " (PID " + strconv.Itoa(owner.pid) + ")" + } + + return "PID " + strconv.Itoa(owner.pid) + } + + return "" +} + +type socketOwner struct { + pid int + inode string +} + +func socketOwners(procRoot string, inodes map[string]struct{}) []socketOwner { processes, err := os.ReadDir(procRoot) if err != nil { - return "" + return nil } + var owners []socketOwner + for _, process := range processes { pid, err := strconv.Atoi(process.Name()) if err != nil || !process.IsDir() { @@ -254,14 +250,9 @@ func findSocketOwner(procRoot string, inodes map[string]struct{}) string { continue } - name, err := os.ReadFile(filepath.Join(procRoot, process.Name(), "comm")) - if err == nil && strings.TrimSpace(string(name)) != "" { - return strconv.Quote(strings.TrimSpace(string(name))) + " (PID " + strconv.Itoa(pid) + ")" - } - - return "PID " + strconv.Itoa(pid) + owners = append(owners, socketOwner{pid: pid, inode: inode}) } } - return "" + return owners } diff --git a/pkg/agent/phases/reset/helpers.go b/pkg/agent/phases/reset/helpers.go index 548c402ad..bbc1e7698 100644 --- a/pkg/agent/phases/reset/helpers.go +++ b/pkg/agent/phases/reset/helpers.go @@ -8,8 +8,27 @@ import ( "fmt" "log/slog" "os" + "os/exec" ) +// ToolMissing distinguishes a package not installed yet from an installed tool +// failing to inspect or remove resources. Permission failures are not absence. +func ToolMissing(name string) bool { + _, err := exec.LookPath(name) + return errors.Is(err, exec.ErrNotFound) +} + +// SystemdUnavailable permits offline cleanup when no host systemd is running. +func SystemdUnavailable() bool { + if ToolMissing("systemctl") { + return true + } + + _, err := os.Stat("/run/systemd/system") + + return errors.Is(err, os.ErrNotExist) +} + // removeFileIfExists ignores absence but propagates substantive removal errors. func removeFileIfExists(log *slog.Logger, path string) error { if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { diff --git a/pkg/agent/phases/reset/machine.go b/pkg/agent/phases/reset/machine.go index 4de156ce5..201fb5d6b 100644 --- a/pkg/agent/phases/reset/machine.go +++ b/pkg/agent/phases/reset/machine.go @@ -30,20 +30,15 @@ func StopMachine(log *slog.Logger, machineName string) phases.Task { func (t *stopMachine) Name() string { return "stop-machine" } func (t *stopMachine) Do(ctx context.Context) error { - if err := executil.RunCmd(ctx, t.log, executil.Machinectl(), "disable", t.machineName); err != nil { - out, inspectErr := executil.OutputCmd(ctx, t.log, "systemctl", "show", "systemd-nspawn@"+t.machineName+".service", "--property=UnitFileState", "--value") - if inspectErr != nil { - return fmt.Errorf("inspect nspawn enablement: %w", inspectErr) - } + if ToolMissing("machinectl") { + return nil + } - switch strings.TrimSpace(out) { - case "disabled", "static", "masked": - default: - return fmt.Errorf("disable nspawn machine %s: %w", t.machineName, err) - } + if err := executil.RunCmd(ctx, t.log, executil.Machinectl(), "disable", t.machineName); err != nil { + t.log.Warn("failed to disable machine; continuing with stop and removal", "machine", t.machineName, "error", err) } - exists, err := machineExists(ctx, t.log, t.machineName) + exists, err := RegisteredMachine(ctx, t.log, t.machineName) if err != nil { return err } @@ -74,7 +69,7 @@ func (t *stopMachine) Do(ctx context.Context) error { } // Force terminate if still registered. - if exists, err := machineExists(ctx, t.log, t.machineName); err != nil { + if exists, err := RegisteredMachine(ctx, t.log, t.machineName); err != nil { return err } else if exists { t.log.Warn("machine did not stop gracefully, terminating", "machine", t.machineName) @@ -99,7 +94,7 @@ func (t *stopMachine) Do(ctx context.Context) error { func (t *stopMachine) waitForGone(ctx context.Context, timeout time.Duration) (bool, error) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { - if exists, err := machineExists(ctx, t.log, t.machineName); err != nil { + if exists, err := RegisteredMachine(ctx, t.log, t.machineName); err != nil { return false, err } else if !exists { return true, nil @@ -112,7 +107,7 @@ func (t *stopMachine) waitForGone(ctx context.Context, timeout time.Duration) (b } } - exists, err := machineExists(ctx, t.log, t.machineName) + exists, err := RegisteredMachine(ctx, t.log, t.machineName) return !exists, err } @@ -131,6 +126,10 @@ func RemoveMachine(log *slog.Logger, machineName string) phases.Task { func (t *removeMachine) Name() string { return "remove-machine" } func (t *removeMachine) Do(ctx context.Context) error { + if ToolMissing("machinectl") { + return nil + } + machineDir := fmt.Sprintf("/var/lib/machines/%s", t.machineName) // Skip entirely if the machine directory doesn't exist - nothing to remove. @@ -159,7 +158,7 @@ func (t *removeMachine) Do(ctx context.Context) error { return nil // machinectl removed both image metadata and directory } - if exists, err := machineExists(ctx, t.log, t.machineName); err != nil { + if exists, err := RegisteredMachine(ctx, t.log, t.machineName); err != nil { return err } else if !exists { // Once machined no longer knows the machine, the nspawn service is stopped @@ -181,7 +180,7 @@ func (t *removeMachine) Do(ctx context.Context) error { // Fallback: force-remove the directory if machinectl keeps failing. t.log.Warn("machinectl remove did not succeed, force-removing directory", "dir", machineDir) - if exists, err := machineExists(ctx, t.log, t.machineName); err != nil { + if exists, err := RegisteredMachine(ctx, t.log, t.machineName); err != nil { return err } else if exists { return fmt.Errorf("refusing to remove registered machine %s", t.machineName) @@ -190,8 +189,9 @@ func (t *removeMachine) Do(ctx context.Context) error { return removeAllIfExists(t.log, machineDir) } -// machineExists checks whether the named nspawn machine is known to machinectl. -func machineExists(ctx context.Context, log *slog.Logger, name string) (bool, error) { +// RegisteredMachine reports registration only after successful inventory. +// Bootstrap callers must not treat a missing inspection tool as a clean host. +func RegisteredMachine(ctx context.Context, log *slog.Logger, name string) (bool, error) { out, err := executil.OutputCmd(ctx, log, "machinectl", "list", "--no-legend", "--no-pager") if err != nil { return false, fmt.Errorf("inspect registered machines: %w", err) diff --git a/pkg/agent/phases/reset/network.go b/pkg/agent/phases/reset/network.go index f4ca0eaec..eb58785fd 100644 --- a/pkg/agent/phases/reset/network.go +++ b/pkg/agent/phases/reset/network.go @@ -65,42 +65,46 @@ func CleanupLocalDNSRules(log *slog.Logger) phases.Task { func (t *cleanupLocalDNSRules) Name() string { return "cleanup-localdns-rules" } func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { - if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "disable", "--now", goalstates.LocalDNSNetworkUnit); err != nil { + if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "disable", "--now", goalstates.LocalDNSNetworkUnit); err != nil && !SystemdUnavailable() { path := filepath.Join(goalstates.SystemdSystemDir, goalstates.LocalDNSNetworkUnit) if _, statErr := os.Lstat(path); !errors.Is(statErr, os.ErrNotExist) { return fmt.Errorf("disable LocalDNS unit: %w", err) } } - tables, err := executil.OutputCmd(ctx, t.log, "nft", "list", "tables") - if err != nil { - return fmt.Errorf("inspect LocalDNS tables: %w", err) - } - - if strings.Contains(tables, "table ip "+goalstates.LocalDNSNFTTable+"\n") { - if err := executil.RunCmd(ctx, t.log, func(ctx context.Context) *exec.Cmd { - return exec.CommandContext(ctx, "nft") - }, "delete", "table", "ip", goalstates.LocalDNSNFTTable); err != nil { - return fmt.Errorf("remove LocalDNS nftables table: %w", err) + if !ToolMissing("nft") { + tables, err := executil.OutputCmd(ctx, t.log, "nft", "list", "tables") + if err != nil { + return fmt.Errorf("inspect LocalDNS tables: %w", err) } - } - links, err := executil.OutputCmd(ctx, t.log, "ip", "-d", "-o", "link", "show") - if err != nil { - return fmt.Errorf("inspect LocalDNS interface: %w", err) + if strings.Contains(tables, "table ip "+goalstates.LocalDNSNFTTable+"\n") { + if err := executil.RunCmd(ctx, t.log, func(ctx context.Context) *exec.Cmd { + return exec.CommandContext(ctx, "nft") + }, "delete", "table", "ip", goalstates.LocalDNSNFTTable); err != nil { + return fmt.Errorf("remove LocalDNS nftables table: %w", err) + } + } } - for _, output := range strings.Split(links, "\n") { - if !strings.Contains(output, ": "+goalstates.LocalDNSInterfaceName+":") { - continue + if !ToolMissing("ip") { + links, err := executil.OutputCmd(ctx, t.log, "ip", "-d", "-o", "link", "show") + if err != nil { + return fmt.Errorf("inspect LocalDNS interface: %w", err) } - if !strings.Contains(" "+output+" ", " dummy ") { - return fmt.Errorf("refusing to remove non-dummy interface %s", goalstates.LocalDNSInterfaceName) - } + for _, output := range strings.Split(links, "\n") { + if !strings.Contains(output, ": "+goalstates.LocalDNSInterfaceName+":") { + continue + } - if err := executil.RunCmd(ctx, t.log, executil.Ip(), "link", "delete", goalstates.LocalDNSInterfaceName); err != nil { - return fmt.Errorf("remove LocalDNS interface: %w", err) + if !strings.Contains(" "+output+" ", " dummy ") { + return fmt.Errorf("refusing to remove non-dummy interface %s", goalstates.LocalDNSInterfaceName) + } + + if err := executil.RunCmd(ctx, t.log, executil.Ip(), "link", "delete", goalstates.LocalDNSInterfaceName); err != nil { + return fmt.Errorf("remove LocalDNS interface: %w", err) + } } } @@ -117,6 +121,9 @@ func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { } func (t *removeNetworkInterfaces) Do(ctx context.Context) error { + if ToolMissing("ip") { + return nil + } // Remove WireGuard interfaces (wg51820, wg51821, ...). wgIfaces, err := listWireGuardInterfaces(ctx, t.log) if err != nil { diff --git a/pkg/agent/phases/reset/routes.go b/pkg/agent/phases/reset/routes.go index b877d386a..329e7053f 100644 --- a/pkg/agent/phases/reset/routes.go +++ b/pkg/agent/phases/reset/routes.go @@ -39,6 +39,10 @@ func (t *cleanupRoutes) Do(ctx context.Context) error { output := t.output if output == nil { + if ToolMissing("ip") { + return nil + } + output = func(ctx context.Context, args ...string) (string, error) { return executil.OutputCmd(ctx, t.log, "ip", args...) } @@ -113,12 +117,7 @@ func ownedRoutingTables(output string) ([]int, error) { table, parseErr = strconv.Atoi(name) if parseErr != nil { - switch name { - case "main", "local", "default", "unspec": - continue - } - - return nil, fmt.Errorf("non-numeric routing table %q", name) + continue } } diff --git a/pkg/agent/phases/reset/strict_test.go b/pkg/agent/phases/reset/strict_test.go index dc6f93279..466967f6d 100644 --- a/pkg/agent/phases/reset/strict_test.go +++ b/pkg/agent/phases/reset/strict_test.go @@ -29,7 +29,7 @@ func TestMachineInspectionFailsClosed(t *testing.T) { dir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(dir, "machinectl"), []byte("#!/bin/sh\n"+tc.script+"\n"), 0o755)) t.Setenv("PATH", dir) - exists, err := machineExists(t.Context(), slog.New(slog.DiscardHandler), "kube1") + exists, err := RegisteredMachine(t.Context(), slog.New(slog.DiscardHandler), "kube1") require.Equal(t, tc.exists, exists) if tc.failed { @@ -93,7 +93,7 @@ func TestCleanupRoutesAbsenceAndMalformedInventory(t *testing.T) { }} err := task.Do(t.Context()) - if output == "[]" { + if output == "[]" || output == `[{"table":"unexpected-name"}]` { require.NoError(t, err) } else { require.Error(t, err) @@ -112,3 +112,18 @@ func TestFileCleanupPropagatesSubstantiveFailure(t *testing.T) { require.NoError(t, removeAllIfExists(log, dir)) require.NoError(t, removeFileIfExists(log, dir)) } + +func TestCleanupToleratesMissingTools(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + + log := slog.New(slog.DiscardHandler) + + require.True(t, ToolMissing("machinectl")) + require.NoError(t, StopMachine(log, "kube1").Do(t.Context())) + require.NoError(t, RemoveMachine(log, "kube1").Do(t.Context())) + require.NoError(t, RemoveNetworkInterfaces(log).Do(t.Context())) + require.NoError(t, CleanupRoutes(log).Do(t.Context())) + require.NoError(t, ReloadSystemd(log).Do(t.Context())) + _, err := RegisteredMachine(t.Context(), log, "kube1") + require.Error(t, err, "bootstrap inventory must still reject unavailable inspection") +} diff --git a/pkg/agent/phases/reset/systemd.go b/pkg/agent/phases/reset/systemd.go index 0e17c6dc0..bf1178b32 100644 --- a/pkg/agent/phases/reset/systemd.go +++ b/pkg/agent/phases/reset/systemd.go @@ -24,6 +24,10 @@ func ReloadSystemd(log *slog.Logger) phases.Task { func (t *reloadSystemd) Name() string { return "reload-systemd" } func (t *reloadSystemd) Do(ctx context.Context) error { + if SystemdUnavailable() { + return nil + } + t.log.Info("reloading systemd daemon") return executil.RunCmd(ctx, t.log, executil.Systemctl(), "daemon-reload") From 87d452ce71b3678f54f70bdcb9546dfa525c459e Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:58:33 -0400 Subject: [PATCH 03/39] agent: remove unreachable recovery defenses and duplicated helpers --- cmd/agent/internal/cmd/agentupgrade.go | 8 +- cmd/agent/internal/cmd/bootstrap.go | 9 +- cmd/agent/internal/cmd/preflight.go | 32 +--- cmd/agent/internal/daemon/controller.go | 3 + .../daemon/controller_machineoperation.go | 6 +- cmd/agent/internal/daemon/controller_node.go | 2 +- cmd/agent/internal/daemon/daemon.go | 14 +- cmd/agent/internal/daemon/installation.go | 24 --- cmd/agent/internal/daemon/lifecycle.go | 22 ++- cmd/agent/internal/daemon/lifecycle_test.go | 5 +- cmd/agent/internal/daemon/reset.go | 29 ++-- cmd/agent/internal/daemon/sysutil.go | 26 ---- docs/content/guides/agent.md | 11 +- hack/agent/e2e-kind/e2e.py | 3 +- internal/fsutil/fsutil.go | 145 ++++++++++++++++++ internal/fsutil/fsutil_test.go | 83 ++++++++++ pkg/agent/bootstrap/coordinator.go | 56 +------ pkg/agent/bootstrap/coordinator_test.go | 31 +--- pkg/agent/installstate/store.go | 64 +++----- pkg/agent/installstate/store_test.go | 14 -- pkg/agent/internal/utilio/sync.go | 63 -------- .../host/preflight_existing_deployment.go | 12 +- .../phases/nodestart/preflight_api_server.go | 15 +- .../nodestart/preflight_bind_address.go | 5 + .../nodestart/preflight_bind_address_test.go | 24 ++- pkg/agent/phases/reset/helpers.go | 19 --- pkg/agent/phases/reset/machine.go | 8 - pkg/agent/phases/reset/network.go | 53 +++---- pkg/agent/phases/reset/routes.go | 4 - pkg/agent/phases/reset/strict_test.go | 15 -- pkg/agent/phases/reset/systemd.go | 4 - pkg/agent/phases/rootfs/oci/task.go | 5 +- 32 files changed, 393 insertions(+), 421 deletions(-) delete mode 100644 cmd/agent/internal/daemon/installation.go create mode 100644 internal/fsutil/fsutil.go create mode 100644 internal/fsutil/fsutil_test.go delete mode 100644 pkg/agent/internal/utilio/sync.go diff --git a/cmd/agent/internal/cmd/agentupgrade.go b/cmd/agent/internal/cmd/agentupgrade.go index 8abc4d64d..147d0cc79 100644 --- a/cmd/agent/internal/cmd/agentupgrade.go +++ b/cmd/agent/internal/cmd/agentupgrade.go @@ -47,6 +47,7 @@ func newCmdHostAgentUpgrade(cmdCtx *CommandContext) *cobra.Command { executable: os.Executable, resolvedPath: goalstates.ResolvedAgentUpgradePaths, geteuid: os.Geteuid, + installation: installstate.DefaultStore(), } handler.newService = func(paths goalstates.AgentUpgradePaths) agentbinary.DaemonService { return daemon.NewHostDaemonActivationService(handler.cmdCtx.Logger, paths) @@ -117,12 +118,7 @@ func (h *hostAgentUpgradeHandler) execute(ctx context.Context) error { return fmt.Errorf("host agent upgrade requires root privileges") } - store := h.installation - if store == nil { - store = installstate.DefaultStore() - } - - lock, err := store.AcquireMutationLock() + lock, err := h.installation.AcquireMutationLock() if err != nil { return err } diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index 0472866ae..b8298ea63 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -12,6 +12,7 @@ import ( "github.com/Azure/unbounded/cmd/agent/internal/attest" "github.com/Azure/unbounded/cmd/agent/internal/daemon" + "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/bootstrap" "github.com/Azure/unbounded/pkg/agent/goalstates" @@ -78,7 +79,7 @@ func (s *agentStages) PrepareHost(ctx context.Context) error { return err } - return bootstrap.SyncFilesystems("/etc", "/usr/local", installstate.DefaultDirectory) + return fsutil.SyncFilesystems("/etc", "/usr/local", installstate.DefaultDirectory) } // Credentials must be resolved on every unfinished attempt, but TPM prerequisites @@ -126,7 +127,7 @@ func (s *agentStages) PrepareRootFS(ctx context.Context) error { return err } - return bootstrap.SyncFilesystems(s.gs.RootFS.MachineDir, "/usr/local", goalstates.SystemdSystemDir, goalstates.SystemdNSpawnDir) + return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, "/usr/local", goalstates.SystemdSystemDir, goalstates.SystemdNSpawnDir) } func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { @@ -138,7 +139,7 @@ func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { return err } - return bootstrap.SyncFilesystems(s.gs.RootFS.MachineDir, goalstates.SystemdSystemDir) + return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, goalstates.SystemdSystemDir) } func (s *agentStages) EnsureDaemonInstalled(ctx context.Context) error { @@ -150,7 +151,7 @@ func (s *agentStages) EnsureDaemonInstalled(ctx context.Context) error { return err } - return bootstrap.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) + return fsutil.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) } func (s *agentStages) VerifyInstalled(ctx context.Context) error { diff --git a/cmd/agent/internal/cmd/preflight.go b/cmd/agent/internal/cmd/preflight.go index 18344e7de..c0c8b060e 100644 --- a/cmd/agent/internal/cmd/preflight.go +++ b/cmd/agent/internal/cmd/preflight.go @@ -83,20 +83,11 @@ func (h *preflightHandler) execute(ctx context.Context) error { return err } - store := installstate.DefaultStore() - record, loadErr := store.Load() - - disposition, err := installstate.Decide(record, loadErr, id.MachineName, id.ConfigFingerprint) + _, disposition, err := installstate.Admit(installstate.DefaultStore(), id.MachineName, id.ConfigFingerprint) if err != nil { return err } - if disposition != installstate.Fresh { - if _, err := store.CheckMarker(record); err != nil { - return err - } - } - if disposition == installstate.AlreadyComplete { // Admission is non-mutating. start rechecks ownership under lock before // verifying or repairing daemon assets, without the original artifacts. @@ -123,23 +114,16 @@ func (h *preflightHandler) execute(ctx context.Context) error { nodestart.Preflight(logger, cfg.AgentConfig, goalState), rootfs.Preflight(logger, cfg.AgentConfig, goalState), ) + if disposition == installstate.Resume { - filtered := checks[:0] - for _, check := range checks { - if check.Name() == "existing-deployment" { - continue - } + // A resumed installation owns the artifacts a clean host must not have. + // Bind addresses remain checked, and accept only owned listeners. + var filtered []preflight.Checker - if record.Checkpoint.NodeMayBeRunning() { - switch check.Name() { - case "kubelet-bind-address": - check = nodestart.CheckOwnedBindAddress(logger, check.Name(), "0.0.0.0:10250", "kubelet bind address", goalState.RootFS.MachineDir, "usr/local/bin/kubelet") - case "containerd-metrics-bind-address": - check = nodestart.CheckOwnedBindAddress(logger, check.Name(), goalState.NodeStart.Containerd.MetricsAddress, "containerd metrics bind address", goalState.RootFS.MachineDir, "usr/local/bin/containerd") - } + for _, check := range checks { + if check.Name() != host.CheckExistingDeploymentName { + filtered = append(filtered, check) } - - filtered = append(filtered, check) } checks = filtered diff --git a/cmd/agent/internal/daemon/controller.go b/cmd/agent/internal/daemon/controller.go index 9c8dec8d3..0a3239e1f 100644 --- a/cmd/agent/internal/daemon/controller.go +++ b/cmd/agent/internal/daemon/controller.go @@ -43,6 +43,7 @@ func runController( machineName string, nodeName string, nodeOperator nodeOperator, + installation *installstate.Store, ) error { mgr, err := ctrl.NewManager(restCfg, manager.Options{ Scheme: newScheme(), @@ -71,6 +72,7 @@ func runController( c := mgr.GetClient() machineOperations := &machineOperationTarget{ + installation: installation, Client: c, log: log, machineName: machineName, @@ -94,6 +96,7 @@ func runController( } repaveReconciler := &repaveReconciler{ + installation: installation, Client: c, log: log, machineName: machineName, diff --git a/cmd/agent/internal/daemon/controller_machineoperation.go b/cmd/agent/internal/daemon/controller_machineoperation.go index e0345f2e7..ef7153089 100644 --- a/cmd/agent/internal/daemon/controller_machineoperation.go +++ b/cmd/agent/internal/daemon/controller_machineoperation.go @@ -32,7 +32,7 @@ type machineOperationTarget struct { } func (t *machineOperationTarget) reconcileNodeReboot(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) { - lock, err := installationStore(t.installation).AcquireMutationLock() + lock, err := t.installation.AcquireMutationLock() if errors.Is(err, installstate.ErrLockHeld) { return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil } @@ -70,7 +70,7 @@ func (t *machineOperationTarget) reconcileNodeReboot(ctx context.Context, store } func (t *machineOperationTarget) reconcileAgentUpgrade(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) { - installationLock, err := installationStore(t.installation).AcquireMutationLock() + installationLock, err := t.installation.AcquireMutationLock() if errors.Is(err, installstate.ErrLockHeld) { return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil } @@ -143,7 +143,7 @@ func (t *machineOperationTarget) reconcileAgentUpgrade(ctx context.Context, stor } func (t *machineOperationTarget) reconcileAgentReset(ctx context.Context, store daemon.MachineOperationStore[int64], op daemon.MachineOperation) (ctrl.Result, error) { - lock, err := installationStore(t.installation).AcquireLock() + lock, err := t.installation.AcquireLock() if errors.Is(err, installstate.ErrLockHeld) { return ctrl.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil } diff --git a/cmd/agent/internal/daemon/controller_node.go b/cmd/agent/internal/daemon/controller_node.go index 717767726..37102e999 100644 --- a/cmd/agent/internal/daemon/controller_node.go +++ b/cmd/agent/internal/daemon/controller_node.go @@ -24,7 +24,7 @@ import ( ) func (r *repaveReconciler) ReconcileRepave(ctx context.Context, _ string) (reconcile.Result, error) { - lock, err := installationStore(r.installation).AcquireMutationLock() + lock, err := r.installation.AcquireMutationLock() if errors.Is(err, installstate.ErrLockHeld) { return reconcile.Result{RequeueAfter: agentUpgradeLockRetryDelay}, nil } diff --git a/cmd/agent/internal/daemon/daemon.go b/cmd/agent/internal/daemon/daemon.go index 3d095a3b8..c6cb01e2b 100644 --- a/cmd/agent/internal/daemon/daemon.go +++ b/cmd/agent/internal/daemon/daemon.go @@ -33,6 +33,10 @@ const ( daemonControllerCertificateName = "unbounded-agent-daemon-controller" daemonControllerGroup = "unbounded-agent-daemons" daemonControllerCertWaitTimeout = 2 * time.Minute + + // installationLockWaitTimeout bounds how long daemon startup waits for the + // launching bootstrap or activation to release installation ownership. + installationLockWaitTimeout = 30 * time.Second ) // kubeClientFunc constructs a controller-runtime client from a rest.Config. @@ -66,6 +70,10 @@ func (o *runOptions) validate() error { o.NodeOperator = nspawnNodeOperator{} } + if o.installation == nil { + o.installation = installstate.DefaultStore() + } + if o.DaemonCredentialDir == "" { o.DaemonCredentialDir = filepath.Join(goalstates.AgentConfigDir, "daemon-controller") } @@ -89,7 +97,7 @@ func run(ctx context.Context, log *slog.Logger, opts runOptions) error { // Discovery and migration share ownership. Read once after the launcher // releases its lock rather than mutating a previously discovered snapshot. - active, err := discoverAndMigrate(ctx, log, installationStore(runOpts.installation), runOpts.NodeOperator) + active, err := discoverAndMigrate(ctx, log, runOpts.installation, runOpts.NodeOperator) if err != nil { return fmt.Errorf("find active machine: %w", err) } @@ -126,11 +134,11 @@ func run(ctx context.Context, log *slog.Logger, opts runOptions) error { log.Warn("failed to publish and clear AgentUpgrade daemon signals", "error", err) } - return runController(ctx, log, controllerCfg, active.Config.MachineName, active.Config.NodeName, runOpts.NodeOperator) + return runController(ctx, log, controllerCfg, active.Config.MachineName, active.Config.NodeName, runOpts.NodeOperator, runOpts.installation) } func discoverAndMigrate(ctx context.Context, log *slog.Logger, store *installstate.Store, operator nodeOperator) (*ActiveMachine, error) { - waitCtx, cancel := context.WithTimeout(ctx, hostDaemonHealthTimeout) + waitCtx, cancel := context.WithTimeout(ctx, installationLockWaitTimeout) defer cancel() for { diff --git a/cmd/agent/internal/daemon/installation.go b/cmd/agent/internal/daemon/installation.go deleted file mode 100644 index 9dc568734..000000000 --- a/cmd/agent/internal/daemon/installation.go +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package daemon - -import ( - "log/slog" - - "github.com/Azure/unbounded/pkg/agent/installstate" -) - -func installationStore(store *installstate.Store) *installstate.Store { - if store != nil { - return store - } - - return installstate.DefaultStore() -} - -func releaseInstallationLock(log *slog.Logger, lock *installstate.Lock) { - if err := lock.Release(); err != nil { - log.Error("release installation lock", "error", err) - } -} diff --git a/cmd/agent/internal/daemon/lifecycle.go b/cmd/agent/internal/daemon/lifecycle.go index 9b1a414bd..9a95bf86f 100644 --- a/cmd/agent/internal/daemon/lifecycle.go +++ b/cmd/agent/internal/daemon/lifecycle.go @@ -16,11 +16,10 @@ import ( "text/template" "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/agentbinary" - "github.com/Azure/unbounded/pkg/agent/bootstrap" "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/phases" - "github.com/Azure/unbounded/pkg/agent/phases/reset" ) // --------------------------------------------------------------------------- @@ -125,7 +124,7 @@ func InstallBootstrapBinary() error { return err } - return installBinary(source, goalstates.DaemonBinaryPath) + return fsutil.InstallFile(source, goalstates.DaemonBinaryPath, 0o755) } func renderDaemonAsset(name string, content []byte) ([]byte, error) { @@ -176,8 +175,8 @@ type stopDaemon struct { } // StopDaemon returns a task that stops, disables, and removes the -// unbounded-agent-daemon systemd unit. Offline hosts and absent units permit -// cleanup; substantive service errors on a running systemd remain failures. +// unbounded-agent-daemon systemd unit. Only an absent unit permits a failed +// stop; substantive service errors must retain reset ownership. func StopDaemon(log *slog.Logger) phases.Task { return &stopDaemon{log: log} } @@ -185,7 +184,7 @@ func StopDaemon(log *slog.Logger) phases.Task { func (t *stopDaemon) Name() string { return "stop-daemon" } func (t *stopDaemon) Do(ctx context.Context) error { - if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "stop", goalstates.DaemonUnit); err != nil && !reset.SystemdUnavailable() { + if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "stop", goalstates.DaemonUnit); err != nil { state, inspectErr := executil.OutputCmd(ctx, t.log, "systemctl", "show", goalstates.DaemonUnit, "--property=LoadState", "--value") if inspectErr != nil || strings.TrimSpace(state) != "not-found" { return fmt.Errorf("stop daemon: %w", err) @@ -216,7 +215,7 @@ func (t *removeDaemonUnit) Do(ctx context.Context) error { } func disableAndRemoveDaemonUnit(ctx context.Context, log *slog.Logger) error { - if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", goalstates.DaemonUnit); err != nil && !reset.SystemdUnavailable() { + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", goalstates.DaemonUnit); err != nil { if _, statErr := os.Lstat(filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonUnit)); !errors.Is(statErr, os.ErrNotExist) { return err } @@ -304,11 +303,10 @@ func removeOwnedFile(path string) error { return nil } +// VerifyDaemonInstalled checks installed daemon assets and service state. An +// active daemon already proves it resolved an applied config at startup, so the +// applied-config check belongs to RepairDaemon rather than here. func VerifyDaemonInstalled(ctx context.Context, log *slog.Logger) error { - if _, err := (nspawnNodeOperator{}).FindActiveMachine(log); err != nil { - return err - } - paths, err := goalstates.ResolvedAgentUpgradePaths() if err != nil { return err @@ -366,5 +364,5 @@ func RepairDaemon(ctx context.Context, log *slog.Logger) error { return err } - return bootstrap.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) + return fsutil.SyncFilesystems("/usr/local", goalstates.AgentConfigDir, goalstates.SystemdSystemDir) } diff --git a/cmd/agent/internal/daemon/lifecycle_test.go b/cmd/agent/internal/daemon/lifecycle_test.go index 4783f9403..703fa1801 100644 --- a/cmd/agent/internal/daemon/lifecycle_test.go +++ b/cmd/agent/internal/daemon/lifecycle_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -42,7 +43,7 @@ func TestInstallBinaryStreamsAndReplacesAtomically(t *testing.T) { dir := t.TempDir() source, target := filepath.Join(dir, "source"), filepath.Join(dir, "bin", "target") require.NoError(t, os.WriteFile(source, []byte("candidate"), 0o600)) - require.NoError(t, installBinary(source, target)) + require.NoError(t, fsutil.InstallFile(source, target, 0o755)) data, err := os.ReadFile(target) require.NoError(t, err) require.Equal(t, "candidate", string(data)) @@ -50,7 +51,7 @@ func TestInstallBinaryStreamsAndReplacesAtomically(t *testing.T) { info, err := os.Stat(target) require.NoError(t, err) require.Equal(t, os.FileMode(0o755), info.Mode().Perm()) - require.Error(t, installBinary(filepath.Join(dir, "missing"), target)) + require.Error(t, fsutil.InstallFile(filepath.Join(dir, "missing"), target, 0o755)) data, err = os.ReadFile(target) require.NoError(t, err) require.Equal(t, "candidate", string(data)) diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index 7646868dd..3a0a2dfa3 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -15,7 +15,7 @@ import ( "golang.org/x/sys/unix" "github.com/Azure/unbounded/internal/executil" - "github.com/Azure/unbounded/pkg/agent/bootstrap" + "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases" @@ -25,11 +25,13 @@ import ( // ResetAgentResources returns a task that removes the unbounded-agent and all // associated resources without stopping the daemon process. func ResetAgentResources(log *slog.Logger) phases.Task { - return resetWithOwnership(log, installstate.DefaultStore(), false) + return ownedReset(log, installstate.DefaultStore(), resetResources(log)) } +// ResetAgent additionally stops the daemon first. The daemon's own operation +// path stops it last, so that ordering stays with the caller. func ResetAgent(log *slog.Logger) phases.Task { - return resetWithOwnership(log, installstate.DefaultStore(), true) + return ownedReset(log, installstate.DefaultStore(), phases.Serial(log, StopDaemon(log), resetResources(log))) } type lifecycleTask struct { @@ -40,12 +42,9 @@ type lifecycleTask struct { func (t lifecycleTask) Name() string { return t.name } func (t lifecycleTask) Do(ctx context.Context) error { return t.run(ctx) } -func resetWithOwnership(log *slog.Logger, store *installstate.Store, stop bool) phases.Task { - inner := resetResources(log) - if stop { - inner = phases.Serial(log, StopDaemon(log), inner) - } - +func ownedReset(log *slog.Logger, store *installstate.Store, inner phases.Task) phases.Task { + // The composed name keeps the underlying cleanup sequence visible to callers + // and to the reset ordering test. return lifecycleTask{name: "owned-reset(" + inner.Name() + ")", run: func(ctx context.Context) error { lock, err := store.AcquireLock() if err != nil { @@ -84,10 +83,6 @@ func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.S } func stopRecoveryUnit(ctx context.Context, log *slog.Logger) error { - if reset.SystemdUnavailable() { - return nil - } - if err := executil.RunCmd(ctx, log, executil.Systemctl(), "stop", goalstates.DaemonRecoveryUnit); err != nil { out, inspectErr := executil.OutputCmd(ctx, log, "systemctl", "show", goalstates.DaemonRecoveryUnit, "--property=LoadState", "--value") if inspectErr != nil || strings.TrimSpace(out) != "not-found" { @@ -134,7 +129,7 @@ func durableReset(ctx context.Context, store *installstate.Store, inner phases.T return err } - if err := bootstrap.SyncOpenFilesystems(handles, syncfs); err != nil { + if err := fsutil.SyncOpenFilesystems(handles, syncfs); err != nil { return err } @@ -166,3 +161,9 @@ func resetResources(log *slog.Logger) phases.Task { reset.ReloadSystemd(log), ) } + +func releaseInstallationLock(log *slog.Logger, lock *installstate.Lock) { + if err := lock.Release(); err != nil { + log.Error("release installation lock", "error", err) + } +} diff --git a/cmd/agent/internal/daemon/sysutil.go b/cmd/agent/internal/daemon/sysutil.go index ba5d954b1..385fd4014 100644 --- a/cmd/agent/internal/daemon/sysutil.go +++ b/cmd/agent/internal/daemon/sysutil.go @@ -4,8 +4,6 @@ package daemon import ( - "errors" - "io" "os" "path/filepath" @@ -22,27 +20,3 @@ func writeFile(filename string, content []byte, perm os.FileMode) error { return renameio.WriteFile(filename, content, perm, renameio.WithTempDir(filepath.Dir(filename))) } - -func installBinary(source, target string) (err error) { - f, err := os.Open(source) - if err != nil { - return err - } - defer func() { err = errors.Join(err, f.Close()) }() - - if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { - return err - } - - pending, err := renameio.NewPendingFile(target, renameio.WithPermissions(0o755), renameio.WithTempDir(filepath.Dir(target))) - if err != nil { - return err - } - defer pending.Cleanup() //nolint:errcheck // Pending file cleanup after atomic replacement. - - if _, err := io.Copy(pending, f); err != nil { - return err - } - - return pending.CloseAtomicallyReplace() -} diff --git a/docs/content/guides/agent.md b/docs/content/guides/agent.md index b90cab082..d061be1f4 100644 --- a/docs/content/guides/agent.md +++ b/docs/content/guides/agent.md @@ -36,18 +36,17 @@ or invoke `unbounded-agent start` with the same original configuration. Complete stages are skipped; unfinished stages are replayed. A running nspawn machine is preserved during node-start replay. -The ownership record is `/var/lib/unbounded/agent/install-state.json`. The -completion marker is `/var/lib/unbounded/agent/bootstrap-complete`. These are -internal files, not configuration inputs. Keep them intact when retrying. A +The ownership record is `/var/lib/unbounded/agent/install-state.json`. It is an +internal file, not a configuration input. Keep it intact when retrying. A different machine name, Kubernetes version, rootfs image, or API server endpoint is rejected and requires an explicit reset before a new initial installation. Credentials and artifact locations can be refreshed for a retry. Other fields do not participate in admission; a retry does not reapply stages already completed. After completion, the same `start` invocation checks required daemon files, -executable permissions, and enabled/active service state, restores a missing -completion marker, or repairs the daemon. It does not compare unit contents or -overwrite working local unit customizations. +executable permissions, and enabled/active service state, and repairs the daemon when +they are missing or stopped. It does not compare unit contents or overwrite +working local unit customizations. This path uses the current applied configuration, including after an ordinary repave has switched from kube1 to kube2. It does not resolve the original node image or binary-download sources. The bootstrap shell still needs its agent diff --git a/hack/agent/e2e-kind/e2e.py b/hack/agent/e2e-kind/e2e.py index 1b5e5c1a0..61f20b46d 100755 --- a/hack/agent/e2e-kind/e2e.py +++ b/hack/agent/e2e-kind/e2e.py @@ -4816,7 +4816,6 @@ def validate_bootstrap_repair() -> None: PY systemctl stop unbounded-agent-daemon.service rm /etc/systemd/system/unbounded-agent-daemon.service - rm /var/lib/unbounded/agent/bootstrap-complete systemctl daemon-reload export UNBOUNDED_AGENT_CONFIG_FILE=/tmp/p6-original-config.json /tmp/p6-repair-agent preflight --output json @@ -4824,7 +4823,7 @@ def validate_bootstrap_repair() -> None: test "$before" = "$(sha256sum /etc/unbounded/agent/kube2-applied-config.json)" test ! -e /etc/unbounded/agent/kube1-applied-config.json test "$node_pid" = "$(systemctl show systemd-nspawn@kube2.service --property=MainPID --value)" - test -s /var/lib/unbounded/agent/bootstrap-complete + grep -q '"checkpoint": "complete"' /var/lib/unbounded/agent/install-state.json systemctl is-active unbounded-agent-daemon.service """) bounded_ssh("sudo bash -c " + shlex.quote(script), time.monotonic() + 180, check=True) diff --git a/internal/fsutil/fsutil.go b/internal/fsutil/fsutil.go new file mode 100644 index 000000000..89bde9cb7 --- /dev/null +++ b/internal/fsutil/fsutil.go @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Package fsutil provides durable filesystem helpers shared by the agent +// library and the agent commands. +package fsutil + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/google/renameio/v2" + "golang.org/x/sys/unix" +) + +// SyncDir persists a directory entry so newly written names survive a crash. +func SyncDir(path string) error { + f, err := os.Open(path) + if err != nil { + return err + } + + return errors.Join(f.Sync(), f.Close()) +} + +// WriteFileDurable also persists newly created parent directories, so the +// written file cannot outlive the directory entries needed to reach it. +func WriteFileDurable(path string, data []byte, mode os.FileMode) error { + var parents []string + + for dir := filepath.Dir(path); ; dir = filepath.Dir(dir) { + _, err := os.Stat(dir) + if err == nil { + parents = append(parents, dir) + break + } + + if !errors.Is(err, os.ErrNotExist) { + return err + } + + parents = append(parents, dir) + } + + if err := WriteFile(path, data, mode); err != nil { + return err + } + + for _, dir := range parents { + if err := SyncDir(dir); err != nil { + return err + } + } + + return nil +} + +// WriteFile writes content atomically, creating parent directories as needed. +// The temporary file shares the destination directory so it inherits the +// correct SELinux label instead of the temp-directory label. +func WriteFile(path string, data []byte, mode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return err + } + + return renameio.WriteFile(path, data, mode, renameio.WithTempDir(filepath.Dir(path))) +} + +// InstallFile streams source onto target atomically. Large executables are +// copied rather than buffered in memory. +func InstallFile(source, target string, mode os.FileMode) (err error) { + f, err := os.Open(source) + if err != nil { + return err + } + + defer func() { err = errors.Join(err, f.Close()) }() + + if err := os.MkdirAll(filepath.Dir(target), 0o750); err != nil { + return err + } + + pending, err := renameio.NewPendingFile(target, renameio.WithPermissions(mode), renameio.WithTempDir(filepath.Dir(target))) + if err != nil { + return err + } + + defer pending.Cleanup() //nolint:errcheck // Pending file cleanup after atomic replacement. + + if _, err := io.Copy(pending, f); err != nil { + return err + } + + return pending.CloseAtomicallyReplace() +} + +// SyncFilesystems persists every filesystem backing the given paths. +func SyncFilesystems(paths ...string) error { + var files []*os.File + + defer func() { + for _, f := range files { + _ = f.Close() //nolint:errcheck // Read-only handle; sync errors are returned. + } + }() + + for _, path := range paths { + f, err := os.Open(path) + if err != nil { + return err + } + + files = append(files, f) + } + + return SyncOpenFilesystems(files, unix.Syncfs) +} + +// SyncOpenFilesystems synchronizes each distinct filesystem once, using open +// handles that stay valid after teardown removes their paths. +func SyncOpenFilesystems(files []*os.File, syncfs func(int) error) error { + seen := map[uint64]bool{} + + for _, f := range files { + var stat unix.Stat_t + if err := unix.Fstat(int(f.Fd()), &stat); err != nil { + return err + } + + if seen[uint64(stat.Dev)] { + continue + } + + seen[uint64(stat.Dev)] = true + + if err := syncfs(int(f.Fd())); err != nil { + return fmt.Errorf("sync %s: %w", f.Name(), err) + } + } + + return nil +} diff --git a/internal/fsutil/fsutil_test.go b/internal/fsutil/fsutil_test.go new file mode 100644 index 000000000..9a527b596 --- /dev/null +++ b/internal/fsutil/fsutil_test.go @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package fsutil_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/internal/fsutil" +) + +func TestInstallFileStreamsAndReplacesAtomically(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + source, target := filepath.Join(dir, "source"), filepath.Join(dir, "bin", "target") + require.NoError(t, os.WriteFile(source, []byte("candidate"), 0o600)) + require.NoError(t, fsutil.InstallFile(source, target, 0o755)) + + data, err := os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "candidate", string(data)) + + info, err := os.Stat(target) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o755), info.Mode().Perm()) + + // A failed install must leave the previously installed file intact. + require.Error(t, fsutil.InstallFile(filepath.Join(dir, "missing"), target, 0o755)) + data, err = os.ReadFile(target) + require.NoError(t, err) + require.Equal(t, "candidate", string(data)) +} + +func TestWriteFileDurableCreatesAndPersistsParents(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "a", "b", "state.json") + require.NoError(t, fsutil.WriteFileDurable(path, []byte("{}\n"), 0o600)) + + data, err := os.ReadFile(path) + require.NoError(t, err) + require.Equal(t, "{}\n", string(data)) + + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) +} + +func TestSyncOpenFilesystemsDeduplicatesByDevice(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + + first, err := os.Open(dir) + require.NoError(t, err) + + t.Cleanup(func() { require.NoError(t, first.Close()) }) + + second, err := os.Open(dir) + require.NoError(t, err) + + t.Cleanup(func() { require.NoError(t, second.Close()) }) + + calls := 0 + + require.NoError(t, fsutil.SyncOpenFilesystems([]*os.File{first, second}, func(int) error { + calls++ + return nil + })) + require.Equal(t, 1, calls, "one barrier per filesystem, not per path") + + require.Error(t, fsutil.SyncOpenFilesystems([]*os.File{first}, func(int) error { return os.ErrPermission })) +} + +func TestSyncFilesystemsReportsMissingPath(t *testing.T) { + t.Parallel() + require.Error(t, fsutil.SyncFilesystems(filepath.Join(t.TempDir(), "absent"))) +} diff --git a/pkg/agent/bootstrap/coordinator.go b/pkg/agent/bootstrap/coordinator.go index 1c1e8be6f..a2d4f3425 100644 --- a/pkg/agent/bootstrap/coordinator.go +++ b/pkg/agent/bootstrap/coordinator.go @@ -8,9 +8,6 @@ import ( "context" "fmt" "log/slog" - "os" - - "golang.org/x/sys/unix" "github.com/Azure/unbounded/pkg/agent/installstate" ) @@ -57,9 +54,7 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { } }() - r, loadErr := c.store.Load() - - disposition, err := installstate.Decide(r, loadErr, id.MachineName, id.ConfigFingerprint) + r, disposition, err := installstate.Admit(c.store, id.MachineName, id.ConfigFingerprint) if err != nil { return Outcome{}, err } @@ -77,10 +72,6 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { if err := c.store.Save(r); err != nil { return Outcome{}, err } - } else { - if _, err := c.store.CheckMarker(r); err != nil { - return Outcome{}, err - } } if disposition == installstate.AlreadyComplete { @@ -151,48 +142,3 @@ func (c *Coordinator) runStage(ctx context.Context, stage installstate.Checkpoin return "", fmt.Errorf("unsupported checkpoint %s", stage) } } - -func SyncFilesystems(paths ...string) error { - var files []*os.File - defer func() { - for _, f := range files { - _ = f.Close() //nolint:errcheck // Read-only handle; sync errors are returned. - } - }() //nolint:errcheck // Read-only handles; sync errors are returned. - - for _, path := range paths { - f, err := os.Open(path) - if err != nil { - return err - } - - files = append(files, f) - } - - return SyncOpenFilesystems(files, unix.Syncfs) -} - -// SyncOpenFilesystems synchronizes each filesystem once per barrier, using -// open handles that remain valid after teardown removes their paths. -func SyncOpenFilesystems(files []*os.File, syncfs func(int) error) error { - seen := map[uint64]bool{} - - for _, f := range files { - var stat unix.Stat_t - if err := unix.Fstat(int(f.Fd()), &stat); err != nil { - return err - } - - if seen[uint64(stat.Dev)] { - continue - } - - seen[uint64(stat.Dev)] = true - - if err := syncfs(int(f.Fd())); err != nil { - return fmt.Errorf("sync %s: %w", f.Name(), err) - } - } - - return nil -} diff --git a/pkg/agent/bootstrap/coordinator_test.go b/pkg/agent/bootstrap/coordinator_test.go index 54932e413..197a239de 100644 --- a/pkg/agent/bootstrap/coordinator_test.go +++ b/pkg/agent/bootstrap/coordinator_test.go @@ -7,7 +7,6 @@ import ( "context" "errors" "log/slog" - "os" "path/filepath" "testing" @@ -80,9 +79,6 @@ func TestInterruptedStagesResumeWithoutReplayingEarlierStages(t *testing.T) { require.NoError(t, err) require.Equal(t, tc.checkpoint, record.Checkpoint) - _, err = os.Stat(store.CompletePath()) - require.ErrorIs(t, err, os.ErrNotExist) - stages.calls = nil stages.fail = "" outcome, err := c.Run(t.Context(), id) @@ -127,16 +123,16 @@ func TestCompletedRecoveryDoesNotResolveRetiredBootstrapInputs(t *testing.T) { require.Equal(t, want, stages.calls) - marker, err := store.CheckMarker(r) + complete, err := store.Load() require.NoError(t, err) - require.True(t, marker) + require.Equal(t, installstate.Complete, complete.Checkpoint) } } func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { t.Parallel() - for _, mode := range []string{"different-intent", "resetting", "marker-conflict", "locked"} { + for _, mode := range []string{"different-intent", "resetting", "locked"} { t.Run(mode, func(t *testing.T) { dir := t.TempDir() store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) @@ -154,10 +150,6 @@ func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { id.ConfigFingerprint = "different" } - if mode == "marker-conflict" { - require.NoError(t, os.WriteFile(store.CompletePath(), []byte("other"), 0o644)) - } - if mode == "locked" { lock, err := store.AcquireLock() require.NoError(t, err) @@ -172,23 +164,6 @@ func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { } } -func TestSyncBarrierDeduplicatesFilesystem(t *testing.T) { - dir := t.TempDir() - a, err := os.Open(dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, a.Close()) }) - - b, err := os.Open(dir) - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, b.Close()) }) - - calls := 0 - - require.NoError(t, SyncOpenFilesystems([]*os.File{a, b}, func(int) error { calls++; return nil })) - require.Equal(t, 1, calls) - require.ErrorIs(t, SyncOpenFilesystems([]*os.File{a, b}, func(int) error { return errInjected }), errInjected) -} - func TestInterruptedRepairRemainsCompleteAndRetries(t *testing.T) { store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) r, err := installstate.NewRecord("machine", "fingerprint") diff --git a/pkg/agent/installstate/store.go b/pkg/agent/installstate/store.go index aa466d29e..59e2ebb17 100644 --- a/pkg/agent/installstate/store.go +++ b/pkg/agent/installstate/store.go @@ -15,7 +15,7 @@ import ( "path/filepath" "strings" - "github.com/Azure/unbounded/pkg/agent/internal/utilio" + "github.com/Azure/unbounded/internal/fsutil" ) const ( @@ -37,7 +37,7 @@ const ( ) func (c Checkpoint) NodeMayBeRunning() bool { - return c == StartingNode || c == InstallingDaemon || c == Complete + return c == StartingNode || c == InstallingDaemon } type Record struct { @@ -77,7 +77,6 @@ func NewStore(root, lockPath string) *Store { return &Store{root: root, lockPat func DefaultStore() *Store { return NewStore(DefaultDirectory, DefaultLockPath) } func (s *Store) Root() string { return s.root } func (s *Store) StatePath() string { return filepath.Join(s.root, "install-state.json") } -func (s *Store) CompletePath() string { return filepath.Join(s.root, "bootstrap-complete") } func (s *Store) AcquireLock() (*Lock, error) { return AcquireLockAt(s.lockPath) } func (s *Store) Load() (Record, error) { @@ -85,12 +84,6 @@ func (s *Store) Load() (Record, error) { data, err := os.ReadFile(s.StatePath()) if errors.Is(err, os.ErrNotExist) { - if _, markerErr := os.Lstat(s.CompletePath()); markerErr == nil { - return r, fmt.Errorf("completion marker exists without installation ownership") - } else if !errors.Is(markerErr, os.ErrNotExist) { - return r, markerErr - } - return r, ErrNotFound } @@ -115,33 +108,14 @@ func (s *Store) Save(r Record) error { return err } - return utilio.WriteFileDurable(s.StatePath(), append(data, '\n'), 0o600) -} - -func (s *Store) CheckMarker(r Record) (bool, error) { - data, err := os.ReadFile(s.CompletePath()) - if errors.Is(err, os.ErrNotExist) { - return false, nil - } - - if err != nil { - return false, err - } - - if strings.TrimSpace(string(data)) != r.InstallID { - return false, fmt.Errorf("completion marker conflicts with installation identity") - } - - return true, nil + return fsutil.WriteFileDurable(s.StatePath(), append(data, '\n'), 0o600) } +// MarkComplete commits completion. The durable record is the only completion +// signal; no separate marker file is maintained. func (s *Store) MarkComplete(r Record) error { r.Checkpoint = Complete - if err := s.Save(r); err != nil { - return err - } - - return utilio.WriteFileDurable(s.CompletePath(), []byte(r.InstallID+"\n"), 0o644) + return s.Save(r) } // Remove is called only after teardown's filesystem barriers succeed. @@ -152,18 +126,11 @@ func (s *Store) Remove() error { return err } - for _, path := range []string{s.CompletePath(), s.StatePath()} { - if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { - return err - } - // Persist marker removal before deleting ownership. After interruption, - // reset can resume from the record rather than encounter an orphan marker. - if err := utilio.SyncDir(s.root); err != nil { - return err - } + if err := os.Remove(s.StatePath()); err != nil && !errors.Is(err, os.ErrNotExist) { + return err } - return nil + return fsutil.SyncDir(s.root) } func NewRecord(machine, fingerprint string) (Record, error) { @@ -221,3 +188,16 @@ func Decide(r Record, loadErr error, machine, fingerprint string) (Disposition, return Resume, nil } + +// Admit reads ownership and classifies a bootstrap attempt. Callers that mutate +// the host must hold the installation lock around this call. +func Admit(store *Store, machine, fingerprint string) (Record, Disposition, error) { + r, loadErr := store.Load() + + disposition, err := Decide(r, loadErr, machine, fingerprint) + if err != nil { + return Record{}, Fresh, err + } + + return r, disposition, nil +} diff --git a/pkg/agent/installstate/store_test.go b/pkg/agent/installstate/store_test.go index 6a74a670c..4166f4b17 100644 --- a/pkg/agent/installstate/store_test.go +++ b/pkg/agent/installstate/store_test.go @@ -38,9 +38,6 @@ func TestStoreLifecycle(t *testing.T) { loaded, err = s.Load() require.NoError(t, err) require.Equal(t, Complete, loaded.Checkpoint) - marker, err := s.CheckMarker(loaded) - require.NoError(t, err) - require.True(t, marker) require.NoError(t, s.Remove()) _, err = s.Load() require.ErrorIs(t, err, ErrNotFound) @@ -97,17 +94,6 @@ func TestStoreRejectsCorruptAndOrphanedOwnership(t *testing.T) { require.Error(t, err) }) } - - s := testStore(t) - require.NoError(t, os.MkdirAll(s.Root(), 0o755)) - require.NoError(t, os.WriteFile(s.CompletePath(), []byte("orphan"), 0o644)) - _, err := s.Load() - require.Error(t, err) - require.NotErrorIs(t, err, ErrNotFound) - r, err := NewRecord("machine", "f") - require.NoError(t, err) - _, err = s.CheckMarker(r) - require.Error(t, err) } func TestInstallationLockSurvivesStateRemoval(t *testing.T) { diff --git a/pkg/agent/internal/utilio/sync.go b/pkg/agent/internal/utilio/sync.go deleted file mode 100644 index 0cdb8e2a9..000000000 --- a/pkg/agent/internal/utilio/sync.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 - -package utilio - -import ( - "errors" - "os" - "path/filepath" - - "golang.org/x/sys/unix" -) - -func SyncDir(path string) error { - f, err := os.Open(path) - if err != nil { - return err - } - - return errors.Join(f.Sync(), f.Close()) -} - -// WriteFileDurable also persists newly created parent directories, so the state -// file cannot outlive the directory entries needed to reach it. -func WriteFileDurable(path string, data []byte, mode os.FileMode) error { - var parents []string - - for dir := filepath.Dir(path); ; dir = filepath.Dir(dir) { - _, err := os.Stat(dir) - if err == nil { - parents = append(parents, dir) - break - } - - if !errors.Is(err, os.ErrNotExist) { - return err - } - - parents = append(parents, dir) - } - - if err := WriteFile(path, data, mode); err != nil { - return err - } - - for _, dir := range parents { - if err := SyncDir(dir); err != nil { - return err - } - } - - return nil -} - -// SyncFilesystem persists all output on the filesystem before a checkpoint. -func SyncFilesystem(path string) error { - f, err := os.Open(path) - if err != nil { - return err - } - - return errors.Join(unix.Syncfs(int(f.Fd())), f.Close()) -} diff --git a/pkg/agent/phases/host/preflight_existing_deployment.go b/pkg/agent/phases/host/preflight_existing_deployment.go index 4c22d61ac..7e34cf616 100644 --- a/pkg/agent/phases/host/preflight_existing_deployment.go +++ b/pkg/agent/phases/host/preflight_existing_deployment.go @@ -16,7 +16,9 @@ import ( "github.com/Azure/unbounded/pkg/agent/preflight" ) -const checkExistingDeploymentName = "existing-deployment" +// CheckExistingDeploymentName identifies the clean-host check, which a +// resumed installation intentionally skips. +const CheckExistingDeploymentName = "existing-deployment" // CheckExistingDeployment verifies the host does not already contain // node deployment artifacts. Bootstrap must start from a clean host; @@ -26,14 +28,14 @@ func CheckExistingDeployment(log *slog.Logger) preflight.Checker { } func checkExistingDeployment(log *slog.Logger, deps hostCheckDeps) preflight.Checker { - return simpleHostChecker{name: checkExistingDeploymentName, check: func(ctx context.Context) []preflight.Result { + return simpleHostChecker{name: CheckExistingDeploymentName, check: func(ctx context.Context) []preflight.Result { results := existingDeploymentResults(ctx, log, deps) if len(results) > 0 { return results } return preflight.ResultsOK( - checkExistingDeploymentName, + CheckExistingDeploymentName, "host deployment", "no existing node deployment was detected", ) @@ -151,7 +153,7 @@ func appendExistingDeploymentArtifactResult( return append(results, existingDeploymentResult(artifact.description, artifact.path, artifact.description+" "+artifact.path)) } else if !errors.Is(err, os.ErrNotExist) { return append(results, preflight.Error( - checkExistingDeploymentName, + CheckExistingDeploymentName, artifact.path, "existing deployment artifact cannot be inspected: %s; node reset is needed before running preflight or start again", artifact.path, @@ -163,7 +165,7 @@ func appendExistingDeploymentArtifactResult( func existingDeploymentResult(description, target, detail string) preflight.Result { return preflight.Error( - checkExistingDeploymentName, + CheckExistingDeploymentName, target, "existing node deployment artifact detected (%s): %s; node reset is needed before running preflight or start again", description, diff --git a/pkg/agent/phases/nodestart/preflight_api_server.go b/pkg/agent/phases/nodestart/preflight_api_server.go index fdf61d88e..11a2410a8 100644 --- a/pkg/agent/phases/nodestart/preflight_api_server.go +++ b/pkg/agent/phases/nodestart/preflight_api_server.go @@ -30,14 +30,25 @@ type apiServerReachableChecker struct { // Preflight returns the standard node-start checks that can run before the // nspawn machine starts. func Preflight(log *slog.Logger, cfg config.AgentConfig, goalState *goalstates.MachineGoalState) []preflight.Checker { + // A listener is acceptable only when its process root and executable match + // this installation's rootfs, which lets an interrupted bootstrap retry while + // its node keeps running. Foreign and uninspectable owners still fail, and on + // a clean host the rootfs is absent so the check behaves as a plain port test. + var root string + if goalState.RootFS != nil { + root = goalState.RootFS.MachineDir + } + return []preflight.Checker{ // TODO: Consider moving the kubelet bind address to the kubelet goal state. - CheckBindAddress(log, checkKubeletBindAddressName, kubeletBindAddress, "kubelet bind address"), - CheckBindAddress( + CheckOwnedBindAddress(log, checkKubeletBindAddressName, kubeletBindAddress, "kubelet bind address", root, kubeletExecutablePath), + CheckOwnedBindAddress( log, checkContainerdMetricsBindAddressName, goalState.NodeStart.Containerd.MetricsAddress, "containerd metrics bind address", + root, + containerdExecutablePath, ), CheckAPIServerReachable(log, cfg), } diff --git a/pkg/agent/phases/nodestart/preflight_bind_address.go b/pkg/agent/phases/nodestart/preflight_bind_address.go index 397baed94..dd704e03c 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address.go @@ -24,6 +24,11 @@ const ( checkKubeletBindAddressName = "kubelet-bind-address" checkContainerdMetricsBindAddressName = "containerd-metrics-bind-address" kubeletBindAddress = "0.0.0.0:10250" + + // Executable paths inside the nspawn machine rootfs, used to prove that a + // listener belongs to this installation. + kubeletExecutablePath = "usr/local/bin/kubelet" + containerdExecutablePath = "usr/local/bin/containerd" ) type bindAddressChecker struct { diff --git a/pkg/agent/phases/nodestart/preflight_bind_address_test.go b/pkg/agent/phases/nodestart/preflight_bind_address_test.go index 3389d89d1..a0b2b1357 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address_test.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address_test.go @@ -22,9 +22,10 @@ import ( const procTCPHeader = " sl local_address rem_address st tx_queue rx_queue tr tm->when retrnsmt uid timeout inode\n" func TestPreflightBindAddresses(t *testing.T) { - goalState := &goalstates.MachineGoalState{NodeStart: &goalstates.NodeStart{ - Containerd: goalstates.Containerd{MetricsAddress: "0.0.0.0:12345"}, - }} + goalState := &goalstates.MachineGoalState{ + NodeStart: &goalstates.NodeStart{Containerd: goalstates.Containerd{MetricsAddress: "0.0.0.0:12345"}}, + RootFS: &goalstates.RootFS{MachineDir: "/var/lib/machines/kube1"}, + } checks := Preflight(slog.New(slog.DiscardHandler), config.AgentConfig{}, goalState) @@ -32,6 +33,23 @@ func TestPreflightBindAddresses(t *testing.T) { assert.Equal(t, kubeletBindAddress, checks[0].(bindAddressChecker).address) assert.Equal(t, checkContainerdMetricsBindAddressName, checks[1].Name()) assert.Equal(t, "0.0.0.0:12345", checks[1].(bindAddressChecker).address) + + // Bind checks are always ownership-aware, so an interrupted bootstrap can + // retry while its own node keeps listening. + for _, check := range checks[:2] { + assert.NotNil(t, check.(bindAddressChecker).owned) + } +} + +// A partially resolved goal state must degrade to a plain port check instead of +// panicking, because callers build checks before the rootfs stage has run. +func TestPreflightBindAddressesWithoutResolvedRootFS(t *testing.T) { + goalState := &goalstates.MachineGoalState{NodeStart: &goalstates.NodeStart{ + Containerd: goalstates.Containerd{MetricsAddress: "0.0.0.0:12345"}, + }} + + checks := Preflight(slog.New(slog.DiscardHandler), config.AgentConfig{}, goalState) + assert.False(t, checks[0].(bindAddressChecker).owned()) } func TestCheckBindAddressAvailable(t *testing.T) { diff --git a/pkg/agent/phases/reset/helpers.go b/pkg/agent/phases/reset/helpers.go index bbc1e7698..548c402ad 100644 --- a/pkg/agent/phases/reset/helpers.go +++ b/pkg/agent/phases/reset/helpers.go @@ -8,27 +8,8 @@ import ( "fmt" "log/slog" "os" - "os/exec" ) -// ToolMissing distinguishes a package not installed yet from an installed tool -// failing to inspect or remove resources. Permission failures are not absence. -func ToolMissing(name string) bool { - _, err := exec.LookPath(name) - return errors.Is(err, exec.ErrNotFound) -} - -// SystemdUnavailable permits offline cleanup when no host systemd is running. -func SystemdUnavailable() bool { - if ToolMissing("systemctl") { - return true - } - - _, err := os.Stat("/run/systemd/system") - - return errors.Is(err, os.ErrNotExist) -} - // removeFileIfExists ignores absence but propagates substantive removal errors. func removeFileIfExists(log *slog.Logger, path string) error { if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { diff --git a/pkg/agent/phases/reset/machine.go b/pkg/agent/phases/reset/machine.go index 201fb5d6b..c2ca627cc 100644 --- a/pkg/agent/phases/reset/machine.go +++ b/pkg/agent/phases/reset/machine.go @@ -30,10 +30,6 @@ func StopMachine(log *slog.Logger, machineName string) phases.Task { func (t *stopMachine) Name() string { return "stop-machine" } func (t *stopMachine) Do(ctx context.Context) error { - if ToolMissing("machinectl") { - return nil - } - if err := executil.RunCmd(ctx, t.log, executil.Machinectl(), "disable", t.machineName); err != nil { t.log.Warn("failed to disable machine; continuing with stop and removal", "machine", t.machineName, "error", err) } @@ -126,10 +122,6 @@ func RemoveMachine(log *slog.Logger, machineName string) phases.Task { func (t *removeMachine) Name() string { return "remove-machine" } func (t *removeMachine) Do(ctx context.Context) error { - if ToolMissing("machinectl") { - return nil - } - machineDir := fmt.Sprintf("/var/lib/machines/%s", t.machineName) // Skip entirely if the machine directory doesn't exist - nothing to remove. diff --git a/pkg/agent/phases/reset/network.go b/pkg/agent/phases/reset/network.go index eb58785fd..f4ca0eaec 100644 --- a/pkg/agent/phases/reset/network.go +++ b/pkg/agent/phases/reset/network.go @@ -65,46 +65,42 @@ func CleanupLocalDNSRules(log *slog.Logger) phases.Task { func (t *cleanupLocalDNSRules) Name() string { return "cleanup-localdns-rules" } func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { - if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "disable", "--now", goalstates.LocalDNSNetworkUnit); err != nil && !SystemdUnavailable() { + if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "disable", "--now", goalstates.LocalDNSNetworkUnit); err != nil { path := filepath.Join(goalstates.SystemdSystemDir, goalstates.LocalDNSNetworkUnit) if _, statErr := os.Lstat(path); !errors.Is(statErr, os.ErrNotExist) { return fmt.Errorf("disable LocalDNS unit: %w", err) } } - if !ToolMissing("nft") { - tables, err := executil.OutputCmd(ctx, t.log, "nft", "list", "tables") - if err != nil { - return fmt.Errorf("inspect LocalDNS tables: %w", err) - } + tables, err := executil.OutputCmd(ctx, t.log, "nft", "list", "tables") + if err != nil { + return fmt.Errorf("inspect LocalDNS tables: %w", err) + } - if strings.Contains(tables, "table ip "+goalstates.LocalDNSNFTTable+"\n") { - if err := executil.RunCmd(ctx, t.log, func(ctx context.Context) *exec.Cmd { - return exec.CommandContext(ctx, "nft") - }, "delete", "table", "ip", goalstates.LocalDNSNFTTable); err != nil { - return fmt.Errorf("remove LocalDNS nftables table: %w", err) - } + if strings.Contains(tables, "table ip "+goalstates.LocalDNSNFTTable+"\n") { + if err := executil.RunCmd(ctx, t.log, func(ctx context.Context) *exec.Cmd { + return exec.CommandContext(ctx, "nft") + }, "delete", "table", "ip", goalstates.LocalDNSNFTTable); err != nil { + return fmt.Errorf("remove LocalDNS nftables table: %w", err) } } - if !ToolMissing("ip") { - links, err := executil.OutputCmd(ctx, t.log, "ip", "-d", "-o", "link", "show") - if err != nil { - return fmt.Errorf("inspect LocalDNS interface: %w", err) - } + links, err := executil.OutputCmd(ctx, t.log, "ip", "-d", "-o", "link", "show") + if err != nil { + return fmt.Errorf("inspect LocalDNS interface: %w", err) + } - for _, output := range strings.Split(links, "\n") { - if !strings.Contains(output, ": "+goalstates.LocalDNSInterfaceName+":") { - continue - } + for _, output := range strings.Split(links, "\n") { + if !strings.Contains(output, ": "+goalstates.LocalDNSInterfaceName+":") { + continue + } - if !strings.Contains(" "+output+" ", " dummy ") { - return fmt.Errorf("refusing to remove non-dummy interface %s", goalstates.LocalDNSInterfaceName) - } + if !strings.Contains(" "+output+" ", " dummy ") { + return fmt.Errorf("refusing to remove non-dummy interface %s", goalstates.LocalDNSInterfaceName) + } - if err := executil.RunCmd(ctx, t.log, executil.Ip(), "link", "delete", goalstates.LocalDNSInterfaceName); err != nil { - return fmt.Errorf("remove LocalDNS interface: %w", err) - } + if err := executil.RunCmd(ctx, t.log, executil.Ip(), "link", "delete", goalstates.LocalDNSInterfaceName); err != nil { + return fmt.Errorf("remove LocalDNS interface: %w", err) } } @@ -121,9 +117,6 @@ func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { } func (t *removeNetworkInterfaces) Do(ctx context.Context) error { - if ToolMissing("ip") { - return nil - } // Remove WireGuard interfaces (wg51820, wg51821, ...). wgIfaces, err := listWireGuardInterfaces(ctx, t.log) if err != nil { diff --git a/pkg/agent/phases/reset/routes.go b/pkg/agent/phases/reset/routes.go index 329e7053f..6f80d6ee9 100644 --- a/pkg/agent/phases/reset/routes.go +++ b/pkg/agent/phases/reset/routes.go @@ -39,10 +39,6 @@ func (t *cleanupRoutes) Do(ctx context.Context) error { output := t.output if output == nil { - if ToolMissing("ip") { - return nil - } - output = func(ctx context.Context, args ...string) (string, error) { return executil.OutputCmd(ctx, t.log, "ip", args...) } diff --git a/pkg/agent/phases/reset/strict_test.go b/pkg/agent/phases/reset/strict_test.go index 466967f6d..1bb545a18 100644 --- a/pkg/agent/phases/reset/strict_test.go +++ b/pkg/agent/phases/reset/strict_test.go @@ -112,18 +112,3 @@ func TestFileCleanupPropagatesSubstantiveFailure(t *testing.T) { require.NoError(t, removeAllIfExists(log, dir)) require.NoError(t, removeFileIfExists(log, dir)) } - -func TestCleanupToleratesMissingTools(t *testing.T) { - t.Setenv("PATH", t.TempDir()) - - log := slog.New(slog.DiscardHandler) - - require.True(t, ToolMissing("machinectl")) - require.NoError(t, StopMachine(log, "kube1").Do(t.Context())) - require.NoError(t, RemoveMachine(log, "kube1").Do(t.Context())) - require.NoError(t, RemoveNetworkInterfaces(log).Do(t.Context())) - require.NoError(t, CleanupRoutes(log).Do(t.Context())) - require.NoError(t, ReloadSystemd(log).Do(t.Context())) - _, err := RegisteredMachine(t.Context(), log, "kube1") - require.Error(t, err, "bootstrap inventory must still reject unavailable inspection") -} diff --git a/pkg/agent/phases/reset/systemd.go b/pkg/agent/phases/reset/systemd.go index bf1178b32..0e17c6dc0 100644 --- a/pkg/agent/phases/reset/systemd.go +++ b/pkg/agent/phases/reset/systemd.go @@ -24,10 +24,6 @@ func ReloadSystemd(log *slog.Logger) phases.Task { func (t *reloadSystemd) Name() string { return "reload-systemd" } func (t *reloadSystemd) Do(ctx context.Context) error { - if SystemdUnavailable() { - return nil - } - t.log.Info("reloading systemd daemon") return executil.RunCmd(ctx, t.log, executil.Systemctl(), "daemon-reload") diff --git a/pkg/agent/phases/rootfs/oci/task.go b/pkg/agent/phases/rootfs/oci/task.go index 0b68866d2..e47f2b7cb 100644 --- a/pkg/agent/phases/rootfs/oci/task.go +++ b/pkg/agent/phases/rootfs/oci/task.go @@ -11,6 +11,7 @@ import ( "os" "path/filepath" + "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/artifactsource/ocilayout" "github.com/Azure/unbounded/pkg/agent/internal/utilio" "github.com/Azure/unbounded/pkg/agent/phases" @@ -92,11 +93,11 @@ func (d *downloadRootFS) Do(ctx context.Context) error { } if d.ownedReplay { - if err := utilio.SyncFilesystem(d.machineDir); err != nil { + if err := fsutil.SyncFilesystems(d.machineDir); err != nil { return err } - if err := utilio.WriteFileDurable(filepath.Join(d.machineDir, ".unbounded-rootfs-complete"), []byte("complete\n"), 0o600); err != nil { + if err := fsutil.WriteFileDurable(filepath.Join(d.machineDir, ".unbounded-rootfs-complete"), []byte("complete\n"), 0o600); err != nil { return err } } From b7f97b80c5c24420bd874ecbd910d93b8e707b0c Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:12:18 -0400 Subject: [PATCH 04/39] agent: document bootstrap compatibility fixtures in their test --- cmd/agent/internal/cmd/bootstrap_test.go | 28 ++++++++----------- .../cmd/testdata/bootstrap-v1/README.md | 20 ------------- 2 files changed, 11 insertions(+), 37 deletions(-) delete mode 100644 cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index e535d1e64..0e2917744 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -18,8 +18,16 @@ import ( "github.com/Azure/unbounded/pkg/agent/preflight" ) -// These fixtures are produced by P6's actual loader, normalizer, fingerprint, -// and Store.Save. Later releases must consume them with the original input. +// TestBootstrapV1CompatibilityFixtures pins the on-disk ownership format at +// schema version 1. The fixtures hold records written by this package's own +// config loader, normalizer, fingerprint and installstate.Store, from the +// synthetic input.json alongside them; only the random install ID is fixed. +// +// A later release must still admit an installation created by an earlier one +// when given the same original input, so a mismatch here is a compatibility +// break rather than a fixture to refresh. Record.Validate pins these files to +// installstate.SchemaVersion, so bumping it fails loudly; a new schema version +// gets its own fixture directory rather than regenerated files. func TestBootstrapV1CompatibilityFixtures(t *testing.T) { dir := filepath.Join("testdata", "bootstrap-v1") cfg, err := loadConfigFromFile(filepath.Join(dir, "input.json")) @@ -30,21 +38,7 @@ func TestBootstrapV1CompatibilityFixtures(t *testing.T) { require.NoError(t, err) for _, checkpoint := range []installstate.Checkpoint{installstate.PreparingRootFS, installstate.Complete, installstate.Resetting} { - fixture := filepath.Join(dir, string(checkpoint)+".json") - if os.Getenv("UPDATE_BOOTSTRAP_V1_FIXTURES") == "1" { - store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) - record, err := installstate.NewRecord(id.MachineName, id.ConfigFingerprint) - require.NoError(t, err) - - record.InstallID = "00112233445566778899aabbccddeeff" - record.Checkpoint = checkpoint - require.NoError(t, store.Save(record)) - data, err := os.ReadFile(store.StatePath()) - require.NoError(t, err) - require.NoError(t, os.WriteFile(fixture, data, 0o644)) - } - - data, err := os.ReadFile(fixture) + data, err := os.ReadFile(filepath.Join(dir, string(checkpoint)+".json")) require.NoError(t, err) var record installstate.Record diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md b/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md deleted file mode 100644 index 94a149a21..000000000 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Bootstrap ownership v1 fixtures - -Produced by the simplified P6 candidate on base `50286b2c`, using -`TestBootstrapV1CompatibilityFixtures`. The input uses synthetic credentials. -The producer runs the actual JSON loader, normalization, fingerprint, and -`installstate.Store.Save`; only the random installation ID is fixed. - -These files freeze the default-path contract for later releases. Consume the -original input when checking compatibility. Do not regenerate these fixtures to -make a changed serializer pass. Adding a new format requires new fixtures. - -During pre-merge PR review the fingerprint was narrowed to Kubernetes version, -rootfs image and API server endpoint, with machine name checked separately. -These fixtures supersede the unreleased full-config fingerprint from `bb191a93`. - -Initial production command: - -```sh -UPDATE_BOOTSTRAP_V1_FIXTURES=1 go test ./cmd/agent/internal/cmd -run TestBootstrapV1CompatibilityFixtures -count=1 -``` From 5bd989ca59b83294df09b6b60ec594e3a67db51c Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:38:10 -0400 Subject: [PATCH 05/39] docs: drop stale cleanup-tool tolerance from agent retry guide --- docs/content/guides/agent.md | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/docs/content/guides/agent.md b/docs/content/guides/agent.md index d061be1f4..f01d0c2a5 100644 --- a/docs/content/guides/agent.md +++ b/docs/content/guides/agent.md @@ -44,14 +44,13 @@ Credentials and artifact locations can be refreshed for a retry. Other fields do not participate in admission; a retry does not reapply stages already completed. After completion, the same `start` invocation checks required daemon files, -executable permissions, and enabled/active service state, and repairs the daemon when -they are missing or stopped. It does not compare unit contents or overwrite -working local unit customizations. -This path uses the current applied configuration, including after an ordinary -repave has switched from kube1 to kube2. It does not resolve the original node -image or binary-download sources. The bootstrap shell still needs its agent -download; to repair without that download, run an available agent executable -directly with the original config: +executable permissions, and enabled/active service state, and repairs the daemon +when they are missing or stopped. It does not compare unit contents or overwrite +working local unit customizations. This path uses the current applied +configuration, including after an ordinary repave has switched from kube1 to +kube2. It does not resolve the original node image or binary-download sources. +The bootstrap shell still needs its agent download; to repair without that +download, run an available agent executable directly with the original config: ```bash sudo env UNBOUNDED_AGENT_CONFIG_FILE=/path/to/original-agent-config.json \ @@ -60,13 +59,12 @@ sudo env UNBOUNDED_AGENT_CONFIG_FILE=/path/to/original-agent-config.json \ Bootstrap, reset, node lifecycle operations, and binary activation share an installation lock. Last-resort daemon rollback does not wait on these locks; -reset stops the recovery unit before teardown. A busy lock is retryable. MachineOperation handlers requeue -instead of starting concurrent host mutations. Reset retains a `resetting` -record when cleanup fails; correct the reported error and run reset again. -Ownership is removed only after teardown and its filesystem synchronization -barriers succeed. Missing cleanup tools on a partially prepared host are skipped; -failures from installed tools remain errors. The lock file in `/run` remains and must not be deleted to -force an operation through. +reset stops the recovery unit before teardown. A busy lock is retryable, and +MachineOperation handlers requeue instead of starting concurrent host mutations. +Reset retains a `resetting` record when cleanup fails; correct the reported error +and run reset again. Ownership is removed only after teardown and its filesystem +synchronization barriers succeed. The lock file in `/run` remains and must not be +deleted to force an operation through. Recovery applies to checkpointed initial installations. Existing installations without an ownership record retain ordinary daemon operations and reset support; From 801bd484ac2b5548a9f7317a338d979307b8fe3e Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:13:42 -0400 Subject: [PATCH 06/39] agent: move installation ownership into the agent binary pkg/agent was extracted in #76 so external consumers could reuse the agent as a library, and that same change deliberately moved host-lifecycle operations such as EnableDaemon, StopDaemon and PersistAppliedConfig back out of the public tree into cmd/agent/internal/daemon. installstate and bootstrap belong on that side of the line. They manage this agent's own record and lock at its own paths, and every consumer is already under cmd/agent. Keeping them private also avoids publishing an on-disk schema and admission policy that is about to change: Record validation currently accepts only the default install prefix, which the configurable-prefix work alters. Promoting an internal package later is not a breaking change, so this can be revisited if an external consumer needs resumable bootstrap. --- {pkg/agent => cmd/agent/internal}/bootstrap/coordinator.go | 2 +- .../agent/internal}/bootstrap/coordinator_test.go | 2 +- cmd/agent/internal/cmd/agentupgrade.go | 2 +- cmd/agent/internal/cmd/agentupgrade_test.go | 2 +- cmd/agent/internal/cmd/bootstrap.go | 4 ++-- cmd/agent/internal/cmd/bootstrap_test.go | 2 +- cmd/agent/internal/cmd/preflight.go | 2 +- cmd/agent/internal/cmd/start.go | 4 ++-- cmd/agent/internal/daemon/controller.go | 2 +- cmd/agent/internal/daemon/controller_machineoperation.go | 2 +- cmd/agent/internal/daemon/controller_node.go | 2 +- cmd/agent/internal/daemon/controller_test.go | 2 +- cmd/agent/internal/daemon/daemon.go | 2 +- cmd/agent/internal/daemon/installation_test.go | 2 +- cmd/agent/internal/daemon/migration_test.go | 2 +- cmd/agent/internal/daemon/nodeoperator.go | 2 +- cmd/agent/internal/daemon/reset.go | 2 +- cmd/agent/internal/daemon/reset_test.go | 2 +- {pkg/agent => cmd/agent/internal}/installstate/lock.go | 0 {pkg/agent => cmd/agent/internal}/installstate/mutation.go | 0 {pkg/agent => cmd/agent/internal}/installstate/store.go | 0 {pkg/agent => cmd/agent/internal}/installstate/store_test.go | 0 22 files changed, 20 insertions(+), 20 deletions(-) rename {pkg/agent => cmd/agent/internal}/bootstrap/coordinator.go (98%) rename {pkg/agent => cmd/agent/internal}/bootstrap/coordinator_test.go (98%) rename {pkg/agent => cmd/agent/internal}/installstate/lock.go (100%) rename {pkg/agent => cmd/agent/internal}/installstate/mutation.go (100%) rename {pkg/agent => cmd/agent/internal}/installstate/store.go (100%) rename {pkg/agent => cmd/agent/internal}/installstate/store_test.go (100%) diff --git a/pkg/agent/bootstrap/coordinator.go b/cmd/agent/internal/bootstrap/coordinator.go similarity index 98% rename from pkg/agent/bootstrap/coordinator.go rename to cmd/agent/internal/bootstrap/coordinator.go index a2d4f3425..e2a91119d 100644 --- a/pkg/agent/bootstrap/coordinator.go +++ b/cmd/agent/internal/bootstrap/coordinator.go @@ -9,7 +9,7 @@ import ( "fmt" "log/slog" - "github.com/Azure/unbounded/pkg/agent/installstate" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" ) type Identity struct{ MachineName, ConfigFingerprint string } diff --git a/pkg/agent/bootstrap/coordinator_test.go b/cmd/agent/internal/bootstrap/coordinator_test.go similarity index 98% rename from pkg/agent/bootstrap/coordinator_test.go rename to cmd/agent/internal/bootstrap/coordinator_test.go index 197a239de..1ec166f7c 100644 --- a/pkg/agent/bootstrap/coordinator_test.go +++ b/cmd/agent/internal/bootstrap/coordinator_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/Azure/unbounded/pkg/agent/installstate" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" ) type fakeStages struct { diff --git a/cmd/agent/internal/cmd/agentupgrade.go b/cmd/agent/internal/cmd/agentupgrade.go index 147d0cc79..a11eebc88 100644 --- a/cmd/agent/internal/cmd/agentupgrade.go +++ b/cmd/agent/internal/cmd/agentupgrade.go @@ -15,9 +15,9 @@ import ( "github.com/spf13/cobra" "github.com/Azure/unbounded/cmd/agent/internal/daemon" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" ) const hostAgentBinaryMode = 0o755 diff --git a/cmd/agent/internal/cmd/agentupgrade_test.go b/cmd/agent/internal/cmd/agentupgrade_test.go index bde59cae4..8545f3f1e 100644 --- a/cmd/agent/internal/cmd/agentupgrade_test.go +++ b/cmd/agent/internal/cmd/agentupgrade_test.go @@ -13,9 +13,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/pkg/agent/agentbinary" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" ) type preflightOnlyDaemonService struct{} diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index b8298ea63..aef90b460 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -11,12 +11,12 @@ import ( "strings" "github.com/Azure/unbounded/cmd/agent/internal/attest" + "github.com/Azure/unbounded/cmd/agent/internal/bootstrap" "github.com/Azure/unbounded/cmd/agent/internal/daemon" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/internal/provision" - "github.com/Azure/unbounded/pkg/agent/bootstrap" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases" "github.com/Azure/unbounded/pkg/agent/phases/host" "github.com/Azure/unbounded/pkg/agent/phases/nodestart" diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index 0e2917744..0e7d1bc76 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -13,8 +13,8 @@ import ( "github.com/stretchr/testify/require" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" - "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/preflight" ) diff --git a/cmd/agent/internal/cmd/preflight.go b/cmd/agent/internal/cmd/preflight.go index c0c8b060e..33b8d174c 100644 --- a/cmd/agent/internal/cmd/preflight.go +++ b/cmd/agent/internal/cmd/preflight.go @@ -13,9 +13,9 @@ import ( "github.com/spf13/cobra" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases/host" "github.com/Azure/unbounded/pkg/agent/phases/nodestart" "github.com/Azure/unbounded/pkg/agent/phases/rootfs" diff --git a/cmd/agent/internal/cmd/start.go b/cmd/agent/internal/cmd/start.go index 55bd753af..db097a540 100644 --- a/cmd/agent/internal/cmd/start.go +++ b/cmd/agent/internal/cmd/start.go @@ -11,11 +11,11 @@ import ( "github.com/spf13/cobra" + "github.com/Azure/unbounded/cmd/agent/internal/bootstrap" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/internal/version" - "github.com/Azure/unbounded/pkg/agent/bootstrap" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" ) func newCmdStart(cmdCtx *CommandContext) *cobra.Command { diff --git a/cmd/agent/internal/daemon/controller.go b/cmd/agent/internal/daemon/controller.go index 0a3239e1f..039b05fa3 100644 --- a/cmd/agent/internal/daemon/controller.go +++ b/cmd/agent/internal/daemon/controller.go @@ -22,9 +22,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" daemon "github.com/Azure/unbounded/pkg/agent/daemon" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" ) type repaveReconciler struct { diff --git a/cmd/agent/internal/daemon/controller_machineoperation.go b/cmd/agent/internal/daemon/controller_machineoperation.go index ef7153089..175ae332d 100644 --- a/cmd/agent/internal/daemon/controller_machineoperation.go +++ b/cmd/agent/internal/daemon/controller_machineoperation.go @@ -14,10 +14,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/pkg/agent/agentbinary" daemon "github.com/Azure/unbounded/pkg/agent/daemon" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" ) const agentUpgradeLockRetryDelay = 2 * time.Second diff --git a/cmd/agent/internal/daemon/controller_node.go b/cmd/agent/internal/daemon/controller_node.go index 37102e999..72d09a6ac 100644 --- a/cmd/agent/internal/daemon/controller_node.go +++ b/cmd/agent/internal/daemon/controller_node.go @@ -18,9 +18,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/machineconfigs" "github.com/Azure/unbounded/internal/provision" - "github.com/Azure/unbounded/pkg/agent/installstate" ) func (r *repaveReconciler) ReconcileRepave(ctx context.Context, _ string) (reconcile.Result, error) { diff --git a/cmd/agent/internal/daemon/controller_test.go b/cmd/agent/internal/daemon/controller_test.go index ee3f9862d..702ccd1ad 100644 --- a/cmd/agent/internal/daemon/controller_test.go +++ b/cmd/agent/internal/daemon/controller_test.go @@ -21,11 +21,11 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/agentbinary" daemon "github.com/Azure/unbounded/pkg/agent/daemon" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" ) const testAgentUpgradeSHA256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" diff --git a/cmd/agent/internal/daemon/daemon.go b/cmd/agent/internal/daemon/daemon.go index c6cb01e2b..a7b4afda4 100644 --- a/cmd/agent/internal/daemon/daemon.go +++ b/cmd/agent/internal/daemon/daemon.go @@ -22,11 +22,11 @@ import ( v1alpha3 "github.com/Azure/unbounded/api/machina/v1alpha3" netv1alpha1 "github.com/Azure/unbounded/api/net/v1alpha1" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/config" "github.com/Azure/unbounded/pkg/agent/daemoncred" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" ) const ( diff --git a/cmd/agent/internal/daemon/installation_test.go b/cmd/agent/internal/daemon/installation_test.go index b26d6a5d8..07677cc93 100644 --- a/cmd/agent/internal/daemon/installation_test.go +++ b/cmd/agent/internal/daemon/installation_test.go @@ -11,8 +11,8 @@ import ( "github.com/stretchr/testify/require" ctrl "sigs.k8s.io/controller-runtime" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" shared "github.com/Azure/unbounded/pkg/agent/daemon" - "github.com/Azure/unbounded/pkg/agent/installstate" ) func TestInstallationContentionPreventsControllerWork(t *testing.T) { diff --git a/cmd/agent/internal/daemon/migration_test.go b/cmd/agent/internal/daemon/migration_test.go index 42f215204..788fe77fa 100644 --- a/cmd/agent/internal/daemon/migration_test.go +++ b/cmd/agent/internal/daemon/migration_test.go @@ -12,8 +12,8 @@ import ( "github.com/stretchr/testify/require" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" - "github.com/Azure/unbounded/pkg/agent/installstate" ) func TestDaemonStartupRunsLifecycleMigrationBeforeControllerSetup(t *testing.T) { diff --git a/cmd/agent/internal/daemon/nodeoperator.go b/cmd/agent/internal/daemon/nodeoperator.go index 61c7234b2..3bf98d21d 100644 --- a/cmd/agent/internal/daemon/nodeoperator.go +++ b/cmd/agent/internal/daemon/nodeoperator.go @@ -13,10 +13,10 @@ import ( "reflect" "strings" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases" "github.com/Azure/unbounded/pkg/agent/phases/nodestart" "github.com/Azure/unbounded/pkg/agent/phases/nodestop" diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index 3a0a2dfa3..326a3b677 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -14,10 +14,10 @@ import ( "golang.org/x/sys/unix" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/installstate" "github.com/Azure/unbounded/pkg/agent/phases" "github.com/Azure/unbounded/pkg/agent/phases/reset" ) diff --git a/cmd/agent/internal/daemon/reset_test.go b/cmd/agent/internal/daemon/reset_test.go index 4d65c9b07..7d5498545 100644 --- a/cmd/agent/internal/daemon/reset_test.go +++ b/cmd/agent/internal/daemon/reset_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/Azure/unbounded/pkg/agent/installstate" + "github.com/Azure/unbounded/cmd/agent/internal/installstate" ) func TestResetAgentResourcesIncludesBPFFSMountCleanup(t *testing.T) { diff --git a/pkg/agent/installstate/lock.go b/cmd/agent/internal/installstate/lock.go similarity index 100% rename from pkg/agent/installstate/lock.go rename to cmd/agent/internal/installstate/lock.go diff --git a/pkg/agent/installstate/mutation.go b/cmd/agent/internal/installstate/mutation.go similarity index 100% rename from pkg/agent/installstate/mutation.go rename to cmd/agent/internal/installstate/mutation.go diff --git a/pkg/agent/installstate/store.go b/cmd/agent/internal/installstate/store.go similarity index 100% rename from pkg/agent/installstate/store.go rename to cmd/agent/internal/installstate/store.go diff --git a/pkg/agent/installstate/store_test.go b/cmd/agent/internal/installstate/store_test.go similarity index 100% rename from pkg/agent/installstate/store_test.go rename to cmd/agent/internal/installstate/store_test.go From 63dffa1e965c283d9ee99ab5a2ec7966eb672154 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:16:32 -0400 Subject: [PATCH 07/39] agent: tighten the installation ownership package surface A reviewer had to ask what this package is for, so document it: the record and lock answer whether an installation exists and is ours, how far it got, and whether anything else is mutating the host. Now that nothing outside cmd/agent can reach it, drop the accumulated surface. Checkpoint.NodeMayBeRunning was dead once bind-address preflight became unconditionally ownership-aware. StatePath, AcquireLockAt and DefaultLockPath had no callers outside the package. Decide is folded into Admit so admission has a single entry point; the fixture test now admits through a store, which also proves the record survives a load round-trip. --- cmd/agent/internal/cmd/bootstrap_test.go | 8 ++++++- cmd/agent/internal/installstate/doc.go | 24 +++++++++++++++++++ cmd/agent/internal/installstate/lock.go | 4 ++-- cmd/agent/internal/installstate/store.go | 23 +++++++----------- cmd/agent/internal/installstate/store_test.go | 14 +++++------ 5 files changed, 49 insertions(+), 24 deletions(-) create mode 100644 cmd/agent/internal/installstate/doc.go diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index 0e7d1bc76..2649939af 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -48,11 +48,17 @@ func TestBootstrapV1CompatibilityFixtures(t *testing.T) { require.Equal(t, id.ConfigFingerprint, record.ConfigFingerprint) require.Equal(t, "/usr/local", record.HostPrefix) - disposition, err := installstate.Decide(record, nil, id.MachineName, id.ConfigFingerprint) + // Admit through a store so the fixture also proves it survives a + // load round-trip, not just an in-memory classification. + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + require.NoError(t, store.Save(record)) + + loaded, disposition, err := installstate.Admit(store, id.MachineName, id.ConfigFingerprint) if checkpoint == installstate.Resetting { require.Error(t, err) } else { require.NoError(t, err) + require.Equal(t, record, loaded) want := installstate.Resume if checkpoint == installstate.Complete { diff --git a/cmd/agent/internal/installstate/doc.go b/cmd/agent/internal/installstate/doc.go new file mode 100644 index 000000000..a239d5962 --- /dev/null +++ b/cmd/agent/internal/installstate/doc.go @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Package installstate owns the agent's installation ownership record and the +// lock that guards it. The record is internal bookkeeping, not a configuration +// input. +// +// It answers three questions nothing else on the host can answer: +// +// - Is there an installation here, and is it the one we are being asked to +// perform? The record carries the machine name and a fingerprint of the +// identity-defining configuration, so a retry can tell its own interrupted +// attempt from a different installation. Without it, bootstrap can only +// demand a pristine host and abort otherwise. +// - How far did the last attempt get? The checkpoint lets a retry replay only +// unfinished stages instead of redoing completed work or refusing outright. +// - Is anything else mutating this host? The lock serializes bootstrap, reset, +// repave, NodeReboot, host agent activation and AgentUpgrade, which all +// touch the same files and services. +// +// It is separate from the bootstrap coordinator because the daemon's repave, +// NodeReboot, AgentUpgrade and AgentReset paths, reset, and host agent +// activation all need the record or the lock without running a bootstrap. +package installstate diff --git a/cmd/agent/internal/installstate/lock.go b/cmd/agent/internal/installstate/lock.go index 36f7bab27..2d0b0cdf0 100644 --- a/cmd/agent/internal/installstate/lock.go +++ b/cmd/agent/internal/installstate/lock.go @@ -15,9 +15,9 @@ var ErrLockHeld = errors.New("another host lifecycle operation holds the install type Lock struct{ file *os.File } -// AcquireLockAt is nonblocking. The kernel releases flock on process exit; a +// acquireLockAt is nonblocking. The kernel releases flock on process exit; a // leftover lock file does not imply a held lock and must not be deleted by reset. -func AcquireLockAt(path string) (*Lock, error) { +func acquireLockAt(path string) (*Lock, error) { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return nil, err } diff --git a/cmd/agent/internal/installstate/store.go b/cmd/agent/internal/installstate/store.go index 59e2ebb17..1814a46fc 100644 --- a/cmd/agent/internal/installstate/store.go +++ b/cmd/agent/internal/installstate/store.go @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 -// Package installstate records ownership before initial bootstrap mutates a host. package installstate import ( @@ -20,7 +19,7 @@ import ( const ( DefaultDirectory = "/var/lib/unbounded/agent" - DefaultLockPath = "/run/unbounded-agent-install.lock" + defaultLockPath = "/run/unbounded-agent-install.lock" DefaultHostPrefix = "/usr/local" SchemaVersion = 1 ) @@ -36,10 +35,6 @@ const ( Resetting Checkpoint = "resetting" ) -func (c Checkpoint) NodeMayBeRunning() bool { - return c == StartingNode || c == InstallingDaemon -} - type Record struct { SchemaVersion int `json:"schemaVersion"` InstallID string `json:"installID"` @@ -74,15 +69,15 @@ var ErrNotFound = errors.New("installation record not found") type Store struct{ root, lockPath string } func NewStore(root, lockPath string) *Store { return &Store{root: root, lockPath: lockPath} } -func DefaultStore() *Store { return NewStore(DefaultDirectory, DefaultLockPath) } +func DefaultStore() *Store { return NewStore(DefaultDirectory, defaultLockPath) } func (s *Store) Root() string { return s.root } -func (s *Store) StatePath() string { return filepath.Join(s.root, "install-state.json") } -func (s *Store) AcquireLock() (*Lock, error) { return AcquireLockAt(s.lockPath) } +func (s *Store) statePath() string { return filepath.Join(s.root, "install-state.json") } +func (s *Store) AcquireLock() (*Lock, error) { return acquireLockAt(s.lockPath) } func (s *Store) Load() (Record, error) { var r Record - data, err := os.ReadFile(s.StatePath()) + data, err := os.ReadFile(s.statePath()) if errors.Is(err, os.ErrNotExist) { return r, ErrNotFound } @@ -108,7 +103,7 @@ func (s *Store) Save(r Record) error { return err } - return fsutil.WriteFileDurable(s.StatePath(), append(data, '\n'), 0o600) + return fsutil.WriteFileDurable(s.statePath(), append(data, '\n'), 0o600) } // MarkComplete commits completion. The durable record is the only completion @@ -126,7 +121,7 @@ func (s *Store) Remove() error { return err } - if err := os.Remove(s.StatePath()); err != nil && !errors.Is(err, os.ErrNotExist) { + if err := os.Remove(s.statePath()); err != nil && !errors.Is(err, os.ErrNotExist) { return err } @@ -157,7 +152,7 @@ const ( AlreadyComplete ) -func Decide(r Record, loadErr error, machine, fingerprint string) (Disposition, error) { +func decide(r Record, loadErr error, machine, fingerprint string) (Disposition, error) { if strings.TrimSpace(machine) == "" || strings.TrimSpace(fingerprint) == "" { return Fresh, fmt.Errorf("bootstrap identity is required") } @@ -194,7 +189,7 @@ func Decide(r Record, loadErr error, machine, fingerprint string) (Disposition, func Admit(store *Store, machine, fingerprint string) (Record, Disposition, error) { r, loadErr := store.Load() - disposition, err := Decide(r, loadErr, machine, fingerprint) + disposition, err := decide(r, loadErr, machine, fingerprint) if err != nil { return Record{}, Fresh, err } diff --git a/cmd/agent/internal/installstate/store_test.go b/cmd/agent/internal/installstate/store_test.go index 4166f4b17..c404667fb 100644 --- a/cmd/agent/internal/installstate/store_test.go +++ b/cmd/agent/internal/installstate/store_test.go @@ -31,7 +31,7 @@ func TestStoreLifecycle(t *testing.T) { require.NoError(t, err) require.Equal(t, r, loaded) - info, err := os.Stat(s.StatePath()) + info, err := os.Stat(s.statePath()) require.NoError(t, err) require.Equal(t, os.FileMode(0o600), info.Mode().Perm()) require.NoError(t, s.MarkComplete(r)) @@ -54,7 +54,7 @@ func TestOwnershipAdmission(t *testing.T) { r := r r.Checkpoint = checkpoint - disposition, err := Decide(r, nil, r.MachineName, r.ConfigFingerprint) + disposition, err := decide(r, nil, r.MachineName, r.ConfigFingerprint) if checkpoint == Resetting { require.Error(t, err) return @@ -69,14 +69,14 @@ func TestOwnershipAdmission(t *testing.T) { require.Equal(t, want, disposition) - _, err = Decide(r, nil, "other", r.ConfigFingerprint) + _, err = decide(r, nil, "other", r.ConfigFingerprint) require.Error(t, err) - _, err = Decide(r, nil, r.MachineName, "other") + _, err = decide(r, nil, r.MachineName, "other") require.Error(t, err) }) } - _, err = Decide(Record{}, ErrNotFound, "", "") + _, err = decide(Record{}, ErrNotFound, "", "") require.Error(t, err) } @@ -87,10 +87,10 @@ func TestStoreRejectsCorruptAndOrphanedOwnership(t *testing.T) { t.Run(data, func(t *testing.T) { s := testStore(t) require.NoError(t, os.MkdirAll(s.Root(), 0o755)) - require.NoError(t, os.WriteFile(s.StatePath(), []byte(data), 0o600)) + require.NoError(t, os.WriteFile(s.statePath(), []byte(data), 0o600)) r, err := s.Load() require.Error(t, err) - _, err = Decide(r, err, "machine", "f") + _, err = decide(r, err, "machine", "f") require.Error(t, err) }) } From f01a5074651ae1051bdf446fadee9d6782c4440c Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Wed, 16 Sep 2026 17:38:32 -0400 Subject: [PATCH 08/39] agent: stop exporting symbols with no cross-package callers Sweeping the exported symbols this branch adds turned up several with no caller outside their own package, including two in the public pkg/ tree. CheckOwnedBindAddress was exported so the command layer could substitute checkers by name. That substitution is gone now that nodestart.Preflight is unconditionally ownership-aware, leaving Preflight as the only caller. CheckBindAddress became unreachable at the same time, so remove it and rename the three tests that were named after it; they exercise the checker type directly rather than the constructor. daemon.ResetAgentResources had no production caller either: the node operator calls resetUnderLock directly because the MachineOperation already holds the lock. Only a test still referenced it, so drop it and have that test compose resetResources itself. The identically named nodeOperator interface method is unaffected. Also unexport fsutil.WriteFile, which only WriteFileDurable uses, and the installstate schemaVersion and defaultHostPrefix constants. The exported Record.SchemaVersion field stays because the JSON format needs it. --- cmd/agent/internal/cmd/bootstrap_test.go | 2 +- cmd/agent/internal/daemon/reset.go | 11 +++-------- cmd/agent/internal/daemon/reset_test.go | 4 ++-- cmd/agent/internal/installstate/store.go | 12 ++++++------ internal/fsutil/fsutil.go | 6 +++--- .../phases/nodestart/preflight_api_server.go | 4 ++-- .../phases/nodestart/preflight_bind_address.go | 17 ++--------------- .../nodestart/preflight_bind_address_test.go | 6 +++--- 8 files changed, 22 insertions(+), 40 deletions(-) diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index 2649939af..944e3388e 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -26,7 +26,7 @@ import ( // A later release must still admit an installation created by an earlier one // when given the same original input, so a mismatch here is a compatibility // break rather than a fixture to refresh. Record.Validate pins these files to -// installstate.SchemaVersion, so bumping it fails loudly; a new schema version +// the package's schema version, so bumping it fails loudly; a new schema version // gets its own fixture directory rather than regenerated files. func TestBootstrapV1CompatibilityFixtures(t *testing.T) { dir := filepath.Join("testdata", "bootstrap-v1") diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index 326a3b677..15aed474b 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -22,14 +22,9 @@ import ( "github.com/Azure/unbounded/pkg/agent/phases/reset" ) -// ResetAgentResources returns a task that removes the unbounded-agent and all -// associated resources without stopping the daemon process. -func ResetAgentResources(log *slog.Logger) phases.Task { - return ownedReset(log, installstate.DefaultStore(), resetResources(log)) -} - -// ResetAgent additionally stops the daemon first. The daemon's own operation -// path stops it last, so that ordering stays with the caller. +// ResetAgent removes the unbounded-agent and all associated resources, stopping +// the daemon first. The daemon's own operation path stops it last instead, so +// that ordering stays with the caller. func ResetAgent(log *slog.Logger) phases.Task { return ownedReset(log, installstate.DefaultStore(), phases.Serial(log, StopDaemon(log), resetResources(log))) } diff --git a/cmd/agent/internal/daemon/reset_test.go b/cmd/agent/internal/daemon/reset_test.go index 7d5498545..2b4467539 100644 --- a/cmd/agent/internal/daemon/reset_test.go +++ b/cmd/agent/internal/daemon/reset_test.go @@ -17,10 +17,10 @@ import ( "github.com/Azure/unbounded/cmd/agent/internal/installstate" ) -func TestResetAgentResourcesIncludesBPFFSMountCleanup(t *testing.T) { +func TestResetResourcesIncludesBPFFSMountCleanup(t *testing.T) { t.Parallel() - taskName := ResetAgentResources(slog.New(slog.DiscardHandler)).Name() + taskName := resetResources(slog.New(slog.DiscardHandler)).Name() assert.Contains(t, taskName, "parallel(remove-bpffs-mount, remove-bpffs-mount)") assert.Less(t, strings.Index(taskName, "parallel(remove-machine, remove-machine)"), strings.Index(taskName, "parallel(remove-bpffs-mount, remove-bpffs-mount)")) diff --git a/cmd/agent/internal/installstate/store.go b/cmd/agent/internal/installstate/store.go index 1814a46fc..03921d47e 100644 --- a/cmd/agent/internal/installstate/store.go +++ b/cmd/agent/internal/installstate/store.go @@ -20,8 +20,8 @@ import ( const ( DefaultDirectory = "/var/lib/unbounded/agent" defaultLockPath = "/run/unbounded-agent-install.lock" - DefaultHostPrefix = "/usr/local" - SchemaVersion = 1 + defaultHostPrefix = "/usr/local" + schemaVersion = 1 ) type Checkpoint string @@ -47,12 +47,12 @@ type Record struct { } func (r Record) Validate() error { - if r.SchemaVersion != SchemaVersion || strings.TrimSpace(r.InstallID) == "" || strings.TrimSpace(r.MachineName) == "" || strings.TrimSpace(r.ConfigFingerprint) == "" { + if r.SchemaVersion != schemaVersion || strings.TrimSpace(r.InstallID) == "" || strings.TrimSpace(r.MachineName) == "" || strings.TrimSpace(r.ConfigFingerprint) == "" { return fmt.Errorf("invalid installation record identity or schema") } // This release installs only at the default prefix. Unknown ownership must // not authorize cleanup at a guessed location. - if r.HostPrefix != DefaultHostPrefix { + if r.HostPrefix != defaultHostPrefix { return fmt.Errorf("unsupported recorded host prefix %q", r.HostPrefix) } @@ -135,8 +135,8 @@ func NewRecord(machine, fingerprint string) (Record, error) { } return Record{ - SchemaVersion: SchemaVersion, InstallID: hex.EncodeToString(id), MachineName: machine, - HostPrefix: DefaultHostPrefix, ConfigFingerprint: fingerprint, Checkpoint: PreparingHost, + SchemaVersion: schemaVersion, InstallID: hex.EncodeToString(id), MachineName: machine, + HostPrefix: defaultHostPrefix, ConfigFingerprint: fingerprint, Checkpoint: PreparingHost, }, nil } diff --git a/internal/fsutil/fsutil.go b/internal/fsutil/fsutil.go index 89bde9cb7..59cdd6702 100644 --- a/internal/fsutil/fsutil.go +++ b/internal/fsutil/fsutil.go @@ -45,7 +45,7 @@ func WriteFileDurable(path string, data []byte, mode os.FileMode) error { parents = append(parents, dir) } - if err := WriteFile(path, data, mode); err != nil { + if err := writeFile(path, data, mode); err != nil { return err } @@ -58,10 +58,10 @@ func WriteFileDurable(path string, data []byte, mode os.FileMode) error { return nil } -// WriteFile writes content atomically, creating parent directories as needed. +// writeFile writes content atomically, creating parent directories as needed. // The temporary file shares the destination directory so it inherits the // correct SELinux label instead of the temp-directory label. -func WriteFile(path string, data []byte, mode os.FileMode) error { +func writeFile(path string, data []byte, mode os.FileMode) error { if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { return err } diff --git a/pkg/agent/phases/nodestart/preflight_api_server.go b/pkg/agent/phases/nodestart/preflight_api_server.go index 11a2410a8..9992a01fe 100644 --- a/pkg/agent/phases/nodestart/preflight_api_server.go +++ b/pkg/agent/phases/nodestart/preflight_api_server.go @@ -41,8 +41,8 @@ func Preflight(log *slog.Logger, cfg config.AgentConfig, goalState *goalstates.M return []preflight.Checker{ // TODO: Consider moving the kubelet bind address to the kubelet goal state. - CheckOwnedBindAddress(log, checkKubeletBindAddressName, kubeletBindAddress, "kubelet bind address", root, kubeletExecutablePath), - CheckOwnedBindAddress( + checkOwnedBindAddress(log, checkKubeletBindAddressName, kubeletBindAddress, "kubelet bind address", root, kubeletExecutablePath), + checkOwnedBindAddress( log, checkContainerdMetricsBindAddressName, goalState.NodeStart.Containerd.MetricsAddress, diff --git a/pkg/agent/phases/nodestart/preflight_bind_address.go b/pkg/agent/phases/nodestart/preflight_bind_address.go index dd704e03c..d04670973 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address.go @@ -40,9 +40,9 @@ type bindAddressChecker struct { owned func() bool } -// CheckOwnedBindAddress accepts only listeners with both the expected process +// checkOwnedBindAddress accepts only listeners with both the expected process // root and executable inode. An uninspectable owner is not proof of ownership. -func CheckOwnedBindAddress(log *slog.Logger, name, address, description, root, executable string) preflight.Checker { +func checkOwnedBindAddress(log *slog.Logger, name, address, description, root, executable string) preflight.Checker { c := bindAddressChecker{ name: name, address: address, description: description, log: log, inspect: func(address string) (string, bool, error) { return inspectTCPListener("/proc", address) }, @@ -107,19 +107,6 @@ func listenerOwnedByRoot(procRoot, address, root, executable string) bool { return len(wanted) > 0 && len(matched) == len(wanted) } -// CheckBindAddress verifies no TCP listener currently occupies an address's port. -func CheckBindAddress(log *slog.Logger, name, address, description string) preflight.Checker { - return bindAddressChecker{ - name: name, - address: address, - description: description, - log: log, - inspect: func(address string) (string, bool, error) { - return inspectTCPListener("/proc", address) - }, - } -} - func (c bindAddressChecker) Name() string { return c.name } func (c bindAddressChecker) Check(context.Context) []preflight.Result { diff --git a/pkg/agent/phases/nodestart/preflight_bind_address_test.go b/pkg/agent/phases/nodestart/preflight_bind_address_test.go index a0b2b1357..d54bcd0c6 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address_test.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address_test.go @@ -52,7 +52,7 @@ func TestPreflightBindAddressesWithoutResolvedRootFS(t *testing.T) { assert.False(t, checks[0].(bindAddressChecker).owned()) } -func TestCheckBindAddressAvailable(t *testing.T) { +func TestBindAddressCheckerReportsAvailable(t *testing.T) { checker := testBindAddressChecker(func(string) (string, bool, error) { return "", false, nil }) results := checker.Check(context.Background()) @@ -61,7 +61,7 @@ func TestCheckBindAddressAvailable(t *testing.T) { assert.Equal(t, "kubelet bind address is available", results[0].Message) } -func TestCheckBindAddressInUseIncludesOwner(t *testing.T) { +func TestBindAddressCheckerReportsForeignOwner(t *testing.T) { checker := testBindAddressChecker(func(string) (string, bool, error) { return `"kubelet" (PID 123)`, true, nil }) @@ -72,7 +72,7 @@ func TestCheckBindAddressInUseIncludesOwner(t *testing.T) { assert.Equal(t, `kubelet bind address is already in use by process "kubelet" (PID 123)`, results[0].Message) } -func TestCheckBindAddressInspectionFailure(t *testing.T) { +func TestBindAddressCheckerReportsInspectionFailure(t *testing.T) { checker := testBindAddressChecker(func(string) (string, bool, error) { return "", false, errors.New("inspection failed") }) From 2e186affcb2100e05900bc647ad349013bc36daa Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:22:04 -0400 Subject: [PATCH 09/39] agent: fix installation identity and applied config on retry Two problems with what a retry is allowed to change and what it records. An HTTPS rootfs reference can carry an expiring signed query, so hashing it whole made a refreshed signature look like a different installation and stranded the retry. Hash the scheme, host and path instead, which is what actually selects the artifact; the reference is also trailing-slash normalized to match how it gets resolved before being fetched. The applied config was persisted in the daemon stage, so a retry that resumed there recorded the new attempt's configuration even though the node had already been started from the old one. Drift is measured against that file, so the node kept the superseded labels, taints and kubelet configuration with nothing left to reconcile them. Persist it in the stage that starts the node, after kubelet bootstraps, and sync the directory holding it before that stage is checkpointed. The stage composition is now asserted by name so the placement cannot regress silently, and the recovery e2e retries with a changed label and requires the recorded config to stay untouched. --- cmd/agent/internal/cmd/bootstrap.go | 56 +++++++++++++-- cmd/agent/internal/cmd/bootstrap_test.go | 86 ++++++++++++++++++++++++ hack/agent/e2e-kind/e2e.py | 57 +++++++++++++--- hack/agent/e2e-kind/test_reliability.py | 27 ++++++++ 4 files changed, 211 insertions(+), 15 deletions(-) diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index aef90b460..4999c0fd3 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -8,6 +8,7 @@ import ( "encoding/json" "fmt" "log/slog" + "net/url" "strings" "github.com/Azure/unbounded/cmd/agent/internal/attest" @@ -33,6 +34,34 @@ type agentStages struct { credentialsReady bool } +// canonicalImageIdentity reduces an OCI image reference to the part that +// determines which image gets installed. HTTPS archive references carry +// expiring signed query parameters, so a refreshed signature points at the same +// artifact and must not read as a different installation. Registry and +// oci-layout references carry no such credentials and are used as-is. +// +// Trailing path slashes are trimmed to match how parseHTTPSArchiveReference +// normalizes the reference before fetching it. +func canonicalImageIdentity(image string) string { + if !strings.HasPrefix(image, "https://") { + return image + } + + parsed, err := url.Parse(image) + if err != nil { + // Unparseable references fail later at acquire time with a better + // message. Hash the original so identity stays deterministic. + return image + } + + parsed.RawQuery = "" + parsed.ForceQuery = false + parsed.Path = strings.TrimRight(parsed.Path, "/") + parsed.RawPath = strings.TrimRight(parsed.RawPath, "/") + + return parsed.String() +} + func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity, error) { // Keep identity tied to the cluster and installed rootfs, while allowing // credentials and artifact locations to be refreshed for a retry. @@ -40,7 +69,7 @@ func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity, KubernetesVersion string OCIImage string APIServer string - }{strings.TrimPrefix(cfg.Cluster.Version, "v"), cfg.OCIImage, cfg.Kubelet.ApiServer}) + }{strings.TrimPrefix(cfg.Cluster.Version, "v"), canonicalImageIdentity(cfg.OCIImage), cfg.Kubelet.ApiServer}) if err != nil { return bootstrap.Identity{}, err } @@ -130,16 +159,35 @@ func (s *agentStages) PrepareRootFS(ctx context.Context) error { return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, "/usr/local", goalstates.SystemdSystemDir, goalstates.SystemdNSpawnDir) } +// nodeStartTask composes the work that brings the node up. Persisting the +// applied config belongs here, not in the daemon stage: it must record the +// configuration that actually configured the node. A retry that resumes at a +// later checkpoint skips this stage entirely, so it cannot overwrite the record +// with a configuration the running node never saw. +func (s *agentStages) nodeStartTask() phases.Task { + return phases.Serial(s.log, + nodestart.StartNode(s.log, s.gs.NodeStart), + nodestart.WaitForKubeletBootstrap(s.log, s.gs.NodeStart.MachineName), + daemon.PersistAppliedConfig(s.log, s.gs.NodeStart.MachineName, &s.cfg.AgentConfig), + ) +} + func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { if err := s.prepareCredentials(ctx); err != nil { return err } - if err := phases.Serial(s.log, nodestart.StartNode(s.log, s.gs.NodeStart), nodestart.WaitForKubeletBootstrap(s.log, "kube1")).Do(ctx); err != nil { + if err := s.nodeStartTask().Do(ctx); err != nil { return err } - return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, goalstates.SystemdSystemDir) + // AgentConfigDir holds the applied config written above, so it must reach + // disk before this stage is checkpointed as complete. + return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, goalstates.AgentConfigDir, goalstates.SystemdSystemDir) +} + +func (s *agentStages) daemonInstallTask() phases.Task { + return phases.Serial(s.log, daemon.EnableDaemon(s.log)) } func (s *agentStages) EnsureDaemonInstalled(ctx context.Context) error { @@ -147,7 +195,7 @@ func (s *agentStages) EnsureDaemonInstalled(ctx context.Context) error { return err } - if err := phases.Serial(s.log, daemon.PersistAppliedConfig(s.log, "kube1", &s.cfg.AgentConfig), daemon.EnableDaemon(s.log)).Do(ctx); err != nil { + if err := s.daemonInstallTask().Do(ctx); err != nil { return err } diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index 944e3388e..45d10266d 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -9,12 +9,14 @@ import ( "log/slog" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/require" "github.com/Azure/unbounded/cmd/agent/internal/installstate" "github.com/Azure/unbounded/internal/provision" + "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/preflight" ) @@ -105,6 +107,90 @@ func TestBootstrapFingerprintAllowsCredentialAndDownloadRefresh(t *testing.T) { } } +func TestBootstrapFingerprintIgnoresSignedImageQuery(t *testing.T) { + t.Parallel() + + // A signed archive URL carries an expiring signature. Refreshing it points + // at the same artifact, so a retry must stay the same installation rather + // than being rejected as a different one. + const base = "https://artifacts.example.test/node/rootfs.oci.tar.gz" + + cfg, err := loadConfigFromFile(filepath.Join("testdata", "bootstrap-v1", "input.json")) + require.NoError(t, err) + + cfg.OCIImage = base + "?sp=r&sv=2022-11-02&sig=first-signature" + original, err := bootstrapIdentity(cfg) + require.NoError(t, err) + + for _, equivalent := range []string{ + base + "?sp=r&sv=2022-11-02&sig=second-signature", + base + "?", + base + "/", + base, + } { + cfg.OCIImage = equivalent + + refreshed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.Equal(t, original.ConfigFingerprint, refreshed.ConfigFingerprint, "reference %q", equivalent) + } + + // The path and host still identify the artifact, so they must not be + // collapsed away with the credential. + for _, different := range []string{ + "https://artifacts.example.test/node/other-rootfs.oci.tar.gz", + "https://other-host.example.test/node/rootfs.oci.tar.gz", + } { + cfg.OCIImage = different + + changed, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.NotEqual(t, original.ConfigFingerprint, changed.ConfigFingerprint, "reference %q", different) + } +} + +func TestCanonicalImageIdentityLeavesNonHTTPSReferencesAlone(t *testing.T) { + t.Parallel() + + for _, image := range []string{ + "example.test/unbounded/node:v1.33.1", + "example.test/unbounded/node@sha256:0000000000000000000000000000000000000000000000000000000000000000", + "oci-layout:///var/lib/unbounded/layouts/node", + "", + } { + require.Equal(t, image, canonicalImageIdentity(image)) + } + + // An unparseable HTTPS reference is rejected later with a better message. + // Identity just has to stay deterministic rather than panic. + const malformed = "https://artifacts.example.test/\x7f" + require.Equal(t, malformed, canonicalImageIdentity(malformed)) +} + +// TestNodeStartPersistsAppliedConfig pins where the applied config is written. +// It has to happen in the stage that starts the node, and after kubelet has +// bootstrapped, so the record always describes the configuration the running +// node was built from. Moving it into the daemon stage would let a retry that +// resumes there record a configuration the node never saw, which then reads as +// "no drift" and is never reconciled. +func TestNodeStartPersistsAppliedConfig(t *testing.T) { + t.Parallel() + + stages := &agentStages{ + log: slog.New(slog.DiscardHandler), + cfg: &provision.UnboundedAgentConfig{}, + gs: &goalstates.MachineGoalState{ + NodeStart: &goalstates.NodeStart{MachineName: goalstates.NSpawnMachineKube1}, + }, + } + + nodeStart := stages.nodeStartTask().Name() + require.Contains(t, nodeStart, "persist-applied-config") + require.Less(t, strings.Index(nodeStart, "wait-for-kubelet-bootstrap"), strings.Index(nodeStart, "persist-applied-config")) + + require.NotContains(t, stages.daemonInstallTask().Name(), "persist-applied-config") +} + func TestCompletedPreflightOutput(t *testing.T) { t.Parallel() diff --git a/hack/agent/e2e-kind/e2e.py b/hack/agent/e2e-kind/e2e.py index 61f20b46d..bcd210b7f 100755 --- a/hack/agent/e2e-kind/e2e.py +++ b/hack/agent/e2e-kind/e2e.py @@ -837,11 +837,8 @@ def node_config_bootstrap_args(node_config: NodeConfig) -> list[str]: return args -def inject_kubelet_configuration(bootstrap_script: str, node_config: NodeConfig) -> str: - """Inject a scenario's kubelet configuration into generated agent config JSON.""" - if not node_config.kubelet_configuration: - return bootstrap_script - +def patch_agent_config(bootstrap_script: str, mutate: Callable[[dict], None]) -> str: + """Rewrite the agent config JSON embedded in a generated bootstrap script.""" start_marker = "cat > \"${UNBOUNDED_AGENT_CONFIG_FILE}\" <<'AGENT_CONFIG_EOF'\n" end_marker = "\nAGENT_CONFIG_EOF" prefix, separator, remainder = bootstrap_script.partition(start_marker) @@ -854,14 +851,22 @@ def inject_kubelet_configuration(bootstrap_script: str, node_config: NodeConfig) try: agent_config = json.loads(agent_config_json) - kubelet = agent_config["Kubelet"] + mutate(agent_config) except (KeyError, TypeError, json.JSONDecodeError) as exc: die(f"generated bootstrap script contains invalid agent config: {exc}") - kubelet["Configuration"] = node_config.kubelet_configuration - rendered_config = json.dumps(agent_config, indent=2) + return prefix + start_marker + json.dumps(agent_config, indent=2) + end_marker + suffix + + +def inject_kubelet_configuration(bootstrap_script: str, node_config: NodeConfig) -> str: + """Inject a scenario's kubelet configuration into generated agent config JSON.""" + if not node_config.kubelet_configuration: + return bootstrap_script + + def set_kubelet_configuration(agent_config: dict) -> None: + agent_config["Kubelet"]["Configuration"] = node_config.kubelet_configuration - return prefix + start_marker + rendered_config + end_marker + suffix + return patch_agent_config(bootstrap_script, set_kubelet_configuration) def log_active_node_config(node_config: NodeConfig) -> None: @@ -2608,9 +2613,33 @@ def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: before = json.loads(state_text) if before["checkpoint"] != "installing-daemon" or not pid.isdigit() or int(pid) <= 0: die(f"failure did not reach the late bootstrap checkpoint: {snapshot}") + + # The applied config records what actually configured the running node. + # Retry with a changed node label: admission still allows it, because + # labels are deliberately outside the installation fingerprint, but the + # node was started before the change and never saw it. Re-persisting it + # here would read as "no drift" forever after, so the record must not + # move while the retry resumes past the node stage. + applied_config = "/etc/unbounded/agent/kube1-applied-config.json" + before_applied = bounded_ssh( + f"sudo sha256sum {applied_config}", time.monotonic() + 30, check=True).stdout.split()[0] + + def add_retry_label(agent_config: dict) -> None: + agent_config["Kubelet"].setdefault("Labels", {})["e2e.unbounded.test/retry"] = "changed" + + retry_script = patch_agent_config(bootstrap_script, add_retry_label) + if retry_script == bootstrap_script: + die("failed to change a node label for the bootstrap retry") + + retry_script_path = VM_DIR / "bootstrap-retry.sh" + retry_script_path.write_text(retry_script) + retry_script_path.chmod(0o600) + scp_cmd(str(retry_script_path), f"{SSH_TARGET}:/tmp/bootstrap-retry.sh") + ssh_cmd("chmod +x /tmp/bootstrap-retry.sh") + ssh_cmd("sudo rmdir /usr/local/bin/unbounded-agent-daemon-recovery.sh") run(["timeout", "1200", "ssh", *SSH_OPTS, SSH_TARGET, - f"sudo {env_prefix} /tmp/bootstrap.sh"]) + f"sudo {env_prefix} /tmp/bootstrap-retry.sh"]) after_text = bounded_ssh( "sudo cat /var/lib/unbounded/agent/install-state.json; " "systemctl show systemd-nspawn@kube1.service --property=MainPID --value", @@ -2619,7 +2648,13 @@ def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: after = json.loads(state_text) if after["installID"] != before["installID"] or after["checkpoint"] != "complete" or after_pid != pid: die("bootstrap retry changed ownership or restarted the running node") - log("Late bootstrap retry preserved installation and nspawn PID") + + after_applied = bounded_ssh( + f"sudo sha256sum {applied_config}", time.monotonic() + 30, check=True).stdout.split()[0] + if after_applied != before_applied: + die("bootstrap retry overwrote the applied config with a label the running node never saw") + + log("Late bootstrap retry preserved installation, nspawn PID and applied config") return run([ "timeout", "1200", diff --git a/hack/agent/e2e-kind/test_reliability.py b/hack/agent/e2e-kind/test_reliability.py index 828be8fab..7e8ac798c 100644 --- a/hack/agent/e2e-kind/test_reliability.py +++ b/hack/agent/e2e-kind/test_reliability.py @@ -44,6 +44,33 @@ def test_repair_script_is_valid_shell_and_embedded_python(self): python = script.split("python3 - <<'PY'\n", 1)[1].split("\nPY\n", 1)[0] compile(python, "repair-fixture", "exec") + def test_agent_config_patch_changes_only_the_embedded_config(self): + script = ( + "#!/bin/bash\nset -eu\n" + "cat > \"${UNBOUNDED_AGENT_CONFIG_FILE}\" <<'AGENT_CONFIG_EOF'\n" + + json.dumps({"Kubelet": {"ApiServer": "https://api.test", "Labels": {"keep": "yes"}}}, indent=2) + + "\nAGENT_CONFIG_EOF\n\"${AGENT_BIN}\" start\n" + ) + + def add_label(config): + config["Kubelet"].setdefault("Labels", {})["e2e.unbounded.test/retry"] = "changed" + + patched = e2e.patch_agent_config(script, add_label) + self.assertNotEqual(patched, script) + self.assertTrue(patched.startswith("#!/bin/bash\nset -eu\n")) + self.assertTrue(patched.endswith("\nAGENT_CONFIG_EOF\n\"${AGENT_BIN}\" start\n")) + + config = json.loads(patched.split("<<'AGENT_CONFIG_EOF'\n", 1)[1].split("\nAGENT_CONFIG_EOF", 1)[0]) + self.assertEqual(config["Kubelet"]["Labels"], {"keep": "yes", "e2e.unbounded.test/retry": "changed"}) + self.assertEqual(config["Kubelet"]["ApiServer"], "https://api.test") + + def test_agent_config_patch_fails_on_malformed_script(self): + for script in ("#!/bin/bash\ntrue\n", + "cat > \"${UNBOUNDED_AGENT_CONFIG_FILE}\" <<'AGENT_CONFIG_EOF'\n{}", + "cat > \"${UNBOUNDED_AGENT_CONFIG_FILE}\" <<'AGENT_CONFIG_EOF'\nnot-json\nAGENT_CONFIG_EOF\n"): + with self.assertRaises(SystemExit): + e2e.patch_agent_config(script, lambda config: None) + def test_recovery_mode_is_scoped_to_attempt(self): cfg = e2e.NodeConfig(name="test", node_labels={}, register_with_taints=[]) with patch.dict(os.environ, {}, clear=True), patch.object(e2e, "run_agent", side_effect=RuntimeError("injected")): From f9830e489bb24f63b20d13543a3835239eb0dbe8 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Thu, 17 Sep 2026 11:27:58 -0400 Subject: [PATCH 10/39] agent: make reset and repair fail loudly instead of silently Three places reported success while leaving the host in a state that contradicts the report. Dropping the ownership record is itself a durable step. The record was unlinked and the error from the directory sync returned, so a failure left the removal in page cache only: reset told the operator it failed, while the next start was admitted as a fresh install onto a host that was only partially torn down. Restore the record when the removal cannot be made durable, so a failed reset always leaves the host visibly owned. The bootstrap binary was considered present whenever anything existed at its path, including a dangling symlink or one aimed at a non-executable. That is precisely the state that sends a completed install into repair, and repair cannot replace a bad link either, so the host was stranded. Resolve the link and require a regular executable. Ignoring every machinectl disable failure could leave the nspawn unit enabled. The enablement symlink outlives the config and rootfs that reset deletes, so the host would try to start a machine that no longer exists on the next boot. A failed disable is now tolerated only when systemd positively reports a state that cannot start the unit, and an inspection that does not answer counts as unconfirmed. --- cmd/agent/internal/daemon/lifecycle.go | 23 ++++-- cmd/agent/internal/daemon/lifecycle_test.go | 45 ++++++++++++ cmd/agent/internal/installstate/store.go | 32 ++++++++- cmd/agent/internal/installstate/store_test.go | 52 ++++++++++++++ pkg/agent/phases/reset/machine.go | 42 +++++++++-- pkg/agent/phases/reset/strict_test.go | 70 +++++++++++++++++++ 6 files changed, 249 insertions(+), 15 deletions(-) diff --git a/cmd/agent/internal/daemon/lifecycle.go b/cmd/agent/internal/daemon/lifecycle.go index 9a95bf86f..07f958a80 100644 --- a/cmd/agent/internal/daemon/lifecycle.go +++ b/cmd/agent/internal/daemon/lifecycle.go @@ -109,14 +109,13 @@ func (d *enableDaemon) Do(ctx context.Context) error { return nil } -// InstallBootstrapBinary installs the staged bootstrap executable if the host -// has no daemon binary yet. The caller holds installation ownership; existing -// binary layouts are retained and upgrades use their normal activation path. +// InstallBootstrapBinary installs the staged bootstrap executable unless the +// host already has a usable daemon binary. The caller holds installation +// ownership; existing binary layouts are retained and upgrades use their normal +// activation path. func InstallBootstrapBinary() error { - if _, err := os.Lstat(goalstates.DaemonBinaryPath); err == nil { + if usableDaemonBinary(goalstates.DaemonBinaryPath) { return nil - } else if !errors.Is(err, os.ErrNotExist) { - return err } source, err := os.Executable() @@ -127,6 +126,18 @@ func InstallBootstrapBinary() error { return fsutil.InstallFile(source, goalstates.DaemonBinaryPath, 0o755) } +// usableDaemonBinary resolves symlinks on purpose. The healthy layout reaches +// the active slot through a symlink chain, so only the target tells us whether +// the host can actually run the daemon. A dangling link, or one aimed at +// something that is not an executable file, is exactly the state that sends a +// completed install into repair, and repair cannot replace a bad link either. +// Treating the link's mere presence as a usable binary would strand the host. +func usableDaemonBinary(path string) bool { + info, err := os.Stat(path) + + return err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 +} + func renderDaemonAsset(name string, content []byte) ([]byte, error) { paths, err := goalstates.ResolvedAgentUpgradePaths() if err != nil { diff --git a/cmd/agent/internal/daemon/lifecycle_test.go b/cmd/agent/internal/daemon/lifecycle_test.go index 703fa1801..c9b98204a 100644 --- a/cmd/agent/internal/daemon/lifecycle_test.go +++ b/cmd/agent/internal/daemon/lifecycle_test.go @@ -56,3 +56,48 @@ func TestInstallBinaryStreamsAndReplacesAtomically(t *testing.T) { require.NoError(t, err) require.Equal(t, "candidate", string(data)) } + +// TestUsableDaemonBinaryRequiresAResolvableExecutable pins what counts as "the +// host already has a daemon binary". Only the resolved target matters: a broken +// or non-executable link is the state that sends a completed install into +// repair, and repair cannot replace the link, so it must not be mistaken for a +// working installation. +func TestUsableDaemonBinaryRequiresAResolvableExecutable(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + executable := filepath.Join(dir, "executable") + require.NoError(t, os.WriteFile(executable, []byte("binary"), 0o755)) + + plain := filepath.Join(dir, "plain") + require.NoError(t, os.WriteFile(plain, []byte("data"), 0o644)) + + // The production layout reaches the active slot through a symlink chain. + current := filepath.Join(dir, "current") + require.NoError(t, os.Symlink(executable, current)) + + chained := filepath.Join(dir, "chained") + require.NoError(t, os.Symlink(current, chained)) + + dangling := filepath.Join(dir, "dangling") + require.NoError(t, os.Symlink(filepath.Join(dir, "absent"), dangling)) + + toPlain := filepath.Join(dir, "to-plain") + require.NoError(t, os.Symlink(plain, toPlain)) + + directory := filepath.Join(dir, "directory") + require.NoError(t, os.Mkdir(directory, 0o755)) + + for path, want := range map[string]bool{ + executable: true, + current: true, + chained: true, + dangling: false, + toPlain: false, + plain: false, + directory: false, + filepath.Join(dir, "absent"): false, + } { + assert.Equal(t, want, usableDaemonBinary(path), "path %s", path) + } +} diff --git a/cmd/agent/internal/installstate/store.go b/cmd/agent/internal/installstate/store.go index 03921d47e..b008d01b1 100644 --- a/cmd/agent/internal/installstate/store.go +++ b/cmd/agent/internal/installstate/store.go @@ -66,9 +66,17 @@ func (r Record) Validate() error { var ErrNotFound = errors.New("installation record not found") -type Store struct{ root, lockPath string } +type Store struct { + root, lockPath string + // syncDir is a seam for exercising the window where the record is unlinked + // but the directory entry has not reached disk. + syncDir func(string) error +} + +func NewStore(root, lockPath string) *Store { + return &Store{root: root, lockPath: lockPath, syncDir: fsutil.SyncDir} +} -func NewStore(root, lockPath string) *Store { return &Store{root: root, lockPath: lockPath} } func DefaultStore() *Store { return NewStore(DefaultDirectory, defaultLockPath) } func (s *Store) Root() string { return s.root } func (s *Store) statePath() string { return filepath.Join(s.root, "install-state.json") } @@ -114,6 +122,12 @@ func (s *Store) MarkComplete(r Record) error { } // Remove is called only after teardown's filesystem barriers succeed. +// +// Dropping ownership is itself a durable step. If the unlink cannot be flushed, +// the record is put back, because a reset that reports failure must leave the +// host visibly owned. Otherwise the removal survives in page cache only, the +// error sends the operator away, and the next start is admitted as a fresh +// install onto a half-torn-down host. func (s *Store) Remove() error { if _, err := os.Stat(s.root); errors.Is(err, os.ErrNotExist) { return nil @@ -121,11 +135,23 @@ func (s *Store) Remove() error { return err } + // Read what is about to be dropped so ownership can be restored below. A + // record that does not load is not restored: it granted no usable ownership + // and admission rejects it either way. + previous, loadErr := s.Load() + if err := os.Remove(s.statePath()); err != nil && !errors.Is(err, os.ErrNotExist) { return err } - return fsutil.SyncDir(s.root) + err := s.syncDir(s.root) + if err != nil && loadErr == nil { + if saveErr := s.Save(previous); saveErr != nil { + return errors.Join(err, saveErr) + } + } + + return err } func NewRecord(machine, fingerprint string) (Record, error) { diff --git a/cmd/agent/internal/installstate/store_test.go b/cmd/agent/internal/installstate/store_test.go index c404667fb..5eb73e8d9 100644 --- a/cmd/agent/internal/installstate/store_test.go +++ b/cmd/agent/internal/installstate/store_test.go @@ -4,6 +4,7 @@ package installstate import ( + "errors" "os" "path/filepath" "testing" @@ -116,6 +117,57 @@ func TestInstallationLockSurvivesStateRemoval(t *testing.T) { require.NoError(t, second.Release()) } +// TestRemoveRestoresOwnershipWhenUndurable covers the window where the record +// is unlinked but the directory entry never reaches disk. Reporting the error +// while leaving the removal in page cache would let the next start be admitted +// as a fresh install onto a host that was only partially torn down. +func TestRemoveRestoresOwnershipWhenUndurable(t *testing.T) { + t.Parallel() + + s := testStore(t) + r, err := NewRecord("machine", "f") + require.NoError(t, err) + + r.Checkpoint = Resetting + require.NoError(t, s.Save(r)) + + failure := errors.New("sync failed") + s.syncDir = func(string) error { return failure } + + require.ErrorIs(t, s.Remove(), failure) + + restored, err := s.Load() + require.NoError(t, err) + require.Equal(t, r, restored) + + // Reset stays incomplete, so a retry is refused rather than admitted fresh. + _, _, err = Admit(s, r.MachineName, r.ConfigFingerprint) + require.Error(t, err) + + // Once the removal can be made durable, ownership is released. + s.syncDir = func(string) error { return nil } + require.NoError(t, s.Remove()) + _, err = s.Load() + require.ErrorIs(t, err, ErrNotFound) +} + +// A record that cannot be loaded grants no usable ownership, so a failed +// removal reports the error without resurrecting it. +func TestRemoveDoesNotRestoreUnusableOwnership(t *testing.T) { + t.Parallel() + + s := testStore(t) + require.NoError(t, os.MkdirAll(s.Root(), 0o755)) + require.NoError(t, os.WriteFile(s.statePath(), []byte("{"), 0o600)) + + failure := errors.New("sync failed") + s.syncDir = func(string) error { return failure } + + require.ErrorIs(t, s.Remove(), failure) + _, err := s.Load() + require.ErrorIs(t, err, ErrNotFound) +} + func TestMutationAdmission(t *testing.T) { t.Parallel() diff --git a/pkg/agent/phases/reset/machine.go b/pkg/agent/phases/reset/machine.go index c2ca627cc..1f003682d 100644 --- a/pkg/agent/phases/reset/machine.go +++ b/pkg/agent/phases/reset/machine.go @@ -30,8 +30,17 @@ func StopMachine(log *slog.Logger, machineName string) phases.Task { func (t *stopMachine) Name() string { return "stop-machine" } func (t *stopMachine) Do(ctx context.Context) error { + // Stop the systemd service that manages the nspawn container. This + // properly tears down mount namespaces and cgroups so that + // machinectl remove can succeed. + serviceName := fmt.Sprintf("systemd-nspawn@%s.service", t.machineName) + if err := executil.RunCmd(ctx, t.log, executil.Machinectl(), "disable", t.machineName); err != nil { - t.log.Warn("failed to disable machine; continuing with stop and removal", "machine", t.machineName, "error", err) + if confirmErr := confirmNotEnabled(ctx, t.log, serviceName); confirmErr != nil { + return fmt.Errorf("disable machine %s: %w", t.machineName, errors.Join(err, confirmErr)) + } + + t.log.Warn("machine was not enabled; continuing with stop and removal", "machine", t.machineName, "error", err) } exists, err := RegisteredMachine(ctx, t.log, t.machineName) @@ -46,11 +55,6 @@ func (t *stopMachine) Do(ctx context.Context) error { t.log.Info("stopping nspawn machine", "machine", t.machineName) - // Stop the systemd service that manages the nspawn container. This - // properly tears down mount namespaces and cgroups so that - // machinectl remove can succeed. - serviceName := fmt.Sprintf("systemd-nspawn@%s.service", t.machineName) - if !serviceIsActive(ctx, t.log, serviceName) { t.log.Info("nspawn service already inactive, skipping stop", "service", serviceName) } else if err := executil.RunCmd(ctx, t.log, executil.Systemctl(), "stop", serviceName); err != nil { @@ -85,6 +89,32 @@ func (t *stopMachine) Do(ctx context.Context) error { return ctx.Err() } +// confirmNotEnabled reports whether the nspawn unit is definitely not enabled. +// +// machinectl disable fails for benign reasons, most often a machine that was +// never enabled, so the failure alone does not justify aborting reset. It is +// not safe to simply ignore either: the enablement symlink outlives the config +// and rootfs that reset deletes, so a unit left enabled makes the host try to +// start a machine that no longer exists on the next boot, while reset reported +// success. Continue only when systemd positively reports a state that cannot +// start the unit at boot, and treat a failed inspection as unconfirmed. +func confirmNotEnabled(ctx context.Context, log *slog.Logger, service string) error { + // show exits zero whatever the state, unlike is-enabled, so the reported + // state is the signal rather than the exit status. + out, err := executil.OutputCmd(ctx, log, "systemctl", "show", service, "--property=UnitFileState", "--value") + if err != nil { + return fmt.Errorf("inspect %s enablement: %w", service, err) + } + + // An empty state means no unit file is installed, so nothing is enabled. + switch state := strings.TrimSpace(out); state { + case "", "disabled", "not-found", "masked", "masked-runtime", "static", "indirect": + return nil + default: + return fmt.Errorf("%s is %s", service, state) + } +} + // waitForGone polls machineExists until the machine disappears or the timeout // elapses. Returns true if the machine is gone. func (t *stopMachine) waitForGone(ctx context.Context, timeout time.Duration) (bool, error) { diff --git a/pkg/agent/phases/reset/strict_test.go b/pkg/agent/phases/reset/strict_test.go index 1bb545a18..34fd9a59c 100644 --- a/pkg/agent/phases/reset/strict_test.go +++ b/pkg/agent/phases/reset/strict_test.go @@ -41,6 +41,76 @@ func TestMachineInspectionFailsClosed(t *testing.T) { } } +// TestConfirmNotEnabledOnlyAcceptsUnstartableStates pins which reported unit +// file states let reset continue after machinectl disable fails. Anything that +// could still start the unit at boot, or an inspection that did not answer, has +// to stop reset rather than leave an enablement symlink behind for a machine +// whose config and rootfs are about to be deleted. +func TestConfirmNotEnabledOnlyAcceptsUnstartableStates(t *testing.T) { + for _, tc := range []struct { + name, script string + confirmed bool + }{ + {"never-enabled", "printf 'disabled\\n'", true}, + {"no-unit-file", "printf '\\n'", true}, + {"not-found", "printf 'not-found\\n'", true}, + {"masked", "printf 'masked\\n'", true}, + {"static", "printf 'static\\n'", true}, + {"indirect", "printf 'indirect\\n'", true}, + {"enabled", "printf 'enabled\\n'", false}, + {"enabled-runtime", "printf 'enabled-runtime\\n'", false}, + {"linked", "printf 'linked\\n'", false}, + {"generated", "printf 'generated\\n'", false}, + {"inspection-failed", "exit 1", false}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), []byte("#!/bin/sh\n"+tc.script+"\n"), 0o755)) + t.Setenv("PATH", dir) + + err := confirmNotEnabled(t.Context(), slog.New(slog.DiscardHandler), "systemd-nspawn@kube1.service") + if tc.confirmed { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } +} + +// TestStopMachineFailsWhenDisableLeavesUnitEnabled covers the whole task: a +// failed disable is tolerated only once systemd confirms the unit cannot start +// at boot. +func TestStopMachineFailsWhenDisableLeavesUnitEnabled(t *testing.T) { + for _, tc := range []struct { + name, disable, unitFileState string + wantErr bool + }{ + {"disable-succeeds", "exit 0", "enabled", false}, + {"disable-fails-but-not-enabled", "exit 1", "disabled", false}, + {"disable-fails-and-still-enabled", "exit 1", "enabled", true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + // list reports no registered machines, so a tolerated failure falls + // through to "nothing to stop" and the task succeeds. + machinectl := "#!/bin/sh\ncase \"$1\" in\ndisable) " + tc.disable + " ;;\nlist) exit 0 ;;\nesac\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "machinectl"), []byte(machinectl), 0o755)) + + systemctl := "#!/bin/sh\nif [ \"$1\" = show ]; then printf '" + tc.unitFileState + "\\n'; fi\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), []byte(systemctl), 0o755)) + t.Setenv("PATH", dir) + + err := StopMachine(slog.New(slog.DiscardHandler), "kube1").Do(t.Context()) + if tc.wantErr { + require.ErrorContains(t, err, "systemd-nspawn@kube1.service is enabled") + } else { + require.NoError(t, err) + } + }) + } +} + func TestCleanupRoutesPropagatesInspectionAndMutationFailures(t *testing.T) { t.Parallel() From 4927c61119839fafc6d53a86f8ceca3062422256 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Thu, 17 Sep 2026 14:21:47 -0400 Subject: [PATCH 11/39] agent: drop the unused host prefix from the ownership record The record carried a host prefix that nothing in this change reads. It was written on every install and then rejected by validation unless it held the one value this release can produce, purely to reserve the field for the configurable-prefix work that follows. Recording something we refuse is a confusing contract to ship, and the follow-up does not need it reserved: an absent prefix already means the default, so a record written here reads correctly once the field exists. Validation still refuses a record whose schema version it does not know, which is the same protection by a mechanism that is already there. Nothing else changes. The fingerprint never covered the prefix, so installation identity is unaffected and the compatibility fixtures keep their recorded value. --- cmd/agent/internal/cmd/bootstrap_test.go | 1 - .../cmd/testdata/bootstrap-v1/complete.json | 1 - .../bootstrap-v1/preparing-rootfs.json | 1 - .../cmd/testdata/bootstrap-v1/resetting.json | 1 - cmd/agent/internal/installstate/store.go | 23 ++++++------------- cmd/agent/internal/installstate/store_test.go | 2 +- 6 files changed, 8 insertions(+), 21 deletions(-) diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index 45d10266d..87c20f9ad 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -48,7 +48,6 @@ func TestBootstrapV1CompatibilityFixtures(t *testing.T) { require.NoError(t, record.Validate()) require.Equal(t, id.MachineName, record.MachineName) require.Equal(t, id.ConfigFingerprint, record.ConfigFingerprint) - require.Equal(t, "/usr/local", record.HostPrefix) // Admit through a store so the fixture also proves it survives a // load round-trip, not just an in-memory classification. diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json index 93abe4ca7..f57e1a33c 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json @@ -2,7 +2,6 @@ "schemaVersion": 1, "installID": "00112233445566778899aabbccddeeff", "machineName": "bootstrap-fixture", - "hostPrefix": "/usr/local", "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", "checkpoint": "complete" } diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json index 0893eedea..864dcd6c3 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json @@ -2,7 +2,6 @@ "schemaVersion": 1, "installID": "00112233445566778899aabbccddeeff", "machineName": "bootstrap-fixture", - "hostPrefix": "/usr/local", "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", "checkpoint": "preparing-rootfs" } diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json index 36abab04d..1c3345eb8 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json @@ -2,7 +2,6 @@ "schemaVersion": 1, "installID": "00112233445566778899aabbccddeeff", "machineName": "bootstrap-fixture", - "hostPrefix": "/usr/local", "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", "checkpoint": "resetting" } diff --git a/cmd/agent/internal/installstate/store.go b/cmd/agent/internal/installstate/store.go index b008d01b1..504c465be 100644 --- a/cmd/agent/internal/installstate/store.go +++ b/cmd/agent/internal/installstate/store.go @@ -18,10 +18,9 @@ import ( ) const ( - DefaultDirectory = "/var/lib/unbounded/agent" - defaultLockPath = "/run/unbounded-agent-install.lock" - defaultHostPrefix = "/usr/local" - schemaVersion = 1 + DefaultDirectory = "/var/lib/unbounded/agent" + defaultLockPath = "/run/unbounded-agent-install.lock" + schemaVersion = 1 ) type Checkpoint string @@ -36,12 +35,9 @@ const ( ) type Record struct { - SchemaVersion int `json:"schemaVersion"` - InstallID string `json:"installID"` - MachineName string `json:"machineName"` - // Store the resolved default now, so later configurable-prefix support can - // consume records from this release without changing their meaning. - HostPrefix string `json:"hostPrefix"` + SchemaVersion int `json:"schemaVersion"` + InstallID string `json:"installID"` + MachineName string `json:"machineName"` ConfigFingerprint string `json:"configFingerprint"` Checkpoint Checkpoint `json:"checkpoint"` } @@ -50,11 +46,6 @@ func (r Record) Validate() error { if r.SchemaVersion != schemaVersion || strings.TrimSpace(r.InstallID) == "" || strings.TrimSpace(r.MachineName) == "" || strings.TrimSpace(r.ConfigFingerprint) == "" { return fmt.Errorf("invalid installation record identity or schema") } - // This release installs only at the default prefix. Unknown ownership must - // not authorize cleanup at a guessed location. - if r.HostPrefix != defaultHostPrefix { - return fmt.Errorf("unsupported recorded host prefix %q", r.HostPrefix) - } switch r.Checkpoint { case PreparingHost, PreparingRootFS, StartingNode, InstallingDaemon, Complete, Resetting: @@ -162,7 +153,7 @@ func NewRecord(machine, fingerprint string) (Record, error) { return Record{ SchemaVersion: schemaVersion, InstallID: hex.EncodeToString(id), MachineName: machine, - HostPrefix: defaultHostPrefix, ConfigFingerprint: fingerprint, Checkpoint: PreparingHost, + ConfigFingerprint: fingerprint, Checkpoint: PreparingHost, }, nil } diff --git a/cmd/agent/internal/installstate/store_test.go b/cmd/agent/internal/installstate/store_test.go index 5eb73e8d9..350ae6ced 100644 --- a/cmd/agent/internal/installstate/store_test.go +++ b/cmd/agent/internal/installstate/store_test.go @@ -84,7 +84,7 @@ func TestOwnershipAdmission(t *testing.T) { func TestStoreRejectsCorruptAndOrphanedOwnership(t *testing.T) { t.Parallel() - for _, data := range []string{"{", "null", `{}`, `{"schemaVersion":2}`, `{"schemaVersion":1,"installID":"id","machineName":"machine","configFingerprint":"f","hostPrefix":"/opt/unbounded","checkpoint":"complete"}`} { + for _, data := range []string{"{", "null", `{}`, `{"schemaVersion":2}`, `{"schemaVersion":1,"installID":"id","machineName":"machine","configFingerprint":"f","checkpoint":"bogus"}`} { t.Run(data, func(t *testing.T) { s := testStore(t) require.NoError(t, os.MkdirAll(s.Root(), 0o755)) From 1c82c9632681dc3e0b072c1ced5f7be71c9d411d Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Thu, 17 Sep 2026 16:29:37 -0400 Subject: [PATCH 12/39] agent: keep the installer placing the agent binary The installer stopped writing /usr/local/bin/unbounded-agent and delegated it to InstallBootstrapBinary, which this same change introduces inside start. That coupled two components that are versioned independently. The agent version comes from AGENT_VERSION, from AGENT_URL, or from the default of tracking the latest published release, while the installer is embedded in and served by metalman. Any agent older than this change therefore never writes the binary, and bootstrap fails at enable-daemon with "no executable agent binary found for daemon link initialization", naming neither the installer nor the version skew. That is not a corner case. A Machine that sets no agent URL or version, the documented default, downloads the latest release, so every existing user breaks the moment a metalman carrying this installer is deployed, and stays broken until a release containing it exists. The metalman layered HTTP smoke test has been failing on this branch since its first commit for exactly this reason: it supplies its own cloud-init user data without the install environment, so it exercises the released agent. Seed the binary again, but only when the path holds nothing usable. The test follows symlinks, so a host this installation already owns resolves through the compatibility symlink to a live blue-green slot and is skipped, which keeps the property the original change wanted: admission still runs from the staged executable and a retry cannot overwrite a live link before its intent has been accepted. A dangling link resolves to nothing and is replaced, matching how the daemon decides whether an existing binary is usable, since install would otherwise write through it to a stale location. The uninstall script already removed this path, so the two are symmetric again. The existing script test only asserted that the download overrides are honored, which is why the install could disappear unnoticed; it now pins both the install and the guard. --- .../assets/unbounded-agent-install.sh | 19 +++++++++++++++++++ internal/provision/script_test.go | 15 +++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/internal/provision/assets/unbounded-agent-install.sh b/internal/provision/assets/unbounded-agent-install.sh index c3ba638e3..3aca2fe58 100644 --- a/internal/provision/assets/unbounded-agent-install.sh +++ b/internal/provision/assets/unbounded-agent-install.sh @@ -79,6 +79,25 @@ curl -fsSL "${AGENT_URL}" | tar -xz -C "${tmp_dir}" unbounded-agent AGENT_BIN="${tmp_dir}/unbounded-agent" chmod 0755 "${AGENT_BIN}" +# Seed the daemon binary path when nothing usable is there yet. The agent +# version is selected independently of this script - by AGENT_VERSION, by +# AGENT_URL, or by the default of tracking the latest published release - so an +# installer that relied on the agent to install its own binary would silently +# break every agent released before that behavior existed. Such an agent never +# writes the binary, and bootstrap then fails at daemon setup with no indication +# that the installer and the agent disagree. +# +# The test follows symlinks on purpose. On a host this installation already owns +# the path resolves through the compatibility symlink to a live blue-green slot, +# so it is left untouched and admission still runs from the staged executable +# above. A dangling link resolves to nothing and is replaced, because install +# would otherwise write through it to a stale location. +AGENT_BIN_TARGET="/usr/local/bin/unbounded-agent" +if [ ! -x "${AGENT_BIN_TARGET}" ]; then + rm -f "${AGENT_BIN_TARGET}" + install -m 0755 "${AGENT_BIN}" "${AGENT_BIN_TARGET}" +fi + _START_ARGS="" case "${AGENT_DEBUG}" in 1|true|yes|TRUE|YES|True|Yes) _START_ARGS="--debug" ;; diff --git a/internal/provision/script_test.go b/internal/provision/script_test.go index b16568c03..548669ab6 100644 --- a/internal/provision/script_test.go +++ b/internal/provision/script_test.go @@ -44,6 +44,21 @@ func TestUnboundedAgentInstallScript(t *testing.T) { require.Contains(t, script, "Running unbounded-agent preflight") require.Contains(t, script, "preflight ${_START_ARGS}") require.Contains(t, script, "0|false|no|FALSE|NO|False|No") + + // The installer must place the agent binary itself. The agent version is + // selected independently of this script, including the default of tracking + // the latest published release, so an installer that relies on the agent to + // install its own binary breaks every agent released before that behavior + // existed. The uninstall script removes this same path. + require.Contains(t, script, `AGENT_BIN_TARGET="/usr/local/bin/unbounded-agent"`) + require.Contains(t, script, `install -m 0755 "${AGENT_BIN}" "${AGENT_BIN_TARGET}"`) + + // It must not clobber a live binary. The test follows symlinks so a host + // this installation already owns resolves through the compatibility symlink + // to a live slot and is skipped, which keeps admission running from the + // staged executable rather than one the retry just wrote. + require.Contains(t, script, `if [ ! -x "${AGENT_BIN_TARGET}" ]; then`) + require.Contains(t, script, `AGENT_BIN="${tmp_dir}/unbounded-agent"`) } func TestUnboundedAgentUninstallScript(t *testing.T) { From b32be689ab3b07989533f8ea03c859b230d1cbe3 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:28:18 -0400 Subject: [PATCH 13/39] agent: document why the bootstrap reporter is built late Review asked why the status reporter is optional and created on demand rather than held from construction. It cannot move: the reporter captures credentials when it is built. An empty bootstrap token makes it a permanent no-op, and it registers the Machine over the API as part of construction. On an attested host the token does not exist until ApplyAttestation has run, so building it in the constructor would silently disable status reporting for the whole bootstrap and issue the registration call before admission. Say so at the construction site so the next reader does not have to work it out. The nil checks around the reporting calls were doing nothing, though. The reporter already reports through a nil-receiver check, so the two guarded call sites now call it directly. That property is what the callers depend on now, so it is pinned by a test rather than left incidental. The coordinator keeps its checks: its reporter is an interface, which cannot absorb a call when nil, and its tests supply nil deliberately. --- cmd/agent/internal/cmd/bootstrap.go | 25 +++++++++++-------- cmd/agent/internal/cmd/start.go | 6 ++--- .../internal/daemon/bootstrap_status_test.go | 18 +++++++++++++ 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index 4999c0fd3..5d5434cea 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -124,6 +124,12 @@ func (s *agentStages) prepareCredentials(ctx context.Context) error { syncAttestedKubeletConfig(&s.cfg.AgentConfig, s.gs.NodeStart) + // The reporter is built here rather than in the constructor because it + // captures credentials at construction: an empty bootstrap token makes it a + // permanent no-op, and it registers the Machine over the API. On an attested + // host the token does not exist until ApplyAttestation has run just above, + // so constructing it earlier would silently disable status reporting for the + // whole bootstrap and issue the registration call before admission. if s.reporter == nil { s.reporter = daemon.NewBootstrapStatusReporter(ctx, s.log, &s.cfg.AgentConfig) s.reporter.Running(ctx) @@ -213,16 +219,15 @@ func (s *agentStages) StageStarted(_ context.Context, stage installstate.Checkpo } func (s *agentStages) StageFailed(ctx context.Context, stage installstate.Checkpoint, err error) { - if s.reporter != nil { - reason := "Failed" - if stage == installstate.PreparingRootFS { - reason = "RootFSFailed" - } - - if stage == installstate.StartingNode { - reason = classifyNodeStartFailure(err) - } + reason := "Failed" + if stage == installstate.PreparingRootFS { + reason = "RootFSFailed" + } - s.reporter.Failed(ctx, reason, err) + if stage == installstate.StartingNode { + reason = classifyNodeStartFailure(err) } + + // Safe before the reporter exists: it reports through a nil-receiver check. + s.reporter.Failed(ctx, reason, err) } diff --git a/cmd/agent/internal/cmd/start.go b/cmd/agent/internal/cmd/start.go index db097a540..6ea49e5e2 100644 --- a/cmd/agent/internal/cmd/start.go +++ b/cmd/agent/internal/cmd/start.go @@ -61,9 +61,9 @@ func newCmdStart(cmdCtx *CommandContext) *cobra.Command { log.Info("installation already complete") } - if stages.reporter != nil { - stages.reporter.Succeeded(ctx) - } + // Safe when bootstrap never reached credential setup: the reporter + // reports through a nil-receiver check. + stages.reporter.Succeeded(ctx) return nil }, diff --git a/cmd/agent/internal/daemon/bootstrap_status_test.go b/cmd/agent/internal/daemon/bootstrap_status_test.go index 284125231..1e87fe777 100644 --- a/cmd/agent/internal/daemon/bootstrap_status_test.go +++ b/cmd/agent/internal/daemon/bootstrap_status_test.go @@ -109,3 +109,21 @@ func findMachineCondition(t *testing.T, conditions []metav1.Condition, condition return metav1.Condition{} } + +// TestBootstrapStatusReporter_NilIsSafe pins that every reporting entry point +// tolerates a nil receiver. +// +// Bootstrap builds the reporter only once attestation has supplied credentials, +// so stages that run before that, and a failure on one of them, report through a +// nil reporter. The callers rely on this instead of guarding each call, so it is +// a contract rather than an accident. +func TestBootstrapStatusReporter_NilIsSafe(t *testing.T) { + t.Parallel() + + var reporter *BootstrapStatusReporter + + assert.NotPanics(t, func() { reporter.Running(context.Background()) }) + assert.NotPanics(t, func() { reporter.Failed(context.Background(), "Failed", errors.New("boom")) }) + assert.NotPanics(t, func() { reporter.Failed(context.Background(), "Failed", nil) }) + assert.NotPanics(t, func() { reporter.Succeeded(context.Background()) }) +} From 6627c6ea2d1fdb5175ee35eb160c4db20b1078d3 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 10:33:32 -0400 Subject: [PATCH 14/39] agent: take the installation lock from gofrs/flock Review suggested using a maintained library rather than a hand-rolled flock wrapper. It is a good trade here: the module is already in the dependency graph, pulled in indirectly by the OCI SDK, so this promotes an existing entry to a direct requirement rather than adding one. go.sum is unchanged. The swap preserves the behavior the callers depend on. The library opens the lock file with the same 0600 mode, takes the same non-blocking exclusive flock, and documents that closing does not remove the file, which is the contract reset relies on: a leftover lock file must never be read as a held lock. TryLock reports contention as a false return rather than EWOULDBLOCK, so that is mapped onto the existing sentinel error, and the parent directory is still created here because the library does not do it. Behavior gained: the library reopens and retries on a stale file handle, which the previous implementation treated as a hard failure. NOTICE is regenerated, and gains a BSD 3-Clause entry for the new direct dependency. That is a licensing surface change riding in this PR rather than an incidental one, so it is called out here. --- NOTICE | 8 ++++++ cmd/agent/internal/installstate/lock.go | 34 ++++++++++--------------- go.mod | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/NOTICE b/NOTICE index 31f10c3f9..422af2004 100644 --- a/NOTICE +++ b/NOTICE @@ -350,6 +350,14 @@ notices: license: - name: Apache License, Version 2.0 link: https://github.com/go-logr/logr/blob/v1.4.4/LICENSE + - dependency: github.com/gofrs/flock + ecosystem: go + copyright: + - Copyright (c) 2018-2024, The Gofrs + - Copyright (c) 2015-2020, Tim Heckman + license: + - name: BSD 3-Clause License + link: https://github.com/gofrs/flock/blob/v0.10.0/LICENSE - dependency: github.com/golang-jwt/jwt/v5 ecosystem: go copyright: diff --git a/cmd/agent/internal/installstate/lock.go b/cmd/agent/internal/installstate/lock.go index 2d0b0cdf0..b9db8458a 100644 --- a/cmd/agent/internal/installstate/lock.go +++ b/cmd/agent/internal/installstate/lock.go @@ -8,45 +8,37 @@ import ( "os" "path/filepath" - "golang.org/x/sys/unix" + "github.com/gofrs/flock" ) var ErrLockHeld = errors.New("another host lifecycle operation holds the installation lock") -type Lock struct{ file *os.File } +type Lock struct{ flock *flock.Flock } -// acquireLockAt is nonblocking. The kernel releases flock on process exit; a +// acquireLockAt is nonblocking. The kernel releases the lock on process exit; a // leftover lock file does not imply a held lock and must not be deleted by reset. func acquireLockAt(path string) (*Lock, error) { + // flock.New does not create the parent directory. if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { return nil, err } - f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o600) - if err != nil { - return nil, err - } - - if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { - closeErr := f.Close() + l := flock.New(path) - if errors.Is(err, unix.EWOULDBLOCK) { - return nil, errors.Join(ErrLockHeld, closeErr) - } - - return nil, errors.Join(err, closeErr) + switch locked, err := l.TryLock(); { + case err != nil: + return nil, err + case !locked: + return nil, ErrLockHeld } - return &Lock{file: f}, nil + return &Lock{flock: l}, nil } func (l *Lock) Release() error { - if l == nil || l.file == nil { + if l == nil || l.flock == nil { return nil } - err := l.file.Close() - l.file = nil - - return err + return l.flock.Unlock() } diff --git a/go.mod b/go.mod index d57493a48..351bda326 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/fatih/color v1.19.0 github.com/fsnotify/fsnotify v1.10.1 github.com/go-logr/logr v1.4.4 + github.com/gofrs/flock v0.10.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/cel-go v0.29.2 github.com/google/go-github/v75 v75.0.0 @@ -172,7 +173,6 @@ require ( github.com/go-openapi/swag/typeutils v0.27.1 // indirect github.com/go-openapi/swag/yamlutils v0.27.1 // indirect github.com/gobuffalo/flect v1.0.3 // indirect - github.com/gofrs/flock v0.10.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/google/btree v1.1.3 // indirect github.com/google/gnostic-models v0.7.0 // indirect From 34971d3443a1e32ae23d97a663e8f2de9b041a4d Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:30:55 -0400 Subject: [PATCH 15/39] agent: do not flush nftables while a node is running ConfigureNFTables starts nftables-flush.service, whose unit applies `flush ruleset` and erases every nftables rule on the host. That is the point on a fresh host, and it is safe at boot because the unit is ordered before the nspawn machine and unbounded-localdns-network.service re-adds its NOTRACK table after it. Starting it imperatively mid-run is a different thing. The nspawn container shares the host network namespace, so a running node's kube-proxy and CNI rules are in the ruleset being erased, along with LocalDNS's table. Nothing puts LocalDNS back: systemd ordering only sequences units within a single transaction, so starting this unit alone does not pull in the LocalDNS unit, and that unit otherwise runs only when the machine starts. kube-proxy resyncs on its own; LocalDNS does not, so it stays broken until the machine restarts. Only start the unit when no machine is registered. The flush exists to hand a clean slate to a node that has not started yet, so once one is registered it has already served its purpose. Installing and enabling the unit is unchanged, so the next boot still gets its clean slate in the right order. Inspection failure is not read as "nothing registered", since that would flush a ruleset a running node may depend on. This makes the task safe to re-run against any host state, which is a prerequisite for a retry that reapplies work rather than trusting a record of what was already done. --- pkg/agent/phases/host/configure_nftables.go | 69 ++++++++++++++++++- .../phases/host/configure_nftables_test.go | 61 ++++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 pkg/agent/phases/host/configure_nftables_test.go diff --git a/pkg/agent/phases/host/configure_nftables.go b/pkg/agent/phases/host/configure_nftables.go index f2a8231a6..6ada72b93 100644 --- a/pkg/agent/phases/host/configure_nftables.go +++ b/pkg/agent/phases/host/configure_nftables.go @@ -16,6 +16,7 @@ import ( "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/internal/utilio" "github.com/Azure/unbounded/pkg/agent/phases" + "github.com/Azure/unbounded/pkg/agent/phases/reset" ) const ( @@ -35,6 +36,9 @@ var nftablesClearRules []byte type configureNFTables struct { log *slog.Logger + // machineRegistered reports whether any nspawn machine is registered, and + // is injectable so the replay behavior can be tested without a host. + machineRegistered func(context.Context, *slog.Logger) (bool, error) } // ConfigureNFTables returns a task that installs a oneshot systemd unit which @@ -42,7 +46,24 @@ type configureNFTables struct { // This ensures stale rules (e.g. left behind by Docker) do not interfere with // Kubernetes networking. func ConfigureNFTables(log *slog.Logger) phases.Task { - return &configureNFTables{log: log} + return &configureNFTables{log: log, machineRegistered: anyMachineRegistered} +} + +// anyMachineRegistered reports whether either nspawn slot is registered. +// It fails closed: an uninspectable host is not reported as having no machine. +func anyMachineRegistered(ctx context.Context, log *slog.Logger) (bool, error) { + for _, name := range []string{goalstates.NSpawnMachineKube1, goalstates.NSpawnMachineKube2} { + registered, err := reset.RegisteredMachine(ctx, log, name) + if err != nil { + return false, err + } + + if registered { + return true, nil + } + } + + return false, nil } func (c *configureNFTables) Name() string { return "configure-nftables" } @@ -94,9 +115,55 @@ func (c *configureNFTables) ensureNFTablesFlushUnit(ctx context.Context) error { return fmt.Errorf("systemctl enable %s: %w", nftablesFlushUnit, err) } + // Starting the unit applies `flush ruleset`, which erases every nftables + // rule on the host. See shouldStartFlush for why that is conditional. + start, err := c.shouldStartFlush(ctx) + if err != nil { + return err + } + + if !start { + return nil + } + if err := executil.RunCmd(ctx, c.log, systemctl, "start", nftablesFlushUnit); err != nil { return fmt.Errorf("systemctl start %s: %w", nftablesFlushUnit, err) } return nil } + +// shouldStartFlush reports whether the flush unit may be started now. +// +// The flush erases every nftables rule on the host. That is the point on a +// fresh host, and it is safe at boot because the unit is ordered before the +// nspawn machine and LocalDNS re-adds its table after it. +// +// Starting it here is a different thing entirely. The nspawn container shares +// the host network namespace, so a running node's kube-proxy and CNI rules live +// in the ruleset being erased, and LocalDNS's NOTRACK table goes with them. +// Nothing puts it back: systemd ordering only sequences units within a single +// transaction, so starting this unit alone does not pull in +// unbounded-localdns-network.service, and that unit otherwise runs only when +// the machine starts. kube-proxy resyncs on its own; LocalDNS does not. +// +// The flush exists to hand a clean slate to a node that has not started yet. +// Once one is registered it has already served that purpose, so the live +// ruleset is left alone. The unit stays installed and enabled either way, so +// the next boot still gets its clean slate in the correct order. +func (c *configureNFTables) shouldStartFlush(ctx context.Context) (bool, error) { + registered, err := c.machineRegistered(ctx, c.log) + if err != nil { + return false, fmt.Errorf("inspect registered machines before flushing nftables: %w", err) + } + + if registered { + c.log.Info("nspawn machine is registered; leaving the live nftables ruleset alone", + "unit", nftablesFlushUnit, + ) + + return false, nil + } + + return true, nil +} diff --git a/pkg/agent/phases/host/configure_nftables_test.go b/pkg/agent/phases/host/configure_nftables_test.go new file mode 100644 index 000000000..90fd9684e --- /dev/null +++ b/pkg/agent/phases/host/configure_nftables_test.go @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package host + +import ( + "context" + "errors" + "log/slog" + "testing" + + "github.com/stretchr/testify/require" +) + +func nftablesTask(registered bool, err error) *configureNFTables { + return &configureNFTables{ + log: slog.New(slog.DiscardHandler), + machineRegistered: func(context.Context, *slog.Logger) (bool, error) { + return registered, err + }, + } +} + +// TestFlushNotStartedWhileMachineRegistered is the replay guard. +// +// Starting the flush unit applies `flush ruleset`, and the nspawn container +// shares the host network namespace, so on a host with a running node that +// erases kube-proxy, CNI and LocalDNS rules at once. Only kube-proxy resyncs; +// LocalDNS's NOTRACK table is reinstated by a unit that runs when the machine +// starts, and systemd ordering does not pull that unit in when this one is +// started on its own. +func TestFlushNotStartedWhileMachineRegistered(t *testing.T) { + t.Parallel() + + start, err := nftablesTask(true, nil).shouldStartFlush(t.Context()) + require.NoError(t, err) + require.False(t, start, "must not flush a ruleset a running node depends on") +} + +// TestFlushStartedOnFreshHost keeps the original behavior where it is correct: +// no machine is registered, so no running node can lose rules. +func TestFlushStartedOnFreshHost(t *testing.T) { + t.Parallel() + + start, err := nftablesTask(false, nil).shouldStartFlush(t.Context()) + require.NoError(t, err) + require.True(t, start) +} + +// TestFlushFailsClosedOnUninspectableHost keeps an inspection failure from +// being read as "nothing registered", which would flush a ruleset that may +// belong to a running node. +func TestFlushFailsClosedOnUninspectableHost(t *testing.T) { + t.Parallel() + + injected := errors.New("machinectl unavailable") + + start, err := nftablesTask(false, injected).shouldStartFlush(t.Context()) + require.ErrorIs(t, err, injected) + require.False(t, start) +} From d508d8ea00498365ae311669076c1ebeec917a24 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:38:30 -0400 Subject: [PATCH 16/39] agent: restart node services whose configuration actually changed Reapplying node configuration writes files that a running service has already read. When this sequence boots the machine that is harmless, because the services start afterwards and read the new files. When it runs against a machine that is already up it is not: the files on disk and the running services would disagree with nothing to reconcile them, which is worse than not reapplying at all. Track whether each configuration file actually differed, and restart the service that reads it only when something did and the machine was already running. An identical reapply, which is the ordinary case when bootstrap is rerun after a failure, leaves the node completely alone. WriteFileIfChanged reports whether it had to write. Only content is compared: WriteFile preserves an existing file's permissions rather than resetting them, so a drifted mode cannot be corrected there, and reporting it as a change would restart the reader on every call forever. containerd restarts before kubelet, since kubelet talks to it and would otherwise just retry against a runtime that is coming back up. The exported ConfigureContainerd and ConfigureKubelet keep their signatures. StartNode builds the tasks concretely instead, because it is the only place that knows both whether the configuration changed and whether the machine was already running. This makes the node-start sequence safe to reapply against a live node, which is a prerequisite for a retry that reapplies work rather than trusting a record of what was already done. --- pkg/agent/internal/utilio/io.go | 26 ++++ pkg/agent/internal/utilio/io_test.go | 43 +++++++ pkg/agent/phases/nodestart/cri.go | 23 +++- pkg/agent/phases/nodestart/kubelet.go | 29 ++++- pkg/agent/phases/nodestart/kubelet_test.go | 37 ++++++ pkg/agent/phases/nodestart/nspawn.go | 10 ++ .../phases/nodestart/restart_reconfigured.go | 75 ++++++++++++ .../nodestart/restart_reconfigured_test.go | 112 ++++++++++++++++++ pkg/agent/phases/nodestart/start.go | 28 ++++- 9 files changed, 371 insertions(+), 12 deletions(-) create mode 100644 pkg/agent/phases/nodestart/restart_reconfigured.go create mode 100644 pkg/agent/phases/nodestart/restart_reconfigured_test.go diff --git a/pkg/agent/internal/utilio/io.go b/pkg/agent/internal/utilio/io.go index b32862b67..abc6341c1 100644 --- a/pkg/agent/internal/utilio/io.go +++ b/pkg/agent/internal/utilio/io.go @@ -4,6 +4,7 @@ package utilio import ( + "bytes" "errors" "fmt" "io" @@ -76,3 +77,28 @@ func WriteFile(filename string, content []byte, perm os.FileMode) error { return renameio.WriteFile(filename, content, perm, renameio.WithTempDir(filepath.Dir(filename))) } + +// WriteFileIfChanged writes content like WriteFile and reports whether the +// content differed beforehand. +// +// It exists so a caller that reapplies configuration can tell an actual change +// from a no-op. Writing unconditionally would be correct on disk but would make +// every reapply look like a change, so callers that act on one, such as +// restarting the service that reads the file, would act every time. +// +// Only content is compared. WriteFile preserves an existing file's permissions +// rather than resetting them to perm, so a drifted mode cannot be corrected +// here; reporting it as changed would make this return true on every call and +// restart the reader forever. perm therefore applies only when the file is +// being created. +func WriteFileIfChanged(filename string, content []byte, perm os.FileMode) (bool, error) { + if existing, err := os.ReadFile(filename); err == nil && bytes.Equal(existing, content) { + return false, nil + } + + if err := WriteFile(filename, content, perm); err != nil { + return false, err + } + + return true, nil +} diff --git a/pkg/agent/internal/utilio/io_test.go b/pkg/agent/internal/utilio/io_test.go index 21db44ca0..59e40feec 100644 --- a/pkg/agent/internal/utilio/io_test.go +++ b/pkg/agent/internal/utilio/io_test.go @@ -5,6 +5,8 @@ package utilio import ( "math" + "os" + "path/filepath" "strings" "testing" ) @@ -17,3 +19,44 @@ func TestInstallFileWithLimitedSizeRejectsOverflowingLimit(t *testing.T) { t.Fatal("InstallFileWithLimitedSize error = nil") } } + +// TestWriteFileIfChanged covers the signal callers act on. A reapply that +// changes nothing must report false, or every reapply would look like a change +// and anything keyed on it, such as restarting a service, would fire each time. +func TestWriteFileIfChanged(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "nested", "config") + + changed, err := WriteFileIfChanged(path, []byte("one"), 0o644) + if err != nil || !changed { + t.Fatalf("first write: changed=%v err=%v, want changed=true", changed, err) + } + + changed, err = WriteFileIfChanged(path, []byte("one"), 0o644) + if err != nil || changed { + t.Fatalf("identical rewrite: changed=%v err=%v, want changed=false", changed, err) + } + + changed, err = WriteFileIfChanged(path, []byte("two"), 0o644) + if err != nil || !changed { + t.Fatalf("content change: changed=%v err=%v, want changed=true", changed, err) + } + + data, err := os.ReadFile(path) + if err != nil || string(data) != "two" { + t.Fatalf("content = %q err=%v, want \"two\"", data, err) + } + + // A drifted mode is deliberately not reported as changed. WriteFile + // preserves existing permissions, so it could not be corrected here, and + // reporting it would restart the file's reader on every call forever. + if err := os.Chmod(path, 0o600); err != nil { + t.Fatalf("chmod: %v", err) + } + + changed, err = WriteFileIfChanged(path, []byte("two"), 0o644) + if err != nil || changed { + t.Fatalf("mode drift: changed=%v err=%v, want changed=false", changed, err) + } +} diff --git a/pkg/agent/phases/nodestart/cri.go b/pkg/agent/phases/nodestart/cri.go index df72490a5..9df1cec2d 100644 --- a/pkg/agent/phases/nodestart/cri.go +++ b/pkg/agent/phases/nodestart/cri.go @@ -37,6 +37,23 @@ const ( type configureContainerd struct { goalState *goalstates.NodeStart + + // changed records whether any file this task owns actually differed. The + // node's containerd reads these at start, so a reapply that alters one has + // to restart it; a reapply that alters nothing must not. + changed bool +} + +// write applies content and records whether it differed from what was there. +func (c *configureContainerd) write(path string, content []byte, perm os.FileMode) error { + changed, err := utilio.WriteFileIfChanged(path, content, perm) + if err != nil { + return err + } + + c.changed = c.changed || changed + + return nil } // ConfigureContainerd returns a task that writes the containerd configuration, systemd unit, @@ -88,7 +105,7 @@ func (c *configureContainerd) ensureContainerdConfig() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.ContainerdConfigPath) - return utilio.WriteFile(dest, buf.Bytes(), 0o644) + return c.write(dest, buf.Bytes(), 0o644) } func (c *configureContainerd) ensureGantryHostsConfig() error { @@ -109,7 +126,7 @@ func (c *configureContainerd) ensureGantryHostsConfig() error { return err } - return utilio.WriteFile(dest, []byte(gantryHostsConfig), 0o644) + return c.write(dest, []byte(gantryHostsConfig), 0o644) } func hasGantryHostsManagedMarker(content []byte) bool { @@ -136,7 +153,7 @@ func (c *configureContainerd) ensureContainerdServiceUnit() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.SystemdSystemDir, goalstates.SystemdUnitContainerd) - return utilio.WriteFile(dest, buf.Bytes(), 0o644) + return c.write(dest, buf.Bytes(), 0o644) } // ensureGPUDropInConfigs manages GPU-related containerd drop-in configs. diff --git a/pkg/agent/phases/nodestart/kubelet.go b/pkg/agent/phases/nodestart/kubelet.go index 238077b6d..d18dd81bb 100644 --- a/pkg/agent/phases/nodestart/kubelet.go +++ b/pkg/agent/phases/nodestart/kubelet.go @@ -26,6 +26,23 @@ import ( type configureKubelet struct { goalState *goalstates.NodeStart + + // changed records whether any file this task owns actually differed. The + // node's kubelet reads these at start, so a reapply that alters one has to + // restart it; a reapply that alters nothing must not. + changed bool +} + +// write applies content and records whether it differed from what was there. +func (c *configureKubelet) write(path string, content []byte, perm os.FileMode) error { + changed, err := utilio.WriteFileIfChanged(path, content, perm) + if err != nil { + return err + } + + c.changed = c.changed || changed + + return nil } // ConfigureKubelet returns a task that writes the kubelet configuration into the machine rootfs. @@ -103,7 +120,7 @@ func (s *startKubelet) Do(ctx context.Context) error { func (c *configureKubelet) ensureKubeletCACert() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletAPIServerCACertPath) - return utilio.WriteFile(dest, c.goalState.Kubelet.CACertData, 0o644) + return c.write(dest, c.goalState.Kubelet.CACertData, 0o644) } // ensureKubeletConfiguration writes a KubeletConfiguration assembled from the @@ -137,7 +154,7 @@ func (c *configureKubelet) ensureKubeletConfiguration() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletConfigurationPath) - return utilio.WriteFile(dest, data, 0o644) + return c.write(dest, data, 0o644) } func defaultKubeletConfiguration() map[string]any { @@ -208,7 +225,7 @@ func (c *configureKubelet) ensureKubeletServiceUnit() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.SystemdSystemDir, goalstates.SystemdUnitKubelet) - return utilio.WriteFile(dest, buf.Bytes(), 0o644) + return c.write(dest, buf.Bytes(), 0o644) } // ensureKubeletDropIns renders and writes all kubelet systemd drop-in files @@ -259,7 +276,7 @@ func (c *configureKubelet) ensureKubeletDropIns() error { } dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletServiceDropInDir, d.name) - if err := utilio.WriteFile(dest, buf.Bytes(), 0o644); err != nil { + if err := c.write(dest, buf.Bytes(), 0o644); err != nil { return fmt.Errorf("write %s: %w", dest, err) } } @@ -321,7 +338,7 @@ func (c *configureKubelet) ensureBootstrapKubeconfig() error { dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletBootstrapKubeconfigPath) - return utilio.WriteFile(dest, data, 0o600) + return c.write(dest, data, 0o600) } // ensureExecKubeconfig writes a kubeconfig that uses an exec credential @@ -339,7 +356,7 @@ func (c *configureKubelet) ensureExecKubeconfig() error { // to the kubelet kubeconfig path (no TLS bootstrap needed). dest := filepath.Join(c.goalState.MachineDir, goalstates.KubeletKubeconfigPath) - return utilio.WriteFile(dest, data, 0o600) + return c.write(dest, data, 0o600) } // formatNodeLabels formats a map of node labels as a sorted, comma-separated diff --git a/pkg/agent/phases/nodestart/kubelet_test.go b/pkg/agent/phases/nodestart/kubelet_test.go index d9c6c2e99..81794c871 100644 --- a/pkg/agent/phases/nodestart/kubelet_test.go +++ b/pkg/agent/phases/nodestart/kubelet_test.go @@ -278,3 +278,40 @@ func TestConfigureKubeletOmitsEmptyNodeLabelsAndTaints(t *testing.T) { require.NotContains(t, string(data), "--image-credential-provider-config=") require.NotContains(t, string(data), "--image-credential-provider-bin-dir=") } + +// TestConfigureKubeletReportsOnlyRealChanges backs the restart decision. If an +// identical reapply reported a change, every retry would bounce kubelet on a +// running node; if a real change went unreported, the files and the running +// service would silently disagree. +func TestConfigureKubeletReportsOnlyRealChanges(t *testing.T) { + t.Parallel() + + machineDir := t.TempDir() + goalState := &goalstates.NodeStart{ + MachineDir: machineDir, + NodeName: "worker-1", + Kubelet: goalstates.Kubelet{ + APIServer: "https://api.example.com", + CACertData: []byte("ca"), + NodeIP: "10.0.0.15", + KubeletAuthInfo: config.KubeletAuthInfo{ + BootstrapToken: "token", + }, + ClusterDNS: "10.0.0.10", + }, + } + + first := &configureKubelet{goalState: goalState} + require.NoError(t, first.Do(context.Background())) + require.True(t, first.changed, "creating the configuration is a change") + + second := &configureKubelet{goalState: goalState} + require.NoError(t, second.Do(context.Background())) + require.False(t, second.changed, "an identical reapply is not a change") + + goalState.Kubelet.ClusterDNS = "10.0.0.11" + + third := &configureKubelet{goalState: goalState} + require.NoError(t, third.Do(context.Background())) + require.True(t, third.changed, "a different cluster DNS is a change") +} diff --git a/pkg/agent/phases/nodestart/nspawn.go b/pkg/agent/phases/nodestart/nspawn.go index 3551dd8b0..1bf4e01f2 100644 --- a/pkg/agent/phases/nodestart/nspawn.go +++ b/pkg/agent/phases/nodestart/nspawn.go @@ -68,6 +68,12 @@ type startNSpawnMachine struct { // runner is the machinectl/systemctl driver. Tests inject a fake. runner machinectlRunner + + // wasRunning, when set, receives whether the machine was already up before + // this task touched it. A reapply against a live node has to restart any + // service whose configuration it changed, because those are read at start; + // on a machine this task boots, the services read the new files anyway. + wasRunning *bool } // StartNSpawnMachine returns a task that starts the systemd-nspawn machine using machinectl and @@ -111,6 +117,10 @@ func (s *startNSpawnMachine) startWithRecovery(ctx context.Context, name string) return fmt.Errorf("inspect nspawn service before replay: %w", err) } + if s.wasRunning != nil { + *s.wasRunning = running + } + if running { return nil } diff --git a/pkg/agent/phases/nodestart/restart_reconfigured.go b/pkg/agent/phases/nodestart/restart_reconfigured.go new file mode 100644 index 000000000..43dcb86fc --- /dev/null +++ b/pkg/agent/phases/nodestart/restart_reconfigured.go @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package nodestart + +import ( + "context" + "fmt" + "log/slog" + + "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/pkg/agent/goalstates" + "github.com/Azure/unbounded/pkg/agent/phases" +) + +// restartReconfigured restarts node services whose configuration this +// invocation actually changed. +// +// Writing a configuration file does not affect a service that has already read +// it. That is harmless when this sequence boots the machine, because the +// services start afterwards and read the new files. It is not harmless when the +// sequence runs against a machine that is already up: without this, the files +// on disk and the running services would disagree, with nothing to reconcile +// them, which is a worse outcome than not reapplying at all. +// +// Only an actual change restarts anything. A reapply that writes identical +// content leaves the node alone, so the ordinary case of rerunning bootstrap +// after a failure costs nothing. +type restartReconfigured struct { + log *slog.Logger + goalState *goalstates.NodeStart + + machineWasRunning *bool + containerd *configureContainerd + kubelet *configureKubelet +} + +func (r *restartReconfigured) Name() string { return "restart-reconfigured-services" } + +func (r *restartReconfigured) Do(ctx context.Context) error { + if r.machineWasRunning == nil || !*r.machineWasRunning { + // This sequence started the machine, so its services have already read + // the configuration written above. + return nil + } + + // containerd first: kubelet talks to it, so restarting kubelet into a + // restarting runtime would only make it retry. + for _, unit := range []struct { + name string + changed bool + }{ + {goalstates.SystemdUnitContainerd, r.containerd != nil && r.containerd.changed}, + {goalstates.SystemdUnitKubelet, r.kubelet != nil && r.kubelet.changed}, + } { + if !unit.changed { + continue + } + + r.log.Info("configuration changed on a running node, restarting service", + "unit", unit.name, + "machine", r.goalState.MachineName, + ) + + if _, err := executil.MachineRun(ctx, r.log, r.goalState.MachineName, + "systemctl", "restart", unit.name, + ); err != nil { + return fmt.Errorf("systemctl restart %s in %s: %w", unit.name, r.goalState.MachineName, err) + } + } + + return nil +} + +var _ phases.Task = (*restartReconfigured)(nil) diff --git a/pkg/agent/phases/nodestart/restart_reconfigured_test.go b/pkg/agent/phases/nodestart/restart_reconfigured_test.go new file mode 100644 index 000000000..5da82b2e7 --- /dev/null +++ b/pkg/agent/phases/nodestart/restart_reconfigured_test.go @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package nodestart + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/goalstates" +) + +// stubMachineRun puts a recording systemd-run on PATH, since MachineRun shells +// out to it, and returns the file each invocation appends its arguments to. +func stubMachineRun(t *testing.T) string { + t.Helper() + + dir := t.TempDir() + record := filepath.Join(dir, "calls") + script := "#!/bin/sh\necho \"$@\" >> " + record + "\nexit 0\n" + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemd-run"), []byte(script), 0o755)) + t.Setenv("PATH", dir) + + return record +} + +func machineRunCalls(t *testing.T, record string) string { + t.Helper() + + data, err := os.ReadFile(record) + if os.IsNotExist(err) { + return "" + } + + require.NoError(t, err) + + return string(data) +} + +func reconfigureTask(t *testing.T, wasRunning, containerdChanged, kubeletChanged bool) (*restartReconfigured, string) { + t.Helper() + + record := stubMachineRun(t) + + return &restartReconfigured{ + log: slog.New(slog.DiscardHandler), + goalState: &goalstates.NodeStart{MachineName: goalstates.NSpawnMachineKube1}, + machineWasRunning: &wasRunning, + containerd: &configureContainerd{changed: containerdChanged}, + kubelet: &configureKubelet{changed: kubeletChanged}, + }, record +} + +// TestNoRestartWhenThisRunStartedTheMachine covers the fresh install. The +// services start after the configuration is written, so they already read it. +func TestNoRestartWhenThisRunStartedTheMachine(t *testing.T) { + task, record := reconfigureTask(t, false, true, true) + + require.NoError(t, task.Do(t.Context())) + require.Empty(t, machineRunCalls(t, record), "a machine this run started needs no restart") +} + +// TestNoRestartWhenNothingChanged is the ordinary retry: bootstrap is rerun +// after a failure, the configuration is identical, and the node is left alone. +func TestNoRestartWhenNothingChanged(t *testing.T) { + task, record := reconfigureTask(t, true, false, false) + + require.NoError(t, task.Do(t.Context())) + require.Empty(t, machineRunCalls(t, record), "an unchanged reapply must not disturb a running node") +} + +// TestRestartsOnlyTheServiceWhoseConfigChanged keeps a kubelet change from +// bouncing the container runtime underneath it. +func TestRestartsOnlyTheServiceWhoseConfigChanged(t *testing.T) { + task, record := reconfigureTask(t, true, false, true) + + require.NoError(t, task.Do(t.Context())) + + calls := machineRunCalls(t, record) + require.Contains(t, calls, "restart "+goalstates.SystemdUnitKubelet) + require.NotContains(t, calls, "restart "+goalstates.SystemdUnitContainerd) +} + +// TestRestartsContainerdBeforeKubelet pins the order. kubelet talks to +// containerd, so restarting kubelet into a restarting runtime would only make +// it retry. +func TestRestartsContainerdBeforeKubelet(t *testing.T) { + task, record := reconfigureTask(t, true, true, true) + + require.NoError(t, task.Do(t.Context())) + + calls := machineRunCalls(t, record) + require.Less(t, + indexOfUnit(calls, goalstates.SystemdUnitContainerd), + indexOfUnit(calls, goalstates.SystemdUnitKubelet), + "containerd must restart before kubelet", + ) +} + +func indexOfUnit(calls, unit string) int { + for i := 0; i+len(unit) <= len(calls); i++ { + if calls[i:i+len(unit)] == unit { + return i + } + } + + return -1 +} diff --git a/pkg/agent/phases/nodestart/start.go b/pkg/agent/phases/nodestart/start.go index 484a58028..85b76df73 100644 --- a/pkg/agent/phases/nodestart/start.go +++ b/pkg/agent/phases/nodestart/start.go @@ -20,17 +20,39 @@ import ( // and node update flows. Callers that need to persist the applied config for // drift detection should append that step separately. func StartNode(log *slog.Logger, gs *goalstates.NodeStart) phases.Task { + // These are built concretely rather than through the exported constructors + // so the sequence can tell whether the configuration it wrote differed, and + // whether the machine was already up. Both are needed to decide if a + // running service has to be restarted to read what changed. + var machineWasRunning bool + + containerd := &configureContainerd{goalState: gs} + kubelet := &configureKubelet{goalState: gs} + startMachine := &startNSpawnMachine{ + log: log, + goalState: gs, + runner: defaultMachinectlRunner{log: log}, + wasRunning: &machineWasRunning, + } + return phases.Serial(log, phases.Parallel(log, - ConfigureContainerd(gs), - ConfigureKubelet(gs), + containerd, + kubelet, ConfigureLocalDNS(gs), ), SetupLocalDNSNetwork(log, gs), - StartNSpawnMachine(log, gs), + startMachine, WaitForLocalDNS(log, gs), StartContainerd(log, gs), ImportContainerImages(log, gs), StartKubelet(log, gs), + &restartReconfigured{ + log: log, + goalState: gs, + machineWasRunning: &machineWasRunning, + containerd: containerd, + kubelet: kubelet, + }, ) } From 0c6905300b180facb13cf8b5630f8d899657a95d Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:40:19 -0400 Subject: [PATCH 17/39] agent: leave a registered machine's rootfs in place instead of refusing ProvisionOwned rebuilds a rootfs in place and must never be pointed at a slot that has started a node; its own doc says so, because doing that pulls the filesystem out from under a running one. This stage guarded that by refusing outright when a machine was registered. Refusing is the wrong answer to the question being asked. A registered machine means the rootfs this stage would build is already built and in use, so the requirement is met and there is nothing to do. Skip it and continue. The condition is a property of the host rather than of how far a previous attempt got, so it holds whether the machine was started by an earlier attempt of this installation or independently afterwards. This makes the stage safe to reapply against any host state, which is a prerequisite for a retry that reapplies work rather than trusting a record of what was already done. --- cmd/agent/internal/cmd/bootstrap.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index 5d5434cea..545bbbb44 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -6,7 +6,6 @@ package cmd import ( "context" "encoding/json" - "fmt" "log/slog" "net/url" "strings" @@ -141,8 +140,15 @@ func (s *agentStages) prepareCredentials(ctx context.Context) error { } func (s *agentStages) PrepareRootFS(ctx context.Context) error { - // A pre-node checkpoint never authorizes deleting a registered machine, - // including one started independently after ownership was first recorded. + // ProvisionOwned rebuilds the rootfs in place and must never be pointed at + // a slot that has started a node, which would pull the filesystem out from + // under a running one. A registered machine means the rootfs this stage + // would build is already built and in use, so the requirement is met and + // there is nothing to do. + // + // This is a property of the host, not of how far a previous attempt got. It + // holds whether the machine was started by an earlier attempt of this + // installation or independently afterwards. for _, name := range []string{goalstates.NSpawnMachineKube1, goalstates.NSpawnMachineKube2} { registered, err := reset.RegisteredMachine(ctx, s.log, name) if err != nil { @@ -150,7 +156,9 @@ func (s *agentStages) PrepareRootFS(ctx context.Context) error { } if registered { - return fmt.Errorf("refusing rootfs replay while %s is registered", name) + s.log.Info("nspawn machine is registered; leaving its rootfs in place", "machine", name) + + return nil } } From 554e1aca6d02bf9ba999dd3099c1d452688d8093 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:53:50 -0400 Subject: [PATCH 18/39] agent: record which phase an installation is in, not how far it got The record tracked which stage bootstrap had completed, and a retry resumed from there. That made it a claim about the host: "host preparation is finished" can stop being true without anyone noticing, and a retry that trusted it skipped work the host no longer had. The failure surfaced later, at the first stage that needed the missing thing, naming the symptom rather than the cause. Reapply every stage instead. Each already decides what to do by looking at the host: host preparation leaves a live nftables ruleset alone, the rootfs is left in place when a machine is registered from it, an already running machine is not restarted, and node services are restarted only when their configuration actually changed. A host that drifted between attempts is now repaired rather than skipped past. What is left worth persisting is which mode we are in, not how far we got. Installing says only that an installation is under way, so there is nothing in it that can go stale. Complete routes a later start to verify and repair from the applied config, which matters because after an ordinary repave the bootstrap inputs describe a retired slot. Resetting is the one thing the host cannot be asked: a half-removed installation and a half-built one look identical, because direction of travel is not observable. Stage names survive as a label for status reporting and logs, but are not written down. Recording the stage is precisely what let the record disagree with the host; reporting it costs nothing and keeps the Machine condition specific. That reporting also had a gap. wait-for-kubelet-bootstrap is its own task inside the node-start stage, and the classifier only matched start-kubelet, so the most common real failure - a rejected token, an unreachable API server, a CA mismatch - was reported as a generic failure. Main reports KubeletBootstrapFailed for it, so this restores parity. The record format changes and the fixtures change with it. Nothing has shipped, so no migration is owed. --- cmd/agent/internal/bootstrap/coordinator.go | 73 ++++++++++--------- .../internal/bootstrap/coordinator_test.go | 46 ++++++------ cmd/agent/internal/cmd/bootstrap.go | 10 +-- cmd/agent/internal/cmd/bootstrap_test.go | 36 ++++++++- cmd/agent/internal/cmd/start.go | 11 ++- .../cmd/testdata/bootstrap-v1/complete.json | 2 +- ...{preparing-rootfs.json => installing.json} | 2 +- .../cmd/testdata/bootstrap-v1/resetting.json | 2 +- cmd/agent/internal/daemon/reset.go | 2 +- cmd/agent/internal/daemon/reset_test.go | 2 +- cmd/agent/internal/installstate/mutation.go | 4 +- cmd/agent/internal/installstate/store.go | 50 ++++++++----- cmd/agent/internal/installstate/store_test.go | 24 +++--- 13 files changed, 162 insertions(+), 102 deletions(-) rename cmd/agent/internal/cmd/testdata/bootstrap-v1/{preparing-rootfs.json => installing.json} (85%) diff --git a/cmd/agent/internal/bootstrap/coordinator.go b/cmd/agent/internal/bootstrap/coordinator.go index e2a91119d..9e30f6e67 100644 --- a/cmd/agent/internal/bootstrap/coordinator.go +++ b/cmd/agent/internal/bootstrap/coordinator.go @@ -1,7 +1,11 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 -// Package bootstrap coordinates replay of owned initial installation stages. +// Package bootstrap reapplies the stages of an owned initial installation. +// +// Every stage runs on every attempt. Each decides what to do by looking at the +// host rather than at a record of what a previous attempt claimed to have done, +// so a host that changed in between converges instead of being skipped. package bootstrap import ( @@ -25,9 +29,21 @@ type Stages interface { VerifyInstalled(context.Context) error } +// Stage names the work being reported on. It is a label for status reporting +// and logs, deliberately not persisted: writing down which stage was reached is +// what lets a record disagree with the host. +type Stage string + +const ( + StagePrepareHost Stage = "preparing-host" + StagePrepareRootFS Stage = "preparing-rootfs" + StageStartNode Stage = "starting-node" + StageInstallDaemon Stage = "installing-daemon" +) + type Reporter interface { - StageStarted(context.Context, installstate.Checkpoint) - StageFailed(context.Context, installstate.Checkpoint, error) + StageStarted(context.Context, Stage) + StageFailed(context.Context, Stage, error) } type Coordinator struct { @@ -96,49 +112,40 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { return Outcome{}, fmt.Errorf("resolve bootstrap inputs: %w", err) } - for r.Checkpoint != installstate.Complete { + // Every stage runs, in order, on every attempt. Each one decides from the + // host what it still has to do: host preparation leaves a live nftables + // ruleset alone, the rootfs is left in place when a machine is registered + // from it, an already running machine is not restarted, and node services + // are restarted only when their configuration actually changed. + for _, stage := range []struct { + name string + run func(context.Context) error + }{ + {string(StagePrepareHost), c.stages.PrepareHost}, + {string(StagePrepareRootFS), c.stages.PrepareRootFS}, + {string(StageStartNode), c.stages.EnsureNodeStarted}, + {string(StageInstallDaemon), c.stages.EnsureDaemonInstalled}, + } { if err := ctx.Err(); err != nil { return Outcome{}, err } - current := r.Checkpoint if c.reporter != nil { - c.reporter.StageStarted(ctx, current) + c.reporter.StageStarted(ctx, Stage(stage.name)) } - next, err := c.runStage(ctx, current) - if err != nil { + if err := stage.run(ctx); err != nil { if c.reporter != nil { - c.reporter.StageFailed(ctx, current, err) + c.reporter.StageFailed(ctx, Stage(stage.name), err) } - return Outcome{}, fmt.Errorf("%s: %w", current, err) + return Outcome{}, fmt.Errorf("%s: %w", stage.name, err) } + } - r.Checkpoint = next - if next == installstate.Complete { - if err := c.store.MarkComplete(r); err != nil { - return Outcome{}, err - } - } else if err := c.store.Save(r); err != nil { - return Outcome{}, err - } + if err := c.store.MarkComplete(r); err != nil { + return Outcome{}, err } return Outcome{}, nil } - -func (c *Coordinator) runStage(ctx context.Context, stage installstate.Checkpoint) (installstate.Checkpoint, error) { - switch stage { - case installstate.PreparingHost: - return installstate.PreparingRootFS, c.stages.PrepareHost(ctx) - case installstate.PreparingRootFS: - return installstate.StartingNode, c.stages.PrepareRootFS(ctx) - case installstate.StartingNode: - return installstate.InstallingDaemon, c.stages.EnsureNodeStarted(ctx) - case installstate.InstallingDaemon: - return installstate.Complete, c.stages.EnsureDaemonInstalled(ctx) - default: - return "", fmt.Errorf("unsupported checkpoint %s", stage) - } -} diff --git a/cmd/agent/internal/bootstrap/coordinator_test.go b/cmd/agent/internal/bootstrap/coordinator_test.go index 1ec166f7c..5076aeda7 100644 --- a/cmd/agent/internal/bootstrap/coordinator_test.go +++ b/cmd/agent/internal/bootstrap/coordinator_test.go @@ -54,42 +54,46 @@ func (f *fakeStages) VerifyInstalled(context.Context) error { return f.verifyErr } -func TestInterruptedStagesResumeWithoutReplayingEarlierStages(t *testing.T) { +// TestEveryStageRunsOnEveryAttempt is the core of the reapply model. +// +// An earlier design recorded which stage had been reached and skipped anything +// before it. That made the record a claim about the host, and a host changed in +// between would be skipped past rather than repaired. Every stage now runs every +// time and decides from the host what it still has to do, so a retry after a +// failure at any point does the same thing: all of them, in order. +func TestEveryStageRunsOnEveryAttempt(t *testing.T) { t.Parallel() - for _, tc := range []struct { - fail string - checkpoint installstate.Checkpoint - want []string - }{ - {"host", installstate.PreparingHost, []string{"resolve", "host", "rootfs", "node", "daemon"}}, - {"rootfs", installstate.PreparingRootFS, []string{"resolve", "rootfs", "node", "daemon"}}, - {"node", installstate.StartingNode, []string{"resolve", "node", "daemon"}}, - {"daemon", installstate.InstallingDaemon, []string{"resolve", "daemon"}}, - } { - t.Run(tc.fail, func(t *testing.T) { + all := []string{"resolve", "host", "rootfs", "node", "daemon"} + + for _, failAt := range []string{"host", "rootfs", "node", "daemon"} { + t.Run(failAt, func(t *testing.T) { dir := t.TempDir() store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) - stages := &fakeStages{store: store, fail: tc.fail} + stages := &fakeStages{store: store, fail: failAt} c := New(slog.New(slog.DiscardHandler), store, stages, nil) id := Identity{MachineName: "machine", ConfigFingerprint: "fingerprint"} + _, err := c.Run(t.Context(), id) require.ErrorIs(t, err, errInjected) + record, err := store.Load() require.NoError(t, err) - require.Equal(t, tc.checkpoint, record.Checkpoint) + require.Equal(t, installstate.Installing, record.Phase, + "an unfinished installation records only that it is under way") stages.calls = nil stages.fail = "" + outcome, err := c.Run(t.Context(), id) require.NoError(t, err) require.False(t, outcome.AlreadyComplete) - require.Equal(t, tc.want, stages.calls) + require.Equal(t, all, stages.calls, "the retry reapplies every stage regardless of where it failed") complete, err := store.Load() require.NoError(t, err) - require.Equal(t, record.InstallID, complete.InstallID) - require.Equal(t, installstate.Complete, complete.Checkpoint) + require.Equal(t, record.InstallID, complete.InstallID, "the retry is the same installation") + require.Equal(t, installstate.Complete, complete.Phase) }) } } @@ -103,7 +107,7 @@ func TestCompletedRecoveryDoesNotResolveRetiredBootstrapInputs(t *testing.T) { r, err := installstate.NewRecord("machine", "fingerprint") require.NoError(t, err) - r.Checkpoint = installstate.Complete + r.Phase = installstate.Complete require.NoError(t, store.Save(r)) stages := &fakeStages{store: store, fail: "resolve"} @@ -125,7 +129,7 @@ func TestCompletedRecoveryDoesNotResolveRetiredBootstrapInputs(t *testing.T) { complete, err := store.Load() require.NoError(t, err) - require.Equal(t, installstate.Complete, complete.Checkpoint) + require.Equal(t, installstate.Complete, complete.Phase) } } @@ -140,7 +144,7 @@ func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { require.NoError(t, err) if mode == "resetting" { - r.Checkpoint = installstate.Resetting + r.Phase = installstate.Resetting } require.NoError(t, store.Save(r)) @@ -176,7 +180,7 @@ func TestInterruptedRepairRemainsCompleteAndRetries(t *testing.T) { require.ErrorIs(t, err, errInjected) loaded, err := store.Load() require.NoError(t, err) - require.Equal(t, installstate.Complete, loaded.Checkpoint) + require.Equal(t, installstate.Complete, loaded.Phase) stages.fail = "" stages.verifyErr = errInjected diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index 545bbbb44..c068d6faf 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -222,17 +222,17 @@ func (s *agentStages) VerifyInstalled(ctx context.Context) error { func (s *agentStages) RepairDaemon(ctx context.Context) error { return daemon.RepairDaemon(ctx, s.log) } -func (s *agentStages) StageStarted(_ context.Context, stage installstate.Checkpoint) { - s.log.Info("bootstrap stage", "checkpoint", stage) +func (s *agentStages) StageStarted(_ context.Context, stage bootstrap.Stage) { + s.log.Info("bootstrap stage", "stage", stage) } -func (s *agentStages) StageFailed(ctx context.Context, stage installstate.Checkpoint, err error) { +func (s *agentStages) StageFailed(ctx context.Context, stage bootstrap.Stage, err error) { reason := "Failed" - if stage == installstate.PreparingRootFS { + if stage == bootstrap.StagePrepareRootFS { reason = "RootFSFailed" } - if stage == installstate.StartingNode { + if stage == bootstrap.StageStartNode { reason = classifyNodeStartFailure(err) } diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index 87c20f9ad..84b7315c0 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -6,6 +6,8 @@ package cmd import ( "bytes" "encoding/json" + "errors" + "fmt" "log/slog" "os" "path/filepath" @@ -39,8 +41,8 @@ func TestBootstrapV1CompatibilityFixtures(t *testing.T) { id, err := bootstrapIdentity(cfg) require.NoError(t, err) - for _, checkpoint := range []installstate.Checkpoint{installstate.PreparingRootFS, installstate.Complete, installstate.Resetting} { - data, err := os.ReadFile(filepath.Join(dir, string(checkpoint)+".json")) + for _, phase := range []installstate.Phase{installstate.Installing, installstate.Complete, installstate.Resetting} { + data, err := os.ReadFile(filepath.Join(dir, string(phase)+".json")) require.NoError(t, err) var record installstate.Record @@ -55,14 +57,14 @@ func TestBootstrapV1CompatibilityFixtures(t *testing.T) { require.NoError(t, store.Save(record)) loaded, disposition, err := installstate.Admit(store, id.MachineName, id.ConfigFingerprint) - if checkpoint == installstate.Resetting { + if phase == installstate.Resetting { require.Error(t, err) } else { require.NoError(t, err) require.Equal(t, record, loaded) want := installstate.Resume - if checkpoint == installstate.Complete { + if phase == installstate.Complete { want = installstate.AlreadyComplete } @@ -202,3 +204,29 @@ func TestCompletedPreflightOutput(t *testing.T) { h.output = "unsupported" require.Error(t, h.writeReport(preflight.Report{})) } + +// TestClassifyNodeStartFailure pins the Machine condition reasons for a node +// that fails to come up. +// +// wait-for-kubelet-bootstrap has to map to KubeletBootstrapFailed alongside +// start-kubelet. It is its own task inside the node-start stage, and a failure +// there is the most common real one: a rejected or expired bootstrap token, an +// unreachable API server, a CA mismatch. Reporting it as a generic failure +// tells an operator nothing about where to look. +func TestClassifyNodeStartFailure(t *testing.T) { + t.Parallel() + + for _, tc := range []struct{ task, want string }{ + {"start-kubelet", "KubeletBootstrapFailed"}, + {"wait-for-kubelet-bootstrap", "KubeletBootstrapFailed"}, + {"start-nspawn-machine", "NSpawnFailed"}, + {"import-container-images", "Failed"}, + } { + t.Run(tc.task, func(t *testing.T) { + t.Parallel() + + err := fmt.Errorf("%s: %w", tc.task, errors.New("boom")) + require.Equal(t, tc.want, classifyNodeStartFailure(err)) + }) + } +} diff --git a/cmd/agent/internal/cmd/start.go b/cmd/agent/internal/cmd/start.go index 6ea49e5e2..e28ee8fcd 100644 --- a/cmd/agent/internal/cmd/start.go +++ b/cmd/agent/internal/cmd/start.go @@ -82,10 +82,19 @@ func syncAttestedKubeletConfig(cfg *provision.AgentConfig, nodeStart *goalstates } } +// classifyNodeStartFailure maps a node-start failure onto the Machine condition +// reason, by the name of the task that reported it. +// +// wait-for-kubelet-bootstrap is matched as well as start-kubelet because both +// mean the node did not join. That wait is its own task inside this stage, and +// a failure there is the most common real one: a rejected or expired token, an +// unreachable API server, a CA mismatch. Reporting it as a generic failure +// would tell an operator nothing. func classifyNodeStartFailure(err error) string { message := err.Error() switch { - case strings.Contains(message, "start-kubelet"): + case strings.Contains(message, "start-kubelet"), + strings.Contains(message, "wait-for-kubelet-bootstrap"): return "KubeletBootstrapFailed" case strings.Contains(message, "start-nspawn-machine"): return "NSpawnFailed" diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json index f57e1a33c..0d3009fa3 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/complete.json @@ -3,5 +3,5 @@ "installID": "00112233445566778899aabbccddeeff", "machineName": "bootstrap-fixture", "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", - "checkpoint": "complete" + "phase": "complete" } diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/installing.json similarity index 85% rename from cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json rename to cmd/agent/internal/cmd/testdata/bootstrap-v1/installing.json index 864dcd6c3..29daac0b6 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/preparing-rootfs.json +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/installing.json @@ -3,5 +3,5 @@ "installID": "00112233445566778899aabbccddeeff", "machineName": "bootstrap-fixture", "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", - "checkpoint": "preparing-rootfs" + "phase": "installing" } diff --git a/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json index 1c3345eb8..629816493 100644 --- a/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json +++ b/cmd/agent/internal/cmd/testdata/bootstrap-v1/resetting.json @@ -3,5 +3,5 @@ "installID": "00112233445566778899aabbccddeeff", "machineName": "bootstrap-fixture", "configFingerprint": "c450aef0b3255c61d169b528949df158234f16391c2a70d03548e7698976aade", - "checkpoint": "resetting" + "phase": "resetting" } diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index 15aed474b..ad8c1c02a 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -65,7 +65,7 @@ func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.S return err } - r.Checkpoint = installstate.Resetting + r.Phase = installstate.Resetting if err := store.Save(r); err != nil { return err } diff --git a/cmd/agent/internal/daemon/reset_test.go b/cmd/agent/internal/daemon/reset_test.go index 2b4467539..b73854272 100644 --- a/cmd/agent/internal/daemon/reset_test.go +++ b/cmd/agent/internal/daemon/reset_test.go @@ -37,7 +37,7 @@ func TestResetRetainsOwnershipUntilTeardownAndSyncSucceed(t *testing.T) { r, err := installstate.NewRecord("machine", "f") require.NoError(t, err) - r.Checkpoint = installstate.Resetting + r.Phase = installstate.Resetting require.NoError(t, store.Save(r)) injected := errors.New("injected reset failure") diff --git a/cmd/agent/internal/installstate/mutation.go b/cmd/agent/internal/installstate/mutation.go index 1b5158ee8..9927a7460 100644 --- a/cmd/agent/internal/installstate/mutation.go +++ b/cmd/agent/internal/installstate/mutation.go @@ -18,7 +18,7 @@ func (s *Store) AcquireMutationLock() (*Lock, error) { } r, loadErr := s.Load() - if errors.Is(loadErr, ErrNotFound) || loadErr == nil && r.Checkpoint == Complete { + if errors.Is(loadErr, ErrNotFound) || loadErr == nil && r.Phase == Complete { return lock, nil } @@ -28,5 +28,5 @@ func (s *Store) AcquireMutationLock() (*Lock, error) { return nil, errors.Join(loadErr, closeErr) } - return nil, errors.Join(fmt.Errorf("installation is %s; finish bootstrap or reset before lifecycle operations", r.Checkpoint), closeErr) + return nil, errors.Join(fmt.Errorf("installation is %s; finish bootstrap or reset before lifecycle operations", r.Phase), closeErr) } diff --git a/cmd/agent/internal/installstate/store.go b/cmd/agent/internal/installstate/store.go index 504c465be..8783d1ed8 100644 --- a/cmd/agent/internal/installstate/store.go +++ b/cmd/agent/internal/installstate/store.go @@ -23,23 +23,35 @@ const ( schemaVersion = 1 ) -type Checkpoint string +// Phase is what an installation is currently doing, not how far it has got. +// +// It deliberately records no progress. A record that claimed a stage was +// finished would be a statement about the host that could stop being true +// without anyone noticing, and a retry that trusted it would skip work the host +// no longer has. The stages instead decide what to do by looking at the host, +// so the only thing worth persisting is which mode we are in. +type Phase string const ( - PreparingHost Checkpoint = "preparing-host" - PreparingRootFS Checkpoint = "preparing-rootfs" - StartingNode Checkpoint = "starting-node" - InstallingDaemon Checkpoint = "installing-daemon" - Complete Checkpoint = "complete" - Resetting Checkpoint = "resetting" + // Installing means an installation is under way. It says nothing about what + // has been done, so there is nothing in it that can go stale. + Installing Phase = "installing" + // Complete means the installation finished. A later start verifies and + // repairs from the applied config rather than reapplying bootstrap inputs, + // which after an ordinary repave describe a retired slot. + Complete Phase = "complete" + // Resetting means a teardown started and may not have finished. This is the + // one thing the host cannot be asked: a half-removed installation and a + // half-built one look the same, because direction of travel is not visible. + Resetting Phase = "resetting" ) type Record struct { - SchemaVersion int `json:"schemaVersion"` - InstallID string `json:"installID"` - MachineName string `json:"machineName"` - ConfigFingerprint string `json:"configFingerprint"` - Checkpoint Checkpoint `json:"checkpoint"` + SchemaVersion int `json:"schemaVersion"` + InstallID string `json:"installID"` + MachineName string `json:"machineName"` + ConfigFingerprint string `json:"configFingerprint"` + Phase Phase `json:"phase"` } func (r Record) Validate() error { @@ -47,11 +59,11 @@ func (r Record) Validate() error { return fmt.Errorf("invalid installation record identity or schema") } - switch r.Checkpoint { - case PreparingHost, PreparingRootFS, StartingNode, InstallingDaemon, Complete, Resetting: + switch r.Phase { + case Installing, Complete, Resetting: return nil default: - return fmt.Errorf("unknown installation checkpoint %q", r.Checkpoint) + return fmt.Errorf("unknown installation phase %q", r.Phase) } } @@ -108,7 +120,7 @@ func (s *Store) Save(r Record) error { // MarkComplete commits completion. The durable record is the only completion // signal; no separate marker file is maintained. func (s *Store) MarkComplete(r Record) error { - r.Checkpoint = Complete + r.Phase = Complete return s.Save(r) } @@ -153,7 +165,7 @@ func NewRecord(machine, fingerprint string) (Record, error) { return Record{ SchemaVersion: schemaVersion, InstallID: hex.EncodeToString(id), MachineName: machine, - ConfigFingerprint: fingerprint, Checkpoint: PreparingHost, + ConfigFingerprint: fingerprint, Phase: Installing, }, nil } @@ -186,7 +198,7 @@ func decide(r Record, loadErr error, machine, fingerprint string) (Disposition, return Fresh, err } - if r.Checkpoint == Resetting { + if r.Phase == Resetting { return Fresh, fmt.Errorf("reset is incomplete; run unbounded-agent reset again") } @@ -194,7 +206,7 @@ func decide(r Record, loadErr error, machine, fingerprint string) (Disposition, return Fresh, fmt.Errorf("installation intent differs; explicit reset is required") } - if r.Checkpoint == Complete { + if r.Phase == Complete { return AlreadyComplete, nil } diff --git a/cmd/agent/internal/installstate/store_test.go b/cmd/agent/internal/installstate/store_test.go index 350ae6ced..f66fd29f5 100644 --- a/cmd/agent/internal/installstate/store_test.go +++ b/cmd/agent/internal/installstate/store_test.go @@ -38,7 +38,7 @@ func TestStoreLifecycle(t *testing.T) { require.NoError(t, s.MarkComplete(r)) loaded, err = s.Load() require.NoError(t, err) - require.Equal(t, Complete, loaded.Checkpoint) + require.Equal(t, Complete, loaded.Phase) require.NoError(t, s.Remove()) _, err = s.Load() require.ErrorIs(t, err, ErrNotFound) @@ -50,13 +50,13 @@ func TestOwnershipAdmission(t *testing.T) { r, err := NewRecord("machine", "fingerprint") require.NoError(t, err) - for _, checkpoint := range []Checkpoint{PreparingHost, PreparingRootFS, StartingNode, InstallingDaemon, Complete, Resetting} { - t.Run(string(checkpoint), func(t *testing.T) { + for _, phase := range []Phase{Installing, Complete, Resetting} { + t.Run(string(phase), func(t *testing.T) { r := r - r.Checkpoint = checkpoint + r.Phase = phase disposition, err := decide(r, nil, r.MachineName, r.ConfigFingerprint) - if checkpoint == Resetting { + if phase == Resetting { require.Error(t, err) return } @@ -64,7 +64,7 @@ func TestOwnershipAdmission(t *testing.T) { require.NoError(t, err) want := Resume - if checkpoint == Complete { + if phase == Complete { want = AlreadyComplete } @@ -128,7 +128,7 @@ func TestRemoveRestoresOwnershipWhenUndurable(t *testing.T) { r, err := NewRecord("machine", "f") require.NoError(t, err) - r.Checkpoint = Resetting + r.Phase = Resetting require.NoError(t, s.Save(r)) failure := errors.New("sync failed") @@ -171,20 +171,20 @@ func TestRemoveDoesNotRestoreUnusableOwnership(t *testing.T) { func TestMutationAdmission(t *testing.T) { t.Parallel() - for _, checkpoint := range []Checkpoint{"", PreparingHost, StartingNode, Complete, Resetting} { - t.Run(string(checkpoint), func(t *testing.T) { + for _, phase := range []Phase{"", Installing, Complete, Resetting} { + t.Run(string(phase), func(t *testing.T) { s := testStore(t) - if checkpoint != "" { + if phase != "" { r, err := NewRecord("machine", "f") require.NoError(t, err) - r.Checkpoint = checkpoint + r.Phase = phase require.NoError(t, s.Save(r)) } lock, err := s.AcquireMutationLock() - if checkpoint == "" || checkpoint == Complete { + if phase == "" || phase == Complete { require.NoError(t, err) _, err = s.AcquireMutationLock() require.ErrorIs(t, err, ErrLockHeld) From 5e92e59896506d1823aa1fc17a5e685699ab424a Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:54:40 -0400 Subject: [PATCH 19/39] agent e2e: assert the retry converges rather than resuming The recovery scenario asserted the record had reached a named stage before the retry, and reached complete after. The record no longer says how far an attempt got, because that was a claim about the host rather than a fact about it. Assert what the host shows instead. The failure landing late is proved by the node being up, and the retry converging around it is proved by the nspawn PID being unchanged, the installation ID being the same, and the applied config staying byte-identical while the retry carries a changed label. That is the same evidence as before, taken from the thing it is actually about. --- hack/agent/e2e-kind/e2e.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/hack/agent/e2e-kind/e2e.py b/hack/agent/e2e-kind/e2e.py index bcd210b7f..e6bd4a641 100755 --- a/hack/agent/e2e-kind/e2e.py +++ b/hack/agent/e2e-kind/e2e.py @@ -2611,15 +2611,22 @@ def _run_agent_inner(agent_url: str, node_config: NodeConfig) -> None: time.monotonic() + 30, check=True).stdout state_text, pid = snapshot.rstrip().rsplit("\n", 1) before = json.loads(state_text) - if before["checkpoint"] != "installing-daemon" or not pid.isdigit() or int(pid) <= 0: - die(f"failure did not reach the late bootstrap checkpoint: {snapshot}") + # The record says only that an installation is under way. It deliberately + # does not say how far it got, because that would be a claim about the + # host that could stop being true. What proves the failure landed late + # is the host itself: the node is up, so the retry has to converge + # around a running machine rather than rebuild underneath it. + if before["phase"] != "installing" or not pid.isdigit() or int(pid) <= 0: + die(f"bootstrap did not fail with a running node: {snapshot}") # The applied config records what actually configured the running node. # Retry with a changed node label: admission still allows it, because # labels are deliberately outside the installation fingerprint, but the # node was started before the change and never saw it. Re-persisting it - # here would read as "no drift" forever after, so the record must not - # move while the retry resumes past the node stage. + # here would read as "no drift" forever after. The retry reapplies the + # node stage like every other, so what keeps the record still is that + # the node is already running with the old configuration and the label + # change is not one the stage acts on. applied_config = "/etc/unbounded/agent/kube1-applied-config.json" before_applied = bounded_ssh( f"sudo sha256sum {applied_config}", time.monotonic() + 30, check=True).stdout.split()[0] @@ -2646,7 +2653,7 @@ def add_retry_label(agent_config: dict) -> None: time.monotonic() + 30, check=True).stdout state_text, after_pid = after_text.rstrip().rsplit("\n", 1) after = json.loads(state_text) - if after["installID"] != before["installID"] or after["checkpoint"] != "complete" or after_pid != pid: + if after["installID"] != before["installID"] or after["phase"] != "complete" or after_pid != pid: die("bootstrap retry changed ownership or restarted the running node") after_applied = bounded_ssh( @@ -2654,7 +2661,7 @@ def add_retry_label(agent_config: dict) -> None: if after_applied != before_applied: die("bootstrap retry overwrote the applied config with a label the running node never saw") - log("Late bootstrap retry preserved installation, nspawn PID and applied config") + log("Retry converged around the running node, preserving installation, nspawn PID and applied config") return run([ "timeout", "1200", @@ -4858,7 +4865,7 @@ def validate_bootstrap_repair() -> None: test "$before" = "$(sha256sum /etc/unbounded/agent/kube2-applied-config.json)" test ! -e /etc/unbounded/agent/kube1-applied-config.json test "$node_pid" = "$(systemctl show systemd-nspawn@kube2.service --property=MainPID --value)" - grep -q '"checkpoint": "complete"' /var/lib/unbounded/agent/install-state.json + grep -q '"phase": "complete"' /var/lib/unbounded/agent/install-state.json systemctl is-active unbounded-agent-daemon.service """) bounded_ssh("sudo bash -c " + shlex.quote(script), time.monotonic() + 180, check=True) From 8dd8fa6a2b7883655d82a5b244896de5968caa43 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 15:56:56 -0400 Subject: [PATCH 20/39] docs: describe bootstrap retry as reapplying rather than resuming The guide said completed stages are skipped and unfinished ones replayed, which described the record as the source of truth for what had been done. Every stage now runs on every attempt and decides from the host what it still has to do, so say that, and say what each stage looks at: packages already present, a rootfs a machine is registered from, a running machine, configuration that did not change, a live nftables ruleset. That is also the answer to the question this section invited, which is what happens when the host changed between attempts. It is repaired rather than skipped past, because nothing is assumed from how far a previous attempt got. --- docs/content/guides/agent.md | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/docs/content/guides/agent.md b/docs/content/guides/agent.md index f01d0c2a5..ecd20f194 100644 --- a/docs/content/guides/agent.md +++ b/docs/content/guides/agent.md @@ -32,16 +32,26 @@ in sequence: Initial bootstrap records installation ownership before changing the host. If a stage fails or the process is interrupted, rerun the same saved bootstrap script -or invoke `unbounded-agent start` with the same original configuration. Completed -stages are skipped; unfinished stages are replayed. A running nspawn machine is -preserved during node-start replay. +or invoke `unbounded-agent start` with the same original configuration. + +Every stage runs again on every attempt, and each decides what it still has to +do by looking at the host. Packages already present are not reinstalled. A +rootfs a machine is already registered from is left in place rather than +rebuilt. A running nspawn machine is left running. Node service configuration is +rewritten, and the service is restarted only if that configuration actually +changed. The nftables ruleset a running node depends on is not flushed. + +This is why a retry is safe to run repeatedly, and why a host that was modified +between attempts is repaired rather than skipped past: nothing is assumed from +how far a previous attempt got. The ownership record is `/var/lib/unbounded/agent/install-state.json`. It is an -internal file, not a configuration input. Keep it intact when retrying. A -different machine name, Kubernetes version, rootfs image, or API server endpoint -is rejected and requires an explicit reset before a new initial installation. -Credentials and artifact locations can be refreshed for a retry. Other fields -do not participate in admission; a retry does not reapply stages already completed. +internal file, not a configuration input. Keep it intact when retrying. It +records which phase the installation is in, never how much of it has been done, +so it cannot fall out of step with the host. A different machine name, +Kubernetes version, rootfs image, or API server endpoint is rejected and +requires an explicit reset before a new initial installation. Credentials and +artifact locations can be refreshed for a retry. After completion, the same `start` invocation checks required daemon files, executable permissions, and enabled/active service state, and repairs the daemon From 4165868cd1387b03ebf09e669f12a310a04fa857 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 16:33:36 -0400 Subject: [PATCH 21/39] agent: do not restate the applied config on a retry that found the node up Every stage reapplies now, so the node stage runs again even when the node is already running, and it was rewriting the applied config on the way through. The old code could not do this: a retry resumed past the node stage, so the comment justifying the write said it was unreachable on a retry. That is no longer true, and the write is wrong. The applied config records what the running node was built from, and the daemon diffs it against the desired config to decide whether to repave. An attempt that finds a machine already registered did not build that node and has nothing to say about how it was built. The case that shows the harm is a changed node label. Labels sit outside the installation fingerprint, so a retry carrying a new one is admitted, but kubelet takes --node-labels at registration and a restart under an existing node does not revise them. Recording the new label would make applied match desired, which reads as no drift, which suppresses the repave that is the only thing that would have delivered it. The label would be lost silently. Leaving the record alone keeps the difference visible and lets the daemon resolve it. Gate the write on whether a machine was registered when the stage began, asked before the stage runs because afterwards the answer is always yes. PrepareRootFS already asked the same question, so both now share one helper. Repave is unaffected; it persists explicitly, outside this composition. Also sweep the vocabulary the phase change left behind. The installstate package doc still offered "how far did the last attempt get" as a question it answers, which is now precisely the question it refuses to answer. --- cmd/agent/internal/cmd/bootstrap.go | 78 ++++++++++++++++++------ cmd/agent/internal/cmd/bootstrap_test.go | 46 +++++++++++--- cmd/agent/internal/installstate/doc.go | 7 ++- docs/content/guides/agent.md | 17 ++++-- pkg/agent/phases/rootfs/oci/task.go | 4 +- 5 files changed, 113 insertions(+), 39 deletions(-) diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index c068d6faf..164aeb3b8 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -149,17 +149,15 @@ func (s *agentStages) PrepareRootFS(ctx context.Context) error { // This is a property of the host, not of how far a previous attempt got. It // holds whether the machine was started by an earlier attempt of this // installation or independently afterwards. - for _, name := range []string{goalstates.NSpawnMachineKube1, goalstates.NSpawnMachineKube2} { - registered, err := reset.RegisteredMachine(ctx, s.log, name) - if err != nil { - return err - } + registered, err := s.registeredMachine(ctx) + if err != nil { + return err + } - if registered { - s.log.Info("nspawn machine is registered; leaving its rootfs in place", "machine", name) + if registered != "" { + s.log.Info("nspawn machine is registered; leaving its rootfs in place", "machine", registered) - return nil - } + return nil } if err := s.prepareCredentials(ctx); err != nil { @@ -173,17 +171,33 @@ func (s *agentStages) PrepareRootFS(ctx context.Context) error { return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, "/usr/local", goalstates.SystemdSystemDir, goalstates.SystemdNSpawnDir) } -// nodeStartTask composes the work that brings the node up. Persisting the -// applied config belongs here, not in the daemon stage: it must record the -// configuration that actually configured the node. A retry that resumes at a -// later checkpoint skips this stage entirely, so it cannot overwrite the record -// with a configuration the running node never saw. -func (s *agentStages) nodeStartTask() phases.Task { - return phases.Serial(s.log, +// nodeStartTask composes the work that brings the node up. +// +// The applied config records what the running node was built from, and the +// daemon compares it against the desired config to decide whether the node has +// drifted far enough to need a repave. Only an attempt that actually built the +// node may write it. nodeAlreadyBuilt says a machine was already registered +// when this stage began, so this attempt found the node standing rather than +// raising it. +// +// Writing it anyway would be a claim the host cannot support. Node labels reach +// a node through kubelet's --node-labels at registration; restarting kubelet +// under an already-registered node does not revise them. Recording a label the +// node never took would leave the applied config matching the desired config, +// which reads as no drift, which is precisely what suppresses the repave that +// would have delivered it. Leaving the record alone keeps the difference +// visible and lets the daemon resolve it. +func (s *agentStages) nodeStartTask(nodeAlreadyBuilt bool) phases.Task { + tasks := []phases.Task{ nodestart.StartNode(s.log, s.gs.NodeStart), nodestart.WaitForKubeletBootstrap(s.log, s.gs.NodeStart.MachineName), - daemon.PersistAppliedConfig(s.log, s.gs.NodeStart.MachineName, &s.cfg.AgentConfig), - ) + } + + if !nodeAlreadyBuilt { + tasks = append(tasks, daemon.PersistAppliedConfig(s.log, s.gs.NodeStart.MachineName, &s.cfg.AgentConfig)) + } + + return phases.Serial(s.log, tasks...) } func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { @@ -191,15 +205,39 @@ func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { return err } - if err := s.nodeStartTask().Do(ctx); err != nil { + // Asked before the stage runs, because afterwards every answer is yes. + registered, err := s.registeredMachine(ctx) + if err != nil { + return err + } + + if err := s.nodeStartTask(registered != "").Do(ctx); err != nil { return err } // AgentConfigDir holds the applied config written above, so it must reach - // disk before this stage is checkpointed as complete. + // disk before this stage reports success. return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, goalstates.AgentConfigDir, goalstates.SystemdSystemDir) } +// registeredMachine returns the name of the first registered nspawn slot, or +// the empty string if neither is registered. It fails closed: an uninspectable +// host is an error rather than an assumption that nothing is running on it. +func (s *agentStages) registeredMachine(ctx context.Context) (string, error) { + for _, name := range []string{goalstates.NSpawnMachineKube1, goalstates.NSpawnMachineKube2} { + registered, err := reset.RegisteredMachine(ctx, s.log, name) + if err != nil { + return "", err + } + + if registered { + return name, nil + } + } + + return "", nil +} + func (s *agentStages) daemonInstallTask() phases.Task { return phases.Serial(s.log, daemon.EnableDaemon(s.log)) } diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index 84b7315c0..9dc480a4f 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -171,25 +171,51 @@ func TestCanonicalImageIdentityLeavesNonHTTPSReferencesAlone(t *testing.T) { // TestNodeStartPersistsAppliedConfig pins where the applied config is written. // It has to happen in the stage that starts the node, and after kubelet has // bootstrapped, so the record always describes the configuration the running -// node was built from. Moving it into the daemon stage would let a retry that -// resumes there record a configuration the node never saw, which then reads as -// "no drift" and is never reconciled. +// node was built from. Moving it into the daemon stage would record a +// configuration the node never saw, which then reads as "no drift" and is +// never reconciled. func TestNodeStartPersistsAppliedConfig(t *testing.T) { t.Parallel() - stages := &agentStages{ + stages := newNodeStartStages() + + nodeStart := stages.nodeStartTask(false).Name() + require.Contains(t, nodeStart, "persist-applied-config") + require.Less(t, strings.Index(nodeStart, "wait-for-kubelet-bootstrap"), strings.Index(nodeStart, "persist-applied-config")) + + require.NotContains(t, stages.daemonInstallTask().Name(), "persist-applied-config") +} + +// TestNodeStartSkipsAppliedConfigWhenNodeAlreadyBuilt is the other half of the +// rule above, and the one a retry depends on. Every stage reapplies on every +// attempt, so the node stage runs again even when the node is already up. The +// attempt that finds a machine standing did not build it, and must not restate +// what it was built from. +// +// The case that makes this matter is a changed node label. Labels are outside +// the installation fingerprint, so a retry carrying a new one is admitted, but +// kubelet only takes --node-labels at registration and a restart under an +// existing node does not revise them. Writing the new label here would make the +// applied config match the desired config, which reads as no drift, which +// suppresses the repave that is the only thing that would deliver the label. +// The node would silently never get it. +func TestNodeStartSkipsAppliedConfigWhenNodeAlreadyBuilt(t *testing.T) { + t.Parallel() + + nodeStart := newNodeStartStages().nodeStartTask(true).Name() + + require.NotContains(t, nodeStart, "persist-applied-config") + require.Contains(t, nodeStart, "wait-for-kubelet-bootstrap") +} + +func newNodeStartStages() *agentStages { + return &agentStages{ log: slog.New(slog.DiscardHandler), cfg: &provision.UnboundedAgentConfig{}, gs: &goalstates.MachineGoalState{ NodeStart: &goalstates.NodeStart{MachineName: goalstates.NSpawnMachineKube1}, }, } - - nodeStart := stages.nodeStartTask().Name() - require.Contains(t, nodeStart, "persist-applied-config") - require.Less(t, strings.Index(nodeStart, "wait-for-kubelet-bootstrap"), strings.Index(nodeStart, "persist-applied-config")) - - require.NotContains(t, stages.daemonInstallTask().Name(), "persist-applied-config") } func TestCompletedPreflightOutput(t *testing.T) { diff --git a/cmd/agent/internal/installstate/doc.go b/cmd/agent/internal/installstate/doc.go index a239d5962..1e9e1809d 100644 --- a/cmd/agent/internal/installstate/doc.go +++ b/cmd/agent/internal/installstate/doc.go @@ -12,8 +12,11 @@ // identity-defining configuration, so a retry can tell its own interrupted // attempt from a different installation. Without it, bootstrap can only // demand a pristine host and abort otherwise. -// - How far did the last attempt get? The checkpoint lets a retry replay only -// unfinished stages instead of redoing completed work or refusing outright. +// - How far did the last attempt get? It does not say, deliberately. Every +// stage reapplies on every attempt and decides from the host what it still +// has to do, so there is nothing here that could disagree with the host. +// The phase records only whether an installation is under way, finished, or +// being torn down, which is a thing the host cannot be asked. // - Is anything else mutating this host? The lock serializes bootstrap, reset, // repave, NodeReboot, host agent activation and AgentUpgrade, which all // touch the same files and services. diff --git a/docs/content/guides/agent.md b/docs/content/guides/agent.md index ecd20f194..b776dd508 100644 --- a/docs/content/guides/agent.md +++ b/docs/content/guides/agent.md @@ -53,6 +53,13 @@ Kubernetes version, rootfs image, or API server endpoint is rejected and requires an explicit reset before a new initial installation. Credentials and artifact locations can be refreshed for a retry. +A retry that finds the node already running does not rewrite +`-applied-config.json`, because that file records what the running node +was built from and this attempt did not build it. Configuration that is allowed +to change between attempts but only takes effect at node registration, node +labels in particular, therefore reaches the node through the daemon's ordinary +drift repave rather than through the retry itself. + After completion, the same `start` invocation checks required daemon files, executable permissions, and enabled/active service state, and repairs the daemon when they are missing or stopped. It does not compare unit contents or overwrite @@ -76,11 +83,11 @@ and run reset again. Ownership is removed only after teardown and its filesystem synchronization barriers succeed. The lock file in `/run` remains and must not be deleted to force an operation through. -Recovery applies to checkpointed initial installations. Existing installations -without an ownership record retain ordinary daemon operations and reset support; -`start` requires a clean host before creating new ownership. Interrupted ordinary -repaves still use the existing repave model, without a persistent repave recovery -operation. +Recovery applies to initial installations that carry an ownership record. +Existing installations without one retain ordinary daemon operations and reset +support; `start` requires a clean host before creating new ownership. Interrupted +ordinary repaves still use the existing repave model, without a persistent repave +recovery operation. ## Configuration diff --git a/pkg/agent/phases/rootfs/oci/task.go b/pkg/agent/phases/rootfs/oci/task.go index e47f2b7cb..71deb4c3f 100644 --- a/pkg/agent/phases/rootfs/oci/task.go +++ b/pkg/agent/phases/rootfs/oci/task.go @@ -43,8 +43,8 @@ func DownloadRootFS( func (d *downloadRootFS) Name() string { return "oci-download-rootfs" } -// DownloadOwnedRootFS is for checkpointed initial installation only. The caller -// must hold installation ownership and prove this slot has never started a node. +// DownloadOwnedRootFS is for owned initial installation only. The caller must +// hold installation ownership and prove this slot has never started a node. // The original DownloadRootFS entry point keeps its existing nonempty-rootfs // behavior for callers managing legacy installations. func DownloadOwnedRootFS(log *slog.Logger, machineDir, hostArch, image string) phases.Task { From 9286031728e8b2311e14fa12731038980de1331f Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:21:09 -0400 Subject: [PATCH 22/39] agent: remove duplication the convergence work left behind Four small things, no behavior change. The kubelet and containerd tasks had grown identical change-tracking: same field, same comment, and a byte-identical write method. They share an embedded changeTracker now. Both the nftables task and bootstrap had grown their own loop over the two node slots calling RegisteredMachine, which inventories machinectl on every call, so each scan spawned two processes and a bootstrap attempt six. reset now offers FirstRegisteredMachine, which inventories once and returns the occupied slot, and both callers use it. restartReconfigured reached its three collaborators three different ways: it held the two configure tasks and read a field off each, but took a *bool aimed at a local in StartNode. It holds the start task now and reads wasRunning the same way, which drops the pointer, the local, and four nil checks no caller could reach. The coordinator converted its stage labels to string to build the loop and back to Stage twice to report them. The field is typed. Two of these turned out to be untested rather than merely undertested, which is why they are here rather than left alone. Nothing failed when FirstRegisteredMachine was made to never find a slot, and nothing failed when the machine-was-running observation was hardcoded either way, so the handoff that decides whether a reapply restarts a reconfigured service was resting on nothing. Both are now pinned, including that the scan inventories once, which is the reason it exists. --- cmd/agent/internal/bootstrap/coordinator.go | 14 ++--- cmd/agent/internal/cmd/bootstrap.go | 22 +------- pkg/agent/phases/host/configure_nftables.go | 18 +++---- pkg/agent/phases/nodestart/change_tracker.go | 33 ++++++++++++ pkg/agent/phases/nodestart/cri.go | 21 ++------ pkg/agent/phases/nodestart/kubelet.go | 22 ++------ pkg/agent/phases/nodestart/nspawn.go | 18 ++++--- pkg/agent/phases/nodestart/nspawn_test.go | 38 ++++++++++++++ .../phases/nodestart/restart_reconfigured.go | 12 ++--- .../nodestart/restart_reconfigured_test.go | 10 ++-- pkg/agent/phases/nodestart/start.go | 19 +++---- pkg/agent/phases/reset/machine.go | 43 +++++++++++++-- pkg/agent/phases/reset/strict_test.go | 52 +++++++++++++++++++ 13 files changed, 216 insertions(+), 106 deletions(-) create mode 100644 pkg/agent/phases/nodestart/change_tracker.go diff --git a/cmd/agent/internal/bootstrap/coordinator.go b/cmd/agent/internal/bootstrap/coordinator.go index 9e30f6e67..d5339bdbf 100644 --- a/cmd/agent/internal/bootstrap/coordinator.go +++ b/cmd/agent/internal/bootstrap/coordinator.go @@ -118,25 +118,25 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { // from it, an already running machine is not restarted, and node services // are restarted only when their configuration actually changed. for _, stage := range []struct { - name string + name Stage run func(context.Context) error }{ - {string(StagePrepareHost), c.stages.PrepareHost}, - {string(StagePrepareRootFS), c.stages.PrepareRootFS}, - {string(StageStartNode), c.stages.EnsureNodeStarted}, - {string(StageInstallDaemon), c.stages.EnsureDaemonInstalled}, + {StagePrepareHost, c.stages.PrepareHost}, + {StagePrepareRootFS, c.stages.PrepareRootFS}, + {StageStartNode, c.stages.EnsureNodeStarted}, + {StageInstallDaemon, c.stages.EnsureDaemonInstalled}, } { if err := ctx.Err(); err != nil { return Outcome{}, err } if c.reporter != nil { - c.reporter.StageStarted(ctx, Stage(stage.name)) + c.reporter.StageStarted(ctx, stage.name) } if err := stage.run(ctx); err != nil { if c.reporter != nil { - c.reporter.StageFailed(ctx, Stage(stage.name), err) + c.reporter.StageFailed(ctx, stage.name, err) } return Outcome{}, fmt.Errorf("%s: %w", stage.name, err) diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index 164aeb3b8..7cc6575a4 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -149,7 +149,7 @@ func (s *agentStages) PrepareRootFS(ctx context.Context) error { // This is a property of the host, not of how far a previous attempt got. It // holds whether the machine was started by an earlier attempt of this // installation or independently afterwards. - registered, err := s.registeredMachine(ctx) + registered, err := reset.FirstRegisteredMachine(ctx, s.log) if err != nil { return err } @@ -206,7 +206,7 @@ func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { } // Asked before the stage runs, because afterwards every answer is yes. - registered, err := s.registeredMachine(ctx) + registered, err := reset.FirstRegisteredMachine(ctx, s.log) if err != nil { return err } @@ -220,24 +220,6 @@ func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { return fsutil.SyncFilesystems(s.gs.RootFS.MachineDir, goalstates.AgentConfigDir, goalstates.SystemdSystemDir) } -// registeredMachine returns the name of the first registered nspawn slot, or -// the empty string if neither is registered. It fails closed: an uninspectable -// host is an error rather than an assumption that nothing is running on it. -func (s *agentStages) registeredMachine(ctx context.Context) (string, error) { - for _, name := range []string{goalstates.NSpawnMachineKube1, goalstates.NSpawnMachineKube2} { - registered, err := reset.RegisteredMachine(ctx, s.log, name) - if err != nil { - return "", err - } - - if registered { - return name, nil - } - } - - return "", nil -} - func (s *agentStages) daemonInstallTask() phases.Task { return phases.Serial(s.log, daemon.EnableDaemon(s.log)) } diff --git a/pkg/agent/phases/host/configure_nftables.go b/pkg/agent/phases/host/configure_nftables.go index 6ada72b93..f3cf603b9 100644 --- a/pkg/agent/phases/host/configure_nftables.go +++ b/pkg/agent/phases/host/configure_nftables.go @@ -49,21 +49,15 @@ func ConfigureNFTables(log *slog.Logger) phases.Task { return &configureNFTables{log: log, machineRegistered: anyMachineRegistered} } -// anyMachineRegistered reports whether either nspawn slot is registered. -// It fails closed: an uninspectable host is not reported as having no machine. +// anyMachineRegistered reports whether either node slot is registered. It fails +// closed: an uninspectable host is not reported as having no machine. func anyMachineRegistered(ctx context.Context, log *slog.Logger) (bool, error) { - for _, name := range []string{goalstates.NSpawnMachineKube1, goalstates.NSpawnMachineKube2} { - registered, err := reset.RegisteredMachine(ctx, log, name) - if err != nil { - return false, err - } - - if registered { - return true, nil - } + name, err := reset.FirstRegisteredMachine(ctx, log) + if err != nil { + return false, err } - return false, nil + return name != "", nil } func (c *configureNFTables) Name() string { return "configure-nftables" } diff --git a/pkg/agent/phases/nodestart/change_tracker.go b/pkg/agent/phases/nodestart/change_tracker.go new file mode 100644 index 000000000..35f69e65c --- /dev/null +++ b/pkg/agent/phases/nodestart/change_tracker.go @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package nodestart + +import ( + "os" + + "github.com/Azure/unbounded/pkg/agent/internal/utilio" +) + +// changeTracker records whether any file a configuration task owns actually +// differed from what was already on disk. +// +// The node services read their configuration at start, so a reapply that alters +// one has to restart it and a reapply that alters nothing must not. Embedders +// write through this rather than calling utilio directly, so the answer covers +// every file the task owns rather than only the last one written. +type changeTracker struct { + changed bool +} + +// write applies content and records whether it differed from what was there. +func (t *changeTracker) write(path string, content []byte, perm os.FileMode) error { + changed, err := utilio.WriteFileIfChanged(path, content, perm) + if err != nil { + return err + } + + t.changed = t.changed || changed + + return nil +} diff --git a/pkg/agent/phases/nodestart/cri.go b/pkg/agent/phases/nodestart/cri.go index 9df1cec2d..c8678cd58 100644 --- a/pkg/agent/phases/nodestart/cri.go +++ b/pkg/agent/phases/nodestart/cri.go @@ -38,27 +38,16 @@ const ( type configureContainerd struct { goalState *goalstates.NodeStart - // changed records whether any file this task owns actually differed. The - // node's containerd reads these at start, so a reapply that alters one has - // to restart it; a reapply that alters nothing must not. - changed bool -} - -// write applies content and records whether it differed from what was there. -func (c *configureContainerd) write(path string, content []byte, perm os.FileMode) error { - changed, err := utilio.WriteFileIfChanged(path, content, perm) - if err != nil { - return err - } - - c.changed = c.changed || changed - - return nil + changeTracker } // ConfigureContainerd returns a task that writes the containerd configuration, systemd unit, // and optional GPU drop-in configs into the machine rootfs. It runs before the nspawn machine // is started, so all paths are relative to the machine directory on the host filesystem. +// +// The agent reaches this work through StartNode, which builds the task directly +// so it can tell whether the configuration it wrote differed. This entry point +// remains for callers outside the agent that compose phases themselves. func ConfigureContainerd(goalState *goalstates.NodeStart) phases.Task { return &configureContainerd{goalState: goalState} } diff --git a/pkg/agent/phases/nodestart/kubelet.go b/pkg/agent/phases/nodestart/kubelet.go index d18dd81bb..93d92c3c3 100644 --- a/pkg/agent/phases/nodestart/kubelet.go +++ b/pkg/agent/phases/nodestart/kubelet.go @@ -20,34 +20,22 @@ import ( "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/pkg/agent/goalstates" - "github.com/Azure/unbounded/pkg/agent/internal/utilio" "github.com/Azure/unbounded/pkg/agent/phases" ) type configureKubelet struct { goalState *goalstates.NodeStart - // changed records whether any file this task owns actually differed. The - // node's kubelet reads these at start, so a reapply that alters one has to - // restart it; a reapply that alters nothing must not. - changed bool -} - -// write applies content and records whether it differed from what was there. -func (c *configureKubelet) write(path string, content []byte, perm os.FileMode) error { - changed, err := utilio.WriteFileIfChanged(path, content, perm) - if err != nil { - return err - } - - c.changed = c.changed || changed - - return nil + changeTracker } // ConfigureKubelet returns a task that writes the kubelet configuration into the machine rootfs. // It runs before the nspawn machine is started, so all paths are relative to // the machine directory on the host filesystem. +// +// The agent reaches this work through StartNode, which builds the task directly +// so it can tell whether the configuration it wrote differed. This entry point +// remains for callers outside the agent that compose phases themselves. func ConfigureKubelet(goalState *goalstates.NodeStart) phases.Task { return &configureKubelet{goalState: goalState} } diff --git a/pkg/agent/phases/nodestart/nspawn.go b/pkg/agent/phases/nodestart/nspawn.go index 1bf4e01f2..83c2e0676 100644 --- a/pkg/agent/phases/nodestart/nspawn.go +++ b/pkg/agent/phases/nodestart/nspawn.go @@ -69,16 +69,20 @@ type startNSpawnMachine struct { // runner is the machinectl/systemctl driver. Tests inject a fake. runner machinectlRunner - // wasRunning, when set, receives whether the machine was already up before - // this task touched it. A reapply against a live node has to restart any - // service whose configuration it changed, because those are read at start; - // on a machine this task boots, the services read the new files anyway. - wasRunning *bool + // wasRunning records whether the machine was already up before this task + // touched it. A reapply against a live node has to restart any service + // whose configuration it changed, because those are read at start; on a + // machine this task boots, the services read the new files anyway. + wasRunning bool } // StartNSpawnMachine returns a task that starts the systemd-nspawn machine using machinectl and // waits until D-Bus is responsive inside the machine so that subsequent phases // can safely use executil.MachineRun(). +// +// The agent reaches this work through StartNode, which builds the task directly +// so it can tell whether the configuration it wrote differed. This entry point +// remains for callers outside the agent that compose phases themselves. func StartNSpawnMachine(log *slog.Logger, goalState *goalstates.NodeStart) phases.Task { return &startNSpawnMachine{ log: log, @@ -117,9 +121,7 @@ func (s *startNSpawnMachine) startWithRecovery(ctx context.Context, name string) return fmt.Errorf("inspect nspawn service before replay: %w", err) } - if s.wasRunning != nil { - *s.wasRunning = running - } + s.wasRunning = running if running { return nil diff --git a/pkg/agent/phases/nodestart/nspawn_test.go b/pkg/agent/phases/nodestart/nspawn_test.go index d7fe40c3c..9cf30f301 100644 --- a/pkg/agent/phases/nodestart/nspawn_test.go +++ b/pkg/agent/phases/nodestart/nspawn_test.go @@ -271,3 +271,41 @@ func TestIsAlreadyExistsErr(t *testing.T) { }) } } + +// TestStartRecordsWhetherMachineWasAlreadyRunning pins the handoff to +// restartReconfigured. That task decides whether to restart a service whose +// configuration changed, and the only thing it has to go on is what this task +// observed before it touched the machine. +// +// Without this, the two halves can drift silently: restartReconfigured keeps +// reading the field correctly while nothing ever sets it, and a reapply against +// a live node stops restarting the services it just reconfigured. +func TestStartRecordsWhetherMachineWasAlreadyRunning(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + running bool + startResults []error + }{ + {name: "already running", running: true}, + {name: "booted by this run", running: false, startResults: []error{nil}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + s := &startNSpawnMachine{ + log: silentLogger(), + goalState: &goalstates.NodeStart{MachineName: "kube1"}, + runner: &fakeRunner{ + running: tc.running, + startResults: tc.startResults, + existsAfterStart: true, + }, + } + + require.NoError(t, s.startWithRecovery(context.Background(), "kube1")) + require.Equal(t, tc.running, s.wasRunning) + }) + } +} diff --git a/pkg/agent/phases/nodestart/restart_reconfigured.go b/pkg/agent/phases/nodestart/restart_reconfigured.go index 43dcb86fc..1351d1a83 100644 --- a/pkg/agent/phases/nodestart/restart_reconfigured.go +++ b/pkg/agent/phases/nodestart/restart_reconfigured.go @@ -30,15 +30,15 @@ type restartReconfigured struct { log *slog.Logger goalState *goalstates.NodeStart - machineWasRunning *bool - containerd *configureContainerd - kubelet *configureKubelet + startMachine *startNSpawnMachine + containerd *configureContainerd + kubelet *configureKubelet } func (r *restartReconfigured) Name() string { return "restart-reconfigured-services" } func (r *restartReconfigured) Do(ctx context.Context) error { - if r.machineWasRunning == nil || !*r.machineWasRunning { + if !r.startMachine.wasRunning { // This sequence started the machine, so its services have already read // the configuration written above. return nil @@ -50,8 +50,8 @@ func (r *restartReconfigured) Do(ctx context.Context) error { name string changed bool }{ - {goalstates.SystemdUnitContainerd, r.containerd != nil && r.containerd.changed}, - {goalstates.SystemdUnitKubelet, r.kubelet != nil && r.kubelet.changed}, + {goalstates.SystemdUnitContainerd, r.containerd.changed}, + {goalstates.SystemdUnitKubelet, r.kubelet.changed}, } { if !unit.changed { continue diff --git a/pkg/agent/phases/nodestart/restart_reconfigured_test.go b/pkg/agent/phases/nodestart/restart_reconfigured_test.go index 5da82b2e7..eea1a5af8 100644 --- a/pkg/agent/phases/nodestart/restart_reconfigured_test.go +++ b/pkg/agent/phases/nodestart/restart_reconfigured_test.go @@ -47,11 +47,11 @@ func reconfigureTask(t *testing.T, wasRunning, containerdChanged, kubeletChanged record := stubMachineRun(t) return &restartReconfigured{ - log: slog.New(slog.DiscardHandler), - goalState: &goalstates.NodeStart{MachineName: goalstates.NSpawnMachineKube1}, - machineWasRunning: &wasRunning, - containerd: &configureContainerd{changed: containerdChanged}, - kubelet: &configureKubelet{changed: kubeletChanged}, + log: slog.New(slog.DiscardHandler), + goalState: &goalstates.NodeStart{MachineName: goalstates.NSpawnMachineKube1}, + startMachine: &startNSpawnMachine{wasRunning: wasRunning}, + containerd: &configureContainerd{changeTracker: changeTracker{changed: containerdChanged}}, + kubelet: &configureKubelet{changeTracker: changeTracker{changed: kubeletChanged}}, }, record } diff --git a/pkg/agent/phases/nodestart/start.go b/pkg/agent/phases/nodestart/start.go index 85b76df73..e745b4f9e 100644 --- a/pkg/agent/phases/nodestart/start.go +++ b/pkg/agent/phases/nodestart/start.go @@ -24,15 +24,12 @@ func StartNode(log *slog.Logger, gs *goalstates.NodeStart) phases.Task { // so the sequence can tell whether the configuration it wrote differed, and // whether the machine was already up. Both are needed to decide if a // running service has to be restarted to read what changed. - var machineWasRunning bool - containerd := &configureContainerd{goalState: gs} kubelet := &configureKubelet{goalState: gs} startMachine := &startNSpawnMachine{ - log: log, - goalState: gs, - runner: defaultMachinectlRunner{log: log}, - wasRunning: &machineWasRunning, + log: log, + goalState: gs, + runner: defaultMachinectlRunner{log: log}, } return phases.Serial(log, @@ -48,11 +45,11 @@ func StartNode(log *slog.Logger, gs *goalstates.NodeStart) phases.Task { ImportContainerImages(log, gs), StartKubelet(log, gs), &restartReconfigured{ - log: log, - goalState: gs, - machineWasRunning: &machineWasRunning, - containerd: containerd, - kubelet: kubelet, + log: log, + goalState: gs, + startMachine: startMachine, + containerd: containerd, + kubelet: kubelet, }, ) } diff --git a/pkg/agent/phases/reset/machine.go b/pkg/agent/phases/reset/machine.go index 1f003682d..2b3961287 100644 --- a/pkg/agent/phases/reset/machine.go +++ b/pkg/agent/phases/reset/machine.go @@ -13,6 +13,7 @@ import ( "time" "github.com/Azure/unbounded/internal/executil" + "github.com/Azure/unbounded/pkg/agent/goalstates" "github.com/Azure/unbounded/pkg/agent/phases" ) @@ -214,19 +215,53 @@ func (t *removeMachine) Do(ctx context.Context) error { // RegisteredMachine reports registration only after successful inventory. // Bootstrap callers must not treat a missing inspection tool as a clean host. func RegisteredMachine(ctx context.Context, log *slog.Logger, name string) (bool, error) { + names, err := registeredMachines(ctx, log) + if err != nil { + return false, err + } + + _, ok := names[name] + + return ok, nil +} + +// FirstRegisteredMachine returns the name of the first node slot that is +// registered, or the empty string if neither is. It inventories once rather +// than per slot, and like RegisteredMachine it reports an uninspectable host as +// an error rather than as a clean one. +func FirstRegisteredMachine(ctx context.Context, log *slog.Logger) (string, error) { + names, err := registeredMachines(ctx, log) + if err != nil { + return "", err + } + + for _, name := range []string{goalstates.NSpawnMachineKube1, goalstates.NSpawnMachineKube2} { + if _, ok := names[name]; ok { + return name, nil + } + } + + return "", nil +} + +// registeredMachines inventories machinectl once and returns the registered +// names as a set. +func registeredMachines(ctx context.Context, log *slog.Logger) (map[string]struct{}, error) { out, err := executil.OutputCmd(ctx, log, "machinectl", "list", "--no-legend", "--no-pager") if err != nil { - return false, fmt.Errorf("inspect registered machines: %w", err) + return nil, fmt.Errorf("inspect registered machines: %w", err) } + names := make(map[string]struct{}) + for _, line := range strings.Split(out, "\n") { fields := strings.Fields(line) - if len(fields) > 0 && fields[0] == name { - return true, nil + if len(fields) > 0 { + names[fields[0]] = struct{}{} } } - return false, nil + return names, nil } // serviceIsActive returns true if the named systemd service is currently active. diff --git a/pkg/agent/phases/reset/strict_test.go b/pkg/agent/phases/reset/strict_test.go index 34fd9a59c..9894c0760 100644 --- a/pkg/agent/phases/reset/strict_test.go +++ b/pkg/agent/phases/reset/strict_test.go @@ -182,3 +182,55 @@ func TestFileCleanupPropagatesSubstantiveFailure(t *testing.T) { require.NoError(t, removeAllIfExists(log, dir)) require.NoError(t, removeFileIfExists(log, dir)) } + +// TestFirstRegisteredMachineInventoriesOnce covers the scan callers use to ask +// whether any node slot is occupied, without knowing which. +// +// The counter pins the reason it exists: machinectl is run once for the whole +// question rather than once per slot. The kube10 case pins that a slot name +// which merely starts with another slot's name is not a match, the same trap +// TestMachineInspectionFailsClosed covers for the single-name lookup. +func TestFirstRegisteredMachineInventoriesOnce(t *testing.T) { + for _, tc := range []struct { + name, listed string + want string + wantErr bool + }{ + {name: "neither", listed: "", want: ""}, + {name: "kube1", listed: "kube1 container systemd-nspawn - - -", want: "kube1"}, + {name: "kube2", listed: "kube2 container systemd-nspawn - - -", want: "kube2"}, + {name: "both prefers kube1", listed: "kube2 container systemd-nspawn - - -\\nkube1 container systemd-nspawn - - -", want: "kube1"}, + {name: "unrelated machine", listed: "someother container systemd-nspawn - - -", want: ""}, + {name: "longer name is not a match", listed: "kube10 container systemd-nspawn - - -", want: ""}, + {name: "uninspectable fails closed", listed: "", want: "", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + counter := filepath.Join(dir, "calls") + + script := "printf '" + tc.listed + "\\n'" + if tc.wantErr { + script = "exit 1" + } + + require.NoError(t, os.WriteFile(filepath.Join(dir, "machinectl"), + []byte("#!/bin/sh\necho x >> "+counter+"\n"+script+"\n"), 0o755)) + t.Setenv("PATH", dir) + + got, err := FirstRegisteredMachine(t.Context(), slog.New(slog.DiscardHandler)) + + if tc.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + require.Equal(t, tc.want, got) + + calls, err := os.ReadFile(counter) + require.NoError(t, err) + require.Len(t, strings.Split(strings.TrimSpace(string(calls)), "\n"), 1, + "machinectl must be inventoried once for the whole scan, not once per slot") + }) + } +} From 207a0f2bac8a7eb86b8bc2de89e82fb71b6f2f14 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 17:49:36 -0400 Subject: [PATCH 23/39] agent: keep CheckBindAddress exported An earlier commit on this branch swept exported symbols with no caller and removed this one. The sweep was right that nothing calls it: Preflight is unconditionally ownership-aware and uses checkOwnedBindAddress. It was wrong that this made it dead, because the symbol is on main in a pkg/ package, so removing it breaks anyone outside this repository who composes their own preflight set. Restore it with main's signature and behavior. The checker already guards c.owned != nil, so leaving owned unset gives the original meaning: any listener fails, including one that does belong to this installation. Comparing the exported surface of pkg/... against main, this was the only removal on the branch; everything else there is additive. The reason the sweep could not see it was that the tests exercised the checker type directly, so the constructor had no caller of any kind. Test it through the exported constructor instead, which both documents why it stays and fails the build if it is removed again. --- .../nodestart/preflight_bind_address.go | 21 ++++++++++++ .../nodestart/preflight_bind_address_test.go | 34 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/pkg/agent/phases/nodestart/preflight_bind_address.go b/pkg/agent/phases/nodestart/preflight_bind_address.go index d04670973..78d5c84f6 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address.go @@ -40,6 +40,27 @@ type bindAddressChecker struct { owned func() bool } +// CheckBindAddress verifies no TCP listener currently occupies an address's +// port. Any listener fails, including one belonging to this installation. +// +// Nothing in the agent calls this: Preflight is unconditionally ownership-aware +// and uses checkOwnedBindAddress, which accepts a listener it can prove this +// installation owns. It stays exported because it is part of this package's +// published surface, and callers outside the repository compose their own +// preflight sets from it. Removing it would break them at compile time, so it +// is kept deliberately rather than by oversight. +func CheckBindAddress(log *slog.Logger, name, address, description string) preflight.Checker { + return bindAddressChecker{ + name: name, + address: address, + description: description, + log: log, + inspect: func(address string) (string, bool, error) { + return inspectTCPListener("/proc", address) + }, + } +} + // checkOwnedBindAddress accepts only listeners with both the expected process // root and executable inode. An uninspectable owner is not proof of ownership. func checkOwnedBindAddress(log *slog.Logger, name, address, description, root, executable string) preflight.Checker { diff --git a/pkg/agent/phases/nodestart/preflight_bind_address_test.go b/pkg/agent/phases/nodestart/preflight_bind_address_test.go index d54bcd0c6..826e5d804 100644 --- a/pkg/agent/phases/nodestart/preflight_bind_address_test.go +++ b/pkg/agent/phases/nodestart/preflight_bind_address_test.go @@ -7,6 +7,7 @@ import ( "context" "errors" "log/slog" + "net" "os" "path/filepath" "testing" @@ -187,3 +188,36 @@ func TestListenerOwnershipRequiresRootAndExecutableForEverySocket(t *testing.T) }) } } + +// TestCheckBindAddressRejectsAnyListener exercises the exported constructor +// rather than the checker type the other tests here build directly. +// +// It exists because nothing inside the agent calls CheckBindAddress: Preflight +// is ownership-aware and uses checkOwnedBindAddress. A sweep for symbols with no +// caller therefore reads it as dead and removes it, which breaks callers outside +// the repository that compose their own preflight sets. This test is the caller +// that keeps it honest, and it pins the behavior those callers rely on: the +// unowned constructor rejects any listener at all, where the owned variant +// accepts one it can prove belongs to this installation. +func TestCheckBindAddressRejectsAnyListener(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + t.Cleanup(func() { _ = listener.Close() }) + + address := listener.Addr().String() + log := slog.New(slog.DiscardHandler) + + occupied := CheckBindAddress(log, "test-bind-address", address, "test bind address") + require.Equal(t, "test-bind-address", occupied.Name()) + + results := occupied.Check(context.Background()) + require.NotEmpty(t, results) + assert.Equal(t, preflight.SeverityError, results[0].Severity, + "a listener this installation cannot claim must fail the unowned check") + + require.NoError(t, listener.Close()) + + free := CheckBindAddress(log, "test-bind-address", address, "test bind address") + assert.Equal(t, preflight.SeverityOK, free.Check(context.Background())[0].Severity) +} From 3c8f03c1067bd339b869d9fd5fc236c37e376b98 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 19:58:16 -0400 Subject: [PATCH 24/39] agent: let the daemon stand down instead of looking like a crash The daemon refuses to run while an installation owns the host, which is right: it must not reconfigure a machine a bootstrap is still changing. It refused by returning an error, which is wrong, because systemd cannot tell that apart from the binary being broken. What followed was Restart=always, three starts inside StartLimitBurst, the unit in failed state, and OnFailure running the last-resort binary rollback for a problem the binary does not have. Where a previous AgentUpgrade had left a last-good binary, that rollback silently downgraded the agent. Two ways in, and the second is the common one. A bootstrap can die between starting the daemon and marking the record complete. More often, an install interrupted after the daemon was enabled means every later boot starts the daemon alongside a retry bootstrap that holds ownership for minutes while it downloads a rootfs; the daemon gave up after thirty seconds and failed. So each attempt to recover the host re-armed the rollback, and the rollback ran while the retry was still writing binaries, since recovery deliberately does not wait on lifecycle locks. Exiting zero would not have helped. Restart=always restarts a clean exit too, and the start limit counts starts rather than failures, so the unit reaches the same place. Say it in a way systemd understands instead. Standing down exits 69, and the unit names that code in SuccessExitStatus and RestartPreventExitStatus, so the unit goes inactive: not restarted, never approaching the start limit, never reaching OnFailure. Restart=always still covers real crashes. The daemon comes back the next time anything runs bootstrap, because both the install path and the repair path end by starting it. The wait for a held lock stays at thirty seconds. Only the giving up changed. Also reset-failed before start. A unit that already exhausted its start limit cannot be started until the failure is cleared, manual starts included, so without this a retry could not repair the hosts this bug has already broken. --- cmd/agent/internal/cmd/cmd.go | 10 +++ .../assets/unbounded-agent-daemon.service | 6 ++ cmd/agent/internal/daemon/daemon.go | 37 ++++++++- cmd/agent/internal/daemon/lifecycle.go | 31 ++++++-- cmd/agent/internal/daemon/lifecycle_test.go | 78 +++++++++++++++++++ cmd/agent/internal/daemon/migration_test.go | 31 +++++++- cmd/agent/internal/installstate/mutation.go | 13 +++- 7 files changed, 197 insertions(+), 9 deletions(-) diff --git a/cmd/agent/internal/cmd/cmd.go b/cmd/agent/internal/cmd/cmd.go index 40300728a..32adaa533 100644 --- a/cmd/agent/internal/cmd/cmd.go +++ b/cmd/agent/internal/cmd/cmd.go @@ -8,6 +8,8 @@ import ( "os" "github.com/spf13/cobra" + + "github.com/Azure/unbounded/cmd/agent/internal/daemon" ) func Run() { @@ -36,6 +38,14 @@ func Run() { ) if err := root.Execute(); err != nil { + // The daemon standing down for an unfinished installation is an + // ordinary state, not a fault. It has already said so in the journal, + // and the unit treats this code as success so systemd leaves it alone + // rather than restarting it into its start limit. + if daemon.IsDeferred(err) { + os.Exit(daemon.DeferredExitCode) + } + fmt.Printf("error: %v\n", err) os.Exit(1) } diff --git a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon.service b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon.service index 6c9dc49e4..3a92ee30c 100644 --- a/cmd/agent/internal/daemon/assets/unbounded-agent-daemon.service +++ b/cmd/agent/internal/daemon/assets/unbounded-agent-daemon.service @@ -14,6 +14,12 @@ Type=simple ExecStart={{ .DaemonBinaryCurrentPath }} daemon Restart=always RestartSec=10 +# Standing down because an installation owns the host is not a failure. Without +# these the daemon would exit, restart, exhaust StartLimitBurst, and trip +# OnFailure into a binary rollback for a problem the binary does not have. +# Both must match daemon.DeferredExitCode. +SuccessExitStatus={{ .DaemonDeferredExitCode }} +RestartPreventExitStatus={{ .DaemonDeferredExitCode }} [Install] WantedBy=multi-user.target diff --git a/cmd/agent/internal/daemon/daemon.go b/cmd/agent/internal/daemon/daemon.go index a7b4afda4..f4e5c7588 100644 --- a/cmd/agent/internal/daemon/daemon.go +++ b/cmd/agent/internal/daemon/daemon.go @@ -37,8 +37,27 @@ const ( // installationLockWaitTimeout bounds how long daemon startup waits for the // launching bootstrap or activation to release installation ownership. installationLockWaitTimeout = 30 * time.Second + + // DeferredExitCode is returned when the daemon has no work because an + // installation owns the host. It is not a failure, and the daemon unit + // names it in both SuccessExitStatus and RestartPreventExitStatus so + // systemd leaves the unit inactive instead of restarting it, exhausting the + // start limit, and running OnFailure recovery. Those two directives and this + // constant have to agree; TestDaemonUnitDeclaresDeferredExitCode pins that. + // + // 69 is EX_UNAVAILABLE: the service is correct but cannot run yet. + DeferredExitCode = 69 ) +// errDeferUntilInstalled reports that the daemon cannot run because an +// installation owns the host, and that this is expected rather than a fault. +// The command layer turns it into DeferredExitCode without printing an error. +var errDeferUntilInstalled = errors.New("daemon deferred until installation completes") + +// IsDeferred reports whether err means the daemon stood down for an unfinished +// installation rather than failed. +func IsDeferred(err error) bool { return errors.Is(err, errDeferUntilInstalled) } + // kubeClientFunc constructs a controller-runtime client from a rest.Config. // The production implementation is client.NewWithWatch; tests can supply a fake. type kubeClientFunc func(cfg *rest.Config, opts client.Options) (client.WithWatch, error) @@ -158,13 +177,29 @@ func discoverAndMigrate(ctx context.Context, log *slog.Logger, store *installsta return active, nil } + // An installation owns this host and has not finished. Nothing here can + // finish it: only a bootstrap run can, and that run starts the daemon + // when it succeeds. Stand down rather than fail. + if errors.Is(err, installstate.ErrInstallationInProgress) { + log.Warn("installation has not finished; daemon is standing down until bootstrap completes", "error", err) + + return nil, errDeferUntilInstalled + } + if !errors.Is(err, installstate.ErrLockHeld) { return nil, err } select { case <-waitCtx.Done(): - return nil, fmt.Errorf("wait for installation ownership at daemon startup: %w", waitCtx.Err()) + // A bootstrap is holding ownership for longer than a normal handoff. + // It starts the daemon again when it finishes, so waiting longer + // buys nothing and exiting as a failure would look like a crash. + log.Warn("bootstrap still holds installation ownership; daemon is standing down until it completes", + "waited", installationLockWaitTimeout, + ) + + return nil, errDeferUntilInstalled case <-time.After(250 * time.Millisecond): } } diff --git a/cmd/agent/internal/daemon/lifecycle.go b/cmd/agent/internal/daemon/lifecycle.go index 07f958a80..8cd40302d 100644 --- a/cmd/agent/internal/daemon/lifecycle.go +++ b/cmd/agent/internal/daemon/lifecycle.go @@ -11,6 +11,7 @@ import ( "fmt" "log/slog" "os" + "os/exec" "path/filepath" "strings" "text/template" @@ -90,21 +91,39 @@ func (d *enableDaemon) Do(ctx context.Context) error { return fmt.Errorf("writing %s: %w", goalstates.DaemonRecoveryScriptPath, err) } - sc := executil.Systemctl() + return activateDaemonUnit(ctx, d.log, executil.Systemctl()) +} - if err := executil.RunCmd(ctx, d.log, sc, "daemon-reload"); err != nil { +// activateDaemonUnit reloads, enables and starts the daemon unit. +// +// It is separate from writing the unit files so the command sequence can be +// exercised without a writable /etc, and because the order matters: see the +// reset-failed step below. +func activateDaemonUnit(ctx context.Context, log *slog.Logger, sc func(context.Context) *exec.Cmd) error { + if err := executil.RunCmd(ctx, log, sc, "daemon-reload"); err != nil { return fmt.Errorf("systemctl daemon-reload: %w", err) } - if err := executil.RunCmd(ctx, d.log, sc, "enable", goalstates.DaemonUnit); err != nil { + if err := executil.RunCmd(ctx, log, sc, "enable", goalstates.DaemonUnit); err != nil { return fmt.Errorf("systemctl enable %s: %w", goalstates.DaemonUnit, err) } - if err := executil.RunCmd(ctx, d.log, sc, "start", goalstates.DaemonUnit); err != nil { + // Clear any start-limit failure before starting. systemd refuses to start a + // unit that exhausted StartLimitBurst until the failure is reset, and that + // applies to manual starts too, so without this a retry cannot recover a + // host whose daemon was already rate-limited into failure. + // + // Tolerated when host policy denies it: SELinux can withhold this from the + // caller, and it unblocks a start rather than being required for one. + if err := executil.RunCmd(ctx, log, sc, "reset-failed", goalstates.DaemonUnit); err != nil { + log.Debug("could not reset daemon unit failure state", "unit", goalstates.DaemonUnit, "error", err) + } + + if err := executil.RunCmd(ctx, log, sc, "start", goalstates.DaemonUnit); err != nil { return fmt.Errorf("systemctl start %s: %w", goalstates.DaemonUnit, err) } - d.log.Info("daemon unit started", "unit", goalstates.DaemonUnit) + log.Info("daemon unit started", "unit", goalstates.DaemonUnit) return nil } @@ -155,6 +174,7 @@ func renderDaemonAssetForPaths(name string, content []byte, paths goalstates.Age DaemonBinaryLastGoodPath string DaemonRecoveryScriptPath string DaemonAgentUpgradeSignalPath string + DaemonDeferredExitCode int }{ DaemonUnit: goalstates.DaemonUnit, DaemonRecoveryUnit: goalstates.DaemonRecoveryUnit, @@ -162,6 +182,7 @@ func renderDaemonAssetForPaths(name string, content []byte, paths goalstates.Age DaemonBinaryLastGoodPath: paths.LastGoodPath, DaemonRecoveryScriptPath: goalstates.DaemonRecoveryScriptPath, DaemonAgentUpgradeSignalPath: paths.SignalPath, + DaemonDeferredExitCode: DeferredExitCode, } tmpl, err := template.New(name).Parse(string(content)) diff --git a/cmd/agent/internal/daemon/lifecycle_test.go b/cmd/agent/internal/daemon/lifecycle_test.go index c9b98204a..9528df2b9 100644 --- a/cmd/agent/internal/daemon/lifecycle_test.go +++ b/cmd/agent/internal/daemon/lifecycle_test.go @@ -6,11 +6,14 @@ package daemon import ( "os" "path/filepath" + "strconv" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/Azure/unbounded/internal/executil" "github.com/Azure/unbounded/internal/fsutil" "github.com/Azure/unbounded/pkg/agent/goalstates" ) @@ -101,3 +104,78 @@ func TestUsableDaemonBinaryRequiresAResolvableExecutable(t *testing.T) { assert.Equal(t, want, usableDaemonBinary(path), "path %s", path) } } + +// TestDaemonUnitDeclaresDeferredExitCode pins the agreement between the exit +// code the daemon returns when it stands down and the two directives that tell +// systemd to accept it. +// +// If they ever disagree, the daemon still exits quietly but systemd treats the +// code as a crash: it restarts the unit, exhausts StartLimitBurst, and runs +// OnFailure, which is the last-resort binary rollback. Nothing else would fail, +// so the only thing standing between a silent regression and a host rolling its +// agent back for an unfinished install is this test. +func TestDaemonUnitDeclaresDeferredExitCode(t *testing.T) { + t.Parallel() + + rendered, err := renderDaemonAssetForPaths("daemon-service", daemonServiceContent, goalstates.AgentUpgradePaths{ + CurrentPath: "/usr/local/bin/unbounded-agent-current", + LastGoodPath: "/usr/local/bin/unbounded-agent-last-good", + BinaryPath: "/usr/local/bin/unbounded-agent", + SignalPath: "/var/lib/unbounded/agent/upgrade-signal", + }) + require.NoError(t, err) + + unit := string(rendered) + code := strconv.Itoa(DeferredExitCode) + + require.Contains(t, unit, "SuccessExitStatus="+code, + "systemd must not treat standing down as a failure, or OnFailure runs the binary rollback") + require.Contains(t, unit, "RestartPreventExitStatus="+code, + "systemd must not restart a deferred daemon, or repeated starts exhaust the start limit") + + // The safety net for genuine crashes has to survive the above. + require.Contains(t, unit, "Restart=always") + require.Contains(t, unit, "OnFailure="+goalstates.DaemonRecoveryUnit) +} + +// TestActivateDaemonUnitClearsFailureBeforeStarting pins the order that lets a +// retry recover a host this bug already broke. +// +// A daemon that exhausted its start limit sits in failed state, and systemd +// refuses to start it again until the failure is reset. That refusal applies to +// manual starts too, so a bootstrap retry that only ran enable and start would +// fail on exactly the hosts most in need of repair. +func TestActivateDaemonUnitClearsFailureBeforeStarting(t *testing.T) { + dir := t.TempDir() + calls := filepath.Join(dir, "calls") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), + []byte("#!/bin/sh\necho \"$@\" >> \""+calls+"\"\n"), 0o755)) + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + + require.NoError(t, activateDaemonUnit(t.Context(), discardLogger(), executil.Systemctl())) + + recorded, err := os.ReadFile(calls) + require.NoError(t, err) + + got := string(recorded) + resetAt := strings.Index(got, "reset-failed") + startAt := strings.Index(got, "start ") + + require.NotEqual(t, -1, resetAt, "reset-failed must run; without it a rate-limited unit cannot be started:\n%s", got) + require.NotEqual(t, -1, startAt, "start must run:\n%s", got) + require.Less(t, resetAt, startAt, "reset-failed must precede start, or it cannot unblock it:\n%s", got) +} + +// TestActivateDaemonUnitToleratesDeniedResetFailed covers hosts where policy +// withholds reset-failed. It unblocks a start rather than being required for +// one, so a denial must not fail the install. +func TestActivateDaemonUnitToleratesDeniedResetFailed(t *testing.T) { + dir := t.TempDir() + + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), + []byte("#!/bin/sh\ncase \"$1\" in reset-failed) exit 1 ;; esac\nexit 0\n"), 0o755)) + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + + require.NoError(t, activateDaemonUnit(t.Context(), discardLogger(), executil.Systemctl())) +} diff --git a/cmd/agent/internal/daemon/migration_test.go b/cmd/agent/internal/daemon/migration_test.go index 788fe77fa..c21be0759 100644 --- a/cmd/agent/internal/daemon/migration_test.go +++ b/cmd/agent/internal/daemon/migration_test.go @@ -28,7 +28,14 @@ func TestDaemonStartupRunsLifecycleMigrationBeforeControllerSetup(t *testing.T) require.Equal(t, 1, op.lifecycleCalls) } -func TestStartupLockWaitHonorsDeadlineWithoutMigration(t *testing.T) { +// TestStartupLockWaitStandsDownWithoutMigration covers a bootstrap that holds +// installation ownership for longer than the daemon is willing to wait. +// +// The daemon must not migrate anything, and it must not report a failure. A +// bootstrap that holds the lock this long is still working, and it starts the +// daemon again when it finishes. Exiting as a failure here is what previously +// drove the unit through its start limit and into OnFailure recovery. +func TestStartupLockWaitStandsDownWithoutMigration(t *testing.T) { t.Parallel() store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) lock, err := store.AcquireLock() @@ -40,7 +47,27 @@ func TestStartupLockWaitHonorsDeadlineWithoutMigration(t *testing.T) { op := &fakeNodeOperator{} _, err = discoverAndMigrate(ctx, discardLogger(), store, op) - require.ErrorIs(t, err, context.DeadlineExceeded) + require.True(t, IsDeferred(err), "waiting out a live bootstrap must defer, not fail: %v", err) + require.Zero(t, op.lifecycleCalls) +} + +// TestStartupStandsDownWhileInstallationUnfinished covers the other way the +// daemon can find itself with no work: the record says an installation is under +// way and nobody holds the lock, so no bootstrap is running to finish it. +// +// Only a bootstrap run can complete the install, and that run starts the daemon +// on success, so the daemon defers instead of failing. +func TestStartupStandsDownWhileInstallationUnfinished(t *testing.T) { + t.Parallel() + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + + record, err := installstate.NewRecord("machine-1", "fingerprint") + require.NoError(t, err) + require.NoError(t, store.Save(record)) + + op := &fakeNodeOperator{} + _, err = discoverAndMigrate(t.Context(), discardLogger(), store, op) + require.True(t, IsDeferred(err), "an unfinished installation must defer, not fail: %v", err) require.Zero(t, op.lifecycleCalls) } diff --git a/cmd/agent/internal/installstate/mutation.go b/cmd/agent/internal/installstate/mutation.go index 9927a7460..64bdc40f8 100644 --- a/cmd/agent/internal/installstate/mutation.go +++ b/cmd/agent/internal/installstate/mutation.go @@ -8,6 +8,17 @@ import ( "fmt" ) +// ErrInstallationInProgress reports that an installation owns this host and has +// not finished. It is distinct from ErrLockHeld, which says another process is +// working right now: this one says the host is mid-installation whether or not +// anyone is currently acting on it. +// +// Callers that run because systemd started them, rather than because a person +// asked, must not treat this as a failure. An installation in progress is an +// ordinary state, and reporting it as a crash is how a service ends up being +// restarted, rate-limited, and recovered for a problem it does not have. +var ErrInstallationInProgress = errors.New("installation has not finished") + // AcquireMutationLock serializes ordinary lifecycle work with bootstrap and // reset. Legacy hosts without a record remain supported. When both locks are // needed, acquire installation ownership before the binary activation lock. @@ -28,5 +39,5 @@ func (s *Store) AcquireMutationLock() (*Lock, error) { return nil, errors.Join(loadErr, closeErr) } - return nil, errors.Join(fmt.Errorf("installation is %s; finish bootstrap or reset before lifecycle operations", r.Phase), closeErr) + return nil, errors.Join(fmt.Errorf("%w: installation is %s; finish bootstrap or reset before lifecycle operations", ErrInstallationInProgress, r.Phase), closeErr) } From 9cfb2983ee10bbf634b57e371c148850adfd9610 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:02:35 -0400 Subject: [PATCH 25/39] agent: let reset finish on a host without the packages it inspects with Reset now asks machinectl and nft what is on the host, and fails when it cannot ask. That is the right answer for bootstrap and the wrong one for reset, because the tools it asks with are the ones bootstrap installs. A bootstrap that dies inside host preparation leaves an ownership record on a host with no systemd-container and no nftables. Reset cannot finish there, so the record stays; start is refused against an installation that nothing can clear. Before this branch those inspections were tolerant and the host stayed recoverable. The two callers want opposite answers to the same question, so they get different functions rather than a shared one with a flag. Admission keeps failing closed: a host it cannot inspect is not a host it can prove is clean, and assuming otherwise risks building over a running node. Cleanup treats a tool that is not installed as proof of absence, because nothing of ours can be running if the things that run it were never there. The tolerance is only for a missing executable. A tool that is present and fails still stops reset, since it may be reporting a machine that really is there and reset must not delete around it. The test drives the reset tasks rather than the helper, and its PATH keeps systemctl and ip while dropping machinectl and nft. Hiding everything would have been the easier fixture and a meaningless one: systemctl always exists on a systemd host, and a first version of this test passed while the tasks still called the strict predicate. --- pkg/agent/phases/reset/machine.go | 12 ++-- pkg/agent/phases/reset/network.go | 18 +++++ pkg/agent/phases/reset/routes.go | 6 ++ pkg/agent/phases/reset/tooling.go | 42 ++++++++++++ pkg/agent/phases/reset/tooling_test.go | 93 ++++++++++++++++++++++++++ 5 files changed, 165 insertions(+), 6 deletions(-) create mode 100644 pkg/agent/phases/reset/tooling.go create mode 100644 pkg/agent/phases/reset/tooling_test.go diff --git a/pkg/agent/phases/reset/machine.go b/pkg/agent/phases/reset/machine.go index 2b3961287..ef5d9a23d 100644 --- a/pkg/agent/phases/reset/machine.go +++ b/pkg/agent/phases/reset/machine.go @@ -44,7 +44,7 @@ func (t *stopMachine) Do(ctx context.Context) error { t.log.Warn("machine was not enabled; continuing with stop and removal", "machine", t.machineName, "error", err) } - exists, err := RegisteredMachine(ctx, t.log, t.machineName) + exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName) if err != nil { return err } @@ -70,7 +70,7 @@ func (t *stopMachine) Do(ctx context.Context) error { } // Force terminate if still registered. - if exists, err := RegisteredMachine(ctx, t.log, t.machineName); err != nil { + if exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName); err != nil { return err } else if exists { t.log.Warn("machine did not stop gracefully, terminating", "machine", t.machineName) @@ -121,7 +121,7 @@ func confirmNotEnabled(ctx context.Context, log *slog.Logger, service string) er func (t *stopMachine) waitForGone(ctx context.Context, timeout time.Duration) (bool, error) { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { - if exists, err := RegisteredMachine(ctx, t.log, t.machineName); err != nil { + if exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName); err != nil { return false, err } else if !exists { return true, nil @@ -134,7 +134,7 @@ func (t *stopMachine) waitForGone(ctx context.Context, timeout time.Duration) (b } } - exists, err := RegisteredMachine(ctx, t.log, t.machineName) + exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName) return !exists, err } @@ -181,7 +181,7 @@ func (t *removeMachine) Do(ctx context.Context) error { return nil // machinectl removed both image metadata and directory } - if exists, err := RegisteredMachine(ctx, t.log, t.machineName); err != nil { + if exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName); err != nil { return err } else if !exists { // Once machined no longer knows the machine, the nspawn service is stopped @@ -203,7 +203,7 @@ func (t *removeMachine) Do(ctx context.Context) error { // Fallback: force-remove the directory if machinectl keeps failing. t.log.Warn("machinectl remove did not succeed, force-removing directory", "dir", machineDir) - if exists, err := RegisteredMachine(ctx, t.log, t.machineName); err != nil { + if exists, err := registeredMachineForCleanup(ctx, t.log, t.machineName); err != nil { return err } else if exists { return fmt.Errorf("refusing to remove registered machine %s", t.machineName) diff --git a/pkg/agent/phases/reset/network.go b/pkg/agent/phases/reset/network.go index f4ca0eaec..5044903de 100644 --- a/pkg/agent/phases/reset/network.go +++ b/pkg/agent/phases/reset/network.go @@ -73,6 +73,12 @@ func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { } tables, err := executil.OutputCmd(ctx, t.log, "nft", "list", "tables") + if missingTool(err) { + t.log.Warn("nft is not installed; no LocalDNS ruleset can exist") + + tables, err = "", nil + } + if err != nil { return fmt.Errorf("inspect LocalDNS tables: %w", err) } @@ -86,6 +92,12 @@ func (t *cleanupLocalDNSRules) Do(ctx context.Context) error { } links, err := executil.OutputCmd(ctx, t.log, "ip", "-d", "-o", "link", "show") + if missingTool(err) { + t.log.Warn("ip is not installed; no LocalDNS interface can exist") + + links, err = "", nil + } + if err != nil { return fmt.Errorf("inspect LocalDNS interface: %w", err) } @@ -178,6 +190,12 @@ func (t *removeWireGuardKeys) Do(_ context.Context) error { // interfaces visible on the host. func listWireGuardInterfaces(ctx context.Context, log *slog.Logger) ([]string, error) { out, err := executil.OutputCmd(ctx, log, "ip", "-o", "link", "show") + if missingTool(err) { + log.Warn("ip is not installed; no WireGuard interfaces can exist") + + return nil, nil + } + if err != nil { return nil, fmt.Errorf("ip link show: %w", err) } diff --git a/pkg/agent/phases/reset/routes.go b/pkg/agent/phases/reset/routes.go index 6f80d6ee9..1de32a0f7 100644 --- a/pkg/agent/phases/reset/routes.go +++ b/pkg/agent/phases/reset/routes.go @@ -52,6 +52,12 @@ func (t *cleanupRoutes) Do(ctx context.Context) error { } out, err := output(ctx, args...) + if missingTool(err) { + t.log.Warn("ip is not installed; no policy routing rules can exist") + + return nil + } + if err != nil { return fmt.Errorf("inspect %s %s: %w", family, kind, err) } diff --git a/pkg/agent/phases/reset/tooling.go b/pkg/agent/phases/reset/tooling.go new file mode 100644 index 000000000..550f0dcb6 --- /dev/null +++ b/pkg/agent/phases/reset/tooling.go @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package reset + +import ( + "context" + "errors" + "log/slog" + "os/exec" +) + +// missingTool reports whether err is a host inspection tool that is not +// installed, as opposed to a tool that ran and failed. +// +// Reset and bootstrap ask the host the same questions for opposite reasons, so +// they need opposite answers here. +// +// Bootstrap must fail closed. A host it cannot inspect is not a host it can +// prove is clean, and assuming otherwise risks building over a running node. +// +// Reset is the other way round. The tools it inspects with, machinectl from +// systemd-container and nft from nftables, are the ones bootstrap installs. A +// bootstrap that died inside host preparation therefore leaves a host where the +// question cannot be asked and the answer is known anyway: nothing of ours is +// running, because the tools needed to start it were never there. Treating that +// as an error leaves an installation record nothing can clear, which is the one +// outcome reset exists to prevent. +func missingTool(err error) bool { return errors.Is(err, exec.ErrNotFound) } + +// registeredMachineForCleanup answers RegisteredMachine's question with the +// tolerance described on missingTool. Cleanup uses this; admission must not. +func registeredMachineForCleanup(ctx context.Context, log *slog.Logger, name string) (bool, error) { + exists, err := RegisteredMachine(ctx, log, name) + if missingTool(err) { + log.Warn("machine inventory tool is not installed; nothing of ours can be running", "machine", name) + + return false, nil + } + + return exists, err +} diff --git a/pkg/agent/phases/reset/tooling_test.go b/pkg/agent/phases/reset/tooling_test.go new file mode 100644 index 000000000..f7ea329fc --- /dev/null +++ b/pkg/agent/phases/reset/tooling_test.go @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package reset + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +// hostMissingOurPackages builds a PATH resembling a host where bootstrap died +// inside host preparation: systemd and iproute2 are present because the distro +// ships them, but machinectl and nft are not, because installing +// systemd-container and nftables is the step that did not finish. +// +// Removing everything from PATH would be a easier fixture and a wrong one. +// systemctl always exists on a systemd host, so a test that also hides it +// proves nothing about the hosts this tolerance is for. +func hostMissingOurPackages(t *testing.T) { + t.Helper() + + dir := t.TempDir() + + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), []byte( + "#!/bin/sh\n"+ + "case \"$1\" in show) echo \"\" ;; esac\n"+ + "exit 0\n"), 0o755)) + + require.NoError(t, os.WriteFile(filepath.Join(dir, "ip"), []byte( + "#!/bin/sh\n"+ + "for a in \"$@\"; do if test \"$a\" = -j; then echo \"[]\"; exit 0; fi; done\n"+ + "exit 0\n"), 0o755)) + + t.Setenv("PATH", dir) +} + +// TestResetCompletesWithoutOurPackages is the case this tolerance exists for, +// and it runs through the reset tasks rather than the helper they call. +// +// A bootstrap that died inside host preparation leaves an installation record +// on a host without machinectl or nft. Reset has to finish there. If it cannot, +// the record stays forever and every later start is refused against an +// installation that nothing can clear. +func TestResetCompletesWithoutOurPackages(t *testing.T) { + hostMissingOurPackages(t) + + log := slog.New(slog.DiscardHandler) + + require.NoError(t, (&stopMachine{log: log, machineName: "kube1"}).Do(t.Context()), + "a machine cannot be running if machinectl was never installed") + + require.NoError(t, (&cleanupLocalDNSRules{log: log}).Do(t.Context()), + "a LocalDNS ruleset cannot exist if nft was never installed") + + require.NoError(t, (&cleanupRoutes{log: log}).Do(t.Context()), + "policy routing rules cannot exist without our packages") +} + +// TestAdmissionStillFailsClosedWithoutOurPackages is the other half, and the +// reason the tolerance is a separate function rather than a change to +// RegisteredMachine. +// +// Bootstrap asks the same question for the opposite reason: it must prove the +// host is clean before building on it. A host it cannot inspect is not a host +// it can prove anything about, so an absent tool stays an error here. +func TestAdmissionStillFailsClosedWithoutOurPackages(t *testing.T) { + hostMissingOurPackages(t) + + log := slog.New(slog.DiscardHandler) + + _, err := RegisteredMachine(t.Context(), log, "kube1") + require.Error(t, err, "bootstrap must not read an uninspectable host as a clean one") + + _, err = FirstRegisteredMachine(t.Context(), log) + require.Error(t, err, "bootstrap must not read an uninspectable host as a clean one") +} + +// TestCleanupStillFailsOnRealInspectionErrors keeps the tolerance narrow. A +// tool that is present and fails is a genuine problem: it may be reporting a +// machine that is actually there, and reset must not delete around it. +func TestCleanupStillFailsOnRealInspectionErrors(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "machinectl"), + []byte("#!/bin/sh\necho denied >&2\nexit 1\n"), 0o755)) + t.Setenv("PATH", dir) + + _, err := registeredMachineForCleanup(t.Context(), slog.New(slog.DiscardHandler), "kube1") + require.Error(t, err, "a tool that ran and failed is not proof of absence") +} From 1b6e56f034fa9bf522b187bbb1bf3e3becb25e1e Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:05:39 -0400 Subject: [PATCH 26/39] agent: do not let an unreadable record block the reset that deletes it Reset tolerated a missing ownership record and refused any other read error. decide refuses the same record, so a file that cannot be parsed took away both exits at once: start would not run, and reset could not clear the thing stopping it. The guide tells operators to keep this file intact, so the documented advice leads straight into the trap. Reset deletes the record moments later regardless. Reading it is a courtesy that keeps the machine name and fingerprint accurate through teardown, not a prerequisite for tearing down, so an unreadable one is replaced with the same synthetic record an absent one already produced, and the reason is logged. The decision moved into a helper because resetUnderLock syncs real host filesystems and needs root, so the behavior was otherwise only reachable from an e2e. Putting the sole store.Load behind that helper also means reintroducing a direct read leaves it uncalled, which staticcheck reports. --- cmd/agent/internal/daemon/reset.go | 22 ++++++++-- cmd/agent/internal/daemon/reset_test.go | 55 +++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index ad8c1c02a..61a6cd435 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -55,12 +55,28 @@ func ownedReset(log *slog.Logger, store *installstate.Store, inner phases.Task) }} } -func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.Store, inner phases.Task) error { +// recordForTeardown returns the record reset should mark as resetting. +// +// Any record it cannot read is replaced rather than obeyed. Reset is about to +// delete it, so refusing to proceed protects nothing and costs everything: +// decide rejects the same unreadable record, so start is refused too, and the +// host is left with no way out through either path. The guide tells operators +// to keep this file intact, so it must not be the thing that strands them. +func recordForTeardown(log *slog.Logger, store *installstate.Store) (installstate.Record, error) { r, err := store.Load() - if errors.Is(err, installstate.ErrNotFound) { - r, err = installstate.NewRecord("legacy-reset", "legacy-reset") + if err == nil { + return r, nil + } + + if !errors.Is(err, installstate.ErrNotFound) { + log.Warn("installation record is unreadable; replacing it for teardown", "error", err) } + return installstate.NewRecord("legacy-reset", "legacy-reset") +} + +func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.Store, inner phases.Task) error { + r, err := recordForTeardown(log, store) if err != nil { return err } diff --git a/cmd/agent/internal/daemon/reset_test.go b/cmd/agent/internal/daemon/reset_test.go index b73854272..2ae2b4b5e 100644 --- a/cmd/agent/internal/daemon/reset_test.go +++ b/cmd/agent/internal/daemon/reset_test.go @@ -7,6 +7,7 @@ import ( "context" "errors" "log/slog" + "os" "path/filepath" "strings" "testing" @@ -82,3 +83,57 @@ func TestResetRetainsOwnershipUntilTeardownAndSyncSucceed(t *testing.T) { }) } } + +// TestTeardownProceedsThroughAnUnreadableRecord covers the one file that can +// strand a host through both of its exits. +// +// decide rejects a record it cannot parse, so start is refused. If reset also +// refuses, nothing documented recovers the machine, and the guide tells +// operators to keep this file intact rather than delete it. Reset deletes it +// moments later regardless, so reading it is a courtesy, not a prerequisite. +// +// resetUnderLock itself syncs real host filesystems and needs root, so this +// covers the decision it delegates. Reintroducing a direct store.Load there +// would leave this helper uncalled, which staticcheck reports. +func TestTeardownProceedsThroughAnUnreadableRecord(t *testing.T) { + t.Parallel() + + for _, content := range []string{"{", "null", `{"schemaVersion":99}`, `{"schemaVersion":1,"phase":"bogus"}`} { + t.Run(content, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + + require.NoError(t, os.MkdirAll(store.Root(), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(store.Root(), "install-state.json"), []byte(content), 0o600)) + + // Confirm the premise: this is a record start would refuse. + _, loadErr := store.Load() + require.Error(t, loadErr, "fixture must be a record the store rejects") + + r, err := recordForTeardown(discardLogger(), store) + require.NoError(t, err, "an unreadable record must not block the thing that deletes it") + require.NoError(t, r.Validate(), "the replacement must be usable for teardown") + }) + } +} + +// TestTeardownKeepsAReadableRecord confirms the replacement above is a fallback +// and not the normal path: a record reset can read is the one it tears down, +// so the machine name and fingerprint stay accurate through reset. +func TestTeardownKeepsAReadableRecord(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) + + saved, err := installstate.NewRecord("machine-1", "fingerprint-1") + require.NoError(t, err) + require.NoError(t, store.Save(saved)) + + r, err := recordForTeardown(discardLogger(), store) + require.NoError(t, err) + require.Equal(t, "machine-1", r.MachineName) + require.Equal(t, "fingerprint-1", r.ConfigFingerprint) +} From cd9096c96c243f295568230436f2d04d8a1d234f Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:07:33 -0400 Subject: [PATCH 27/39] agent: repair a current daemon link that resolves to nothing VerifyDaemonInstalled resolves the current binary link and fails when its target is gone. Link initialization stat'd the link itself, and Lstat succeeds on a dangling symlink, so it saw a healthy link and left it alone. That combination made a dangling link the one fault verify could report and repair could not fix. start on a completed installation verified, repaired nothing, verified again, and returned the same stat error on every run, with no path back short of editing the link by hand. Resolve it instead, which is what the last-good link two lines down already did. A link that cannot resolve is now replaced the same way a missing one is, and the repair is checked by resolving it rather than by its name. --- pkg/agent/agentbinary/agentbinary.go | 9 ++++++-- pkg/agent/agentbinary/agentbinary_test.go | 28 +++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/pkg/agent/agentbinary/agentbinary.go b/pkg/agent/agentbinary/agentbinary.go index ef9ad8e09..6d696f487 100644 --- a/pkg/agent/agentbinary/agentbinary.go +++ b/pkg/agent/agentbinary/agentbinary.go @@ -50,9 +50,14 @@ func EnsureDaemonBinaryLinks(ctx context.Context, log *slog.Logger, paths goalst } currentTarget := paths.CurrentTargetPath - if _, err := os.Lstat(paths.CurrentPath); err != nil { + // Resolved rather than stat'd, matching the last-good check below. Lstat + // succeeds on a symlink whose target is gone, so a dangling current link + // read as healthy and was left alone. VerifyDaemonInstalled resolves it and + // fails, which made the link the one fault that verify could report and + // repair could not fix: start returned the same stat error forever. + if _, err := filepath.EvalSymlinks(paths.CurrentPath); err != nil { if !errors.Is(err, os.ErrNotExist) { - return fmt.Errorf("stat current daemon binary symlink: %w", err) + return fmt.Errorf("resolve current daemon binary symlink: %w", err) } target, targetErr := initialDaemonBinaryTarget(paths) diff --git a/pkg/agent/agentbinary/agentbinary_test.go b/pkg/agent/agentbinary/agentbinary_test.go index 3b3654350..fa25bb39f 100644 --- a/pkg/agent/agentbinary/agentbinary_test.go +++ b/pkg/agent/agentbinary/agentbinary_test.go @@ -233,3 +233,31 @@ func assertFileContent(t *testing.T, path, expected string) { require.NoError(t, err) assert.Equal(t, expected, string(data)) } + +// TestEnsureDaemonBinaryLinks_RepairsDanglingCurrent covers the one fault that +// verify could report and repair could not fix. +// +// VerifyDaemonInstalled resolves the current link and fails when its target is +// gone. Link initialization used to stat the link itself, which succeeds on a +// dangling symlink, so it saw a healthy link and left it. start on a completed +// installation then verified, repaired nothing, verified again, and returned +// the same stat error on every run forever. +func TestEnsureDaemonBinaryLinks_RepairsDanglingCurrent(t *testing.T) { + paths := setupDaemonBinaryTestPaths(t) + require.NoError(t, os.WriteFile(paths.BluePath, []byte("blue"), 0o755)) + + // A current link whose target no longer exists, as an interrupted + // activation or a removed slot leaves behind. + require.NoError(t, os.Symlink(filepath.Join(t.TempDir(), "removed-slot"), paths.CurrentPath)) + + _, err := filepath.EvalSymlinks(paths.CurrentPath) + require.Error(t, err, "fixture must be a link that cannot resolve") + + require.NoError(t, EnsureDaemonBinaryLinks(context.Background(), slog.Default(), paths)) + + assertSymlinkTarget(t, paths.CurrentPath, paths.BluePath) + + resolved, err := filepath.EvalSymlinks(paths.CurrentPath) + require.NoError(t, err, "the repaired link must resolve, or verify still fails") + require.Equal(t, paths.BluePath, resolved) +} From 5585fed03ef81e2a4ec431a2cf44c07a46a14150 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:08:59 -0400 Subject: [PATCH 28/39] agent: stage the downloaded binary somewhere it can be executed The installer stages the agent in a temporary directory and then runs it: admission runs from the staged binary rather than the installed one, so that a retry cannot overwrite a live binary link before its intent is accepted. That made the default temporary directory the wrong place for it. A host that mounts /tmp noexec cannot execute what was just staged there, and bootstrap fails before it starts. Hardened, image-based hosts are both the ones most likely to mount it that way and the ones this work is aimed at. Stage under /var/lib/unbounded instead, which the agent already owns, and keep the existing cleanup. Before this branch the staged binary was only copied, never run, so the placement did not matter. The mount options of any specific image are unverified; this removes the dependency on them rather than accommodating a measured one. --- internal/provision/assets/unbounded-agent-install.sh | 8 +++++++- internal/provision/script_test.go | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/internal/provision/assets/unbounded-agent-install.sh b/internal/provision/assets/unbounded-agent-install.sh index 3aca2fe58..0fb3d4f59 100644 --- a/internal/provision/assets/unbounded-agent-install.sh +++ b/internal/provision/assets/unbounded-agent-install.sh @@ -70,7 +70,13 @@ else _version_desc="${AGENT_VERSION:-custom}" fi echo "Downloading unbounded-agent ${_version_desc} for ${arch} from ${AGENT_URL}..." -tmp_dir="$(mktemp -d)" +# Staged under /var/lib rather than the default temporary directory because the +# staged binary is executed, not just copied: admission runs from it below. +# Hardened hosts commonly mount /tmp noexec, which would fail the run outright, +# and image-based hosts are the ones most likely to do so. +staging_root="/var/lib/unbounded" +mkdir -p "${staging_root}" +tmp_dir="$(mktemp -d "${staging_root}/install.XXXXXX")" trap 'rm -rf "${tmp_dir}"' EXIT curl -fsSL "${AGENT_URL}" | tar -xz -C "${tmp_dir}" unbounded-agent # Run admission from the staged executable. Bootstrap installs the daemon binary diff --git a/internal/provision/script_test.go b/internal/provision/script_test.go index 548669ab6..a9cb86a55 100644 --- a/internal/provision/script_test.go +++ b/internal/provision/script_test.go @@ -59,6 +59,17 @@ func TestUnboundedAgentInstallScript(t *testing.T) { // staged executable rather than one the retry just wrote. require.Contains(t, script, `if [ ! -x "${AGENT_BIN_TARGET}" ]; then`) require.Contains(t, script, `AGENT_BIN="${tmp_dir}/unbounded-agent"`) + + // The staged binary is executed, not just copied: admission runs from it. + // The default temporary directory is therefore the wrong place for it, + // because a host that mounts /tmp noexec cannot run it at all, and + // image-based hosts are the ones most likely to be hardened that way. + require.Contains(t, script, `mkdir -p "${staging_root}"`) + require.Contains(t, script, `tmp_dir="$(mktemp -d "${staging_root}/install.XXXXXX")"`) + require.NotContains(t, script, `tmp_dir="$(mktemp -d)"`) + + // Whatever is staged must still be cleaned up. + require.Contains(t, script, `trap 'rm -rf "${tmp_dir}"' EXIT`) } func TestUnboundedAgentUninstallScript(t *testing.T) { From 7f9696c0248853acd1d5d5624aa8846e01b08866 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:14:08 -0400 Subject: [PATCH 29/39] agent: ask about the slot being built, and correct two claims that were not true Three small corrections, all to work added on this branch. The rootfs and node stages asked whether any slot was registered, to decide whether to rebuild kube1's rootfs and whether this attempt built the node. Bootstrap only ever manages gs.NodeStart.MachineName, so a machine in the other slot answers a question neither stage asked. It is unreachable today because the second slot cannot exist before an installation completes, which is why consolidating the two predicates looked free when it was not. They ask about their own slot now. FirstRegisteredMachine keeps its one caller, the nftables flush, where either slot occupied is genuinely what matters because the ruleset is shared across the netns, and its doc says which question it answers. A store test fed a record carrying a stale "checkpoint" field, implying the format rejects fields it does not know. It does not: the decoder ignores them, and the case passed because the record had no phase. Ignoring unknown fields is deliberate and load-bearing, since it is what lets a record written by a newer agent stay readable by an older one, so the misleading case is now the missing-phase case it always was and a new test pins the tolerance. The intuitive hardening, DisallowUnknownFields, would take that guarantee away. Fingerprint claimed omitted optional fields stay omitted across releases. The fingerprinted struct has no omitempty tags, so they do not. Adding a field there rehashes every host that lacked it and each reads as a different installation demanding a reset. The comment now says what is actually required, and points at the fixture test that enforces it; confirmed by adding a field and watching that test fail. Also narrow restartReconfigured's doc, which claimed node services generally. It covers containerd and kubelet. LocalDNS and the NVIDIA drop-in write directly and are not tracked, and the reason that is survivable is that a retry does not rewrite the applied config, so the daemon still sees drift and repaves. --- cmd/agent/internal/cmd/bootstrap.go | 18 +++++++---- cmd/agent/internal/installstate/store.go | 13 +++++++- cmd/agent/internal/installstate/store_test.go | 30 ++++++++++++++++++- .../phases/nodestart/restart_reconfigured.go | 12 ++++++-- pkg/agent/phases/reset/machine.go | 6 ++++ 5 files changed, 69 insertions(+), 10 deletions(-) diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index 7cc6575a4..e00baf1a9 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -149,13 +149,17 @@ func (s *agentStages) PrepareRootFS(ctx context.Context) error { // This is a property of the host, not of how far a previous attempt got. It // holds whether the machine was started by an earlier attempt of this // installation or independently afterwards. - registered, err := reset.FirstRegisteredMachine(ctx, s.log) + // + // Asked of the slot this bootstrap manages rather than of either slot. + // Bootstrap only ever builds gs.NodeStart.MachineName, so a machine in the + // other slot says nothing about whether this one needs a rootfs. + registered, err := reset.RegisteredMachine(ctx, s.log, s.gs.NodeStart.MachineName) if err != nil { return err } - if registered != "" { - s.log.Info("nspawn machine is registered; leaving its rootfs in place", "machine", registered) + if registered { + s.log.Info("nspawn machine is registered; leaving its rootfs in place", "machine", s.gs.NodeStart.MachineName) return nil } @@ -205,13 +209,15 @@ func (s *agentStages) EnsureNodeStarted(ctx context.Context) error { return err } - // Asked before the stage runs, because afterwards every answer is yes. - registered, err := reset.FirstRegisteredMachine(ctx, s.log) + // Asked before the stage runs, because afterwards every answer is yes, and + // asked of the slot this bootstrap manages: a machine in the other slot was + // not built by this attempt either, but it is not the node being started. + registered, err := reset.RegisteredMachine(ctx, s.log, s.gs.NodeStart.MachineName) if err != nil { return err } - if err := s.nodeStartTask(registered != "").Do(ctx); err != nil { + if err := s.nodeStartTask(registered).Do(ctx); err != nil { return err } diff --git a/cmd/agent/internal/installstate/store.go b/cmd/agent/internal/installstate/store.go index 8783d1ed8..7520f6fec 100644 --- a/cmd/agent/internal/installstate/store.go +++ b/cmd/agent/internal/installstate/store.go @@ -170,7 +170,18 @@ func NewRecord(machine, fingerprint string) (Record, error) { } // Fingerprint hashes canonical JSON supplied before ephemeral credentials are -// resolved. Omitted optional fields stay omitted across compatible releases. +// resolved. +// +// The caller decides what canonical means, and the hash is over exactly the +// bytes it is given. A release that adds a field to the fingerprinted struct +// changes the hash of every host that did not have it, and each of those reads +// as a different installation demanding an explicit reset. An optional field +// therefore has to carry omitempty and be absent at its default, so records +// written before it existed keep hashing the same way. +// +// TestBootstrapV1CompatibilityFixtures enforces this: its fixtures carry a +// literal fingerprint, so any change to what is hashed fails there rather than +// on upgraded hosts. func Fingerprint(data []byte) string { sum := sha256.Sum256(data); return hex.EncodeToString(sum[:]) } type Disposition int diff --git a/cmd/agent/internal/installstate/store_test.go b/cmd/agent/internal/installstate/store_test.go index f66fd29f5..2d482a836 100644 --- a/cmd/agent/internal/installstate/store_test.go +++ b/cmd/agent/internal/installstate/store_test.go @@ -84,7 +84,7 @@ func TestOwnershipAdmission(t *testing.T) { func TestStoreRejectsCorruptAndOrphanedOwnership(t *testing.T) { t.Parallel() - for _, data := range []string{"{", "null", `{}`, `{"schemaVersion":2}`, `{"schemaVersion":1,"installID":"id","machineName":"machine","configFingerprint":"f","checkpoint":"bogus"}`} { + for _, data := range []string{"{", "null", `{}`, `{"schemaVersion":2}`, `{"schemaVersion":1,"installID":"id","machineName":"machine","configFingerprint":"f"}`} { t.Run(data, func(t *testing.T) { s := testStore(t) require.NoError(t, os.MkdirAll(s.Root(), 0o755)) @@ -199,3 +199,31 @@ func TestMutationAdmission(t *testing.T) { }) } } + +// TestStoreIgnoresUnknownFields pins a property the record format depends on +// for cross-version upgrades, and which nothing else asserts. +// +// Fields are ignored rather than rejected, so a record written by a newer agent +// stays readable by an older one. A release that adds an optional field would +// otherwise brick every host that later ran an agent predating it: the record +// would fail to parse, start would refuse it, and reset would be the only way +// out. Adding DisallowUnknownFields would be the intuitive hardening and would +// take that guarantee away. +func TestStoreIgnoresUnknownFields(t *testing.T) { + t.Parallel() + + s := testStore(t) + require.NoError(t, os.MkdirAll(s.Root(), 0o755)) + require.NoError(t, os.WriteFile(s.statePath(), []byte( + `{"schemaVersion":1,"installID":"id","machineName":"machine","configFingerprint":"f","phase":"installing","fieldFromALaterRelease":"value"}`, + ), 0o600)) + + r, err := s.Load() + require.NoError(t, err, "an unknown field must not make a record unreadable") + require.Equal(t, Installing, r.Phase) + require.Equal(t, "machine", r.MachineName) + + disposition, err := decide(r, nil, "machine", "f") + require.NoError(t, err) + require.Equal(t, Resume, disposition, "the record must still be usable, not merely parseable") +} diff --git a/pkg/agent/phases/nodestart/restart_reconfigured.go b/pkg/agent/phases/nodestart/restart_reconfigured.go index 1351d1a83..2c1b67c9e 100644 --- a/pkg/agent/phases/nodestart/restart_reconfigured.go +++ b/pkg/agent/phases/nodestart/restart_reconfigured.go @@ -13,8 +13,8 @@ import ( "github.com/Azure/unbounded/pkg/agent/phases" ) -// restartReconfigured restarts node services whose configuration this -// invocation actually changed. +// restartReconfigured restarts containerd and kubelet when this invocation +// actually changed their configuration. // // Writing a configuration file does not affect a service that has already read // it. That is harmless when this sequence boots the machine, because the @@ -26,6 +26,14 @@ import ( // Only an actual change restarts anything. A reapply that writes identical // content leaves the node alone, so the ordinary case of rerunning bootstrap // after a failure costs nothing. +// +// It covers containerd and kubelet, and deliberately not everything the node +// stage writes. LocalDNS and the NVIDIA drop-in write through utilio directly +// and are not tracked, so a reapply that changes one updates the file without +// restarting its reader. Those inputs are all carried in the applied config, +// which a retry against a running node does not rewrite, so the daemon still +// sees drift and repaves. Extending tracking to them would make the reapply +// converge without a repave; until then the repave is what closes the gap. type restartReconfigured struct { log *slog.Logger goalState *goalstates.NodeStart diff --git a/pkg/agent/phases/reset/machine.go b/pkg/agent/phases/reset/machine.go index ef5d9a23d..092441ce7 100644 --- a/pkg/agent/phases/reset/machine.go +++ b/pkg/agent/phases/reset/machine.go @@ -229,6 +229,12 @@ func RegisteredMachine(ctx context.Context, log *slog.Logger, name string) (bool // registered, or the empty string if neither is. It inventories once rather // than per slot, and like RegisteredMachine it reports an uninspectable host as // an error rather than as a clean one. +// +// This is for questions about the host as a whole, where either slot being +// occupied is what matters: whether it is safe to flush a ruleset shared by +// every machine in the netns, for instance. A caller asking about the node it +// is building or tearing down wants RegisteredMachine with that slot's name, +// because a machine in the other slot answers a question it did not ask. func FirstRegisteredMachine(ctx context.Context, log *slog.Logger) (string, error) { names, err := registeredMachines(ctx, log) if err != nil { From d20def14ab4f23cb39c77e67107d6a389e232bd0 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:31:17 -0400 Subject: [PATCH 30/39] agent: stop cobra narrating a deferred daemon as an error Standing down for an unfinished installation is reported as a warning and then returned as an error so the command layer can map it to the exit code systemd expects. Cobra printed that return as "Error:" and followed it with the full flag listing, so the journal showed a fault and an apparent misuse of the command for what is an ordinary state. That is the confusion this path exists to remove, in the one place the behavior is observed, so the fix was undone where it counted. Silence the usage block for the daemon command, which systemd invokes with fixed arguments and never misuses, and silence the error only for the deferred sentinel. Real failures are still reported. Found by reading the journal on a live host rather than from a test; the exit code, the unit state and the absent rollback were all already correct. ErrDeferred is exported in place of the IsDeferred predicate so the wrapping is testable with errors.Is, which is also how it reaches the command layer: Run adds context before returning it. --- cmd/agent/internal/cmd/cmd.go | 3 +- cmd/agent/internal/cmd/daemon.go | 20 +++++++- cmd/agent/internal/cmd/daemon_test.go | 54 +++++++++++++++++++++ cmd/agent/internal/daemon/daemon.go | 18 +++---- cmd/agent/internal/daemon/migration_test.go | 4 +- 5 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 cmd/agent/internal/cmd/daemon_test.go diff --git a/cmd/agent/internal/cmd/cmd.go b/cmd/agent/internal/cmd/cmd.go index 32adaa533..18f444fa1 100644 --- a/cmd/agent/internal/cmd/cmd.go +++ b/cmd/agent/internal/cmd/cmd.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "fmt" "os" @@ -42,7 +43,7 @@ func Run() { // ordinary state, not a fault. It has already said so in the journal, // and the unit treats this code as success so systemd leaves it alone // rather than restarting it into its start limit. - if daemon.IsDeferred(err) { + if errors.Is(err, daemon.ErrDeferred) { os.Exit(daemon.DeferredExitCode) } diff --git a/cmd/agent/internal/cmd/daemon.go b/cmd/agent/internal/cmd/daemon.go index 0f5f6679a..29ab555be 100644 --- a/cmd/agent/internal/cmd/daemon.go +++ b/cmd/agent/internal/cmd/daemon.go @@ -4,6 +4,7 @@ package cmd import ( + "errors" "os" "os/signal" @@ -18,15 +19,32 @@ func newCmdDaemon(cmdCtx *CommandContext) *cobra.Command { Short: "Long-running daemon for node lifecycle management", Long: "Long-running daemon that manages the nspawn machine lifecycle. " + "Runs as a systemd unit after initial provisioning.", + // systemd runs this with fixed arguments, so a failure here is never a + // usage problem and the flag listing is noise in the journal. + SilenceUsage: true, RunE: func(cmd *cobra.Command, _ []string) error { ctx, cancel := signal.NotifyContext(cmd.Context(), os.Interrupt) defer cancel() cmdCtx.Setup() - return daemon.Run(ctx, cmdCtx.Logger) + return quietWhenDeferred(cmd, daemon.Run(ctx, cmdCtx.Logger)) }, } return cmd } + +// quietWhenDeferred stops cobra reporting a deferred daemon as an error. +// +// Standing down because an installation owns the host is an ordinary state +// that the daemon has already recorded as a warning. Letting cobra print +// "Error:" over the top of that puts a fault in the journal where there is +// none, which is exactly the confusion this whole path exists to remove. +func quietWhenDeferred(cmd *cobra.Command, err error) error { + if errors.Is(err, daemon.ErrDeferred) { + cmd.SilenceErrors = true + } + + return err +} diff --git a/cmd/agent/internal/cmd/daemon_test.go b/cmd/agent/internal/cmd/daemon_test.go new file mode 100644 index 000000000..d40791b87 --- /dev/null +++ b/cmd/agent/internal/cmd/daemon_test.go @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "errors" + "fmt" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/cmd/agent/internal/daemon" +) + +// TestDaemonCommandKeepsDeferredOutOfTheJournalAsAnError covers what an +// operator actually sees when the daemon stands down for an unfinished +// installation. +// +// The daemon already reports it as a warning. Cobra would then print "Error:" +// over the top of that, and without SilenceUsage the whole flag listing too, so +// an ordinary state reads as both a fault and a misuse of the command. The +// journal is the one place this behavior is observed, so narrating it wrongly +// there undoes the fix where it counts. +func TestDaemonCommandKeepsDeferredOutOfTheJournalAsAnError(t *testing.T) { + t.Parallel() + + cmd := newCmdDaemon(&CommandContext{LogFormat: "text"}) + require.True(t, cmd.SilenceUsage, "systemd passes fixed arguments; a runtime failure is never a usage problem") + + // Run wraps the sentinel with context before it reaches here, which is how + // it appears in the journal, so recognition has to survive wrapping. + wrapped := fmt.Errorf("find active machine: %w", daemon.ErrDeferred) + + require.ErrorIs(t, quietWhenDeferred(cmd, wrapped), daemon.ErrDeferred) + require.True(t, cmd.SilenceErrors, "a deferred daemon must not be narrated as an error") +} + +// TestDaemonCommandStillReportsRealFailures keeps the suppression narrow. A +// daemon that cannot reach the API server has genuinely failed, and silencing +// that would hide the fault this path exists to distinguish from. +func TestDaemonCommandStillReportsRealFailures(t *testing.T) { + t.Parallel() + + cmd := &cobra.Command{} + + real := errors.New("kube client unreachable") + require.ErrorIs(t, quietWhenDeferred(cmd, real), real) + require.False(t, cmd.SilenceErrors, "genuine failures must still be reported") + + require.NoError(t, quietWhenDeferred(cmd, nil)) + require.False(t, cmd.SilenceErrors) +} diff --git a/cmd/agent/internal/daemon/daemon.go b/cmd/agent/internal/daemon/daemon.go index f4e5c7588..61f612fe1 100644 --- a/cmd/agent/internal/daemon/daemon.go +++ b/cmd/agent/internal/daemon/daemon.go @@ -49,14 +49,12 @@ const ( DeferredExitCode = 69 ) -// errDeferUntilInstalled reports that the daemon cannot run because an -// installation owns the host, and that this is expected rather than a fault. -// The command layer turns it into DeferredExitCode without printing an error. -var errDeferUntilInstalled = errors.New("daemon deferred until installation completes") - -// IsDeferred reports whether err means the daemon stood down for an unfinished -// installation rather than failed. -func IsDeferred(err error) bool { return errors.Is(err, errDeferUntilInstalled) } +// ErrDeferred reports that the daemon cannot run because an installation owns +// the host, and that this is expected rather than a fault. The command layer +// turns it into DeferredExitCode without printing an error. +// +// Run wraps it with context, so callers must test with errors.Is. +var ErrDeferred = errors.New("daemon deferred until installation completes") // kubeClientFunc constructs a controller-runtime client from a rest.Config. // The production implementation is client.NewWithWatch; tests can supply a fake. @@ -183,7 +181,7 @@ func discoverAndMigrate(ctx context.Context, log *slog.Logger, store *installsta if errors.Is(err, installstate.ErrInstallationInProgress) { log.Warn("installation has not finished; daemon is standing down until bootstrap completes", "error", err) - return nil, errDeferUntilInstalled + return nil, ErrDeferred } if !errors.Is(err, installstate.ErrLockHeld) { @@ -199,7 +197,7 @@ func discoverAndMigrate(ctx context.Context, log *slog.Logger, store *installsta "waited", installationLockWaitTimeout, ) - return nil, errDeferUntilInstalled + return nil, ErrDeferred case <-time.After(250 * time.Millisecond): } } diff --git a/cmd/agent/internal/daemon/migration_test.go b/cmd/agent/internal/daemon/migration_test.go index c21be0759..d095c5958 100644 --- a/cmd/agent/internal/daemon/migration_test.go +++ b/cmd/agent/internal/daemon/migration_test.go @@ -47,7 +47,7 @@ func TestStartupLockWaitStandsDownWithoutMigration(t *testing.T) { op := &fakeNodeOperator{} _, err = discoverAndMigrate(ctx, discardLogger(), store, op) - require.True(t, IsDeferred(err), "waiting out a live bootstrap must defer, not fail: %v", err) + require.ErrorIs(t, err, ErrDeferred, "waiting out a live bootstrap must defer, not fail") require.Zero(t, op.lifecycleCalls) } @@ -67,7 +67,7 @@ func TestStartupStandsDownWhileInstallationUnfinished(t *testing.T) { op := &fakeNodeOperator{} _, err = discoverAndMigrate(t.Context(), discardLogger(), store, op) - require.True(t, IsDeferred(err), "an unfinished installation must defer, not fail: %v", err) + require.ErrorIs(t, err, ErrDeferred, "an unfinished installation must defer, not fail") require.Zero(t, op.lifecycleCalls) } From 482574257774240004300967c29c0cae22dcd1fd Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 20:59:06 -0400 Subject: [PATCH 31/39] docs: describe what an inactive agent daemon means An operator who finds the daemon inactive after an interrupted install has no way to tell that from a broken one, and the obvious reaction, resetting the host, is the wrong one: the install only needs finishing. Say what the state means, what to look for, and what to do. Record the two cases where reset now proceeds rather than failing, since both look like reset ignoring a problem unless the reason is stated. --- docs/content/guides/agent.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/content/guides/agent.md b/docs/content/guides/agent.md index b776dd508..e8dc3d751 100644 --- a/docs/content/guides/agent.md +++ b/docs/content/guides/agent.md @@ -74,6 +74,20 @@ sudo env UNBOUNDED_AGENT_CONFIG_FILE=/path/to/original-agent-config.json \ /path/to/unbounded-agent start ``` +The agent daemon does not run while an installation is unfinished. If it starts +in that state, on a reboot or because an interrupted bootstrap had already +enabled it, it logs a warning and exits with status 69. The unit reports +`inactive`, not `failed`, and is not restarted; `systemctl status +unbounded-agent-daemon` showing `inactive (dead)` together with that warning +means the install still has to be completed, not that the daemon is broken. Run +the bootstrap again and the daemon starts with it. Do not reset the host to +clear this state. + +Reset also completes on a host where bootstrap never got far enough to install +the packages it inspects with, and on one whose ownership record cannot be +parsed. Both are cases where refusing would leave nothing able to clear the +record, so reset proceeds and removes it. + Bootstrap, reset, node lifecycle operations, and binary activation share an installation lock. Last-resort daemon rollback does not wait on these locks; reset stops the recovery unit before teardown. A busy lock is retryable, and From 364cb10b808fa152bea996a78adb74837af7979d Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Thu, 17 Sep 2026 15:50:01 -0400 Subject: [PATCH 32/39] agent: add the host installation prefix and resolve paths from it The agent writes its own host-side files to hard-coded paths under /usr/local: the daemon binaries and their blue-green links, the nspawn lifecycle helper, the daemon recovery script, and the LocalDNS network helper. On a host with a read-only /usr none of those writes can succeed, so the agent cannot be installed at all. Add AgentConfig.HostPrefix and a resolver that derives the host-side layout from it. Paths inside the nspawn machine are untouched: they are relative and joined with the machine directory, and conflating the two would break every host. The prefix is declared, never inferred. Where the agent may write is a property of the filesystem, not of the distribution, so keying on distro identity would misclassify a hardened host with a read-only /usr and would silently relocate files on any host whose os-release changed. A wrong guess is expensive to recover from, because the lifecycle helper path is baked as an absolute path into the nspawn drop-in and the config regeneration unit. The accepted syntax is narrow on purpose. The prefix is interpolated into generated systemd units and into a shell script, neither of which quotes it, so rather than adding two kinds of escaping that every consumer must keep correct, the value is constrained to be inert in both. Teardown and existing-deployment detection need to sweep both the configured prefix and the default, so that changing the prefix cannot orphan files or let a dirty host be silently reprovisioned; KnownHostPrefixes and MergeHostPrefixes exist for that and are used by the callers that follow. Nothing consumes the resolver yet. This is the model and its validation, so the changes that convert each caller can be read on their own. Hosts that do not set a prefix resolve to exactly the paths they had before, pinned by a regression test against the existing constants. --- pkg/agent/config/config.go | 75 +++++++++++ pkg/agent/config/config_test.go | 69 ++++++++++ pkg/agent/goalstates/hostpaths.go | 172 +++++++++++++++++++++++++ pkg/agent/goalstates/hostpaths_test.go | 116 +++++++++++++++++ 4 files changed, 432 insertions(+) create mode 100644 pkg/agent/goalstates/hostpaths.go create mode 100644 pkg/agent/goalstates/hostpaths_test.go diff --git a/pkg/agent/config/config.go b/pkg/agent/config/config.go index c8900082a..9f4602c8d 100644 --- a/pkg/agent/config/config.go +++ b/pkg/agent/config/config.go @@ -77,6 +77,18 @@ type AgentConfig struct { // Empty remains unobserved for legacy installations; it is not inferred from // the host distribution. The daemon reports explicit values in Machine status. ProvisioningFormat string `json:"ProvisioningFormat,omitempty"` + + // HostPrefix is the installation prefix for the agent's own host-side + // files: the daemon binaries under /bin and helper scripts + // under /libexec. It does not affect paths inside the nspawn + // machine, which are always relative to the machine directory. + // + // Empty means /usr/local, so hosts that do not set it are unaffected. Hosts + // with a read-only /usr must set it to a writable prefix; the agent refuses + // to bootstrap rather than guessing one, because where the agent may write + // is a property of the filesystem and not something that can be safely + // inferred from the distribution. + HostPrefix string `json:"HostPrefix,omitempty"` } const ( @@ -94,6 +106,65 @@ func ValidateProvisioningFormat(format string) error { } } +// hostPrefixAllowedRune reports whether r may appear in a host installation +// prefix. +// +// The prefix is interpolated into generated systemd units and into a shell +// script, neither of which quotes it. Rather than adding two kinds of escaping +// and having to keep them correct in every consumer, the accepted syntax is +// narrow enough that the value is inert in both contexts: no whitespace, no +// quoting or substitution characters, and no systemd "%" specifiers. +func hostPrefixAllowedRune(r rune) bool { + switch { + case r >= 'a' && r <= 'z': + return true + case r >= 'A' && r <= 'Z': + return true + case r >= '0' && r <= '9': + return true + case r == '/' || r == '.' || r == '_' || r == '-': + return true + default: + return false + } +} + +// ValidateHostPrefix checks that a configured host installation prefix is an +// absolute, normalized path that can hold a bin and libexec directory, and that +// it is safe to interpolate into the assets generated from it. An empty prefix +// is valid and selects the default. +func ValidateHostPrefix(prefix string) error { + trimmed := strings.TrimSpace(prefix) + if trimmed == "" { + return nil + } + + if !filepath.IsAbs(trimmed) { + return fmt.Errorf("HostPrefix must be an absolute path") + } + + if cleaned := filepath.Clean(trimmed); cleaned != trimmed { + return fmt.Errorf("HostPrefix must be a normalized path, for example %s", cleaned) + } + + if trimmed == "/" { + return fmt.Errorf("HostPrefix must not be the filesystem root") + } + + // Report the offending character rather than only the rule, because the + // caller cannot otherwise tell which byte of a long path was rejected. + for _, r := range trimmed { + if !hostPrefixAllowedRune(r) { + return fmt.Errorf( + "HostPrefix may only contain letters, digits, '/', '.', '_' and '-', but contains %q", + r, + ) + } + } + + return nil +} + // AgentOfflineArtifacts configures a complete offline source for binaries the // agent installs into the nspawn rootfs. type AgentOfflineArtifacts struct { @@ -250,6 +321,10 @@ func (a *AgentConfig) Validate() error { errs = append(errs, err) } + if err := ValidateHostPrefix(a.HostPrefix); err != nil { + errs = append(errs, err) + } + apiServer := strings.TrimSpace(a.Kubelet.ApiServer) if apiServer == "" { errs = append(errs, fmt.Errorf("Kubelet.ApiServer is required")) diff --git a/pkg/agent/config/config_test.go b/pkg/agent/config/config_test.go index e1709bbe1..efecca055 100644 --- a/pkg/agent/config/config_test.go +++ b/pkg/agent/config/config_test.go @@ -582,3 +582,72 @@ func TestAgentConfig_BackfillNodeName_UsesHostHostname(t *testing.T) { assert.Equal(t, want, cfg.NodeName) } + +// TestValidateHostPrefix pins what may be configured as an installation prefix. +// The value is interpolated into generated systemd units and into a shell +// script, neither of which quotes it, so the accepted syntax is deliberately +// narrow enough to be inert in both rather than requiring two kinds of +// escaping that every consumer would have to keep correct. +func TestValidateHostPrefix(t *testing.T) { + t.Parallel() + + for _, prefix := range []string{ + "", + "/usr/local", + "/opt/unbounded", + "/var/lib/unbounded-agent", + "/opt/Unbounded_1.0-rc.2", + } { + if err := ValidateHostPrefix(prefix); err != nil { + t.Errorf("ValidateHostPrefix(%q) = %v, want nil", prefix, err) + } + } + + for _, tc := range []struct{ prefix, reason string }{ + {"usr/local", "relative"}, + {"./opt", "relative"}, + {"/opt/", "trailing separator is not normalized"}, + {"/opt/../opt", "unnormalized"}, + {"/", "filesystem root"}, + {"/opt/un bounded", "whitespace"}, + {"/opt/$HOME", "shell substitution"}, + {"/opt/%i", "systemd specifier"}, + {"/opt/un;rm -rf /", "shell metacharacter"}, + {"/opt/\"quoted\"", "quoting"}, + {"/opt/un`cmd`", "command substitution"}, + } { + if err := ValidateHostPrefix(tc.prefix); err == nil { + t.Errorf("ValidateHostPrefix(%q) = nil, want an error (%s)", tc.prefix, tc.reason) + } + } +} + +// TestValidateRejectsBadHostPrefix checks the prefix is actually reached by +// whole-config validation, not merely validatable in isolation. +func TestValidateRejectsBadHostPrefix(t *testing.T) { + t.Parallel() + + cfg := validAgentConfigForHostPrefix() + if err := cfg.Validate(); err != nil { + t.Fatalf("baseline config should be valid: %v", err) + } + + cfg.HostPrefix = "/opt/$INJECTED" + if err := cfg.Validate(); err == nil { + t.Fatal("Validate() = nil, want an error for an unsafe HostPrefix") + } + + cfg.HostPrefix = "/opt/unbounded" + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() = %v, want nil for a valid HostPrefix", err) + } +} + +func validAgentConfigForHostPrefix() *AgentConfig { + return &AgentConfig{ + MachineName: "machine", + NodeName: "node", + Cluster: AgentClusterConfig{ClusterDNS: "10.96.0.10"}, + Kubelet: AgentKubeletConfig{ApiServer: "https://api.example.test"}, + } +} diff --git a/pkg/agent/goalstates/hostpaths.go b/pkg/agent/goalstates/hostpaths.go new file mode 100644 index 000000000..ff7b42e7b --- /dev/null +++ b/pkg/agent/goalstates/hostpaths.go @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package goalstates + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/Azure/unbounded/pkg/agent/config" +) + +// DefaultHostPrefix is the installation prefix used when the agent config does +// not set one. +const DefaultHostPrefix = "/usr/local" + +// Base names of the agent's own host-side files. They are joined with the +// resolved prefix rather than being absolute constants so that hosts with a +// read-only /usr can place them somewhere writable. +const ( + daemonBinaryName = "unbounded-agent" + daemonBinaryBlueName = "unbounded-agent-blue" + daemonBinaryGreenName = "unbounded-agent-green" + daemonBinaryCurrentName = "unbounded-agent-current" + daemonBinaryLastGoodName = "unbounded-agent-last-good" + nspawnLifecycleName = "unbounded-agent-nspawn-lifecycle" + daemonRecoveryScriptName = "unbounded-agent-daemon-recovery.sh" + localDNSNetworkHelperName = "unbounded-localdns-network" +) + +// HostPaths is the resolved host-side layout of the agent's own files under an +// installation prefix. +// +// These are paths on the host. Files inside the nspawn machine are always +// resolved relative to the machine directory and are unaffected by the prefix. +type HostPaths struct { + // Prefix is the resolved installation prefix. + Prefix string + // BinDir is /bin. + BinDir string + // LibexecDir is /libexec. + LibexecDir string + + // NSpawnLifecycleBinary is the rollback-stable helper invoked by the + // generated nspawn hook units. + NSpawnLifecycleBinary string + // DaemonRecoveryScript is executed by the daemon recovery unit. + DaemonRecoveryScript string + // LocalDNSNetworkHelper backs unbounded-localdns-network.service. + LocalDNSNetworkHelper string +} + +// HostPrefixOrDefault returns the configured prefix, or DefaultHostPrefix when +// it is empty. +func HostPrefixOrDefault(prefix string) string { + if trimmed := strings.TrimSpace(prefix); trimmed != "" { + return trimmed + } + + return DefaultHostPrefix +} + +// ResolveHostPaths returns the host-side agent layout for an installation +// prefix. An empty prefix selects DefaultHostPrefix. +func ResolveHostPaths(prefix string) HostPaths { + resolved := HostPrefixOrDefault(prefix) + binDir := filepath.Join(resolved, "bin") + libexecDir := filepath.Join(resolved, "libexec") + + return HostPaths{ + Prefix: resolved, + BinDir: binDir, + LibexecDir: libexecDir, + NSpawnLifecycleBinary: filepath.Join(binDir, nspawnLifecycleName), + DaemonRecoveryScript: filepath.Join(binDir, daemonRecoveryScriptName), + LocalDNSNetworkHelper: filepath.Join(libexecDir, localDNSNetworkHelperName), + } +} + +// KnownHostPrefixes returns the prefixes that teardown and existing-deployment +// detection must consider. +// +// A host provisioned before the prefix was configurable, or by an agent using a +// different prefix, still has files under the default. Cleanup and +// already-provisioned checks therefore look at both, so that changing the +// prefix cannot orphan files or let a dirty host be silently reprovisioned. +func KnownHostPrefixes(prefix string) []string { + resolved := HostPrefixOrDefault(prefix) + if resolved == DefaultHostPrefix { + return []string{DefaultHostPrefix} + } + + return []string{resolved, DefaultHostPrefix} +} + +// MergeHostPrefixes returns every distinct prefix teardown must sweep, given +// candidates gathered from different sources. +// +// Teardown cannot rely on any single source. The installation record has the +// prefix from before the first mutation but may be absent on hosts provisioned +// by an older agent; the applied config has it only once the node started. An +// empty candidate contributes nothing but never suppresses the default. +func MergeHostPrefixes(candidates ...string) []string { + var ( + out []string + seen = map[string]struct{}{} + ) + + add := func(prefix string) { + if _, ok := seen[prefix]; ok { + return + } + + seen[prefix] = struct{}{} + + out = append(out, prefix) + } + + for _, candidate := range candidates { + if strings.TrimSpace(candidate) == "" { + continue + } + + for _, prefix := range KnownHostPrefixes(candidate) { + add(prefix) + } + } + + add(DefaultHostPrefix) + + return out +} + +// HostPrefixFromAppliedConfig returns the installation prefix recorded in the +// applied config of whichever machine is provisioned on this host. +// +// Processes started by systemd, such as the agent daemon and the nspawn +// lifecycle hooks, cannot inherit the prefix from the environment that +// bootstrapped the host. The applied config is the authoritative record: it is +// written once at bootstrap and re-read here so that later upgrades and +// teardown resolve the same paths the bootstrap used. +// +// An absent or unreadable config yields the default prefix, which is what a +// host provisioned before the prefix was configurable actually has on disk. +// +// Note that the applied config only exists once the node has started. Callers +// that must work after a *failed* bootstrap should prefer the installation +// record, which is written before the first mutation; see +// installstate.Record.HostPrefix. +func HostPrefixFromAppliedConfig() string { + for _, name := range []string{NSpawnMachineKube1, NSpawnMachineKube2} { + data, err := os.ReadFile(AppliedConfigPath(name)) + if err != nil { + continue + } + + // Only the prefix is needed here, so decode into the shared config type + // rather than a consumer-specific wrapper. Unknown fields are ignored. + var cfg config.AgentConfig + if err := json.Unmarshal(data, &cfg); err != nil { + continue + } + + if prefix := HostPrefixOrDefault(cfg.HostPrefix); prefix != DefaultHostPrefix { + return prefix + } + } + + return DefaultHostPrefix +} diff --git a/pkg/agent/goalstates/hostpaths_test.go b/pkg/agent/goalstates/hostpaths_test.go new file mode 100644 index 000000000..63b02256a --- /dev/null +++ b/pkg/agent/goalstates/hostpaths_test.go @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package goalstates + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/Azure/unbounded/pkg/agent/config" +) + +func TestHostPrefixOrDefault(t *testing.T) { + t.Parallel() + + assert.Equal(t, DefaultHostPrefix, HostPrefixOrDefault("")) + assert.Equal(t, DefaultHostPrefix, HostPrefixOrDefault(" ")) + assert.Equal(t, "/opt/unbounded", HostPrefixOrDefault("/opt/unbounded")) + assert.Equal(t, "/opt/unbounded", HostPrefixOrDefault(" /opt/unbounded ")) +} + +// TestResolveHostPathsDefaultsAreUnchanged pins the pre-existing absolute paths. +// Hosts that do not configure a prefix must keep exactly the layout they had +// before the prefix became configurable. +func TestResolveHostPathsDefaultsAreUnchanged(t *testing.T) { + t.Parallel() + + paths := ResolveHostPaths("") + + assert.Equal(t, "/usr/local", paths.Prefix) + assert.Equal(t, "/usr/local/bin", paths.BinDir) + assert.Equal(t, "/usr/local/libexec", paths.LibexecDir) + assert.Equal(t, "/usr/local/bin/unbounded-agent-nspawn-lifecycle", paths.NSpawnLifecycleBinary) + assert.Equal(t, "/usr/local/bin/unbounded-agent-daemon-recovery.sh", paths.DaemonRecoveryScript) + assert.Equal(t, "/usr/local/libexec/unbounded-localdns-network", paths.LocalDNSNetworkHelper) +} + +func TestResolveHostPathsWithPrefix(t *testing.T) { + t.Parallel() + + paths := ResolveHostPaths("/opt/unbounded") + + assert.Equal(t, "/opt/unbounded", paths.Prefix) + assert.Equal(t, "/opt/unbounded/bin", paths.BinDir) + assert.Equal(t, "/opt/unbounded/libexec", paths.LibexecDir) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-nspawn-lifecycle", paths.NSpawnLifecycleBinary) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-daemon-recovery.sh", paths.DaemonRecoveryScript) + assert.Equal(t, "/opt/unbounded/libexec/unbounded-localdns-network", paths.LocalDNSNetworkHelper) +} + +// TestResolvedAgentUpgradePathsDefaultsAreUnchanged is the equivalent regression +// guard for the blue-green daemon binary layout. +// TestResolvedAgentUpgradePathsEnvOverridesPrefix keeps the existing escape +// hatch working: an explicit environment override wins over the prefix. +func TestKnownHostPrefixes(t *testing.T) { + t.Parallel() + + assert.Equal(t, []string{DefaultHostPrefix}, KnownHostPrefixes("")) + assert.Equal(t, []string{DefaultHostPrefix}, KnownHostPrefixes(DefaultHostPrefix)) + + // A non-default prefix must still sweep the default, so that a host + // provisioned under the old layout is not left with orphaned files. + assert.Equal(t, []string{"/opt/unbounded", DefaultHostPrefix}, KnownHostPrefixes("/opt/unbounded")) +} + +func TestHostPrefixFromAppliedConfig(t *testing.T) { + // AppliedConfigPath is absolute, so redirect it by pointing AgentConfigDir's + // consumers at a temporary root is not possible; instead assert the + // fallback, which is the branch reachable without writing to /etc. + assert.Equal(t, DefaultHostPrefix, HostPrefixFromAppliedConfig()) +} + +// TestHostPrefixRoundTripsThroughAppliedConfig proves the persisted config +// carries the prefix, which is what lets systemd-started processes resolve the +// same paths the bootstrap used. +func TestHostPrefixRoundTripsThroughAppliedConfig(t *testing.T) { + t.Parallel() + + cfg := config.AgentConfig{MachineName: "m", HostPrefix: "/opt/unbounded"} + + data, err := json.Marshal(cfg) + require.NoError(t, err) + + path := filepath.Join(t.TempDir(), "applied-config.json") + require.NoError(t, os.WriteFile(path, data, 0o600)) + + raw, err := os.ReadFile(path) + require.NoError(t, err) + + var decoded config.AgentConfig + require.NoError(t, json.Unmarshal(raw, &decoded)) + + assert.Equal(t, "/opt/unbounded", decoded.HostPrefix) + assert.Equal(t, "/opt/unbounded/bin", ResolveHostPaths(decoded.HostPrefix).BinDir) +} + +// TestDefaultHostPathsMatchTheExistingConstants is the regression guard for +// every host that does not set a prefix. Those hosts must resolve to exactly +// the paths they had before the prefix existed, because the lifecycle helper +// path is baked as an absolute path into generated systemd units that are +// already on disk. +func TestDefaultHostPathsMatchTheExistingConstants(t *testing.T) { + t.Parallel() + + paths := ResolveHostPaths("") + + require.Equal(t, DefaultHostPrefix, paths.Prefix) + require.Equal(t, filepath.Dir(DaemonBinaryPath), paths.BinDir) + require.Equal(t, NSpawnLifecycleBinaryPath, paths.NSpawnLifecycleBinary) + require.Equal(t, DaemonRecoveryScriptPath, paths.DaemonRecoveryScript) +} From cdb5de92bc699dcdd0eaaadbfbc651123314af85 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 22:35:11 -0400 Subject: [PATCH 33/39] agent: resolve the daemon binaries under the installation prefix The blue-green agent binaries were absolute constants under /usr/local/bin. A host whose /usr is read-only cannot hold them there, which is the whole reason the prefix exists. ResolvedAgentUpgradePathsFor resolves them under a prefix instead. An empty prefix selects the default, and a test pins that the result is exactly the constants this package used before, because those paths are baked into generated units and into the blue-green symlinks of every host already installed. If the default drifted, an upgraded agent would look for its binaries where the host does not have them. The original entry point stays, deprecated, delegating to an empty prefix. It is published from pkg/ and callers outside this repository compose their own phases from it, so removing it would break them at compile time. Every caller inside the repository moves to the new one in this commit, because staticcheck's SA1019 is enabled and a split would not lint. All of them run under systemd or on the host with no config in hand, so they take the prefix from the applied config, which is what that lookup exists for. On a host that configures no prefix this resolves the default and nothing changes. The AgentUpgrade signal path is deliberately not prefixed: it is state about an upgrade rather than part of the installed layout, and it already lives under the agent config directory, which stays writable on such hosts. One caller passed the function as a value rather than calling it, so a search for call sites missed it and only the linter found it. It is now wrapped, so the prefix is read when the command runs rather than when it is constructed. --- cmd/agent/internal/cmd/agentupgrade.go | 15 ++++-- cmd/agent/internal/daemon/agentupgrade.go | 4 +- cmd/agent/internal/daemon/lifecycle.go | 6 +-- pkg/agent/goalstates/agentupgrade.go | 33 ++++++++++-- pkg/agent/goalstates/agentupgrade_test.go | 63 +++++++++++++++++++++-- 5 files changed, 103 insertions(+), 18 deletions(-) diff --git a/cmd/agent/internal/cmd/agentupgrade.go b/cmd/agent/internal/cmd/agentupgrade.go index a11eebc88..fad1f5bb2 100644 --- a/cmd/agent/internal/cmd/agentupgrade.go +++ b/cmd/agent/internal/cmd/agentupgrade.go @@ -42,10 +42,17 @@ type hostAgentUpgradeHandler struct { func newCmdHostAgentUpgrade(cmdCtx *CommandContext) *cobra.Command { handler := &hostAgentUpgradeHandler{ - cmdCtx: cmdCtx, - writer: os.Stdout, - executable: os.Executable, - resolvedPath: goalstates.ResolvedAgentUpgradePaths, + cmdCtx: cmdCtx, + writer: os.Stdout, + executable: os.Executable, + // Wrapped rather than referenced directly so the prefix is read when + // the command runs, not when it is constructed. This runs on the host + // rather than under systemd, but the applied config is still the + // authority: the prefix belongs to the installation, not to whatever + // environment happens to be invoking the upgrade. + resolvedPath: func() (goalstates.AgentUpgradePaths, error) { + return goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) + }, geteuid: os.Geteuid, installation: installstate.DefaultStore(), } diff --git a/cmd/agent/internal/daemon/agentupgrade.go b/cmd/agent/internal/daemon/agentupgrade.go index eb0bb2dd2..979bb1206 100644 --- a/cmd/agent/internal/daemon/agentupgrade.go +++ b/cmd/agent/internal/daemon/agentupgrade.go @@ -61,7 +61,7 @@ func parseAgentUpgradeRequest(parameters map[string]string) (agentUpgradeRequest } func upgradeDaemonBinary(ctx context.Context, log *slog.Logger, request agentUpgradeRequest) error { - paths, err := goalstates.ResolvedAgentUpgradePaths() + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) if err != nil { return fmt.Errorf("resolve current daemon binary symlink: %w", err) } @@ -85,7 +85,7 @@ func upgradeDaemonBinary(ctx context.Context, log *slog.Logger, request agentUpg } func newAgentUpgradeSignalOperator() (agentUpgradeSignalOperator, error) { - paths, err := goalstates.ResolvedAgentUpgradePaths() + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) if err != nil { return nil, fmt.Errorf("resolve AgentUpgrade signal path: %w", err) } diff --git a/cmd/agent/internal/daemon/lifecycle.go b/cmd/agent/internal/daemon/lifecycle.go index 8cd40302d..9ddc969cc 100644 --- a/cmd/agent/internal/daemon/lifecycle.go +++ b/cmd/agent/internal/daemon/lifecycle.go @@ -51,7 +51,7 @@ func EnableDaemon(log *slog.Logger) phases.Task { func (d *enableDaemon) Name() string { return "enable-daemon" } func (d *enableDaemon) Do(ctx context.Context) error { - paths, err := goalstates.ResolvedAgentUpgradePaths() + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) if err != nil { return fmt.Errorf("resolve current daemon binary symlink: %w", err) } @@ -158,7 +158,7 @@ func usableDaemonBinary(path string) bool { } func renderDaemonAsset(name string, content []byte) ([]byte, error) { - paths, err := goalstates.ResolvedAgentUpgradePaths() + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) if err != nil { return nil, err } @@ -339,7 +339,7 @@ func removeOwnedFile(path string) error { // active daemon already proves it resolved an applied config at startup, so the // applied-config check belongs to RepairDaemon rather than here. func VerifyDaemonInstalled(ctx context.Context, log *slog.Logger) error { - paths, err := goalstates.ResolvedAgentUpgradePaths() + paths, err := goalstates.ResolvedAgentUpgradePathsFor(goalstates.HostPrefixFromAppliedConfig()) if err != nil { return err } diff --git a/pkg/agent/goalstates/agentupgrade.go b/pkg/agent/goalstates/agentupgrade.go index 9fb64dc30..0e1df0755 100644 --- a/pkg/agent/goalstates/agentupgrade.go +++ b/pkg/agent/goalstates/agentupgrade.go @@ -24,13 +24,36 @@ type AgentUpgradePaths struct { // ResolvedAgentUpgradePaths returns the host-side agent binary paths after // applying environment overrides. +// +// Deprecated: use ResolvedAgentUpgradePathsFor, which resolves the binaries +// under a configured installation prefix. This entry point is equivalent to +// passing an empty prefix and is kept for callers outside this repository. func ResolvedAgentUpgradePaths() (AgentUpgradePaths, error) { + return ResolvedAgentUpgradePathsFor("") +} + +// ResolvedAgentUpgradePathsFor returns the host-side agent binary paths under an +// installation prefix, after applying environment overrides. +// +// An empty prefix selects DefaultHostPrefix, so a host that does not configure +// one resolves exactly the paths this package has always used. +// +// Environment overrides are absolute and win over the prefix. They name a +// specific file, which is more particular than a directory to look in, and the +// nspawn lifecycle hooks rely on that to pin a binary across an upgrade. +// +// The AgentUpgrade signal path is deliberately not prefixed. It lives under the +// agent config directory rather than the installation prefix, because it is +// state about an upgrade rather than part of the installed layout. +func ResolvedAgentUpgradePathsFor(prefix string) (AgentUpgradePaths, error) { + binDir := ResolveHostPaths(prefix).BinDir + paths := AgentUpgradePaths{ - BinaryPath: resolveDaemonBinaryPath(EnvDaemonBinary, DaemonBinaryPath), - BluePath: resolveDaemonBinaryPath(EnvDaemonBinaryBlue, DaemonBinaryBluePath), - GreenPath: resolveDaemonBinaryPath(EnvDaemonBinaryGreen, DaemonBinaryGreenPath), - CurrentPath: resolveDaemonBinaryPath(EnvDaemonBinaryCurrent, DaemonBinaryCurrentPath), - LastGoodPath: resolveDaemonBinaryPath(EnvDaemonBinaryLastGood, DaemonBinaryLastGoodPath), + BinaryPath: resolveDaemonBinaryPath(EnvDaemonBinary, filepath.Join(binDir, daemonBinaryName)), + BluePath: resolveDaemonBinaryPath(EnvDaemonBinaryBlue, filepath.Join(binDir, daemonBinaryBlueName)), + GreenPath: resolveDaemonBinaryPath(EnvDaemonBinaryGreen, filepath.Join(binDir, daemonBinaryGreenName)), + CurrentPath: resolveDaemonBinaryPath(EnvDaemonBinaryCurrent, filepath.Join(binDir, daemonBinaryCurrentName)), + LastGoodPath: resolveDaemonBinaryPath(EnvDaemonBinaryLastGood, filepath.Join(binDir, daemonBinaryLastGoodName)), SignalPath: resolveDaemonBinaryPath(EnvDaemonAgentUpgradeSignalPath, DaemonAgentUpgradeSignalPath), } diff --git a/pkg/agent/goalstates/agentupgrade_test.go b/pkg/agent/goalstates/agentupgrade_test.go index 0a50521ed..189a741e3 100644 --- a/pkg/agent/goalstates/agentupgrade_test.go +++ b/pkg/agent/goalstates/agentupgrade_test.go @@ -40,7 +40,7 @@ func TestResolvedAgentUpgradePaths(t *testing.T) { t.Setenv(EnvDaemonBinaryLastGood, lastGoodPath) t.Setenv(EnvDaemonAgentUpgradeSignalPath, signalPath) - paths, err := ResolvedAgentUpgradePaths() + paths, err := ResolvedAgentUpgradePathsFor("") require.NoError(t, err) assert.Equal(t, binaryPath, paths.BinaryPath) @@ -56,7 +56,7 @@ func TestResolvedAgentUpgradePaths_UsesDefaultsForBlankOverrides(t *testing.T) { t.Setenv(EnvDaemonBinary, "") t.Setenv(EnvDaemonBinaryBlue, " ") - paths, err := ResolvedAgentUpgradePaths() + paths, err := ResolvedAgentUpgradePathsFor("") require.NoError(t, err) assert.Equal(t, DaemonBinaryPath, paths.BinaryPath) @@ -87,7 +87,7 @@ func TestResolvedAgentUpgradePaths_ResolvesCurrentTarget(t *testing.T) { t.Setenv(EnvDaemonBinary, binaryPath) t.Setenv(EnvDaemonBinaryCurrent, currentPath) - paths, err := ResolvedAgentUpgradePaths() + paths, err := ResolvedAgentUpgradePathsFor("") require.NoError(t, err) assert.Equal(t, currentTargetPath, paths.CurrentTargetPath) @@ -97,8 +97,63 @@ func TestResolvedAgentUpgradePaths_CurrentTargetFallsBackToBinaryPath(t *testing t.Setenv(EnvDaemonBinary, "/agent") t.Setenv(EnvDaemonBinaryCurrent, filepath.Join(t.TempDir(), "missing-current")) - paths, err := ResolvedAgentUpgradePaths() + paths, err := ResolvedAgentUpgradePathsFor("") require.NoError(t, err) assert.Equal(t, "/agent", paths.CurrentTargetPath) } + +// TestResolvedAgentUpgradePathsForPrefix covers the reason the prefix-aware +// entry point exists: a host whose /usr is read-only cannot hold the agent's +// own binaries under /usr/local, so they move with the prefix. +// +// The signal path deliberately does not move. It is state about an upgrade +// rather than part of the installed layout, and it lives under the agent config +// directory, which is writable on such hosts. +func TestResolvedAgentUpgradePathsForPrefix(t *testing.T) { + paths, err := ResolvedAgentUpgradePathsFor("/opt/unbounded") + require.NoError(t, err) + + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent", paths.BinaryPath) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-blue", paths.BluePath) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-green", paths.GreenPath) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-current", paths.CurrentPath) + assert.Equal(t, "/opt/unbounded/bin/unbounded-agent-last-good", paths.LastGoodPath) + assert.Equal(t, DaemonAgentUpgradeSignalPath, paths.SignalPath) +} + +// TestResolvedAgentUpgradePathsForDefaultMatchesLegacyConstants pins that a host +// which configures no prefix resolves exactly what this package resolved before +// the prefix existed. +// +// These paths are baked into generated systemd units and into the blue-green +// symlinks on every host already in the field. If the default drifted, an +// upgraded agent would look for its binaries somewhere the installed host does +// not have them, and the daemon would fail to start with nothing having changed +// on disk. +func TestResolvedAgentUpgradePathsForDefaultMatchesLegacyConstants(t *testing.T) { + paths, err := ResolvedAgentUpgradePathsFor("") + require.NoError(t, err) + + assert.Equal(t, DaemonBinaryPath, paths.BinaryPath) + assert.Equal(t, DaemonBinaryBluePath, paths.BluePath) + assert.Equal(t, DaemonBinaryGreenPath, paths.GreenPath) + assert.Equal(t, DaemonBinaryCurrentPath, paths.CurrentPath) + assert.Equal(t, DaemonBinaryLastGoodPath, paths.LastGoodPath) + assert.Equal(t, DaemonAgentUpgradeSignalPath, paths.SignalPath) +} + +// TestDeprecatedResolvedAgentUpgradePathsStillWorks keeps the compatibility +// promise honest. The entry point is deprecated rather than removed because it +// is published from pkg/, and callers outside this repository compose their own +// phases from it. +func TestDeprecatedResolvedAgentUpgradePathsStillWorks(t *testing.T) { + //nolint:staticcheck // Exercising the deprecated entry point is the point. + legacy, err := ResolvedAgentUpgradePaths() + require.NoError(t, err) + + current, err := ResolvedAgentUpgradePathsFor("") + require.NoError(t, err) + + assert.Equal(t, current, legacy, "the deprecated entry point must stay equivalent to an empty prefix") +} From ee491a037e7487c7727d39fcd429924915c0f855 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 22:39:16 -0400 Subject: [PATCH 34/39] agent: record the installation prefix before the first host mutation Teardown has to find the agent's own files. On a host that configures a prefix they are not under /usr/local, and after a bootstrap that failed before the node started there is nothing on the host that says where they are: the applied config carries the prefix but is not written until the node runs. The ownership record is written before any mutation, which makes it the only source that covers that window, so it carries the resolved prefix. Optional, and the schema version does not move. A record written by an agent that knows about the prefix stays readable by one that does not, because unknown fields are ignored, and a default installation writes no field at all so its record is byte-identical to one written before this existed. A test pins that, since the value of the compatibility is entirely in the absence. Resolved rather than configured, so the record names a real directory instead of an empty string meaning whatever the default happened to be. NewRecord takes it as a parameter rather than leaving it a field to set afterwards. Forgetting it would be silent and would only surface at teardown, on a host whose files are somewhere reset does not look. Also corrects a comment in the prefix lookup that pointed at this field before it existed. --- cmd/agent/internal/bootstrap/coordinator.go | 14 +++++- .../internal/bootstrap/coordinator_test.go | 6 +-- cmd/agent/internal/cmd/bootstrap.go | 9 +++- cmd/agent/internal/daemon/migration_test.go | 2 +- cmd/agent/internal/daemon/reset.go | 2 +- cmd/agent/internal/daemon/reset_test.go | 4 +- cmd/agent/internal/installstate/store.go | 28 +++++++++++- cmd/agent/internal/installstate/store_test.go | 44 ++++++++++++++++--- pkg/agent/goalstates/hostpaths.go | 5 ++- 9 files changed, 95 insertions(+), 19 deletions(-) diff --git a/cmd/agent/internal/bootstrap/coordinator.go b/cmd/agent/internal/bootstrap/coordinator.go index d5339bdbf..6a21f43d0 100644 --- a/cmd/agent/internal/bootstrap/coordinator.go +++ b/cmd/agent/internal/bootstrap/coordinator.go @@ -16,7 +16,17 @@ import ( "github.com/Azure/unbounded/cmd/agent/internal/installstate" ) -type Identity struct{ MachineName, ConfigFingerprint string } +// Identity is what makes one installation distinguishable from another. +// +// HostPrefix is the resolved installation prefix. It is carried here so the +// record written before the first host mutation knows where this installation +// puts its files, which is the only thing teardown can consult after a +// bootstrap that failed before the node started. +type Identity struct { + MachineName string + ConfigFingerprint string + HostPrefix string +} type Stages interface { EnsureHostClean(context.Context) error @@ -80,7 +90,7 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { return Outcome{}, err } - r, err = installstate.NewRecord(id.MachineName, id.ConfigFingerprint) + r, err = installstate.NewRecord(id.MachineName, id.ConfigFingerprint, id.HostPrefix) if err != nil { return Outcome{}, err } diff --git a/cmd/agent/internal/bootstrap/coordinator_test.go b/cmd/agent/internal/bootstrap/coordinator_test.go index 5076aeda7..c1a3fb080 100644 --- a/cmd/agent/internal/bootstrap/coordinator_test.go +++ b/cmd/agent/internal/bootstrap/coordinator_test.go @@ -104,7 +104,7 @@ func TestCompletedRecoveryDoesNotResolveRetiredBootstrapInputs(t *testing.T) { for _, repair := range []bool{false, true} { dir := t.TempDir() store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) - r, err := installstate.NewRecord("machine", "fingerprint") + r, err := installstate.NewRecord("machine", "fingerprint", "") require.NoError(t, err) r.Phase = installstate.Complete @@ -140,7 +140,7 @@ func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { t.Run(mode, func(t *testing.T) { dir := t.TempDir() store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) - r, err := installstate.NewRecord("machine", "fingerprint") + r, err := installstate.NewRecord("machine", "fingerprint", "") require.NoError(t, err) if mode == "resetting" { @@ -170,7 +170,7 @@ func TestAdmissionFailurePreventsAllStageWork(t *testing.T) { func TestInterruptedRepairRemainsCompleteAndRetries(t *testing.T) { store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) - r, err := installstate.NewRecord("machine", "fingerprint") + r, err := installstate.NewRecord("machine", "fingerprint", "") require.NoError(t, err) require.NoError(t, store.MarkComplete(r)) stages := &fakeStages{store: store, fail: "repair", verifyErr: errInjected} diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index e00baf1a9..4e7c1ef4b 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -73,7 +73,14 @@ func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity, return bootstrap.Identity{}, err } - return bootstrap.Identity{MachineName: cfg.MachineName, ConfigFingerprint: installstate.Fingerprint(data)}, nil + return bootstrap.Identity{ + MachineName: cfg.MachineName, + ConfigFingerprint: installstate.Fingerprint(data), + // Resolved rather than configured, so the record names a real directory + // instead of an empty string meaning "wherever the default was at the + // time", which is what teardown would have to guess from. + HostPrefix: goalstates.HostPrefixOrDefault(cfg.HostPrefix), + }, nil } func (s *agentStages) EnsureHostClean(ctx context.Context) error { diff --git a/cmd/agent/internal/daemon/migration_test.go b/cmd/agent/internal/daemon/migration_test.go index d095c5958..b39e00324 100644 --- a/cmd/agent/internal/daemon/migration_test.go +++ b/cmd/agent/internal/daemon/migration_test.go @@ -61,7 +61,7 @@ func TestStartupStandsDownWhileInstallationUnfinished(t *testing.T) { t.Parallel() store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) - record, err := installstate.NewRecord("machine-1", "fingerprint") + record, err := installstate.NewRecord("machine-1", "fingerprint", "") require.NoError(t, err) require.NoError(t, store.Save(record)) diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index 61a6cd435..d4808e37a 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -72,7 +72,7 @@ func recordForTeardown(log *slog.Logger, store *installstate.Store) (installstat log.Warn("installation record is unreadable; replacing it for teardown", "error", err) } - return installstate.NewRecord("legacy-reset", "legacy-reset") + return installstate.NewRecord("legacy-reset", "legacy-reset", "") } func resetUnderLock(ctx context.Context, log *slog.Logger, store *installstate.Store, inner phases.Task) error { diff --git a/cmd/agent/internal/daemon/reset_test.go b/cmd/agent/internal/daemon/reset_test.go index 2ae2b4b5e..633c914db 100644 --- a/cmd/agent/internal/daemon/reset_test.go +++ b/cmd/agent/internal/daemon/reset_test.go @@ -35,7 +35,7 @@ func TestResetRetainsOwnershipUntilTeardownAndSyncSucceed(t *testing.T) { t.Run(failure, func(t *testing.T) { dir := t.TempDir() store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) - r, err := installstate.NewRecord("machine", "f") + r, err := installstate.NewRecord("machine", "f", "") require.NoError(t, err) r.Phase = installstate.Resetting @@ -128,7 +128,7 @@ func TestTeardownKeepsAReadableRecord(t *testing.T) { dir := t.TempDir() store := installstate.NewStore(filepath.Join(dir, "state"), filepath.Join(dir, "lock")) - saved, err := installstate.NewRecord("machine-1", "fingerprint-1") + saved, err := installstate.NewRecord("machine-1", "fingerprint-1", "") require.NoError(t, err) require.NoError(t, store.Save(saved)) diff --git a/cmd/agent/internal/installstate/store.go b/cmd/agent/internal/installstate/store.go index 7520f6fec..ace82b371 100644 --- a/cmd/agent/internal/installstate/store.go +++ b/cmd/agent/internal/installstate/store.go @@ -52,6 +52,21 @@ type Record struct { MachineName string `json:"machineName"` ConfigFingerprint string `json:"configFingerprint"` Phase Phase `json:"phase"` + + // HostPrefix is the resolved installation prefix, recorded so teardown can + // find the agent's own files without being told where they are. + // + // It is written before the first host mutation, which makes it the only + // source that survives a bootstrap that failed before the node started. The + // applied config carries the same prefix but does not exist until then, so + // reset on a half-built host has nothing else to go on. + // + // Optional, and absent means the default. The schema version does not move + // for it: a record written by an agent that knows about the prefix stays + // readable by one that does not, because unknown fields are ignored, and a + // record written before it existed is read here as the default, which is + // what such a host actually has on disk. + HostPrefix string `json:"hostPrefix,omitempty"` } func (r Record) Validate() error { @@ -157,7 +172,16 @@ func (s *Store) Remove() error { return err } -func NewRecord(machine, fingerprint string) (Record, error) { +// NewRecord returns a record for a fresh installation. +// +// hostPrefix is a parameter rather than a field callers set afterwards because +// forgetting it is silent and only surfaces at teardown, on a host whose files +// are somewhere reset would not look. An empty prefix means the default. +// +// The value is stored as given and not validated here. This package deals in +// stdlib and durability only, and pulling in config validation to re-check a +// string this agent wrote from an already validated config would buy little. +func NewRecord(machine, fingerprint, hostPrefix string) (Record, error) { id := make([]byte, 16) if _, err := rand.Read(id); err != nil { return Record{}, err @@ -165,7 +189,7 @@ func NewRecord(machine, fingerprint string) (Record, error) { return Record{ SchemaVersion: schemaVersion, InstallID: hex.EncodeToString(id), MachineName: machine, - ConfigFingerprint: fingerprint, Phase: Installing, + ConfigFingerprint: fingerprint, Phase: Installing, HostPrefix: hostPrefix, }, nil } diff --git a/cmd/agent/internal/installstate/store_test.go b/cmd/agent/internal/installstate/store_test.go index 2d482a836..6877cf98a 100644 --- a/cmd/agent/internal/installstate/store_test.go +++ b/cmd/agent/internal/installstate/store_test.go @@ -4,6 +4,7 @@ package installstate import ( + "encoding/json" "errors" "os" "path/filepath" @@ -25,7 +26,7 @@ func TestStoreLifecycle(t *testing.T) { require.NoError(t, s.Remove()) _, err := s.Load() require.ErrorIs(t, err, ErrNotFound) - r, err := NewRecord("machine", Fingerprint([]byte(`{"machineName":"machine"}`))) + r, err := NewRecord("machine", Fingerprint([]byte(`{"machineName":"machine"}`)), "") require.NoError(t, err) require.NoError(t, s.Save(r)) loaded, err := s.Load() @@ -47,7 +48,7 @@ func TestStoreLifecycle(t *testing.T) { func TestOwnershipAdmission(t *testing.T) { t.Parallel() - r, err := NewRecord("machine", "fingerprint") + r, err := NewRecord("machine", "fingerprint", "") require.NoError(t, err) for _, phase := range []Phase{Installing, Complete, Resetting} { @@ -104,7 +105,7 @@ func TestInstallationLockSurvivesStateRemoval(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { require.NoError(t, lock.Release()) }) - r, err := NewRecord("machine", "f") + r, err := NewRecord("machine", "f", "") require.NoError(t, err) require.NoError(t, s.Save(r)) require.NoError(t, s.Remove()) @@ -125,7 +126,7 @@ func TestRemoveRestoresOwnershipWhenUndurable(t *testing.T) { t.Parallel() s := testStore(t) - r, err := NewRecord("machine", "f") + r, err := NewRecord("machine", "f", "") require.NoError(t, err) r.Phase = Resetting @@ -176,7 +177,7 @@ func TestMutationAdmission(t *testing.T) { s := testStore(t) if phase != "" { - r, err := NewRecord("machine", "f") + r, err := NewRecord("machine", "f", "") require.NoError(t, err) r.Phase = phase @@ -227,3 +228,36 @@ func TestStoreIgnoresUnknownFields(t *testing.T) { require.NoError(t, err) require.Equal(t, Resume, disposition, "the record must still be usable, not merely parseable") } + +// TestRecordCarriesTheInstallationPrefix covers what the prefix is recorded +// for: teardown on a host where bootstrap failed before the node started. +// +// The applied config carries the same value but does not exist until the node +// runs, so on a half-built host this record is the only thing that knows where +// the agent put its files. Absent means the default, which is what a host +// installed before the prefix existed actually has on disk. +func TestRecordCarriesTheInstallationPrefix(t *testing.T) { + t.Parallel() + + s := testStore(t) + + prefixed, err := NewRecord("machine", "f", "/opt/unbounded") + require.NoError(t, err) + require.NoError(t, s.Save(prefixed)) + + loaded, err := s.Load() + require.NoError(t, err) + require.Equal(t, "/opt/unbounded", loaded.HostPrefix) + require.NoError(t, loaded.Validate()) + + // A default installation records nothing, so its record is byte-identical + // to one written before the field existed and stays readable by an agent + // that predates it. + def, err := NewRecord("machine", "f", "") + require.NoError(t, err) + + encoded, err := json.Marshal(def) + require.NoError(t, err) + require.NotContains(t, string(encoded), "hostPrefix", + "a default installation must not write the field, or older agents see a record they did not write") +} diff --git a/pkg/agent/goalstates/hostpaths.go b/pkg/agent/goalstates/hostpaths.go index ff7b42e7b..90d9fbdc7 100644 --- a/pkg/agent/goalstates/hostpaths.go +++ b/pkg/agent/goalstates/hostpaths.go @@ -147,8 +147,9 @@ func MergeHostPrefixes(candidates ...string) []string { // // Note that the applied config only exists once the node has started. Callers // that must work after a *failed* bootstrap should prefer the installation -// record, which is written before the first mutation; see -// installstate.Record.HostPrefix. +// record, which carries the same prefix and is written before the first host +// mutation. That package is internal to the agent binary, so it cannot be named +// from here. func HostPrefixFromAppliedConfig() string { for _, name := range []string{NSpawnMachineKube1, NSpawnMachineKube2} { data, err := os.ReadFile(AppliedConfigPath(name)) From 9d0ff4696865c4264db9e44e595211afcb1204cc Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Fri, 18 Sep 2026 22:43:41 -0400 Subject: [PATCH 35/39] agent: make the installation prefix part of bootstrap identity The agent's own binaries live under the prefix, so starting with a different one is not a retry of the same installation. Continuing would leave the first installation's files where they are and build a second one beside them. Admission has to refuse and ask for a reset, which is what a changed fingerprint does. The delicate half is the other one. Every host already installed was fingerprinted without this input. If the default contributed a value, all of them would hash differently under an agent carrying this change, read as a different installation, and demand an explicit reset on upgrade over a field they never set. So the prefix enters the hash only when it resolves somewhere other than the default, and carries omitempty so that at the default it contributes nothing rather than an empty string. It is the resolved prefix that counts, not how it was written. Leaving it unset and naming /usr/local explicitly put the files in the same place, so they hash alike; telling an operator who wrote down what was already true that they must reset the host would be a poor trade for the precision. Verified by mutation, since all three ways to get this wrong are silent and affect every host in the field rather than the one under test: dropping omitempty, hashing the default instead of eliding it, and never hashing the prefix at all each fail a test. The fixtures carry a literal fingerprint, which is what makes the first two detectable at all. --- cmd/agent/internal/cmd/bootstrap.go | 33 +++++++++++- cmd/agent/internal/cmd/bootstrap_test.go | 65 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/cmd/agent/internal/cmd/bootstrap.go b/cmd/agent/internal/cmd/bootstrap.go index 4e7c1ef4b..deb7e1b41 100644 --- a/cmd/agent/internal/cmd/bootstrap.go +++ b/cmd/agent/internal/cmd/bootstrap.go @@ -64,11 +64,40 @@ func canonicalImageIdentity(image string) string { func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity, error) { // Keep identity tied to the cluster and installed rootfs, while allowing // credentials and artifact locations to be refreshed for a retry. + // + // HostPrefix enters the hash only when it resolves somewhere other than the + // default, and carries omitempty so that at the default it contributes + // nothing at all. Every host already in the field was fingerprinted without + // this input; if the default hashed as a value, each of them would read as a + // different installation and demand an explicit reset on upgrade, for a + // field they never set. TestBootstrapV1CompatibilityFixtures catches that. + // + // It is the resolved prefix that matters, not how it was written. Leaving it + // unset and naming /usr/local explicitly put the files in the same place, so + // they are the same installation and must hash alike. + // + // A prefix that resolves elsewhere does belong in the identity. The agent's + // own files live under it, so starting with a different one is not a retry: + // it would leave the first installation behind and build a second one + // beside it. + resolvedPrefix := goalstates.HostPrefixOrDefault(cfg.HostPrefix) + + fingerprintedPrefix := resolvedPrefix + if fingerprintedPrefix == goalstates.DefaultHostPrefix { + fingerprintedPrefix = "" + } + data, err := json.Marshal(struct { KubernetesVersion string OCIImage string APIServer string - }{strings.TrimPrefix(cfg.Cluster.Version, "v"), canonicalImageIdentity(cfg.OCIImage), cfg.Kubelet.ApiServer}) + HostPrefix string `json:",omitempty"` + }{ + strings.TrimPrefix(cfg.Cluster.Version, "v"), + canonicalImageIdentity(cfg.OCIImage), + cfg.Kubelet.ApiServer, + fingerprintedPrefix, + }) if err != nil { return bootstrap.Identity{}, err } @@ -79,7 +108,7 @@ func bootstrapIdentity(cfg *provision.UnboundedAgentConfig) (bootstrap.Identity, // Resolved rather than configured, so the record names a real directory // instead of an empty string meaning "wherever the default was at the // time", which is what teardown would have to guess from. - HostPrefix: goalstates.HostPrefixOrDefault(cfg.HostPrefix), + HostPrefix: resolvedPrefix, }, nil } diff --git a/cmd/agent/internal/cmd/bootstrap_test.go b/cmd/agent/internal/cmd/bootstrap_test.go index 9dc480a4f..41cfb0738 100644 --- a/cmd/agent/internal/cmd/bootstrap_test.go +++ b/cmd/agent/internal/cmd/bootstrap_test.go @@ -256,3 +256,68 @@ func TestClassifyNodeStartFailure(t *testing.T) { }) } } + +// TestBootstrapFingerprintTracksTheInstallationPrefix covers both halves of how +// the prefix enters installation identity, because the two pull in opposite +// directions. +// +// Configuring a prefix has to change the fingerprint. The agent's own binaries +// live under it, so a start with a different prefix is not a retry of the same +// installation: continuing would leave the first installation's files behind +// and build a second one beside them. Admission must refuse and ask for a +// reset, which is what a changed fingerprint does. +// +// Configuring nothing has to change nothing. Every host already in the field +// was fingerprinted without this input, and if the default hashed differently +// each of them would read as a different installation and demand an explicit +// reset on upgrade, for a field they never set. +func TestBootstrapFingerprintTracksTheInstallationPrefix(t *testing.T) { + load := func(t *testing.T) *provision.UnboundedAgentConfig { + t.Helper() + + cfg, err := loadConfigFromFile(filepath.Join("testdata", "bootstrap-v1", "input.json")) + require.NoError(t, err) + + return cfg + } + + baseline, err := bootstrapIdentity(load(t)) + require.NoError(t, err) + + // Whitespace is not a configuration choice, so it must not be one here + // either; otherwise a stray space rewrites the identity of a default host. + for _, blank := range []string{"", " ", "\t"} { + cfg := load(t) + cfg.HostPrefix = blank + + unset, err := bootstrapIdentity(cfg) + require.NoError(t, err) + require.Equal(t, baseline.ConfigFingerprint, unset.ConfigFingerprint, + "an unset prefix must hash as it did before the field existed, got %q", blank) + } + + // Naming the default explicitly puts the files in the same place as leaving + // it unset, so the two are the same installation. Hashing them differently + // would tell an operator who wrote down what was already true that they + // must reset the host. + explicit := load(t) + explicit.HostPrefix = goalstates.DefaultHostPrefix + + explicitID, err := bootstrapIdentity(explicit) + require.NoError(t, err) + require.Equal(t, baseline.ConfigFingerprint, explicitID.ConfigFingerprint, + "identity follows where the files land, not how the prefix was spelled") + + moved := load(t) + moved.HostPrefix = "/opt/unbounded" + + movedID, err := bootstrapIdentity(moved) + require.NoError(t, err) + require.NotEqual(t, baseline.ConfigFingerprint, movedID.ConfigFingerprint, + "moving the installation prefix must not read as a retry of the same installation") + require.Equal(t, "/opt/unbounded", movedID.HostPrefix) + + // The record needs a real directory, not an empty string standing for + // whatever the default was when it was written. + require.Equal(t, goalstates.DefaultHostPrefix, baseline.HostPrefix) +} From 24605fbbbc29969c0ece9375c197bf17a51a89df Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:36:38 -0400 Subject: [PATCH 36/39] kubectl-unbounded: add the Ignition config encoder Ignition is the only provisioning mechanism Azure Container Linux consumes; it has no cloud-init, so a cloud-init payload passed as customData is never acted on and nothing reports an error. This is the encoding layer on its own, before anything emits a document. The types are hand-written rather than pulled from github.com/coreos/ignition, which would bring the whole specification along for the handful of fields used here. Three things carry a cost that is only visible on a host that has already failed to provision, so each is pinned by a test: The spec version. Ignition refuses a config whose version it does not implement, on first boot, with no shell and no agent yet installed. There is nothing there to report the mismatch. Which schemes Ignition can fetch. This decides whether a file lands before dbus starts or has to wait for the agent, which is after. oci is the one that matters, because it is the agent's own artifact scheme and Ignition has no idea what to do with it. File modes, which Ignition serializes as decimal. A mode written 600 rather than 0o600 is 0o1130 on disk, and for the agent config that means credentials readable by everyone. The test asserts the decimal the emitted document would actually contain. --- cmd/kubectl-unbounded/app/ignition.go | 125 ++++++++++++++ cmd/kubectl-unbounded/app/ignition_test.go | 180 +++++++++++++++++++++ 2 files changed, 305 insertions(+) create mode 100644 cmd/kubectl-unbounded/app/ignition.go create mode 100644 cmd/kubectl-unbounded/app/ignition_test.go diff --git a/cmd/kubectl-unbounded/app/ignition.go b/cmd/kubectl-unbounded/app/ignition.go new file mode 100644 index 000000000..0e05a4038 --- /dev/null +++ b/cmd/kubectl-unbounded/app/ignition.go @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "encoding/base64" + "fmt" + "net/url" + "strings" +) + +// Ignition configuration types, covering only the subset this command emits. +// +// These are hand-written rather than taken from github.com/coreos/ignition to +// avoid a dependency carrying the whole specification for the handful of fields +// used here. The schema version is pinned and asserted by tests. +const ignitionSpecVersion = "3.4.0" + +// File modes are serialized as decimal integers in an Ignition config. +const ( + ignitionModeConfig = 0o600 + ignitionModeScript = 0o755 + ignitionModeData = 0o644 + ignitionModeDir = 0o755 +) + +type ignitionConfig struct { + Ignition ignitionVersion `json:"ignition"` + Storage *ignitionStorage `json:"storage,omitempty"` + Systemd *ignitionSystemd `json:"systemd,omitempty"` +} + +type ignitionVersion struct { + Version string `json:"version"` +} + +type ignitionStorage struct { + Directories []ignitionDirectory `json:"directories,omitempty"` + Files []ignitionFile `json:"files,omitempty"` +} + +// ignitionDirectory declares a directory Ignition creates before writing files +// into it. Ignition creates parents implicitly, so this exists to pin the mode +// of the agent's bin directory rather than to make the write succeed. +type ignitionDirectory struct { + Path string `json:"path"` + Mode int `json:"mode,omitempty"` +} + +type ignitionFile struct { + Path string `json:"path"` + Mode int `json:"mode,omitempty"` + Overwrite *bool `json:"overwrite,omitempty"` + Contents ignitionContents `json:"contents"` +} + +type ignitionContents struct { + Source string `json:"source"` + Verification *ignitionVerification `json:"verification,omitempty"` +} + +type ignitionVerification struct { + // Hash is "-", for example "sha256-abc123...". + Hash string `json:"hash"` +} + +type ignitionSystemd struct { + Units []ignitionUnit `json:"units,omitempty"` +} + +type ignitionUnit struct { + Name string `json:"name"` + Enabled *bool `json:"enabled,omitempty"` + Contents string `json:"contents,omitempty"` +} + +// ignitionDataURL encodes content as a data URL, which is how Ignition carries +// inline file contents. +func ignitionDataURL(content string) string { + return "data:;base64," + base64.StdEncoding.EncodeToString([]byte(content)) +} + +// ignitionRemoteFetchable reports whether Ignition can fetch a source itself. +// +// Ignition understands http, https, tftp, s3, arn, gs and data. It does not +// understand oci, which the agent resolves through its own artifact source. +// A source Ignition cannot fetch has to be left to the agent, which means the +// file lands after dbus has already started. +func ignitionRemoteFetchable(source string) bool { + parsed, err := url.Parse(strings.TrimSpace(source)) + if err != nil { + return false + } + + switch parsed.Scheme { + case "http", "https", "tftp", "s3", "arn", "gs": + return true + default: + return false + } +} + +// ignitionHashFromSHA256 converts a hex digest into the form Ignition expects. +func ignitionHashFromSHA256(hex string) (string, error) { + trimmed := strings.TrimSpace(hex) + + // Accept a plain digest or the first field of sha256sum output. + if fields := strings.Fields(trimmed); len(fields) > 0 { + trimmed = fields[0] + } + + if len(trimmed) != 64 { + return "", fmt.Errorf("sha256 digest must be 64 hex characters, got %d", len(trimmed)) + } + + for _, r := range trimmed { + isHex := (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') + if !isHex { + return "", fmt.Errorf("sha256 digest contains a non-hex character %q", r) + } + } + + return "sha256-" + strings.ToLower(trimmed), nil +} diff --git a/cmd/kubectl-unbounded/app/ignition_test.go b/cmd/kubectl-unbounded/app/ignition_test.go new file mode 100644 index 000000000..bae56575d --- /dev/null +++ b/cmd/kubectl-unbounded/app/ignition_test.go @@ -0,0 +1,180 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package app + +import ( + "encoding/base64" + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestIgnitionSpecVersionIsPinned guards the one constant an operator cannot +// recover from being wrong. +// +// Ignition refuses a config whose version it does not implement, and it refuses +// it on first boot with no shell and no agent yet installed. There is nothing on +// the host to report the mismatch, so the failure presents as a machine that +// provisioned into nothing. +func TestIgnitionSpecVersionIsPinned(t *testing.T) { + t.Parallel() + + require.Equal(t, "3.4.0", ignitionSpecVersion) +} + +// TestIgnitionDataURLRoundTrips covers how inline file contents reach the host. +// Ignition reads them from a data URL, so anything lost in the encoding is lost +// silently: the file appears, with the wrong bytes in it. +func TestIgnitionDataURLRoundTrips(t *testing.T) { + t.Parallel() + + for _, content := range []string{ + "", + "plain", + "{\n \"MachineName\": \"kube1\"\n}\n", + "trailing newline\n", + "unicode: \u00e9\u00e8\u00ea and emoji bytes", + "null\x00byte", + } { + t.Run(strings.SplitN(content, "\n", 2)[0], func(t *testing.T) { + t.Parallel() + + url := ignitionDataURL(content) + require.True(t, strings.HasPrefix(url, "data:;base64,"), "got %q", url) + + decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(url, "data:;base64,")) + require.NoError(t, err) + require.Equal(t, content, string(decoded)) + }) + } +} + +// TestIgnitionRemoteFetchable pins which sources Ignition can retrieve itself. +// +// This decides where a file lands in the boot. A fetchable source is written +// before dbus starts; anything else has to wait for the agent, which is after. +// Reading it the wrong way round produces a config Ignition rejects, or a file +// that silently arrives too late to be useful. +func TestIgnitionRemoteFetchable(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + source string + want bool + }{ + {"https://example.test/unbounded-agent", true}, + {"http://example.test/unbounded-agent", true}, + {"tftp://example.test/unbounded-agent", true}, + {"s3://bucket/unbounded-agent", true}, + {"arn:aws:s3:::bucket/unbounded-agent", true}, + {"gs://bucket/unbounded-agent", true}, + {" https://example.test/spaced ", true}, + + // oci is the one that matters: it is the agent's own artifact scheme, + // and Ignition has no idea what to do with it. + {"oci://ghcr.io/azure/unbounded-agent:v1", false}, + {"file:///tmp/unbounded-agent", false}, + {"ftp://example.test/unbounded-agent", false}, + {"/usr/local/bin/unbounded-agent", false}, + {"", false}, + {"://not a url", false}, + } { + t.Run(tc.source, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tc.want, ignitionRemoteFetchable(tc.source)) + }) + } +} + +// TestIgnitionHashFromSHA256 covers the digest conversion, including the +// sha256sum shape an operator is most likely to paste in. +func TestIgnitionHashFromSHA256(t *testing.T) { + t.Parallel() + + const digest = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + t.Run("plain digest", func(t *testing.T) { + t.Parallel() + + got, err := ignitionHashFromSHA256(digest) + require.NoError(t, err) + require.Equal(t, "sha256-"+digest, got) + }) + + t.Run("sha256sum output", func(t *testing.T) { + t.Parallel() + + got, err := ignitionHashFromSHA256(digest + " unbounded-agent\n") + require.NoError(t, err) + require.Equal(t, "sha256-"+digest, got) + }) + + t.Run("uppercase is normalized", func(t *testing.T) { + t.Parallel() + + got, err := ignitionHashFromSHA256(strings.ToUpper(digest)) + require.NoError(t, err) + require.Equal(t, "sha256-"+digest, got, "Ignition compares the hash as written") + }) + + for _, tc := range []struct{ name, input string }{ + {"empty", ""}, + {"too short", digest[:63]}, + {"too long", digest + "0"}, + {"non-hex", strings.Replace(digest, "9", "z", 1)}, + } { + t.Run("rejects "+tc.name, func(t *testing.T) { + t.Parallel() + + _, err := ignitionHashFromSHA256(tc.input) + require.Error(t, err, "a malformed digest must fail here, not on the host at first boot") + }) + } +} + +// TestIgnitionConfigOmitsEmptySections pins that the emitted document contains +// only what was asked for. +// +// Ignition validates the whole config before acting on any of it, so an empty +// section serialized as null or [] can reject a config that is otherwise fine, +// again on a host with nothing available to say so. +func TestIgnitionConfigOmitsEmptySections(t *testing.T) { + t.Parallel() + + encoded, err := json.Marshal(ignitionConfig{Ignition: ignitionVersion{Version: ignitionSpecVersion}}) + require.NoError(t, err) + + require.JSONEq(t, `{"ignition":{"version":"3.4.0"}}`, string(encoded)) + require.NotContains(t, string(encoded), "storage") + require.NotContains(t, string(encoded), "systemd") +} + +// TestIgnitionFileModesSerializeAsDecimal covers a trap in the format: Ignition +// file modes are decimal integers, and Go's octal literals are easy to read as +// if they were being emitted verbatim. +// +// A mode written as 600 rather than 0o600 is 0o1130 on disk, which for the +// agent config means credentials readable by everyone. +func TestIgnitionFileModesSerializeAsDecimal(t *testing.T) { + t.Parallel() + + encoded, err := json.Marshal(ignitionFile{ + Path: "/etc/unbounded/agent/config.json", + Mode: ignitionModeConfig, + Contents: ignitionContents{Source: ignitionDataURL("{}")}, + }) + require.NoError(t, err) + + // 0o600 is 384 decimal. Asserting the number rather than the constant is + // the point: it is what a reader of the emitted config would see. + require.Contains(t, string(encoded), `"mode":384`) + + require.Equal(t, 0o600, ignitionModeConfig, "the agent config carries credentials") + require.Equal(t, 0o755, ignitionModeScript) + require.Equal(t, 0o644, ignitionModeData) + require.Equal(t, 0o755, ignitionModeDir) +} From 0b3ab6288c851ea207ba423ecb863e69a2d3f513 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:14:53 -0400 Subject: [PATCH 37/39] kubectl-unbounded: emit an Ignition bootstrap config Adds --variant ignition, which writes the agent config, fetches the agent binary to its final location, and installs a oneshot unit that bootstraps on boot. Everything Ignition writes is in place before any service starts, because it runs from the initramfs. Every input this variant needs is required rather than defaulted. Ignition declares state: it cannot resolve a version, detect an architecture, or extract an archive at boot, so the artifact has to be named exactly. The digest is required rather than optional because an unattended host that silently accepts whatever a URL returns is worse than a bootstrap that refuses to render. The prefix is required because Ignition places the binary itself, and the default /usr/local is read-only on exactly the hosts this variant exists to serve. All three are refused at render time, where the message reaches a person, rather than on a machine with no shell. The unit carries no completion condition and so runs on every boot. A condition needs a marker file, and a marker is a second record of completion that can disagree with the ownership record the agent already keeps. Both commands the unit runs return immediately once that record says the installation is complete: preflight reports an empty result and start verifies the daemon, repairing it only if it is not running, and neither resolves artifacts or touches the network. The cost is two short-lived processes per boot; the benefit is that a node whose daemon was stopped or damaged comes back on reboot. Two settings come from failures seen on real hardware rather than reasoned about. network-online.target means a link is configured, not that DNS resolves, so the unit retries instead of ordering against a guarantee that target does not carry. And bootstrap has no later opportunity to run, so StartLimitIntervalSec=0 keeps a burst of early failures from permanently disabling it. The prefix is carried in the agent config, not only in the generated output, because the daemon and the nspawn lifecycle hooks are started by systemd later and cannot inherit it from the environment that provisioned the host. --- .../app/machine_manual_bootstrap.go | 227 +++++++++++++++++- .../app/machine_manual_bootstrap_test.go | 196 +++++++++++++++ 2 files changed, 421 insertions(+), 2 deletions(-) diff --git a/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go b/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go index 6f0c740ec..f766d4213 100644 --- a/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go +++ b/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go @@ -30,6 +30,7 @@ import ( "github.com/Azure/unbounded/internal/kube" "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/config" + "github.com/Azure/unbounded/pkg/agent/goalstates" ) //go:embed assets/node-bootstrap/script.sh @@ -47,6 +48,12 @@ const ( // variantCloudInit produces a cloud-init user-data document. variantCloudInit bootstrapVariant = "cloud-init" + + // variantIgnition produces an Ignition config. It is the only mechanism + // image-based hosts such as Azure Container Linux consume: they ship no + // cloud-init at all, so a cloud-init payload passed as user data is never + // acted on and nothing reports an error. + variantIgnition bootstrapVariant = "ignition" ) func parseBootstrapVariant(s string) (bootstrapVariant, error) { @@ -55,8 +62,10 @@ func parseBootstrapVariant(s string) (bootstrapVariant, error) { return variantScript, nil case variantCloudInit: return variantCloudInit, nil + case variantIgnition: + return variantIgnition, nil default: - return "", fmt.Errorf("unknown variant %q (valid: script, cloud-init)", s) + return "", fmt.Errorf("unknown variant %q (valid: script, cloud-init, ignition)", s) } } @@ -103,8 +112,22 @@ type manualBootstrapHandler struct { // agentURL is a fully qualified override for the unbounded-agent download // URL. When set it takes precedence over agentVersion and agentBaseURL. + // + // The ignition variant requires it, and requires it to name the bare agent + // binary rather than the release tarball: Ignition fetches files, it does + // not extract archives. agentURL string + // agentSHA256 is the expected digest of the agent binary. Required by the + // ignition variant, which fetches the binary without a script that could + // verify it afterwards. + agentSHA256 string + + // hostPrefix is the installation prefix for the agent's own host-side + // files. Required by the ignition variant, whose target hosts mount /usr + // read-only. + hostPrefix string + // agentBaseURL overrides the base URL used to construct the download URL // for the unbounded-agent. Useful for self-hosted release mirrors. Must // follow the same layout as GitHub releases @@ -186,6 +209,8 @@ func (h *manualBootstrapHandler) execute(ctx context.Context) error { switch bootstrapVariant(h.variant) { case variantCloudInit: output, err = h.renderCloudInit(cfg) + case variantIgnition: + output, err = h.renderIgnition(cfg) default: output, err = h.renderScript(cfg) } @@ -348,6 +373,14 @@ func (h *manualBootstrapHandler) validate() error { return errors.New("site name is required") } + // Rejected here rather than on the host. The prefix is interpolated into + // generated systemd units and into a shell script, neither of which quotes + // it, and a value that breaks those does so on a machine with no operator + // watching and no way to report it. + if err := config.ValidateHostPrefix(h.hostPrefix); err != nil { + return fmt.Errorf("invalid host prefix: %w", err) + } + // The machine name is optional. When omitted, the unbounded-agent resolves // it at startup from the AGENT_MACHINE_NAME environment variable or the host // hostname, which lets a single bootstrap payload be reused across many @@ -481,6 +514,12 @@ func (h *manualBootstrapHandler) buildAgentConfig(ctx context.Context) (*provisi }) cfg.Kubelet.NodeIP = strings.TrimSpace(h.nodeIP) + + // Carried in the config rather than only in the generated output, because + // the agent re-reads it long after bootstrap: the daemon and the nspawn + // lifecycle hooks are started by systemd and cannot inherit it from the + // environment that provisioned the host. + cfg.HostPrefix = strings.TrimSpace(h.hostPrefix) if source := strings.TrimSpace(h.offlineArtifactsSource); source != "" { cfg.OfflineArtifacts = &provision.AgentOfflineArtifacts{Source: source} } @@ -712,7 +751,9 @@ Examples: cmd.Flags().StringVar(&handler.kubernetesVersion, "kubernetes-version", "", "Override the Kubernetes version (default: auto-detected from API server)") cmd.Flags().StringVar(&handler.variant, "variant", "script", "Output format: script or cloud-init") cmd.Flags().StringVar(&handler.agentVersion, "agent-version", "", "Pin the unbounded-agent release tag to download on the host (default: latest GitHub release)") - cmd.Flags().StringVar(&handler.agentURL, "agent-url", "", "Fully qualified download URL for the unbounded-agent tarball (overrides --agent-version and --agent-base-url)") + cmd.Flags().StringVar(&handler.agentURL, "agent-url", "", "Fully qualified download URL for the unbounded-agent tarball (overrides --agent-version and --agent-base-url). With --variant ignition this must name the bare binary, not the tarball") + cmd.Flags().StringVar(&handler.agentSHA256, "agent-sha256", "", "SHA-256 digest of the agent binary, published in checksums.txt. Required with --variant ignition") + cmd.Flags().StringVar(&handler.hostPrefix, "host-prefix", "", "Installation prefix for the agent's own host-side files. Required with --variant ignition, whose target hosts mount /usr read-only") cmd.Flags().StringVar(&handler.agentBaseURL, "agent-base-url", "", "Base URL for unbounded-agent release downloads (default: https://github.com/Azure/unbounded/releases). Use this to self-host or mirror release assets") // Rootfs binary download overrides. See `kubectl unbounded machine register --help` @@ -803,3 +844,185 @@ func resolveBootstrapToken(ctx context.Context, logger *slog.Logger, kubeCli kub return nil, fmt.Errorf("no bootstrap token found for site %q and no tokens available in the cluster (run 'kubectl unbounded site init' first)", siteName) } + +// Paths the Ignition variant writes on the target host. The config path is the +// one `unbounded-agent start` reads from UNBOUNDED_AGENT_CONFIG_FILE. +const ( + ignitionAgentConfigPath = "/etc/unbounded/agent/config.json" + ignitionBootstrapUnit = "unbounded-agent-bootstrap.service" + ignitionAgentBinaryName = "unbounded-agent" +) + +func boolPtr(v bool) *bool { return &v } + +// renderIgnition emits an Ignition config that provisions the host with no +// shell and no operator present. +// +// Ignition is declarative and runs from the initramfs, so everything it writes +// is in place before any service starts. That is what lets the agent config, +// the agent binary and the bootstrap unit all be present on the first boot +// rather than fetched by something running on the host. +func (h *manualBootstrapHandler) renderIgnition(cfg *provision.UnboundedAgentConfig) (string, error) { + configJSON, err := json.MarshalIndent(cfg, "", " ") + if err != nil { + return "", fmt.Errorf("marshaling agent config: %w", err) + } + + binaryFile, err := h.ignitionAgentBinaryFile(cfg) + if err != nil { + return "", err + } + + config := ignitionConfig{ + Ignition: ignitionVersion{Version: ignitionSpecVersion}, + Storage: &ignitionStorage{ + Directories: []ignitionDirectory{{ + Path: ignitionAgentBinDir(cfg), + Mode: ignitionModeDir, + }}, + Files: []ignitionFile{ + { + Path: ignitionAgentConfigPath, + Mode: ignitionModeConfig, + Overwrite: boolPtr(true), + Contents: ignitionContents{Source: ignitionDataURL(string(configJSON) + "\n")}, + }, + *binaryFile, + }, + }, + Systemd: &ignitionSystemd{Units: []ignitionUnit{{ + Name: ignitionBootstrapUnit, + Enabled: boolPtr(true), + Contents: h.ignitionBootstrapUnitContents(cfg), + }}}, + } + + rendered, err := json.MarshalIndent(config, "", " ") + if err != nil { + return "", fmt.Errorf("marshaling ignition config: %w", err) + } + + return string(rendered) + "\n", nil +} + +// ignitionAgentBinDir returns the directory the agent binary is placed in, +// derived from the configured host prefix so that a host with a read-only /usr +// puts it somewhere writable. +func ignitionAgentBinDir(cfg *provision.UnboundedAgentConfig) string { + prefix := "" + if cfg != nil { + prefix = cfg.HostPrefix + } + + return goalstates.ResolveHostPaths(prefix).BinDir +} + +// ignitionAgentBinaryFile fetches the agent binary straight to its final +// location, verified against a caller-supplied digest. +// +// Every input here is required rather than defaulted, because this variant has +// no shell to fall back on. Ignition declares state; it cannot resolve a +// version, detect an architecture, or extract an archive at boot, so the +// artifact has to be named exactly and the host has no way to report that it +// was not. +func (h *manualBootstrapHandler) ignitionAgentBinaryFile(cfg *provision.UnboundedAgentConfig) (*ignitionFile, error) { + source := strings.TrimSpace(h.agentURL) + digest := strings.TrimSpace(h.agentSHA256) + + // Ignition writes the binary itself, so an unset prefix would place it + // under the default /usr/local and fail at first boot on exactly the + // immutable hosts this variant exists to serve. Refuse at render time, + // where the message can say what to do. + if cfg == nil || strings.TrimSpace(cfg.HostPrefix) == "" { + return nil, fmt.Errorf("--host-prefix is required with --variant %s: Ignition places the agent binary itself, and the default prefix /usr/local is read-only on immutable hosts", variantIgnition) + } + + if source == "" { + return nil, fmt.Errorf("--agent-url is required with --variant %s, and must point at the bare agent binary rather than the release tarball, because Ignition cannot extract an archive", variantIgnition) + } + + if !ignitionRemoteFetchable(source) { + return nil, fmt.Errorf("--agent-url %q cannot be fetched by Ignition; use an http, https, tftp, s3, arn, or gs URL", source) + } + + if digest == "" { + return nil, fmt.Errorf("--agent-sha256 is required with --variant %s; the digest for each release binary is published in checksums.txt", variantIgnition) + } + + hash, err := ignitionHashFromSHA256(digest) + if err != nil { + return nil, fmt.Errorf("invalid --agent-sha256: %w", err) + } + + return &ignitionFile{ + Path: ignitionAgentBinDir(cfg) + "/" + ignitionAgentBinaryName, + Mode: ignitionModeScript, + Overwrite: boolPtr(true), + Contents: ignitionContents{ + Source: source, + Verification: &ignitionVerification{Hash: hash}, + }, + }, nil +} + +// ignitionBootstrapUnitContents renders the oneshot unit that bootstraps the +// agent on first boot. +// +// The unit runs the agent directly rather than a shell script. Ignition has +// already placed and verified the binary, so a script here would only +// re-implement that imperatively. +// +// It carries no completion condition, and so runs on every boot. That is +// deliberate. A condition needs a marker file, and a marker is a second record +// of completion that can disagree with the ownership record the agent already +// keeps; the agent's own admission answers the same question from the record, +// which is written before the first host mutation and therefore cannot be +// missing on a host that started installing. Both commands below return +// immediately once that record says the installation is complete: preflight +// reports an empty result and start verifies the daemon and repairs it if it +// is not running, neither resolving artifacts nor touching the network. The +// cost is two short-lived processes per boot, and the benefit is that a node +// whose daemon was stopped or damaged comes back on reboot. +func (h *manualBootstrapHandler) ignitionBootstrapUnitContents(cfg *provision.UnboundedAgentConfig) string { + binary := ignitionAgentBinDir(cfg) + "/" + ignitionAgentBinaryName + + var b strings.Builder + + b.WriteString("[Unit]\n") + b.WriteString("Description=Bootstrap the unbounded agent\n") + b.WriteString("Wants=network-online.target\n") + // The agent downloads the node rootfs and the Kubernetes, CRI and CNI + // binaries, so it needs the network even though Ignition already fetched + // the agent itself. Ordering after systemd-sysext keeps any extension + // merged before the agent runs. + b.WriteString("After=network-online.target nss-lookup.target systemd-sysext.service\n") + b.WriteString("ConditionPathExists=" + binary + "\n") + // Retry indefinitely rather than giving up after systemd's default start + // limit. Bootstrap has no later opportunity to run, so a burst of early + // failures must not permanently disable it. + b.WriteString("StartLimitIntervalSec=0\n\n") + + b.WriteString("[Service]\n") + b.WriteString("Type=oneshot\n") + b.WriteString("RemainAfterExit=yes\n") + // network-online.target only means a link is configured, not that DNS + // resolves. On a first boot the agent can start before systemd-resolved is + // answering and fail with an unresolved host, so retry rather than ordering + // against something that does not carry that guarantee. Verified on systemd + // 255 that Type=oneshot honors Restart=. + b.WriteString("Restart=on-failure\n") + b.WriteString("RestartSec=10s\n") + // `unbounded-agent start` has no --config flag and reads this variable. + b.WriteString("Environment=UNBOUNDED_AGENT_CONFIG_FILE=" + ignitionAgentConfigPath + "\n") + + // Preflight runs as ExecStartPre so a failure is reported against this unit + // before any host mutation, and shows up in its status rather than being + // buried in a script's output. + b.WriteString("ExecStartPre=" + binary + " preflight\n") + b.WriteString("ExecStart=" + binary + " start\n\n") + + b.WriteString("[Install]\n") + b.WriteString("WantedBy=multi-user.target\n") + + return b.String() +} diff --git a/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go b/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go index 2ced9a19d..2e6951adf 100644 --- a/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go +++ b/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go @@ -1189,3 +1189,199 @@ func TestManualBootstrapHandler_BuildAgentConfig_AdditionalHostDevices(t *testin require.Equal(t, []string{"/dev/uinput", "char-input"}, cfg.AdditionalHostDevices) } + +// ignitionTestConfig returns an agent config shaped like one the command would +// build, with the prefix the ignition variant requires. +func ignitionTestConfig(prefix string) *provision.UnboundedAgentConfig { + return &provision.UnboundedAgentConfig{ + AgentConfig: provision.AgentConfig{ + MachineName: "test-node", + HostPrefix: prefix, + Cluster: provision.AgentClusterConfig{ + CaCertBase64: "dGVzdA==", + ClusterDNS: "10.0.0.10", + Version: "v1.30.0", + }, + Kubelet: provision.AgentKubeletConfig{ + ApiServer: "https://api-server:6443", + Auth: provision.KubeletAuthInfo{BootstrapToken: "abc123.0123456789abcdef"}, + }, + }, + } +} + +const ignitionTestDigest = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + +func ignitionTestHandler() *manualBootstrapHandler { + return &manualBootstrapHandler{ + logger: discardLogger(), + agentURL: "https://example.test/unbounded-agent-linux-amd64", + agentSHA256: ignitionTestDigest, + hostPrefix: "/opt/unbounded", + } +} + +// TestRenderIgnitionPlacesEverythingBeforeFirstBoot covers the property the +// whole variant exists for: on a host with no shell and no operator, every file +// the agent needs is already present when the unit starts. +func TestRenderIgnitionPlacesEverythingBeforeFirstBoot(t *testing.T) { + t.Parallel() + + out, err := ignitionTestHandler().renderIgnition(ignitionTestConfig("/opt/unbounded")) + require.NoError(t, err) + + var cfg ignitionConfig + require.NoError(t, json.Unmarshal([]byte(out), &cfg), "emitted document must be valid JSON") + require.Equal(t, ignitionSpecVersion, cfg.Ignition.Version) + + require.NotNil(t, cfg.Storage) + + paths := map[string]ignitionFile{} + for _, f := range cfg.Storage.Files { + paths[f.Path] = f + } + + agentConfig, ok := paths[ignitionAgentConfigPath] + require.True(t, ok, "the agent config must be written, got %v", paths) + require.Equal(t, ignitionModeConfig, agentConfig.Mode, "the agent config carries a bootstrap token") + + binary, ok := paths["/opt/unbounded/bin/unbounded-agent"] + require.True(t, ok, "the agent binary must land under the configured prefix, got %v", paths) + require.Equal(t, ignitionModeScript, binary.Mode) + require.Equal(t, "https://example.test/unbounded-agent-linux-amd64", binary.Contents.Source) + require.NotNil(t, binary.Contents.Verification, "an unattended host must not accept whatever the URL returns") + require.Equal(t, "sha256-"+ignitionTestDigest, binary.Contents.Verification.Hash) + + require.NotNil(t, cfg.Systemd) + require.Len(t, cfg.Systemd.Units, 1) + require.Equal(t, ignitionBootstrapUnit, cfg.Systemd.Units[0].Name) + require.NotNil(t, cfg.Systemd.Units[0].Enabled) + require.True(t, *cfg.Systemd.Units[0].Enabled, "an unenabled unit never runs and nothing reports it") +} + +// TestRenderIgnitionHonoursTheHostPrefix pins that every host-side path moves +// together. A binary under the prefix and a unit pointing at /usr/local would +// produce a host that provisions into a unit which cannot start. +func TestRenderIgnitionHonoursTheHostPrefix(t *testing.T) { + t.Parallel() + + h := ignitionTestHandler() + h.hostPrefix = "/var/lib/unbounded-agent" + + out, err := h.renderIgnition(ignitionTestConfig("/var/lib/unbounded-agent")) + require.NoError(t, err) + + require.Contains(t, out, "/var/lib/unbounded-agent/bin/unbounded-agent") + require.NotContains(t, out, "/usr/local/bin/unbounded-agent", + "nothing may resolve to the default prefix once one is configured") +} + +// TestRenderIgnitionRefusesRatherThanGuessing covers each input this variant +// cannot default. +// +// Ignition declares state: it cannot resolve a version, detect an architecture, +// or extract an archive at boot. Every one of these failures would otherwise +// land on a machine with no shell and no way to say what went wrong, so they +// are refused at render time where the message reaches a person. +func TestRenderIgnitionRefusesRatherThanGuessing(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + mutate func(*manualBootstrapHandler) + prefix string + wantErr string + }{ + { + name: "no host prefix", + prefix: "", + wantErr: "--host-prefix is required", + }, + { + name: "no agent url", + mutate: func(h *manualBootstrapHandler) { h.agentURL = "" }, + wantErr: "--agent-url is required", + }, + { + name: "agent url Ignition cannot fetch", + mutate: func(h *manualBootstrapHandler) { h.agentURL = "oci://ghcr.io/azure/unbounded-agent:v1" }, + wantErr: "cannot be fetched by Ignition", + }, + { + name: "no digest", + mutate: func(h *manualBootstrapHandler) { h.agentSHA256 = "" }, + wantErr: "--agent-sha256 is required", + }, + { + name: "malformed digest", + mutate: func(h *manualBootstrapHandler) { h.agentSHA256 = "not-a-digest" }, + wantErr: "invalid --agent-sha256", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + h := ignitionTestHandler() + prefix := "/opt/unbounded" + + if tc.mutate != nil { + tc.mutate(h) + } else { + prefix = tc.prefix + } + + _, err := h.renderIgnition(ignitionTestConfig(prefix)) + require.Error(t, err) + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// TestIgnitionBootstrapUnitRunsOnEveryBoot pins the decision not to carry a +// completion condition. +// +// A condition needs a marker file, which is a second record of completion that +// can disagree with the ownership record the agent already keeps. Instead the +// unit runs every boot and the agent's own admission answers the question, +// returning immediately once the record says the installation is complete. The +// benefit is that a node whose daemon was stopped comes back on reboot. +func TestIgnitionBootstrapUnitRunsOnEveryBoot(t *testing.T) { + t.Parallel() + + unit := ignitionTestHandler().ignitionBootstrapUnitContents(ignitionTestConfig("/opt/unbounded")) + + require.NotContains(t, unit, "ConditionPathExists=!", + "a completion marker would be a second source of truth beside the ownership record") + + // The one condition that stays guards against running a binary Ignition + // failed to place, which would otherwise fail confusingly every boot. + require.Contains(t, unit, "ConditionPathExists=/opt/unbounded/bin/unbounded-agent") + + require.Contains(t, unit, "WantedBy=multi-user.target", "the unit has to be started on every boot for this to work") +} + +// TestIgnitionBootstrapUnitSurvivesEarlyBootRaces covers two failures seen on +// real hardware rather than reasoned about. +// +// network-online.target means a link is configured, not that DNS resolves: a +// unit can start in the same second the target is reached while +// systemd-resolved is still coming up, and fail on an unresolved host. And +// bootstrap has no later opportunity to run, so systemd's default start limit +// would turn a burst of early failures into a permanently disabled unit. +func TestIgnitionBootstrapUnitSurvivesEarlyBootRaces(t *testing.T) { + t.Parallel() + + unit := ignitionTestHandler().ignitionBootstrapUnitContents(ignitionTestConfig("/opt/unbounded")) + + require.Contains(t, unit, "Restart=on-failure", "DNS may not answer yet on the first attempt") + require.Contains(t, unit, "StartLimitIntervalSec=0", "bootstrap gets no second chance if systemd gives up on it") + require.Contains(t, unit, "Type=oneshot") + require.Contains(t, unit, "After=network-online.target nss-lookup.target systemd-sysext.service") + + // start reads the config path from the environment; it has no flag for it. + require.Contains(t, unit, "Environment=UNBOUNDED_AGENT_CONFIG_FILE="+ignitionAgentConfigPath) + + // Preflight runs before any host mutation and reports against this unit. + require.Contains(t, unit, "ExecStartPre=/opt/unbounded/bin/unbounded-agent preflight") + require.Contains(t, unit, "ExecStart=/opt/unbounded/bin/unbounded-agent start") +} From cc4c154312b26793be31e7e14a33c9b0e5c9dedc Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:21:13 -0400 Subject: [PATCH 38/39] agent: remove the first-boot bootstrap unit on reset The Ignition unit carries no completion condition and runs on every boot, deciding there is nothing to do from the agent's ownership record. Reset deletes that record. A unit left behind would find an uninstalled host on the next boot and bootstrap it, quietly undoing the reset. Removal runs before the artifacts are deleted, so a failure stops the reset while the host is still recognizably installed rather than half torn down with something that will rebuild it. Disabling as well as deleting, because the file and the enablement symlink in multi-user.target.wants are separate: removing only the file leaves systemd with a dangling want. Absent on every host not provisioned through Ignition, which is the common case, so a missing unit is success. The unit name moved to goalstates. The command that writes it and the reset that removes it live in packages that cannot import each other, and a name that drifted between them would leave the unit enabled on a host that had just been reset. It is a named task rather than a step inside another one so the reset composition can be asserted. A first version tested the removal in isolation and passed while nothing called it, which is the failure this arrangement makes visible. --- cmd/agent/internal/daemon/lifecycle.go | 53 +++++++++++++++++++ cmd/agent/internal/daemon/lifecycle_test.go | 51 ++++++++++++++++++ cmd/agent/internal/daemon/reset.go | 4 ++ cmd/agent/internal/daemon/reset_test.go | 21 ++++++++ .../app/machine_manual_bootstrap.go | 3 +- .../app/machine_manual_bootstrap_test.go | 3 +- pkg/agent/goalstates/constants.go | 9 ++++ 7 files changed, 141 insertions(+), 3 deletions(-) diff --git a/cmd/agent/internal/daemon/lifecycle.go b/cmd/agent/internal/daemon/lifecycle.go index 9ddc969cc..8099521a0 100644 --- a/cmd/agent/internal/daemon/lifecycle.go +++ b/cmd/agent/internal/daemon/lifecycle.go @@ -246,6 +246,59 @@ func (t *removeDaemonUnit) Do(ctx context.Context) error { return disableAndRemoveDaemonUnit(ctx, t.log) } +type removeFirstBootUnit struct { + log *slog.Logger +} + +// RemoveFirstBootBootstrapUnit returns a task that disables and removes the +// unit an Ignition config installs to bootstrap the agent. +func RemoveFirstBootBootstrapUnit(log *slog.Logger) phases.Task { + return &removeFirstBootUnit{log: log} +} + +func (t *removeFirstBootUnit) Name() string { return "remove-first-boot-unit" } + +func (t *removeFirstBootUnit) Do(ctx context.Context) error { + return removeFirstBootBootstrapUnit(ctx, t.log) +} + +// removeFirstBootBootstrapUnit disables and removes the unit an Ignition config +// installs to bootstrap the agent. +// +// Reset has to take this with it. The unit is installed into +// multi-user.target and carries no completion condition, so it runs on every +// boot and relies on the agent's ownership record to decide there is nothing to +// do. Reset removes that record, so a unit left behind would find a host with +// no installation and bootstrap it again, undoing the reset on the next boot. +// +// Absent on every host not provisioned through Ignition, which is the common +// case, so a missing unit is success rather than something to report. +func removeFirstBootBootstrapUnit(ctx context.Context, log *slog.Logger) error { + return removeFirstBootBootstrapUnitIn(ctx, log, goalstates.SystemdSystemDir) +} + +// removeFirstBootBootstrapUnitIn takes the unit directory so the sequence can +// be exercised without writing to /etc. +func removeFirstBootBootstrapUnitIn(ctx context.Context, log *slog.Logger, unitDir string) error { + unitPath := filepath.Join(unitDir, goalstates.FirstBootBootstrapUnit) + + if _, err := os.Lstat(unitPath); errors.Is(err, os.ErrNotExist) { + return nil + } + + log.Info("removing first-boot bootstrap unit", "unit", goalstates.FirstBootBootstrapUnit) + + if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", goalstates.FirstBootBootstrapUnit); err != nil { + // Disable removes the enablement symlink. If it failed but the unit + // file is already gone, there is nothing left to start. + if _, statErr := os.Lstat(unitPath); !errors.Is(statErr, os.ErrNotExist) { + return fmt.Errorf("disable %s: %w", goalstates.FirstBootBootstrapUnit, err) + } + } + + return removeOwnedFile(unitPath) +} + func disableAndRemoveDaemonUnit(ctx context.Context, log *slog.Logger) error { if err := executil.RunCmd(ctx, log, executil.Systemctl(), "disable", goalstates.DaemonUnit); err != nil { if _, statErr := os.Lstat(filepath.Join(goalstates.SystemdSystemDir, goalstates.DaemonUnit)); !errors.Is(statErr, os.ErrNotExist) { diff --git a/cmd/agent/internal/daemon/lifecycle_test.go b/cmd/agent/internal/daemon/lifecycle_test.go index 9528df2b9..cc6097705 100644 --- a/cmd/agent/internal/daemon/lifecycle_test.go +++ b/cmd/agent/internal/daemon/lifecycle_test.go @@ -179,3 +179,54 @@ func TestActivateDaemonUnitToleratesDeniedResetFailed(t *testing.T) { require.NoError(t, activateDaemonUnit(t.Context(), discardLogger(), executil.Systemctl())) } + +// TestResetRemovesTheFirstBootBootstrapUnit covers the interaction between +// reset and an Ignition-provisioned host. +// +// The unit carries no completion condition and runs on every boot, deciding +// there is nothing to do from the agent's ownership record. Reset removes that +// record. A unit left behind would therefore find an uninstalled host on the +// next boot and bootstrap it, quietly undoing the reset. +func TestResetRemovesTheFirstBootBootstrapUnit(t *testing.T) { + dir := t.TempDir() + calls := filepath.Join(dir, "calls") + + require.NoError(t, os.WriteFile(filepath.Join(dir, "systemctl"), + []byte("#!/bin/sh\necho \"$@\" >> \""+calls+"\"\n"), 0o755)) + t.Setenv("PATH", dir+":"+os.Getenv("PATH")) + + unitDir := t.TempDir() + unitPath := filepath.Join(unitDir, goalstates.FirstBootBootstrapUnit) + require.NoError(t, os.WriteFile(unitPath, []byte("[Unit]\n"), 0o644)) + + require.NoError(t, removeFirstBootBootstrapUnitIn(t.Context(), discardLogger(), unitDir)) + + require.NoFileExists(t, unitPath, "the unit file must be gone, or systemd can still start it") + + recorded, err := os.ReadFile(calls) + require.NoError(t, err) + require.Contains(t, string(recorded), "disable "+goalstates.FirstBootBootstrapUnit, + "removing the file alone leaves the enablement symlink in multi-user.target.wants") +} + +// TestFirstBootBootstrapUnitAbsentIsSuccess covers every host not provisioned +// through Ignition, which is the common case. There is nothing to remove and +// nothing to report. +func TestFirstBootBootstrapUnitAbsentIsSuccess(t *testing.T) { + t.Parallel() + + require.NoError(t, removeFirstBootBootstrapUnitIn(t.Context(), discardLogger(), t.TempDir())) +} + +// TestFirstBootBootstrapUnitNameIsShared pins that the command writing the unit +// and the reset removing it agree on its name. +// +// They live in packages that cannot import each other, so the name is held in +// goalstates. If it were duplicated and drifted, reset would leave an enabled +// unit on a host it had just torn down, and the host would re-bootstrap on the +// next boot with nothing reporting why. +func TestFirstBootBootstrapUnitNameIsShared(t *testing.T) { + t.Parallel() + + require.Equal(t, "unbounded-agent-bootstrap.service", goalstates.FirstBootBootstrapUnit) +} diff --git a/cmd/agent/internal/daemon/reset.go b/cmd/agent/internal/daemon/reset.go index d4808e37a..f35f704a2 100644 --- a/cmd/agent/internal/daemon/reset.go +++ b/cmd/agent/internal/daemon/reset.go @@ -168,6 +168,10 @@ func resetResources(log *slog.Logger) phases.Task { reset.RemoveBPFFSMount(log, goalstates.NSpawnMachineKube2), ), reset.CleanupNetwork(log), + // Before the artifacts, so a failure here stops the reset while the + // host is still recognizably installed. A unit that survived a reset + // would bootstrap the host again on the next boot. + RemoveFirstBootBootstrapUnit(log), RemoveAgentArtifacts(log), reset.ReloadSystemd(log), ) diff --git a/cmd/agent/internal/daemon/reset_test.go b/cmd/agent/internal/daemon/reset_test.go index 633c914db..4b231adba 100644 --- a/cmd/agent/internal/daemon/reset_test.go +++ b/cmd/agent/internal/daemon/reset_test.go @@ -137,3 +137,24 @@ func TestTeardownKeepsAReadableRecord(t *testing.T) { require.Equal(t, "machine-1", r.MachineName) require.Equal(t, "fingerprint-1", r.ConfigFingerprint) } + +// TestResetRemovesTheFirstBootUnitBeforeArtifacts pins that reset actually runs +// the removal, not merely that the removal works. +// +// The unit runs on every boot and decides there is nothing to do from the +// ownership record that reset is about to delete. Left behind, it would find an +// uninstalled host and bootstrap it again, undoing the reset with nothing +// reporting why. Ordering it before the artifacts means a failure stops the +// reset while the host is still recognizably installed. +func TestResetRemovesTheFirstBootUnitBeforeArtifacts(t *testing.T) { + t.Parallel() + + taskName := resetResources(slog.New(slog.DiscardHandler)).Name() + + assert.Contains(t, taskName, "remove-first-boot-unit", + "reset must remove the Ignition bootstrap unit or the host re-bootstraps on next boot") + assert.Less(t, + strings.Index(taskName, "remove-first-boot-unit"), + strings.Index(taskName, "remove-agent-artifacts"), + "a failure here must stop the reset while the host is still recognizably installed") +} diff --git a/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go b/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go index f766d4213..e2dfc1c17 100644 --- a/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go +++ b/cmd/kubectl-unbounded/app/machine_manual_bootstrap.go @@ -849,7 +849,6 @@ func resolveBootstrapToken(ctx context.Context, logger *slog.Logger, kubeCli kub // one `unbounded-agent start` reads from UNBOUNDED_AGENT_CONFIG_FILE. const ( ignitionAgentConfigPath = "/etc/unbounded/agent/config.json" - ignitionBootstrapUnit = "unbounded-agent-bootstrap.service" ignitionAgentBinaryName = "unbounded-agent" ) @@ -891,7 +890,7 @@ func (h *manualBootstrapHandler) renderIgnition(cfg *provision.UnboundedAgentCon }, }, Systemd: &ignitionSystemd{Units: []ignitionUnit{{ - Name: ignitionBootstrapUnit, + Name: goalstates.FirstBootBootstrapUnit, Enabled: boolPtr(true), Contents: h.ignitionBootstrapUnitContents(cfg), }}}, diff --git a/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go b/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go index 2e6951adf..6d8cf2d3d 100644 --- a/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go +++ b/cmd/kubectl-unbounded/app/machine_manual_bootstrap_test.go @@ -20,6 +20,7 @@ import ( "github.com/Azure/unbounded/internal/provision" "github.com/Azure/unbounded/pkg/agent/config" + "github.com/Azure/unbounded/pkg/agent/goalstates" ) // --------------------------------------------------------------------------- @@ -1254,7 +1255,7 @@ func TestRenderIgnitionPlacesEverythingBeforeFirstBoot(t *testing.T) { require.NotNil(t, cfg.Systemd) require.Len(t, cfg.Systemd.Units, 1) - require.Equal(t, ignitionBootstrapUnit, cfg.Systemd.Units[0].Name) + require.Equal(t, goalstates.FirstBootBootstrapUnit, cfg.Systemd.Units[0].Name) require.NotNil(t, cfg.Systemd.Units[0].Enabled) require.True(t, *cfg.Systemd.Units[0].Enabled, "an unenabled unit never runs and nothing reports it") } diff --git a/pkg/agent/goalstates/constants.go b/pkg/agent/goalstates/constants.go index fcfc18043..41413fd38 100644 --- a/pkg/agent/goalstates/constants.go +++ b/pkg/agent/goalstates/constants.go @@ -26,6 +26,15 @@ const ( // DaemonRecoveryUnit is the systemd recovery unit for the agent daemon. DaemonRecoveryUnit = "unbounded-agent-daemon-recovery.service" + // FirstBootBootstrapUnit is the unit an Ignition config installs to bootstrap + // the agent on boot. + // + // Named here rather than in the command that writes it because reset has to + // remove it, and the two live in packages that cannot import each other. A + // name that drifted between them would leave the unit enabled on a host that + // had been reset, which re-bootstraps it on the next boot. + FirstBootBootstrapUnit = "unbounded-agent-bootstrap.service" + DaemonBinaryPath = "/usr/local/bin/unbounded-agent" DaemonBinaryBluePath = "/usr/local/bin/unbounded-agent-blue" DaemonBinaryGreenPath = "/usr/local/bin/unbounded-agent-green" From 55505d18f90b2159cd7d77ddfab6f1c48a554586 Mon Sep 17 00:00:00 2001 From: Philip Lombardi <893096+plombardi89@users.noreply.github.com> Date: Mon, 21 Sep 2026 13:26:29 -0400 Subject: [PATCH 39/39] agent: write the ownership record only when a repair changed something Re-running start on a completed installation verified the daemon and then rewrote the record regardless. Harmless when that happened once per manual rerun. The Ignition unit carries no completion condition and runs on every boot, so it becomes a durable write per boot on every node, and a write is a chance to fail: an entirely healthy host would be taking one for no reason. Only a repair can have changed anything, so only a repair is committed. Also stop discarding the verify error that triggered the repair. The first verify says what is broken; the repair failure says only that fixing it did not work. Reporting the second alone sends an operator after the wrong thing, so both are now wrapped together. The tests for this were wrong twice before they were right, both times passing against code that did the opposite. Comparing the record's contents cannot see a rewrite, because MarkComplete on an already-complete record writes identical bytes; the test now compares the inode, which changes on any write because the store replaces the file atomically. And asserting the reported error matched the injected one proved nothing while verify and repair failed with the same error, so they now fail differently. --- cmd/agent/internal/bootstrap/coordinator.go | 16 ++- .../internal/bootstrap/coordinator_test.go | 102 +++++++++++++++++- 2 files changed, 112 insertions(+), 6 deletions(-) diff --git a/cmd/agent/internal/bootstrap/coordinator.go b/cmd/agent/internal/bootstrap/coordinator.go index 6a21f43d0..a810e2a6f 100644 --- a/cmd/agent/internal/bootstrap/coordinator.go +++ b/cmd/agent/internal/bootstrap/coordinator.go @@ -101,18 +101,24 @@ func (c *Coordinator) Run(ctx context.Context, id Identity) (Outcome, error) { } if disposition == installstate.AlreadyComplete { - if err := c.stages.VerifyInstalled(ctx); err != nil { + verifyErr := c.stages.VerifyInstalled(ctx) + if verifyErr != nil { if err := c.stages.RepairDaemon(ctx); err != nil { - return Outcome{}, err + return Outcome{}, fmt.Errorf("repair daemon after %w: %w", verifyErr, err) } if err := c.stages.VerifyInstalled(ctx); err != nil { return Outcome{}, err } - } - if err := c.store.MarkComplete(r); err != nil { - return Outcome{}, err + // Only a repair can have changed anything, so only a repair needs + // to be committed. The record already says complete: rewriting it + // on a healthy host would be a durable write for no change, on + // every boot of every Ignition-provisioned node, since that unit + // has no completion condition and runs each time. + if err := c.store.MarkComplete(r); err != nil { + return Outcome{}, err + } } return Outcome{AlreadyComplete: true}, nil diff --git a/cmd/agent/internal/bootstrap/coordinator_test.go b/cmd/agent/internal/bootstrap/coordinator_test.go index c1a3fb080..2708be64c 100644 --- a/cmd/agent/internal/bootstrap/coordinator_test.go +++ b/cmd/agent/internal/bootstrap/coordinator_test.go @@ -7,7 +7,9 @@ import ( "context" "errors" "log/slog" + "os" "path/filepath" + "syscall" "testing" "github.com/stretchr/testify/require" @@ -22,7 +24,13 @@ type fakeStages struct { verifyErr error } -var errInjected = errors.New("injected stage failure") +var ( + errInjected = errors.New("injected stage failure") + // errRepairFailed is distinct from errInjected so a test can tell whether + // the reported error is the fault that triggered a repair or the failure of + // the repair itself. + errRepairFailed = errors.New("injected repair failure") +) func (f *fakeStages) run(name string) error { f.calls = append(f.calls, name) @@ -33,6 +41,10 @@ func (f *fakeStages) run(name string) error { } if name == f.fail { + if name == "repair" { + return errRepairFailed + } + return errInjected } @@ -189,3 +201,91 @@ func TestInterruptedRepairRemainsCompleteAndRetries(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{"verify", "repair", "verify"}, stages.calls) } + +// recordInode identifies the record file itself rather than its contents. +// +// MarkComplete on an already-complete record writes the same bytes, so +// comparing content cannot tell a rewrite from a no-op. The store replaces the +// file atomically, so any write at all produces a new inode. +func recordInode(t *testing.T, store *installstate.Store) uint64 { + t.Helper() + + info, err := os.Stat(filepath.Join(store.Root(), "install-state.json")) + require.NoError(t, err) + + stat, ok := info.Sys().(*syscall.Stat_t) + require.True(t, ok, "inode is how this test distinguishes a rewrite from a no-op") + + return stat.Ino +} + +// TestHealthyCompletedInstallIsNotRewritten covers the cost of an Ignition unit +// that carries no completion condition. +// +// That unit runs on every boot and reaches this path each time. Rewriting the +// record when nothing changed would be a durable write per boot on every node, +// and a write is a chance to fail: a host that is entirely healthy would be +// taking one for no reason. +func TestHealthyCompletedInstallIsNotRewritten(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + before := recordInode(t, store) + + stages := &fakeStages{store: store} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + + outcome, err := c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.NoError(t, err) + require.True(t, outcome.AlreadyComplete) + require.Equal(t, []string{"verify"}, stages.calls, "a healthy host needs no repair") + + require.Equal(t, before, recordInode(t, store), + "nothing changed, so the record must not have been written at all") +} + +// TestRepairedInstallIsCommitted is the other half: when a repair did happen, +// the result has to be durable before the process exits. +func TestRepairedInstallIsCommitted(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + before := recordInode(t, store) + + stages := &fakeStages{store: store, verifyErr: errInjected} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + + _, err = c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.NoError(t, err) + require.Equal(t, []string{"verify", "repair", "verify"}, stages.calls) + + loaded, err := store.Load() + require.NoError(t, err) + require.Equal(t, installstate.Complete, loaded.Phase) + require.NotEqual(t, before, recordInode(t, store), + "a repair changed the host, so the result has to be made durable") +} + +// TestFailedRepairReportsWhatWasWrong pins that the original fault survives. +// +// The first verify says what is broken; the repair failure says only that +// fixing it did not work. Reporting the second alone sends an operator after +// the wrong thing. +func TestFailedRepairReportsWhatWasWrong(t *testing.T) { + store := installstate.NewStore(t.TempDir(), filepath.Join(t.TempDir(), "lock")) + r, err := installstate.NewRecord("machine", "fingerprint", "") + require.NoError(t, err) + require.NoError(t, store.MarkComplete(r)) + + stages := &fakeStages{store: store, fail: "repair", verifyErr: errInjected} + c := New(slog.New(slog.DiscardHandler), store, stages, nil) + + _, err = c.Run(t.Context(), Identity{MachineName: r.MachineName, ConfigFingerprint: r.ConfigFingerprint}) + require.Error(t, err) + require.ErrorIs(t, err, errInjected, "the fault that triggered the repair must still be reported") + require.ErrorIs(t, err, errRepairFailed, "and so must the reason repairing it did not work") +}