diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 46de6e0d4..bd6707780 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -466,6 +466,11 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * return nil, err } + spec, err := buildAteomWorkloadSpec(req.GetSpec()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid workload spec: %v", err) + } + // Tell ateom to start the workload. gVisor uses RunscPath; the micro-VM // runtime uses the full RuntimeAssetPaths set. if _, err := client.RunWorkload(ctx, &ateompb.RunWorkloadRequest{ @@ -475,7 +480,7 @@ func (s *AteomHerder) Run(ctx context.Context, req *ateletpb.RunRequest) (resp * ActorTemplateName: req.GetActorTemplateName(), RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, - Spec: buildAteomWorkloadSpec(req.GetSpec()), + Spec: spec, ActorUid: actorUID, EgressGateway: toAteomEgressGateway(req.GetEgressGateway()), CpuMilli: req.GetCpuMilli(), @@ -580,6 +585,11 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe // Tell ateom to take the checkpoint and delete containers. ateom reports the // exact files it wrote so we ship precisely that set (gVisor's image files, // cloud-hypervisor's snapshot set, ...) rather than a hardcoded list. + spec, err := buildAteomWorkloadSpec(req.GetSpec()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid workload spec: %v", err) + } + tAteom := time.Now() resp, err := client.CheckpointWorkload(ctx, &ateompb.CheckpointWorkloadRequest{ Atespace: actorRef.Atespace, @@ -588,7 +598,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe ActorTemplateName: req.GetActorTemplateName(), RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, - Spec: buildAteomWorkloadSpec(req.GetSpec()), + Spec: spec, Scope: toAteomSnapshotScope(req.GetScope()), ActorUid: actorUID, }) @@ -601,7 +611,7 @@ func (s *AteomHerder) Checkpoint(ctx context.Context, req *ateletpb.CheckpointRe } sandboxRec.SnapshotFiles = resp.GetSnapshotFiles() - if len(sandboxRec.SnapshotFiles) == 0 { + if len(sandboxRec.SnapshotFiles) == 0 && shouldHaveSnapshots(req) { return nil, ateerrors.NewGRPCError(ctx, codes.DataLoss, ateerrors.ReasonInvalidCheckpointResult, ateerrors.ActorCrashedMetadata(), errors.New("ateom reported no snapshot files for checkpoint")) } sandboxRec.Atespace = req.GetAtespace() @@ -693,6 +703,20 @@ func (s *AteomHerder) moveLocalCheckpoint(ctx context.Context, req *ateletpb.Che return nil } +// shouldHaveSnapshots returns true if the checkpoint request is expected to produce snapshot files. +func shouldHaveSnapshots(req *ateletpb.CheckpointRequest) bool { + if req.GetScope() != ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA { + return true + } + + for _, vol := range req.GetSpec().GetVolumes() { + if vol.GetType() == ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR { + return true + } + } + return false +} + func (s *AteomHerder) uploadExternalCheckpoint(ctx context.Context, req *ateletpb.CheckpointRequest, checkpointDir string, rec *sandboxAssetsRecord) error { uri, err := resources.ParseSnapshotURI(req.GetExternalConfig().GetSnapshotUri()) if err != nil { @@ -1105,6 +1129,11 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) // Tell ateom to do runsc create + runsc restore for pause container and // all application containers. + spec, err := buildAteomWorkloadSpec(req.GetSpec()) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid workload spec: %v", err) + } + tAteom := time.Now() _, err = client.RestoreWorkload(ctx, &ateompb.RestoreWorkloadRequest{ Atespace: actorRef.Atespace, @@ -1113,7 +1142,7 @@ func (s *AteomHerder) Restore(ctx context.Context, req *ateletpb.RestoreRequest) ActorTemplateName: req.GetActorTemplateName(), RunscPath: runscPathFor(assetPaths), RuntimeAssetPaths: assetPaths, - Spec: buildAteomWorkloadSpec(req.GetSpec()), + Spec: spec, Scope: toAteomSnapshotScope(req.GetScope()), ActorUid: req.GetActorUid(), EgressGateway: toAteomEgressGateway(req.GetEgressGateway()), @@ -1495,32 +1524,50 @@ func (s *AteomHerder) dialAteom(ctx context.Context, targetAteomUid string) (ate // buildAteomWorkloadSpec projects the atelet-facing workload spec onto // the ateom-facing one. -func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) *ateompb.WorkloadSpec { - ddVolumes := make(map[string]bool) +func buildAteomWorkloadSpec(spec *ateletpb.WorkloadSpec) (*ateompb.WorkloadSpec, error) { + volumes := make(map[string]ateletpb.VolumeType) for _, vol := range spec.GetVolumes() { - if vol.GetType() == ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR { - ddVolumes[vol.GetName()] = true + name := vol.GetName() + if _, duplicate := volumes[name]; duplicate { + return nil, fmt.Errorf("duplicate volume name %q in workload spec", name) } + volumes[name] = vol.GetType() } out := &ateompb.WorkloadSpec{} for _, ctr := range spec.GetContainers() { var ddMounts []*ateompb.DurableDirVolumeMount + var csiMounts []*ateompb.VolumeMount for _, vm := range ctr.GetVolumeMounts() { - if ddVolumes[vm.GetName()] { + volName := vm.GetName() + volType, ok := volumes[volName] + if !ok { + return nil, fmt.Errorf("container %q mounts volume %q which is not defined in workload volumes", ctr.GetName(), volName) + } + + switch volType { + case ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR: ddMounts = append(ddMounts, &ateompb.DurableDirVolumeMount{ - VolumeName: vm.GetName(), + VolumeName: volName, MountPath: vm.GetMountPath(), }) + case ateletpb.VolumeType_VOLUME_TYPE_EXTERNAL: + csiMounts = append(csiMounts, &ateompb.VolumeMount{ + VolumeName: volName, + MountPath: vm.GetMountPath(), + }) + default: + return nil, fmt.Errorf("container %q mounts volume %q with unsupported type %v", ctr.GetName(), volName, volType) } } out.Containers = append(out.Containers, &ateompb.Container{ Name: ctr.GetName(), DurableDirVolumeMounts: ddMounts, + CsiVolumeMounts: csiMounts, Readyz: toAteomReadyz(ctr.GetReadyz()), }) } - return out + return out, nil } func toAteomEgressGateway(gateway *ateletpb.EgressGateway) *ateompb.EgressGateway { diff --git a/cmd/atelet/main_test.go b/cmd/atelet/main_test.go index 1bb492db4..50788d63a 100644 --- a/cmd/atelet/main_test.go +++ b/cmd/atelet/main_test.go @@ -693,7 +693,10 @@ func TestBuildAteomWorkloadSpecForwardsReadyz(t *testing.T) { {Name: "without-probe"}, }, } - got := buildAteomWorkloadSpec(in) + got, err := buildAteomWorkloadSpec(in) + if err != nil { + t.Fatalf("buildAteomWorkloadSpec failed: %v", err) + } if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { t.Errorf("buildAteomWorkloadSpec mismatch (-want +got):\n%s", diff) } @@ -736,6 +739,9 @@ func TestBuildAteomWorkloadSpecForwardsDurableDirMounts(t *testing.T) { {VolumeName: "data", MountPath: "/home/counter"}, {VolumeName: "cache", MountPath: "/var/cache"}, }, + CsiVolumeMounts: []*ateompb.VolumeMount{ + {VolumeName: "scratch", MountPath: "/scratch"}, + }, }, { Name: "sidecar", @@ -746,12 +752,88 @@ func TestBuildAteomWorkloadSpecForwardsDurableDirMounts(t *testing.T) { {Name: "no-volumes"}, }, } - got := buildAteomWorkloadSpec(in) + got, err := buildAteomWorkloadSpec(in) + if err != nil { + t.Fatalf("buildAteomWorkloadSpec failed: %v", err) + } if diff := cmp.Diff(want, got, protocmp.Transform()); diff != "" { t.Errorf("buildAteomWorkloadSpec mismatch (-want +got):\n%s", diff) } } +func TestBuildAteomWorkloadSpecValidation(t *testing.T) { + tests := []struct { + name string + in *ateletpb.WorkloadSpec + wantErr string + }{ + { + name: "missing volume definition", + in: &ateletpb.WorkloadSpec{ + Volumes: []*ateletpb.Volume{ + {Name: "data", Type: ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR}, + }, + Containers: []*ateletpb.Container{ + { + Name: "ctr", + VolumeMounts: []*ateletpb.VolumeMount{ + {Name: "missing-vol", MountPath: "/data"}, + }, + }, + }, + }, + wantErr: `container "ctr" mounts volume "missing-vol" which is not defined in workload volumes`, + }, + { + name: "unsupported volume type", + in: &ateletpb.WorkloadSpec{ + Volumes: []*ateletpb.Volume{ + {Name: "data", Type: ateletpb.VolumeType_VOLUME_TYPE_UNSPECIFIED}, + }, + Containers: []*ateletpb.Container{ + { + Name: "ctr", + VolumeMounts: []*ateletpb.VolumeMount{ + {Name: "data", MountPath: "/data"}, + }, + }, + }, + }, + wantErr: `container "ctr" mounts volume "data" with unsupported type VOLUME_TYPE_UNSPECIFIED`, + }, + { + name: "duplicate volume names", + in: &ateletpb.WorkloadSpec{ + Volumes: []*ateletpb.Volume{ + {Name: "data", Type: ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR}, + {Name: "data", Type: ateletpb.VolumeType_VOLUME_TYPE_EXTERNAL}, + }, + Containers: []*ateletpb.Container{ + { + Name: "ctr", + VolumeMounts: []*ateletpb.VolumeMount{ + {Name: "data", MountPath: "/data"}, + }, + }, + }, + }, + wantErr: `duplicate volume name "data" in workload spec`, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := buildAteomWorkloadSpec(tc.in) + if err == nil { + t.Fatal("expected error, got nil") + } + if got, want := err.Error(), tc.wantErr; !strings.Contains(got, want) { + t.Errorf("error mismatch:\nwant: %s\ngot: %s", want, got) + } + }) + } +} + func TestToAteomEgressGateway(t *testing.T) { if got := toAteomEgressGateway(nil); got != nil { t.Fatalf("toAteomEgressGateway(nil) = %v, want nil", got) @@ -1580,3 +1662,72 @@ func TestValidateUploadPausedCheckpointRequest(t *testing.T) { }) } } + +func TestShouldHaveSnapshots(t *testing.T) { + tests := []struct { + name string + req *ateletpb.CheckpointRequest + want bool + }{ + { + name: "full scope always expects snapshots", + req: &ateletpb.CheckpointRequest{ + Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_FULL, + }, + want: true, + }, + { + name: "data scope with durable volumes expects snapshots", + req: &ateletpb.CheckpointRequest{ + Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Spec: &ateletpb.WorkloadSpec{ + Volumes: []*ateletpb.Volume{ + {Name: "durable", Type: ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR}, + }, + }, + }, + want: true, + }, + { + name: "data scope with only CSI volumes does not expect snapshots", + req: &ateletpb.CheckpointRequest{ + Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Spec: &ateletpb.WorkloadSpec{ + Volumes: []*ateletpb.Volume{ + {Name: "csi", Type: ateletpb.VolumeType_VOLUME_TYPE_EXTERNAL}, + }, + }, + }, + want: false, + }, + { + name: "data scope with both durable and CSI volumes expects snapshots", + req: &ateletpb.CheckpointRequest{ + Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Spec: &ateletpb.WorkloadSpec{ + Volumes: []*ateletpb.Volume{ + {Name: "durable", Type: ateletpb.VolumeType_VOLUME_TYPE_DURABLE_DIR}, + {Name: "csi", Type: ateletpb.VolumeType_VOLUME_TYPE_EXTERNAL}, + }, + }, + }, + want: true, + }, + { + name: "data scope with no volumes does not expect snapshots", + req: &ateletpb.CheckpointRequest{ + Scope: ateletpb.SnapshotScope_SNAPSHOT_SCOPE_DATA, + Spec: &ateletpb.WorkloadSpec{}, + }, + want: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := shouldHaveSnapshots(tc.req); got != tc.want { + t.Errorf("shouldHaveSnapshots() = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/cmd/ateom-microvm/checkpoint.go b/cmd/ateom-microvm/checkpoint.go index b065d33c4..60f360ea2 100644 --- a/cmd/ateom-microvm/checkpoint.go +++ b/cmd/ateom-microvm/checkpoint.go @@ -21,7 +21,6 @@ import ( "fmt" "log/slog" "os" - "os/exec" "path/filepath" "time" @@ -90,13 +89,15 @@ func (s *AteomService) CheckpointWorkload(ctx context.Context, req *ateompb.Chec // captures. DATA_ON_GOLDEN is restore-only (a DataOnGolden commit arrives // here as plain DATA) and lands in the default rejection. durable := hasDurableVolumes(req.GetSpec().GetContainers()) + csi := hasCsiVolumes(req.GetSpec().GetContainers()) scope := req.GetScope() switch scope { case ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL: case ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA: - if !durable { + // TODO: Revisit handling for CSI volumes since snapshots are currently quietly ignored. + if !durable && !csi { return nil, status.Error(codes.FailedPrecondition, - "no durable-dir volumes found for a Data-scope snapshot") + "no durable-dir or CSI volumes found for a Data-scope snapshot") } default: return nil, status.Errorf(codes.InvalidArgument, "unsupported snapshot scope: %v", scope) @@ -331,13 +332,10 @@ func (s *AteomService) teardownActor(ctx context.Context, id string, ra *running _ = ra.chCmd.Process.Kill() _, _ = ra.chCmd.Process.Wait() } - // Kill the virtiofsds (after CH, their only client): the merged rootfs - // share's and, when the actor has durable-dir volumes, the durable share's. - for _, cmd := range []*exec.Cmd{ra.vfsdCmd, ra.durableVfsdCmd} { - if cmd != nil && cmd.Process != nil { - _ = cmd.Process.Kill() - _, _ = cmd.Process.Wait() - } + // Kill the virtiofsd (after CH, its only client). + if ra.vfsdCmd != nil && ra.vfsdCmd.Process != nil { + _ = ra.vfsdCmd.Process.Kill() + _, _ = ra.vfsdCmd.Process.Wait() } } diff --git a/cmd/ateom-microvm/csi.go b/cmd/ateom-microvm/csi.go new file mode 100644 index 000000000..4b29d1ac5 --- /dev/null +++ b/cmd/ateom-microvm/csi.go @@ -0,0 +1,82 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/reaper" + "github.com/agent-substrate/substrate/internal/ateompath" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// hasCsiVolumes reports whether any container mounts a CSI volume. +func hasCsiVolumes(containers []*ateompb.Container) bool { + for _, c := range containers { + if len(c.GetCsiVolumeMounts()) > 0 { + return true + } + } + return false +} + +// csiMounts returns the OCI mounts that expose a container's CSI +// volumes at the paths it declared. Each source is that volume's directory +// inside the guest's CSI share, which the agent mounts at sandbox creation. +func csiMounts(mounts []*ateompb.VolumeMount) []specs.Mount { + out := make([]specs.Mount, 0, len(mounts)) + for _, m := range mounts { + out = append(out, specs.Mount{ + Destination: m.GetMountPath(), + Source: kata.GuestCSIVolumeDir(m.GetVolumeName()), + Type: "bind", + Options: []string{"rbind", "rw"}, + }) + } + return out +} + +// stageCsiVolumes bind-mounts the actor's host CSI volumes directory +// into the sandbox's shared virtio-fs tree at SharedDir(actorUID)/csi. +func (s *AteomService) stageCsiVolumes(ctx context.Context, actorUID string) error { + src := ateompath.VolumesDir(actorUID) + if _, err := os.Stat(src); err != nil { + return fmt.Errorf("while checking CSI volumes dir %q: %w", src, err) + } + dst := filepath.Join(kata.SharedDir(actorUID), "csi") + // Drop any stale mount first (lazy if busy), then ensure clean mountpoint. + if err := reaper.Run(exec.Command("umount", dst)); err != nil { + _ = reaper.Run(exec.Command("umount", "-l", dst)) + } + if err := os.MkdirAll(dst, 0o755); err != nil { + return fmt.Errorf("creating %q: %w", dst, err) + } + cmd := exec.CommandContext(ctx, "mount", "--bind", src, dst) + var stderr strings.Builder + cmd.Stderr = &stderr + if err := reaper.Run(cmd); err != nil { + return fmt.Errorf("bind-mounting CSI volumes at %q: %w (%s)", dst, err, strings.TrimSpace(stderr.String())) + } + return nil +} diff --git a/cmd/ateom-microvm/csi_test.go b/cmd/ateom-microvm/csi_test.go new file mode 100644 index 000000000..e3d88ac0a --- /dev/null +++ b/cmd/ateom-microvm/csi_test.go @@ -0,0 +1,114 @@ +//go:build linux + +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/internal/proto/ateompb" + "github.com/google/go-cmp/cmp" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +func TestHasCsiVolumes(t *testing.T) { + tests := []struct { + name string + containers []*ateompb.Container + want bool + }{ + { + name: "empty containers", + containers: nil, + want: false, + }, + { + name: "no CSI volumes", + containers: []*ateompb.Container{ + { + Name: "c1", + DurableDirVolumeMounts: []*ateompb.DurableDirVolumeMount{ + {VolumeName: "data", MountPath: "/data"}, + }, + }, + }, + want: false, + }, + { + name: "has CSI volumes", + containers: []*ateompb.Container{ + { + Name: "c1", + CsiVolumeMounts: []*ateompb.VolumeMount{ + {VolumeName: "csi-vol", MountPath: "/csi"}, + }, + }, + }, + want: true, + }, + { + name: "multiple containers, one has CSI", + containers: []*ateompb.Container{ + { + Name: "c1", + }, + { + Name: "c2", + CsiVolumeMounts: []*ateompb.VolumeMount{ + {VolumeName: "csi-vol", MountPath: "/csi"}, + }, + }, + }, + want: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := hasCsiVolumes(tc.containers); got != tc.want { + t.Errorf("hasCsiVolumes() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestCsiMounts(t *testing.T) { + mounts := []*ateompb.VolumeMount{ + {VolumeName: "vol1", MountPath: "/mnt/vol1"}, + {VolumeName: "vol2", MountPath: "/mnt/vol2"}, + } + + want := []specs.Mount{ + { + Destination: "/mnt/vol1", + Source: kata.GuestCSIVolumeDir("vol1"), + Type: "bind", + Options: []string{"rbind", "rw"}, + }, + { + Destination: "/mnt/vol2", + Source: kata.GuestCSIVolumeDir("vol2"), + Type: "bind", + Options: []string{"rbind", "rw"}, + }, + } + + got := csiMounts(mounts) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("csiMounts() mismatch (-want +got):\n%s", diff) + } +} diff --git a/cmd/ateom-microvm/durable.go b/cmd/ateom-microvm/durable.go index 46088de05..18f997447 100644 --- a/cmd/ateom-microvm/durable.go +++ b/cmd/ateom-microvm/durable.go @@ -25,16 +25,11 @@ package main // ateompath.DurableDirVolumeMountsDir(actorUID) and wipes them when the actor's // directories are reset. // -// ateom exposes that host directory to the guest over a SECOND virtiofsd — -// separate from the kataShared rootfs share because the two have different -// owners and lifecycles: atelet owns this directory (created before boot, -// wiped on actor reset, captured under EVERY snapshot scope), while the -// rootfs uppers are ateom-owned and Full-scope-only. Each volume is a -// subdirectory of the one share, at kata.GuestDurableVolumeDir(volume), -// bind-mounted from there into every container that declares it. An actor may -// have any number of them: they cost a subdirectory each, not a device, so -// nothing here scales with the volume count (gVisor is the runtime that caps -// this at one, via the ActorTemplate CEL rules). +// ateom exposes that host directory to the guest under the single kataShared +// virtio-fs share at SharedDir(actorUID)/durable (in-guest path: +// kata.GuestDurableVolumeDir(volume)), bind-mounted from there into every +// container that declares it. An actor may have any number of them: they cost a +// subdirectory each, not a device, so nothing here scales with the volume count. // // Snapshots carry the contents as a tar of the whole per-actor directory, so // every volume rides along and the layout is reproduced verbatim on restore. @@ -48,8 +43,10 @@ import ( "os" "os/exec" "path/filepath" + "strings" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/kata" + "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/reaper" "github.com/agent-substrate/substrate/cmd/ateom-microvm/internal/tarutil" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/proto/ateompb" @@ -95,45 +92,41 @@ func durableMounts(mounts []*ateompb.DurableDirVolumeMount) []specs.Mount { // The spec is copied rather than mutated so the bundle's on-disk config.json // stays as prepared — only the started container sees the binds. func workloadSpec(c actorContainer) *specs.Spec { - if len(c.durableMounts) == 0 { + if len(c.durableMounts) == 0 && len(c.csiMounts) == 0 { return c.spec } spec := *c.spec - spec.Mounts = append(append([]specs.Mount(nil), c.spec.Mounts...), durableMounts(c.durableMounts)...) - return &spec -} -// durableVirtiofsdLogPath is where the durable-dir share's virtiofsd logs, -// beside the overlay lower's (see virtiofsdLogPath) under the actor's VM dir. -func durableVirtiofsdLogPath(id string) string { - return filepath.Join(kata.VMDir(id), "virtiofsd-durable.log") + var mounts []specs.Mount + mounts = append(mounts, c.spec.Mounts...) + mounts = append(mounts, durableMounts(c.durableMounts)...) + mounts = append(mounts, csiMounts(c.csiMounts)...) + spec.Mounts = mounts + return &spec } -// stageDurableShare starts the virtiofsd serving the actor's durable-dir volumes. -// -// It serves ateompath.DurableDirVolumeMountsDir directly — no bind into the -// kataShared tree — so teardown has nothing extra to unmount. Unlike the RO -// lower's virtiofsd this one runs with cache=auto: the host contents change -// underneath the guest whenever a snapshot is restored into them. -// -// The returned cmd outlives this call (CH talks to it for the VM's lifetime); -// the caller owns it (tracked on runningActor, killed in teardownActor). -func (s *AteomService) stageDurableShare(ctx context.Context, rr resolvedRuntime, actorUID string) (*exec.Cmd, error) { - shared := ateompath.DurableDirVolumeMountsDir(actorUID) - if _, err := os.Stat(shared); err != nil { - return nil, fmt.Errorf("while checking durable-dir volumes dir %q: %w", shared, err) +// stageDurableVolumes bind-mounts the actor's host durable-dir directory +// into the sandbox's shared virtio-fs tree at SharedDir(actorUID)/durable. +func (s *AteomService) stageDurableVolumes(ctx context.Context, actorUID string) error { + src := ateompath.DurableDirVolumeMountsDir(actorUID) + if _, err := os.Stat(src); err != nil { + return fmt.Errorf("while checking durable-dir volumes dir %q: %w", src, err) + } + dst := filepath.Join(kata.SharedDir(actorUID), "durable") + // Drop any stale mount first (lazy if busy), then ensure clean mountpoint. + if err := reaper.Run(exec.Command("umount", dst)); err != nil { + _ = reaper.Run(exec.Command("umount", "-l", dst)) } - log, _ := os.OpenFile(durableVirtiofsdLogPath(actorUID), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) - cmd, err := kata.StartVirtiofsd(ctx, kata.VirtiofsdOptions{ - Binary: rr.virtiofsd, - SocketPath: kata.DurableVirtiofsdSocketPath(actorUID), - SharedDir: shared, - Log: log, - }) - if err != nil { - return nil, fmt.Errorf("while starting durable-dir virtiofsd: %w", err) + if err := os.MkdirAll(dst, 0o755); err != nil { + return fmt.Errorf("creating %q: %w", dst, err) } - return cmd, nil + cmd := exec.CommandContext(ctx, "mount", "--bind", src, dst) + var stderr strings.Builder + cmd.Stderr = &stderr + if err := reaper.Run(cmd); err != nil { + return fmt.Errorf("bind-mounting durable-dir volumes at %q: %w (%s)", dst, err, strings.TrimSpace(stderr.String())) + } + return nil } // tarDurableVolumes archives the actor's durable-dir volumes (dir) into the diff --git a/cmd/ateom-microvm/internal/kata/overlay_linux.go b/cmd/ateom-microvm/internal/kata/overlay_linux.go index fabf4071b..b9976391a 100644 --- a/cmd/ateom-microvm/internal/kata/overlay_linux.go +++ b/cmd/ateom-microvm/internal/kata/overlay_linux.go @@ -49,22 +49,22 @@ const ( typeVirtioFS = "virtiofs" virtioFSDriver = "virtio-fs" // guestSharedDir is where the agent mounts the kataShared tag in the guest; - // per-container rootfs then lives at //rootfs. + // per-container rootfs then lives at //rootfs, durable + // volumes at /durable/, and CSI volumes at + // /csi/. guestSharedDir = "/run/kata-containers/shared/containers/" - - // DurableFsTag is the virtio-fs tag for the actor's WRITABLE durable-dir - // share, served by a second virtiofsd (the kataShared share stays read-only). - DurableFsTag = "ateDurable" - // guestDurableDir is where the agent mounts DurableFsTag in the guest; each - // volume's contents live at / and are bind-mounted - // from there into the containers that declare the volume. - guestDurableDir = "/run/ateom-durable" ) // GuestDurableVolumeDir is the in-guest path holding one durable volume's // contents, i.e. the bind source for that volume's container mount points. func GuestDurableVolumeDir(volumeName string) string { - return guestDurableDir + "/" + volumeName + return guestSharedDir + "durable/" + volumeName +} + +// GuestCSIVolumeDir is the in-guest path holding one CSI volume's +// contents, i.e. the bind source for that volume's container mount points. +func GuestCSIVolumeDir(volumeName string) string { + return guestSharedDir + "csi/" + volumeName } // SharedDir is the host directory virtiofsd serves into the guest as the RO base. @@ -231,29 +231,66 @@ func UnmountMergedRootfs(restoreID, cid string) { } } +// ReconstructSharedDirFromImage bind-mounts a container's OCI image rootfs at +// /rootfs under SharedDir(restoreID) so virtiofsd serves it as the read-only +// lower. LEGACY restores only: guests from retired guest-tmpfs-upper snapshots hold +// this plain image tree open (their overlay upper lives inside the restored guest +// memory), so the share must present the bare image, not a merged overlay. The bind +// copies nothing on the host. cid is stable across the actor's lineage. +func ReconstructSharedDirFromImage(ctx context.Context, bundleRootfs, restoreID, cid string) error { + if cid == "" { + return fmt.Errorf("ReconstructSharedDirFromImage: empty container id") + } + dst := filepath.Join(SharedDir(restoreID), cid, "rootfs") + // Drop any stale bind first (lazy if busy), then ensure a clean mountpoint. Not + // RemoveAll: that would chase a live bind into bundleRootfs. + if err := reaper.Run(exec.Command("umount", dst)); err != nil { + _ = reaper.Run(exec.Command("umount", "-l", dst)) + } + if err := os.MkdirAll(dst, 0o755); err != nil { + return fmt.Errorf("creating shared dir %q: %w", dst, err) + } + cmd := exec.CommandContext(ctx, "mount", "--bind", bundleRootfs, dst) + var stderr strings.Builder + cmd.Stderr = &stderr + if err := reaper.Run(cmd); err != nil { + return fmt.Errorf("bind-mounting image rootfs %q -> %q: %w (%s)", bundleRootfs, dst, err, strings.TrimSpace(stderr.String())) + } + // Ensure the standard OCI mountpoints exist even for minimal images: the container + // mounts /proc,/sys,/dev over them, and find-paths re-opens the lower by path on + // restore, so the layout must match on every node. (Bind still writable; ignore EEXIST.) + for _, d := range []string{"proc", "sys", "dev"} { + _ = os.MkdirAll(filepath.Join(dst, d), 0o755) + } + // Remount read-only: the lower is immutable, so all writes go to the overlay upper + // and it stays byte-identical across reconstructions (required by find-paths migration). + ro := exec.CommandContext(ctx, "mount", "-o", "remount,bind,ro", dst) + var roErr strings.Builder + ro.Stderr = &roErr + if err := reaper.Run(ro); err != nil { + return fmt.Errorf("remounting overlay lower read-only %q: %w (%s)", dst, err, strings.TrimSpace(roErr.String())) + } + return nil +} + +type CreateSandboxOpts struct { + SandboxID string + Hostname string +} + // CreateSandboxForActor creates the guest sandbox with the kataShared virtio-fs mount -// (the merged rootfs trees every container runs on). Mirrors kata startSandbox. -// -// withDurableShare additionally mounts the writable durable-dir share, whose -// per-volume subdirectories the containers bind-mount at their declared paths. -func (a *AgentClient) CreateSandboxForActor(ctx context.Context, sandboxID, hostname string, withDurableShare bool) error { +// (the merged rootfs trees, durable volumes, and CSI volumes every container runs on). +// Mirrors kata startSandbox. +func (a *AgentClient) CreateSandboxForActor(ctx context.Context, opts CreateSandboxOpts) error { storages := []*agentpb.Storage{{ Driver: virtioFSDriver, Source: FsTag, Fstype: typeVirtioFS, MountPoint: guestSharedDir, }} - if withDurableShare { - storages = append(storages, &agentpb.Storage{ - Driver: virtioFSDriver, - Source: DurableFsTag, - Fstype: typeVirtioFS, - MountPoint: guestDurableDir, - }) - } return a.CreateSandbox(ctx, &agentpb.CreateSandboxRequest{ - Hostname: hostname, - SandboxId: sandboxID, + Hostname: opts.Hostname, + SandboxId: opts.SandboxID, Storages: storages, }) } diff --git a/cmd/ateom-microvm/internal/kata/restore.go b/cmd/ateom-microvm/internal/kata/restore.go index c4cfab469..060b1bb2e 100644 --- a/cmd/ateom-microvm/internal/kata/restore.go +++ b/cmd/ateom-microvm/internal/kata/restore.go @@ -35,3 +35,9 @@ func VsockSocketPath(id string) string { return filepath.Join(VMDir(id), "clh.so func DurableVirtiofsdSocketPath(id string) string { return filepath.Join(VMDir(id), "virtiofsd-durable.sock") } + +// CsiVirtiofsdSocketPath is the vhost-user-fs socket for the actor's writable +// CSI volumes share. +func CsiVirtiofsdSocketPath(id string) string { + return filepath.Join(VMDir(id), "virtiofsd-csi.sock") +} diff --git a/cmd/ateom-microvm/restore.go b/cmd/ateom-microvm/restore.go index c962a4e6c..2e030af95 100644 --- a/cmd/ateom-microvm/restore.go +++ b/cmd/ateom-microvm/restore.go @@ -22,7 +22,6 @@ import ( "fmt" "log/slog" "os" - "os/exec" "path/filepath" "strings" "time" @@ -260,7 +259,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, return untarErr } tUpper := time.Now() - vfsdCmd, err := s.stageMergedRootfs(ctx, rr, actorUID, ctrs) + vfsdCmd, err := s.stageMergedRootfs(ctx, rr, actorUID, ctrs, containers) if err != nil { return err } @@ -272,26 +271,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, }() tLowers := time.Now() - - // Restart the durable-dir share's virtiofsd over the contents the caller - // restored. The guest reattaches to it by the socket path rewritten into the - // snapshot config below; find-paths re-opens whatever files it still holds open - // against the same paths, which the restored tar reproduces exactly. - var durableVfsdCmd *exec.Cmd - if hasDurableVolumes(containers) { - if durableVfsdCmd, err = s.stageDurableShare(ctx, rr, actorUID); err != nil { - return err - } - defer func() { - if retErr != nil && durableVfsdCmd.Process != nil { - _ = durableVfsdCmd.Process.Kill() - _, _ = durableVfsdCmd.Process.Wait() - } - }() - } - - tDurable := time.Now() - + tDurable := tLowers // Networking: rebuild the per-activation veth + tap; the snapshot's virtio-net // is fd-backed, so CH needs fresh tap FDs (net_fds) on restore. if err := ateomnet.SetupActorNetwork(ctx, ateomnet.NetworkConfig{ @@ -419,7 +399,7 @@ func (s *AteomService) restoreFullScope(ctx context.Context, p actorBootParams, } ra := &runningActor{ - chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, + chCmd: chCmd, vfsdCmd: vfsdCmd, apiSocket: apiSocket, baseID: srcID, restoreSourceDir: restoreDir, snapshotIsSelfContained: memMode == ch.MemRestoreEager, // Signaling an id the agent does not know fails the whole graceful @@ -492,11 +472,8 @@ func rewriteSnapshotSocketPaths(snapshotDir, id string) error { serial["file"] = filepath.Join(kata.VMDir(id), "serial.log") } } - // Each virtio-fs share is served by its own per-VMDir virtiofsd socket; the - // snapshot recorded the golden actor's, so repoint them at this actor's VMDir. - // Match on the device tag: the shares have separate sockets (the rootfs - // share's and, when the actor has durable-dir volumes, the durable share's), - // and crossing them would hand the guest the wrong filesystem. + // The virtio-fs share is served by its per-VMDir virtiofsd socket; the + // snapshot recorded the golden actor's, so repoint it at this actor's VMDir. if fss, ok := cfg["fs"].([]any); ok { for _, f := range fss { fm, ok := f.(map[string]any) @@ -506,8 +483,12 @@ func rewriteSnapshotSocketPaths(snapshotDir, id string) error { switch tag, _ := fm["tag"].(string); tag { case kata.FsTag: fm["socket"] = kata.VirtiofsdSocketPath(id) - case kata.DurableFsTag: + case "ateDurable": + // Legacy multi-virtiofs snapshot backward compatibility. fm["socket"] = kata.DurableVirtiofsdSocketPath(id) + case "ateCSI": + // Legacy multi-virtiofs snapshot backward compatibility. + fm["socket"] = kata.CsiVirtiofsdSocketPath(id) default: return fmt.Errorf("snapshot config %q has fs device with unknown tag %q", cfgPath, tag) } diff --git a/cmd/ateom-microvm/restore_test.go b/cmd/ateom-microvm/restore_test.go index b16440c0c..1cceaf87c 100644 --- a/cmd/ateom-microvm/restore_test.go +++ b/cmd/ateom-microvm/restore_test.go @@ -85,26 +85,24 @@ func TestRewriteSnapshotSocketPaths(t *testing.T) { } }) - t.Run("each share keeps its own socket", func(t *testing.T) { + t.Run("legacy multi-share snapshots repoint each tag", func(t *testing.T) { // Ordered with the rootfs share last to catch a rewrite that assumes it // comes first, which would hand the guest the wrong filesystem. dir := writeSnapshotConfig(t, []map[string]any{ - {"tag": kata.DurableFsTag, "socket": "/run/vc/vm/golden/virtiofsd-durable.sock"}, + {"tag": "ateDurable", "socket": "/run/vc/vm/golden/virtiofsd-durable.sock"}, {"tag": kata.FsTag, "socket": "/run/vc/vm/golden/virtiofsd.sock"}, }) if err := rewriteSnapshotSocketPaths(dir, id); err != nil { t.Fatalf("rewriteSnapshotSocketPaths: %v", err) } got := readFsSockets(t, dir) - for tag, want := range map[string]string{ - kata.FsTag: kata.VirtiofsdSocketPath(id), - kata.DurableFsTag: kata.DurableVirtiofsdSocketPath(id), - } { - if got[tag] != want { - t.Errorf("%s socket = %q, want %q", tag, got[tag], want) - } + if got[kata.FsTag] != kata.VirtiofsdSocketPath(id) { + t.Errorf("%s socket = %q, want %q", kata.FsTag, got[kata.FsTag], kata.VirtiofsdSocketPath(id)) + } + if got["ateDurable"] != kata.DurableVirtiofsdSocketPath(id) { + t.Errorf("ateDurable socket = %q, want %q", got["ateDurable"], kata.DurableVirtiofsdSocketPath(id)) } - if got[kata.FsTag] == got[kata.DurableFsTag] { + if got[kata.FsTag] == got["ateDurable"] { t.Error("both shares were pointed at the same socket") } }) diff --git a/cmd/ateom-microvm/run.go b/cmd/ateom-microvm/run.go index f38f6b51f..9d62b0512 100644 --- a/cmd/ateom-microvm/run.go +++ b/cmd/ateom-microvm/run.go @@ -59,14 +59,10 @@ type runningActor struct { // ateom owns this CH process (booted at Run or relaunched at Restore). chCmd *exec.Cmd - // vfsdCmd is the virtiofsd serving the overlay RO lower (the CH fs device - // demand-pages from it for the actor's lifetime). ateom owns it; teardownActor + // vfsdCmd is the virtiofsd serving the unified share (merged rootfs overlay, + // durable-dir volumes, and CSI volumes). ateom owns it; teardownActor // kills it after the CH process. vfsdCmd *exec.Cmd - // durableVfsdCmd is the second virtiofsd, serving the actor's writable - // durable-dir volumes. nil when the actor declares none. Owned and torn down - // exactly like vfsdCmd. - durableVfsdCmd *exec.Cmd // apiSocket is the CH api-socket for this ateom-owned VMM. apiSocket string @@ -187,6 +183,9 @@ type actorContainer struct { // durableMounts are the durable-dir volumes this container mounts, and where // (see durable.go). Empty for containers that declare none. durableMounts []*ateompb.DurableDirVolumeMount + // csiMounts are the CSI volumes this container mounts, and where (see csi.go). + // Empty for containers that declare none. + csiMounts []*ateompb.VolumeMount } // resolvedRuntime holds the concrete binary/config paths for a request, taken @@ -471,10 +470,10 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re } // Assemble each container's merged rootfs on the host (overlay of image lower + - // host upper, mounted into the shared dir) + start the ONE virtiofsd that serves - // them. CH connects to it at vm.create and demand-pages for the actor's - // lifetime, so ateom owns the process (killed in teardownActor). - vfsdCmd, err := s.stageMergedRootfs(ctx, rr, actorUID, ctrs) + // host upper, mounted into the shared dir) + durable-dir and CSI volumes (if any), + // and start the ONE virtiofsd that serves them all. CH connects to it at vm.create + // and demand-pages for the actor's lifetime, so ateom owns the process (killed in teardownActor). + vfsdCmd, err := s.stageMergedRootfs(ctx, rr, actorUID, ctrs, containers) if err != nil { return err } @@ -485,23 +484,6 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re } }() - // Durable-dir volumes (if any) share one writable virtio-fs share, served by - // a second virtiofsd from the host directory atelet prepared; each volume is - // a subdirectory of it. - durable := hasDurableVolumes(containers) - var durableVfsdCmd *exec.Cmd - if durable { - if durableVfsdCmd, err = s.stageDurableShare(ctx, rr, actorUID); err != nil { - return err - } - defer func() { - if retErr != nil && durableVfsdCmd.Process != nil { - _ = durableVfsdCmd.Process.Kill() - _, _ = durableVfsdCmd.Process.Wait() - } - }() - } - // Launch a bare VMM (CH + api-socket); ateom owns this process for teardown. apiSocket := filepath.Join(kata.VMDir(actorUID), "clh-api.sock") chCmd, client, err := ch.LaunchVMM(ctx, ch.LaunchVMMOptions{ @@ -521,11 +503,11 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re }() // Assemble the CH VmConfig (kata-compatible cmdline, RO kata image on /dev/vda + - // the virtio-fs devices; no actor virtio-blk disks — rootfs writes land in the + // the virtio-fs device; no actor virtio-blk disks — rootfs writes land in the // host-side overlay upper through the shared mount). serialLog is also read on a // failed agent dial below, so keep it here. serialLog := filepath.Join(kata.VMDir(actorUID), "serial.log") - vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, durable, + vmCfg := buildVMConfig(actorUID, kernel, image, kparams, serialLog, memMiB, vcpus, agentInit(ctx, client.Info())) if err := client.CreateVM(ctx, vmCfg); err != nil { return fmt.Errorf("while creating VM: %w", err) @@ -583,7 +565,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re }() // Post-boot kata-agent setup: sandbox, guest networking, start each container. - if err := s.startActorContainers(ctx, ac, actorUID, vsockPath, ctrs, durable); err != nil { + if err := s.startActorContainers(ctx, ac, actorUID, vsockPath, ctrs); err != nil { return err } tContainers := time.Now() @@ -602,7 +584,7 @@ func (s *AteomService) coldBootActor(ctx context.Context, p actorBootParams) (re slog.Duration("readyz", time.Since(tContainers)), slog.Duration("since_boot", time.Since(tBooted))) - ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, durableVfsdCmd: durableVfsdCmd, apiSocket: apiSocket, baseID: actorUID, guestAgent: ac, workloadIDs: workloadIDs(ctrs)} + ra := &runningActor{chCmd: chCmd, vfsdCmd: vfsdCmd, apiSocket: apiSocket, baseID: actorUID, guestAgent: ac, workloadIDs: workloadIDs(ctrs)} if err := s.activateActorNetworking(p.actorRef.Atespace, p.actorRef.Name, egress); err != nil { return err } @@ -665,6 +647,7 @@ func (s *AteomService) buildActorContainers(actorUID string, containers []*ateom bundleRootfs: bundleRootfs, spec: spec, durableMounts: c.GetDurableDirVolumeMounts(), + csiMounts: c.GetCsiVolumeMounts(), } } return ctrs, nil @@ -672,19 +655,30 @@ func (s *AteomService) buildActorContainers(actorUID string, containers []*ateom // stageMergedRootfs assembles each container's merged rootfs on the host // (overlay: image lower + the actor's rootfs-upper dirs) at virtiofsd's -// find-paths location (SharedDir(id)//rootfs), then starts the ONE -// virtiofsd that serves them all. Must run AFTER CleanupSandboxState (which +// find-paths location (SharedDir(id)//rootfs), stages durable-dir volumes +// and CSI volumes (if any) under SharedDir(id)/durable and SharedDir(id)/csi, +// then starts the ONE virtiofsd that serves them all. Must run AFTER CleanupSandboxState (which // wipes SharedDir) and resetRootfsUpperDir/untarRootfsUpper (which own the // upper contents). The returned virtiofsd cmd outlives this call (CH // demand-pages from it); the caller owns it (tracked on runningActor, killed // in teardownActor). -func (s *AteomService) stageMergedRootfs(ctx context.Context, rr resolvedRuntime, id string, ctrs []actorContainer) (*exec.Cmd, error) { +func (s *AteomService) stageMergedRootfs(ctx context.Context, rr resolvedRuntime, id string, ctrs []actorContainer, containers []*ateompb.Container) (*exec.Cmd, error) { upperBase := rootfsUpperDir(id) for _, c := range ctrs { if err := kata.StageMergedRootfs(ctx, c.bundleRootfs, upperBase, id, c.name); err != nil { return nil, fmt.Errorf("while staging merged rootfs for %q: %w", c.name, err) } } + if hasDurableVolumes(containers) { + if err := s.stageDurableVolumes(ctx, id); err != nil { + return nil, fmt.Errorf("while staging durable-dir volumes: %w", err) + } + } + if hasCsiVolumes(containers) { + if err := s.stageCsiVolumes(ctx, id); err != nil { + return nil, fmt.Errorf("while staging CSI volumes: %w", err) + } + } vfsdLog, _ := os.OpenFile(virtiofsdLogPath(id), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o600) vfsdCmd, err := kata.StartVirtiofsd(ctx, kata.VirtiofsdOptions{ Binary: rr.virtiofsd, @@ -828,10 +822,8 @@ func initParams(agentInit bool) string { // v53, which advances the guest clock across a restore itself; on v52 a restored guest // stays frozen at the instant it was snapshotted. // -// withDurable adds a second virtio-fs device for the actor's writable durable-dir -// volumes (see durable.go), served by its own virtiofsd on the same PCI segment. // The disk-backed rootfs upper share (see rootfsupper.go) is always present. -func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, withDurable, agentInit bool) ch.VmConfig { +func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus int, agentInit bool) ch.VmConfig { console := "ttyS0" if runtime.GOARCH == "arm64" { console = "ttyAMA0" @@ -849,7 +841,7 @@ func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus i Disks: []ch.DiskConfig{ {Path: image, Readonly: true, ImageType: "Raw", NumQueues: int32(vcpus), QueueSize: 1024}, }, - Fs: buildFsConfigs(id, withDurable), + Fs: buildFsConfigs(id), Platform: &ch.PlatformConfig{NumPciSegments: 2}, Rng: &ch.RngConfig{Src: "/dev/urandom"}, Serial: &ch.ConsoleConfig{Mode: "File", File: serialLog}, @@ -857,37 +849,30 @@ func buildVMConfig(id, kernel, image, kparams, serialLog string, memMiB, vcpus i } } -// buildFsConfigs returns the VM's virtio-fs devices: the merged rootfs share, -// plus the writable durable-dir share when the actor has one. Both sit on PCI +// buildFsConfigs returns the VM's virtio-fs device: the unified share hosting +// container rootfs trees, durable volumes, and CSI volumes. Sits on PCI // segment 1 (the segment buildVMConfig reserves for virtio-fs). -func buildFsConfigs(id string, withDurable bool) []ch.FsConfig { - fs := []ch.FsConfig{{ +func buildFsConfigs(id string) []ch.FsConfig { + return []ch.FsConfig{{ Tag: kata.FsTag, Socket: kata.VirtiofsdSocketPath(id), NumQueues: 1, QueueSize: 1024, PciSegment: 1, }} - if withDurable { - fs = append(fs, ch.FsConfig{ - Tag: kata.DurableFsTag, Socket: kata.DurableVirtiofsdSocketPath(id), - NumQueues: 1, QueueSize: 1024, PciSegment: 1, - }) - } - return fs } // startActorContainers performs the post-boot kata-agent setup the shim normally // does at boot: establish the sandbox once (mounting the kataShared virtio-fs base), // configure guest networking (eth0 IP/MAC/MTU + routes) once, then start each // container on its own overlay rootfs. On failure it dumps guest diagnostics. -// -// durable says the actor has durable-dir volumes: the sandbox then also mounts -// the writable durable share, and each container binds the volumes it declared. -func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentClient, id, vsockPath string, ctrs []actorContainer, durable bool) error { +func (s *AteomService) startActorContainers(ctx context.Context, ac *kata.AgentClient, id, vsockPath string, ctrs []actorContainer) error { // Establish the agent sandbox + the kataShared virtio-fs mount (every - // container's merged rootfs). All containers share it, so use the first - // container's hostname. + // container's merged rootfs, durable volumes, and CSI volumes). All containers + // share it, so use the first container's hostname. tStart := time.Now() sbCtx, sbCancel := context.WithTimeout(ctx, 20*time.Second) - err := ac.CreateSandboxForActor(sbCtx, id, ctrs[0].spec.Hostname, durable) + err := ac.CreateSandboxForActor(sbCtx, kata.CreateSandboxOpts{ + SandboxID: id, + Hostname: ctrs[0].spec.Hostname, + }) sbCancel() if err != nil { return fmt.Errorf("while creating agent sandbox: %w", err) @@ -1034,7 +1019,6 @@ func logGuestBootDiagnostics(ctx context.Context, actorUID, serialLog string) { for _, l := range []struct{ name, path string }{ {"serial", serialLog}, {"virtiofsd", virtiofsdLogPath(actorUID)}, - {"virtiofsd-durable", durableVirtiofsdLogPath(actorUID)}, } { b, err := os.ReadFile(l.path) if err != nil || len(b) == 0 { diff --git a/demos/counter/counter-microvm-csi-test.yaml b/demos/counter/counter-microvm-csi-test.yaml new file mode 100644 index 000000000..c2ac98b34 --- /dev/null +++ b/demos/counter/counter-microvm-csi-test.yaml @@ -0,0 +1,67 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: Namespace +metadata: + name: ate-demo-counter-microvm-csi + +--- + +apiVersion: ate.dev/v1alpha1 +kind: WorkerPool +metadata: + name: counter-microvm-csi + namespace: ate-demo-counter-microvm-csi + labels: + workload: counter-microvm-csi +spec: + replicas: 1 + sandboxClass: microvm + sandboxConfigName: microvm + ateomImage: ko://github.com/agent-substrate/substrate/cmd/ateom-microvm + +--- + +apiVersion: ate.dev/v1alpha1 +kind: ActorTemplate +metadata: + name: counter-microvm-csi + namespace: ate-demo-counter-microvm-csi +spec: + sandboxClass: microvm + containers: + - name: counter + image: ko://github.com/agent-substrate/substrate/demos/counter + readyz: + httpGet: + path: /readyz + port: 80 + volumeMounts: + - name: data + mountPath: /home/counter + workerSelector: + matchLabels: + workload: counter-microvm-csi + snapshotsConfig: + onPause: Full + onCommit: Data + onResume: + fromData: Golden + location: gs://ate-snapshots/ate-demo-counter-microvm-csi/ + volumes: + - name: data + externalVolumeTemplate: + capacity: 1Gi + storageClassName: csi-hostpath-sc diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index c28680c87..213461f89 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -399,9 +399,6 @@ func TestMultipleDurableDirLifecycle(t *testing.T) { } func TestExternalVolumeLifecycle(t *testing.T) { - if isMicroVMEnvironment() { - t.Skip("Skipping TestExternalVolumeLifecycle for microVM environment") - } tests := []struct { name string diff --git a/internal/proto/ateompb/ateom.pb.go b/internal/proto/ateompb/ateom.pb.go index c865d0a1c..f6dfc12b6 100644 --- a/internal/proto/ateompb/ateom.pb.go +++ b/internal/proto/ateompb/ateom.pb.go @@ -500,8 +500,10 @@ type Container struct { // durable_dir_volume_mounts are the durable-dir volumes this container // mounts, if any. DurableDirVolumeMounts []*DurableDirVolumeMount `protobuf:"bytes,4,rep,name=durable_dir_volume_mounts,json=durableDirVolumeMounts,proto3" json:"durable_dir_volume_mounts,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // csi_volume_mounts are the CSI volumes this container mounts, if any. + CsiVolumeMounts []*VolumeMount `protobuf:"bytes,5,rep,name=csi_volume_mounts,json=csiVolumeMounts,proto3" json:"csi_volume_mounts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Container) Reset() { @@ -555,6 +557,66 @@ func (x *Container) GetDurableDirVolumeMounts() []*DurableDirVolumeMount { return nil } +func (x *Container) GetCsiVolumeMounts() []*VolumeMount { + if x != nil { + return x.CsiVolumeMounts + } + return nil +} + +// VolumeMount is one volume mounted into a container. +type VolumeMount struct { + state protoimpl.MessageState `protogen:"open.v1"` + VolumeName string `protobuf:"bytes,1,opt,name=volume_name,json=volumeName,proto3" json:"volume_name,omitempty"` + MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VolumeMount) Reset() { + *x = VolumeMount{} + mi := &file_ateom_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VolumeMount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VolumeMount) ProtoMessage() {} + +func (x *VolumeMount) ProtoReflect() protoreflect.Message { + mi := &file_ateom_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. +func (*VolumeMount) Descriptor() ([]byte, []int) { + return file_ateom_proto_rawDescGZIP(), []int{4} +} + +func (x *VolumeMount) GetVolumeName() string { + if x != nil { + return x.VolumeName + } + return "" +} + +func (x *VolumeMount) GetMountPath() string { + if x != nil { + return x.MountPath + } + return "" +} + // DurableDirVolumeMount is one durable-dir volume mounted into a container. type DurableDirVolumeMount struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -569,7 +631,7 @@ type DurableDirVolumeMount struct { func (x *DurableDirVolumeMount) Reset() { *x = DurableDirVolumeMount{} - mi := &file_ateom_proto_msgTypes[4] + mi := &file_ateom_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -581,7 +643,7 @@ func (x *DurableDirVolumeMount) String() string { func (*DurableDirVolumeMount) ProtoMessage() {} func (x *DurableDirVolumeMount) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[4] + mi := &file_ateom_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -594,7 +656,7 @@ func (x *DurableDirVolumeMount) ProtoReflect() protoreflect.Message { // Deprecated: Use DurableDirVolumeMount.ProtoReflect.Descriptor instead. func (*DurableDirVolumeMount) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{4} + return file_ateom_proto_rawDescGZIP(), []int{5} } func (x *DurableDirVolumeMount) GetVolumeName() string { @@ -625,7 +687,7 @@ type Readyz struct { func (x *Readyz) Reset() { *x = Readyz{} - mi := &file_ateom_proto_msgTypes[5] + mi := &file_ateom_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -637,7 +699,7 @@ func (x *Readyz) String() string { func (*Readyz) ProtoMessage() {} func (x *Readyz) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[5] + mi := &file_ateom_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -650,7 +712,7 @@ func (x *Readyz) ProtoReflect() protoreflect.Message { // Deprecated: Use Readyz.ProtoReflect.Descriptor instead. func (*Readyz) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{5} + return file_ateom_proto_rawDescGZIP(), []int{6} } func (x *Readyz) GetHttpGet() *HTTPGetAction { @@ -680,7 +742,7 @@ type HTTPGetAction struct { func (x *HTTPGetAction) Reset() { *x = HTTPGetAction{} - mi := &file_ateom_proto_msgTypes[6] + mi := &file_ateom_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -692,7 +754,7 @@ func (x *HTTPGetAction) String() string { func (*HTTPGetAction) ProtoMessage() {} func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[6] + mi := &file_ateom_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -705,7 +767,7 @@ func (x *HTTPGetAction) ProtoReflect() protoreflect.Message { // Deprecated: Use HTTPGetAction.ProtoReflect.Descriptor instead. func (*HTTPGetAction) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{6} + return file_ateom_proto_rawDescGZIP(), []int{7} } func (x *HTTPGetAction) GetPath() string { @@ -730,7 +792,7 @@ type RunWorkloadResponse struct { func (x *RunWorkloadResponse) Reset() { *x = RunWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[7] + mi := &file_ateom_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -742,7 +804,7 @@ func (x *RunWorkloadResponse) String() string { func (*RunWorkloadResponse) ProtoMessage() {} func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[7] + mi := &file_ateom_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -755,7 +817,7 @@ func (x *RunWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RunWorkloadResponse.ProtoReflect.Descriptor instead. func (*RunWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{7} + return file_ateom_proto_rawDescGZIP(), []int{8} } type CheckpointWorkloadRequest struct { @@ -789,7 +851,7 @@ type CheckpointWorkloadRequest struct { func (x *CheckpointWorkloadRequest) Reset() { *x = CheckpointWorkloadRequest{} - mi := &file_ateom_proto_msgTypes[8] + mi := &file_ateom_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -801,7 +863,7 @@ func (x *CheckpointWorkloadRequest) String() string { func (*CheckpointWorkloadRequest) ProtoMessage() {} func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[8] + mi := &file_ateom_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -814,7 +876,7 @@ func (x *CheckpointWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadRequest.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{8} + return file_ateom_proto_rawDescGZIP(), []int{9} } func (x *CheckpointWorkloadRequest) GetAtespace() string { @@ -899,7 +961,7 @@ type CheckpointWorkloadResponse struct { func (x *CheckpointWorkloadResponse) Reset() { *x = CheckpointWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[9] + mi := &file_ateom_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -911,7 +973,7 @@ func (x *CheckpointWorkloadResponse) String() string { func (*CheckpointWorkloadResponse) ProtoMessage() {} func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[9] + mi := &file_ateom_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -924,7 +986,7 @@ func (x *CheckpointWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CheckpointWorkloadResponse.ProtoReflect.Descriptor instead. func (*CheckpointWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{9} + return file_ateom_proto_rawDescGZIP(), []int{10} } func (x *CheckpointWorkloadResponse) GetSnapshotFiles() []string { @@ -969,7 +1031,7 @@ type RestoreWorkloadRequest struct { func (x *RestoreWorkloadRequest) Reset() { *x = RestoreWorkloadRequest{} - mi := &file_ateom_proto_msgTypes[10] + mi := &file_ateom_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -981,7 +1043,7 @@ func (x *RestoreWorkloadRequest) String() string { func (*RestoreWorkloadRequest) ProtoMessage() {} func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[10] + mi := &file_ateom_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -994,7 +1056,7 @@ func (x *RestoreWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadRequest.ProtoReflect.Descriptor instead. func (*RestoreWorkloadRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{10} + return file_ateom_proto_rawDescGZIP(), []int{11} } func (x *RestoreWorkloadRequest) GetAtespace() string { @@ -1103,7 +1165,7 @@ type RestoreWorkloadResponse struct { func (x *RestoreWorkloadResponse) Reset() { *x = RestoreWorkloadResponse{} - mi := &file_ateom_proto_msgTypes[11] + mi := &file_ateom_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1115,7 +1177,7 @@ func (x *RestoreWorkloadResponse) String() string { func (*RestoreWorkloadResponse) ProtoMessage() {} func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[11] + mi := &file_ateom_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1128,7 +1190,7 @@ func (x *RestoreWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RestoreWorkloadResponse.ProtoReflect.Descriptor instead. func (*RestoreWorkloadResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{11} + return file_ateom_proto_rawDescGZIP(), []int{12} } type GetWorkloadStatsRequest struct { @@ -1144,7 +1206,7 @@ type GetWorkloadStatsRequest struct { func (x *GetWorkloadStatsRequest) Reset() { *x = GetWorkloadStatsRequest{} - mi := &file_ateom_proto_msgTypes[12] + mi := &file_ateom_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1156,7 +1218,7 @@ func (x *GetWorkloadStatsRequest) String() string { func (*GetWorkloadStatsRequest) ProtoMessage() {} func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[12] + mi := &file_ateom_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1169,7 +1231,7 @@ func (x *GetWorkloadStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsRequest.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{12} + return file_ateom_proto_rawDescGZIP(), []int{13} } func (x *GetWorkloadStatsRequest) GetActorUid() string { @@ -1233,7 +1295,7 @@ type WorkloadStatsSample struct { func (x *WorkloadStatsSample) Reset() { *x = WorkloadStatsSample{} - mi := &file_ateom_proto_msgTypes[13] + mi := &file_ateom_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1245,7 +1307,7 @@ func (x *WorkloadStatsSample) String() string { func (*WorkloadStatsSample) ProtoMessage() {} func (x *WorkloadStatsSample) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[13] + mi := &file_ateom_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1258,7 +1320,7 @@ func (x *WorkloadStatsSample) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkloadStatsSample.ProtoReflect.Descriptor instead. func (*WorkloadStatsSample) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{13} + return file_ateom_proto_rawDescGZIP(), []int{14} } func (x *WorkloadStatsSample) GetAtespace() string { @@ -1354,7 +1416,7 @@ type GetWorkloadStatsResponse struct { func (x *GetWorkloadStatsResponse) Reset() { *x = GetWorkloadStatsResponse{} - mi := &file_ateom_proto_msgTypes[14] + mi := &file_ateom_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1366,7 +1428,7 @@ func (x *GetWorkloadStatsResponse) String() string { func (*GetWorkloadStatsResponse) ProtoMessage() {} func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[14] + mi := &file_ateom_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1379,7 +1441,7 @@ func (x *GetWorkloadStatsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadStatsResponse.ProtoReflect.Descriptor instead. func (*GetWorkloadStatsResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{14} + return file_ateom_proto_rawDescGZIP(), []int{15} } func (x *GetWorkloadStatsResponse) GetSample() *WorkloadStatsSample { @@ -1397,7 +1459,7 @@ type GetActiveWorkloadStatsRequest struct { func (x *GetActiveWorkloadStatsRequest) Reset() { *x = GetActiveWorkloadStatsRequest{} - mi := &file_ateom_proto_msgTypes[15] + mi := &file_ateom_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1409,7 +1471,7 @@ func (x *GetActiveWorkloadStatsRequest) String() string { func (*GetActiveWorkloadStatsRequest) ProtoMessage() {} func (x *GetActiveWorkloadStatsRequest) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[15] + mi := &file_ateom_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1422,7 +1484,7 @@ func (x *GetActiveWorkloadStatsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActiveWorkloadStatsRequest.ProtoReflect.Descriptor instead. func (*GetActiveWorkloadStatsRequest) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{15} + return file_ateom_proto_rawDescGZIP(), []int{16} } type GetActiveWorkloadStatsResponse struct { @@ -1444,7 +1506,7 @@ type GetActiveWorkloadStatsResponse struct { func (x *GetActiveWorkloadStatsResponse) Reset() { *x = GetActiveWorkloadStatsResponse{} - mi := &file_ateom_proto_msgTypes[16] + mi := &file_ateom_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1456,7 +1518,7 @@ func (x *GetActiveWorkloadStatsResponse) String() string { func (*GetActiveWorkloadStatsResponse) ProtoMessage() {} func (x *GetActiveWorkloadStatsResponse) ProtoReflect() protoreflect.Message { - mi := &file_ateom_proto_msgTypes[16] + mi := &file_ateom_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1469,7 +1531,7 @@ func (x *GetActiveWorkloadStatsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetActiveWorkloadStatsResponse.ProtoReflect.Descriptor instead. func (*GetActiveWorkloadStatsResponse) Descriptor() ([]byte, []int) { - return file_ateom_proto_rawDescGZIP(), []int{16} + return file_ateom_proto_rawDescGZIP(), []int{17} } func (x *GetActiveWorkloadStatsResponse) GetResult() isGetActiveWorkloadStatsResponse_Result { @@ -1542,11 +1604,17 @@ const file_ateom_proto_rawDesc = "" + "\fWorkloadSpec\x120\n" + "\n" + "containers\x18\x01 \x03(\v2\x10.ateom.ContainerR\n" + - "containers\"\xba\x01\n" + + "containers\"\xfa\x01\n" + "\tContainer\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12%\n" + "\x06readyz\x18\x02 \x01(\v2\r.ateom.ReadyzR\x06readyz\x12W\n" + - "\x19durable_dir_volume_mounts\x18\x04 \x03(\v2\x1c.ateom.DurableDirVolumeMountR\x16durableDirVolumeMountsJ\x04\b\x03\x10\x04R\x13durable_dir_volumes\"W\n" + + "\x19durable_dir_volume_mounts\x18\x04 \x03(\v2\x1c.ateom.DurableDirVolumeMountR\x16durableDirVolumeMounts\x12>\n" + + "\x11csi_volume_mounts\x18\x05 \x03(\v2\x12.ateom.VolumeMountR\x0fcsiVolumeMountsJ\x04\b\x03\x10\x04R\x13durable_dir_volumes\"M\n" + + "\vVolumeMount\x12\x1f\n" + + "\vvolume_name\x18\x01 \x01(\tR\n" + + "volumeName\x12\x1d\n" + + "\n" + + "mount_path\x18\x02 \x01(\tR\tmountPath\"W\n" + "\x15DurableDirVolumeMount\x12\x1f\n" + "\vvolume_name\x18\x01 \x01(\tR\n" + "volumeName\x12\x1d\n" + @@ -1662,7 +1730,7 @@ func file_ateom_proto_rawDescGZIP() []byte { } var file_ateom_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_ateom_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_ateom_proto_msgTypes = make([]protoimpl.MessageInfo, 21) var file_ateom_proto_goTypes = []any{ (SnapshotScope)(0), // 0: ateom.SnapshotScope (SandboxClass)(0), // 1: ateom.SandboxClass @@ -1672,58 +1740,60 @@ var file_ateom_proto_goTypes = []any{ (*EgressGateway)(nil), // 5: ateom.EgressGateway (*WorkloadSpec)(nil), // 6: ateom.WorkloadSpec (*Container)(nil), // 7: ateom.Container - (*DurableDirVolumeMount)(nil), // 8: ateom.DurableDirVolumeMount - (*Readyz)(nil), // 9: ateom.Readyz - (*HTTPGetAction)(nil), // 10: ateom.HTTPGetAction - (*RunWorkloadResponse)(nil), // 11: ateom.RunWorkloadResponse - (*CheckpointWorkloadRequest)(nil), // 12: ateom.CheckpointWorkloadRequest - (*CheckpointWorkloadResponse)(nil), // 13: ateom.CheckpointWorkloadResponse - (*RestoreWorkloadRequest)(nil), // 14: ateom.RestoreWorkloadRequest - (*RestoreWorkloadResponse)(nil), // 15: ateom.RestoreWorkloadResponse - (*GetWorkloadStatsRequest)(nil), // 16: ateom.GetWorkloadStatsRequest - (*WorkloadStatsSample)(nil), // 17: ateom.WorkloadStatsSample - (*GetWorkloadStatsResponse)(nil), // 18: ateom.GetWorkloadStatsResponse - (*GetActiveWorkloadStatsRequest)(nil), // 19: ateom.GetActiveWorkloadStatsRequest - (*GetActiveWorkloadStatsResponse)(nil), // 20: ateom.GetActiveWorkloadStatsResponse - nil, // 21: ateom.RunWorkloadRequest.RuntimeAssetPathsEntry - nil, // 22: ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - nil, // 23: ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry + (*VolumeMount)(nil), // 8: ateom.VolumeMount + (*DurableDirVolumeMount)(nil), // 9: ateom.DurableDirVolumeMount + (*Readyz)(nil), // 10: ateom.Readyz + (*HTTPGetAction)(nil), // 11: ateom.HTTPGetAction + (*RunWorkloadResponse)(nil), // 12: ateom.RunWorkloadResponse + (*CheckpointWorkloadRequest)(nil), // 13: ateom.CheckpointWorkloadRequest + (*CheckpointWorkloadResponse)(nil), // 14: ateom.CheckpointWorkloadResponse + (*RestoreWorkloadRequest)(nil), // 15: ateom.RestoreWorkloadRequest + (*RestoreWorkloadResponse)(nil), // 16: ateom.RestoreWorkloadResponse + (*GetWorkloadStatsRequest)(nil), // 17: ateom.GetWorkloadStatsRequest + (*WorkloadStatsSample)(nil), // 18: ateom.WorkloadStatsSample + (*GetWorkloadStatsResponse)(nil), // 19: ateom.GetWorkloadStatsResponse + (*GetActiveWorkloadStatsRequest)(nil), // 20: ateom.GetActiveWorkloadStatsRequest + (*GetActiveWorkloadStatsResponse)(nil), // 21: ateom.GetActiveWorkloadStatsResponse + nil, // 22: ateom.RunWorkloadRequest.RuntimeAssetPathsEntry + nil, // 23: ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + nil, // 24: ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry } var file_ateom_proto_depIdxs = []int32{ 6, // 0: ateom.RunWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 21, // 1: ateom.RunWorkloadRequest.runtime_asset_paths:type_name -> ateom.RunWorkloadRequest.RuntimeAssetPathsEntry + 22, // 1: ateom.RunWorkloadRequest.runtime_asset_paths:type_name -> ateom.RunWorkloadRequest.RuntimeAssetPathsEntry 5, // 2: ateom.RunWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway 7, // 3: ateom.WorkloadSpec.containers:type_name -> ateom.Container - 9, // 4: ateom.Container.readyz:type_name -> ateom.Readyz - 8, // 5: ateom.Container.durable_dir_volume_mounts:type_name -> ateom.DurableDirVolumeMount - 10, // 6: ateom.Readyz.http_get:type_name -> ateom.HTTPGetAction - 6, // 7: ateom.CheckpointWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 22, // 8: ateom.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry - 0, // 9: ateom.CheckpointWorkloadRequest.scope:type_name -> ateom.SnapshotScope - 6, // 10: ateom.RestoreWorkloadRequest.spec:type_name -> ateom.WorkloadSpec - 23, // 11: ateom.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry - 0, // 12: ateom.RestoreWorkloadRequest.scope:type_name -> ateom.SnapshotScope - 5, // 13: ateom.RestoreWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway - 1, // 14: ateom.WorkloadStatsSample.sandbox_class:type_name -> ateom.SandboxClass - 2, // 15: ateom.WorkloadStatsSample.source:type_name -> ateom.StatsSource - 17, // 16: ateom.GetWorkloadStatsResponse.sample:type_name -> ateom.WorkloadStatsSample - 17, // 17: ateom.GetActiveWorkloadStatsResponse.sample:type_name -> ateom.WorkloadStatsSample - 3, // 18: ateom.GetActiveWorkloadStatsResponse.no_sample_reason:type_name -> ateom.NoSampleReason - 4, // 19: ateom.Ateom.RunWorkload:input_type -> ateom.RunWorkloadRequest - 12, // 20: ateom.Ateom.CheckpointWorkload:input_type -> ateom.CheckpointWorkloadRequest - 14, // 21: ateom.Ateom.RestoreWorkload:input_type -> ateom.RestoreWorkloadRequest - 16, // 22: ateom.Ateom.GetWorkloadStats:input_type -> ateom.GetWorkloadStatsRequest - 19, // 23: ateom.Ateom.GetActiveWorkloadStats:input_type -> ateom.GetActiveWorkloadStatsRequest - 11, // 24: ateom.Ateom.RunWorkload:output_type -> ateom.RunWorkloadResponse - 13, // 25: ateom.Ateom.CheckpointWorkload:output_type -> ateom.CheckpointWorkloadResponse - 15, // 26: ateom.Ateom.RestoreWorkload:output_type -> ateom.RestoreWorkloadResponse - 18, // 27: ateom.Ateom.GetWorkloadStats:output_type -> ateom.GetWorkloadStatsResponse - 20, // 28: ateom.Ateom.GetActiveWorkloadStats:output_type -> ateom.GetActiveWorkloadStatsResponse - 24, // [24:29] is the sub-list for method output_type - 19, // [19:24] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 10, // 4: ateom.Container.readyz:type_name -> ateom.Readyz + 9, // 5: ateom.Container.durable_dir_volume_mounts:type_name -> ateom.DurableDirVolumeMount + 8, // 6: ateom.Container.csi_volume_mounts:type_name -> ateom.VolumeMount + 11, // 7: ateom.Readyz.http_get:type_name -> ateom.HTTPGetAction + 6, // 8: ateom.CheckpointWorkloadRequest.spec:type_name -> ateom.WorkloadSpec + 23, // 9: ateom.CheckpointWorkloadRequest.runtime_asset_paths:type_name -> ateom.CheckpointWorkloadRequest.RuntimeAssetPathsEntry + 0, // 10: ateom.CheckpointWorkloadRequest.scope:type_name -> ateom.SnapshotScope + 6, // 11: ateom.RestoreWorkloadRequest.spec:type_name -> ateom.WorkloadSpec + 24, // 12: ateom.RestoreWorkloadRequest.runtime_asset_paths:type_name -> ateom.RestoreWorkloadRequest.RuntimeAssetPathsEntry + 0, // 13: ateom.RestoreWorkloadRequest.scope:type_name -> ateom.SnapshotScope + 5, // 14: ateom.RestoreWorkloadRequest.egress_gateway:type_name -> ateom.EgressGateway + 1, // 15: ateom.WorkloadStatsSample.sandbox_class:type_name -> ateom.SandboxClass + 2, // 16: ateom.WorkloadStatsSample.source:type_name -> ateom.StatsSource + 18, // 17: ateom.GetWorkloadStatsResponse.sample:type_name -> ateom.WorkloadStatsSample + 18, // 18: ateom.GetActiveWorkloadStatsResponse.sample:type_name -> ateom.WorkloadStatsSample + 3, // 19: ateom.GetActiveWorkloadStatsResponse.no_sample_reason:type_name -> ateom.NoSampleReason + 4, // 20: ateom.Ateom.RunWorkload:input_type -> ateom.RunWorkloadRequest + 13, // 21: ateom.Ateom.CheckpointWorkload:input_type -> ateom.CheckpointWorkloadRequest + 15, // 22: ateom.Ateom.RestoreWorkload:input_type -> ateom.RestoreWorkloadRequest + 17, // 23: ateom.Ateom.GetWorkloadStats:input_type -> ateom.GetWorkloadStatsRequest + 20, // 24: ateom.Ateom.GetActiveWorkloadStats:input_type -> ateom.GetActiveWorkloadStatsRequest + 12, // 25: ateom.Ateom.RunWorkload:output_type -> ateom.RunWorkloadResponse + 14, // 26: ateom.Ateom.CheckpointWorkload:output_type -> ateom.CheckpointWorkloadResponse + 16, // 27: ateom.Ateom.RestoreWorkload:output_type -> ateom.RestoreWorkloadResponse + 19, // 28: ateom.Ateom.GetWorkloadStats:output_type -> ateom.GetWorkloadStatsResponse + 21, // 29: ateom.Ateom.GetActiveWorkloadStats:output_type -> ateom.GetActiveWorkloadStatsResponse + 25, // [25:30] is the sub-list for method output_type + 20, // [20:25] is the sub-list for method input_type + 20, // [20:20] is the sub-list for extension type_name + 20, // [20:20] is the sub-list for extension extendee + 0, // [0:20] is the sub-list for field type_name } func init() { file_ateom_proto_init() } @@ -1732,8 +1802,8 @@ func file_ateom_proto_init() { return } file_ateom_proto_msgTypes[0].OneofWrappers = []any{} - file_ateom_proto_msgTypes[10].OneofWrappers = []any{} - file_ateom_proto_msgTypes[16].OneofWrappers = []any{ + file_ateom_proto_msgTypes[11].OneofWrappers = []any{} + file_ateom_proto_msgTypes[17].OneofWrappers = []any{ (*GetActiveWorkloadStatsResponse_Sample)(nil), (*GetActiveWorkloadStatsResponse_NoSampleReason)(nil), } @@ -1743,7 +1813,7 @@ func file_ateom_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_ateom_proto_rawDesc), len(file_ateom_proto_rawDesc)), NumEnums: 4, - NumMessages: 20, + NumMessages: 21, NumExtensions: 0, NumServices: 1, }, diff --git a/internal/proto/ateompb/ateom.proto b/internal/proto/ateompb/ateom.proto index 9ec80a232..b84c1b09e 100644 --- a/internal/proto/ateompb/ateom.proto +++ b/internal/proto/ateompb/ateom.proto @@ -152,6 +152,15 @@ message Container { // durable_dir_volume_mounts are the durable-dir volumes this container // mounts, if any. repeated DurableDirVolumeMount durable_dir_volume_mounts = 4; + + // csi_volume_mounts are the CSI volumes this container mounts, if any. + repeated VolumeMount csi_volume_mounts = 5; +} + +// VolumeMount is one volume mounted into a container. +message VolumeMount { + string volume_name = 1; + string mount_path = 2; } // DurableDirVolumeMount is one durable-dir volume mounted into a container. diff --git a/manifests/ate-install/generated/ate.dev_actortemplates.yaml b/manifests/ate-install/generated/ate.dev_actortemplates.yaml index 2cb17b29a..66bc74f9d 100644 --- a/manifests/ate-install/generated/ate.dev_actortemplates.yaml +++ b/manifests/ate-install/generated/ate.dev_actortemplates.yaml @@ -329,6 +329,8 @@ spec: OnCommit specifies what to include in the snapshot when a commit is requested. If not provided, the "Full" behavior is used by default. onCommit must be a subset of the onPause content. + Note: Data scope only captures DurableDir-typed volumes; external/CSI + volumes are not snapshotted as they persist independently. For example: - if onPause is "Full", then onCommit can be "Full" or "Data". @@ -342,6 +344,8 @@ spec: description: |- OnPause specifies what to include in the snapshot when the actor is paused. If not provided, the "Full" behavior is used by default. + Note: Data scope only captures DurableDir-typed volumes; external/CSI + volumes are not snapshotted as they persist independently. enum: - Full - Data @@ -422,6 +426,9 @@ spec: == 1' maxItems: 32 type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map workerSelector: description: |- WorkerSelector restricts which worker pools actors from this template may @@ -484,9 +491,6 @@ spec: rule: '!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))' - - message: ExternalVolumes are not supported when sandboxClass is 'microvm' - rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || - !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))' - message: 'onResume.fromData: Golden is not supported when sandboxClass is ''gvisor''' rule: '(has(self.sandboxClass) && self.sandboxClass == ''microvm'') @@ -503,6 +507,10 @@ spec: rule: '!has(self.sandboxClass) || self.sandboxClass != ''microvm'' || !has(self.resources) || !has(self.resources.limits) || !(''memory'' in self.resources.limits) || !quantity(self.resources.limits[''memory'']).isLessThan(quantity(''256Mi''))' + - message: All volume mounts must refer to a volume defined in spec.volumes + rule: '!has(self.containers) || self.containers.all(c, !has(c.volumeMounts) + || c.volumeMounts.all(vm, has(self.volumes) && self.volumes.exists(v, + v.name == vm.name)))' status: description: status is the observed state of ActorTemplate properties: diff --git a/pkg/api/v1alpha1/actortemplate_types.go b/pkg/api/v1alpha1/actortemplate_types.go index 3244ccdc6..f3856d305 100644 --- a/pkg/api/v1alpha1/actortemplate_types.go +++ b/pkg/api/v1alpha1/actortemplate_types.go @@ -235,7 +235,8 @@ const ( // the OCI image (including any attached DurableDir volumes). SnapshotScopeFull SnapshotScope = "Full" // Data captures only the contents of attached volumes that support - // snapshots (currently DurableDir-typed volumes). Process memory and + // snapshots (currently DurableDir-typed volumes; external/CSI volumes + // are not snapshotted as they persist independently). Process memory and // the rest of rootfs are excluded. SnapshotScopeData SnapshotScope = "Data" ) @@ -282,6 +283,8 @@ type SnapshotsConfig struct { // OnPause specifies what to include in the snapshot when the actor is paused. // If not provided, the "Full" behavior is used by default. + // Note: Data scope only captures DurableDir-typed volumes; external/CSI + // volumes are not snapshotted as they persist independently. // // +optional // +kubebuilder:default=Full @@ -290,6 +293,8 @@ type SnapshotsConfig struct { // OnCommit specifies what to include in the snapshot when a commit is requested. // If not provided, the "Full" behavior is used by default. // onCommit must be a subset of the onPause content. + // Note: Data scope only captures DurableDir-typed volumes; external/CSI + // volumes are not snapshotted as they persist independently. // // For example: // - if onPause is "Full", then onCommit can be "Full" or "Data". @@ -311,7 +316,6 @@ type SnapshotsConfig struct { // ActorTemplateSpec defined desired spec of an actor. // // +kubebuilder:validation:XValidation:rule="!has(self.volumes) || self.volumes.all(v, has(self.containers) && self.containers.exists(c, has(c.volumeMounts) && c.volumeMounts.exists(vm, vm.name == v.name)))",message="All volumes defined in spec.volumes must be mounted by at least one container" -// +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.volumes) || !self.volumes.exists(v, has(v.externalVolumeTemplate))",message="ExternalVolumes are not supported when sandboxClass is 'microvm'" // +kubebuilder:validation:XValidation:rule="(has(self.sandboxClass) && self.sandboxClass == 'microvm') || !has(self.snapshotsConfig.onResume) || (has(self.snapshotsConfig.onResume.fromData) ? self.snapshotsConfig.onResume.fromData : 'ColdBoot') != 'Golden'",message="onResume.fromData: Golden is not supported when sandboxClass is 'gvisor'" // +kubebuilder:validation:XValidation:rule="!has(self.resources) || !has(self.resources.requests)",message="spec.resources.requests is not supported; actors are sized by spec.resources.limits only" // +kubebuilder:validation:XValidation:rule="!has(self.resources) || !has(self.resources.claims)",message="spec.resources.claims is not supported" @@ -324,6 +328,7 @@ type SnapshotsConfig struct { // assumes the default reserve; deployments that raise --vmm-mem-reserve-mib rely on // the runtime check. gVisor has no reserve, so this only applies to micro-VM. // +kubebuilder:validation:XValidation:rule="!has(self.sandboxClass) || self.sandboxClass != 'microvm' || !has(self.resources) || !has(self.resources.limits) || !('memory' in self.resources.limits) || !quantity(self.resources.limits['memory']).isLessThan(quantity('256Mi'))",message="For sandboxClass 'microvm', spec.resources.limits.memory must be at least 256Mi (128Mi VMM reserve + 128Mi guest minimum); below this the VM cannot boot" +// +kubebuilder:validation:XValidation:rule="!has(self.containers) || self.containers.all(c, !has(c.volumeMounts) || c.volumeMounts.all(vm, has(self.volumes) && self.volumes.exists(v, v.name == vm.name)))",message="All volume mounts must refer to a volume defined in spec.volumes" type ActorTemplateSpec struct { // Containers is the workload definition. // @@ -369,6 +374,8 @@ type ActorTemplateSpec struct { // // +optional // +kubebuilder:validation:MaxItems=32 + // +listType=map + // +listMapKey=name Volumes []Volume `json:"volumes,omitempty"` // Resources declares the compute resources for each actor of this template. diff --git a/pkg/api/v1alpha1/actortemplate_validation_test.go b/pkg/api/v1alpha1/actortemplate_validation_test.go index 4b9a8e864..554bdc045 100644 --- a/pkg/api/v1alpha1/actortemplate_validation_test.go +++ b/pkg/api/v1alpha1/actortemplate_validation_test.go @@ -1105,7 +1105,7 @@ func TestActorTemplateValidation(t *testing.T) { }, wantErr: false, }, { - name: "Volumes: ExternalVolumeTemplate volume with SandboxClass microvm is invalid", + name: "Volumes: ExternalVolumeTemplate volume with SandboxClass microvm is valid", mutate: func(at *ActorTemplate) { at.Spec.SandboxClass = SandboxClassMicroVM at.Spec.Volumes = []Volume{ @@ -1123,8 +1123,7 @@ func TestActorTemplateValidation(t *testing.T) { {Name: "vol1", MountPath: "/mnt/data"}, } }, - wantErr: true, - errMsg: "ExternalVolumes are not supported when sandboxClass is 'microvm'", + wantErr: false, }, { name: "Volumes: ExternalVolumeTemplate volume with SandboxClass gvisor is valid", mutate: func(at *ActorTemplate) { @@ -1189,6 +1188,28 @@ func TestActorTemplateValidation(t *testing.T) { }, wantErr: true, errMsg: "All volumes defined in spec.volumes must be mounted by at least one container", + }, { + name: "Volumes: volumeMount without volume is invalid", + mutate: func(at *ActorTemplate) { + at.Spec.Containers[0].VolumeMounts = []VolumeMount{ + {Name: "missing-vol", MountPath: "/mnt/data"}, + } + }, + wantErr: true, + errMsg: "All volume mounts must refer to a volume defined in spec.volumes", + }, { + name: "Volumes: duplicate volume names is invalid", + mutate: func(at *ActorTemplate) { + at.Spec.Volumes = []Volume{ + {Name: "vol1", VolumeSource: VolumeSource{DurableDir: &DurableDirVolumeSource{}}}, + {Name: "vol1", VolumeSource: VolumeSource{DurableDir: &DurableDirVolumeSource{}}}, + } + at.Spec.Containers[0].VolumeMounts = []VolumeMount{ + {Name: "vol1", MountPath: "/mnt/data"}, + } + }, + wantErr: true, + errMsg: "vol1", }} for _, tt := range tests {