From c0edd2d7f92838ffe2c81d36adf3b99407949e1d Mon Sep 17 00:00:00 2001 From: Maria Ines Parnisari Date: Tue, 1 Sep 2026 16:50:53 -0700 Subject: [PATCH 1/2] test: fix e2e --- e2e/e2e_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 2ced64a3..2f2293b7 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -24,6 +24,7 @@ import ( "github.com/onsi/gomega/gexec" "github.com/spf13/afero" "go.uber.org/zap" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -414,7 +415,7 @@ func SnapshotFailHandler(message string, callerSkip ...int) { gvrs := []schema.GroupVersionResource{ v1alpha1ClusterGVR, corev1.SchemeGroupVersion.WithResource("pods"), - corev1.SchemeGroupVersion.WithResource("jobs"), + batchv1.SchemeGroupVersion.WithResource("jobs"), corev1.SchemeGroupVersion.WithResource("secrets"), } @@ -465,6 +466,7 @@ func SaveClusterState(directory string, gvrs []schema.GroupVersionResource) { objs, err := c.Resource(gvr).Namespace(n.Name).List(ctx, metav1.ListOptions{}) if err != nil { GinkgoWriter.Println("error fetching", gvr.Resource, "from namespace", n.Name, err) + continue } for _, item := range objs.Items { cluster, err := yaml.Marshal(item) From 6f1c1bf97f197e9aa637ccbabe430628e62cb4d7 Mon Sep 17 00:00:00 2001 From: Maria Ines Parnisari Date: Tue, 1 Sep 2026 18:37:27 -0700 Subject: [PATCH 2/2] test: also save db container logs --- e2e/e2e_test.go | 91 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 87 insertions(+), 4 deletions(-) diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 2f2293b7..d808245b 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -406,9 +406,13 @@ func Provision(name string) (string, func(), error) { return kubeconfig, deprovision, nil } -// SnapshotFailHandler dumps cluster state when a test fails -// It prints SpiceDBClusters, Pods, and Jobs from all namespaces with the prefix -// "test". +// dumpedNamespacePrefixes selects the namespaces saved by SaveClusterState: +// the per-spec test namespaces and the shared database namespaces. +var dumpedNamespacePrefixes = []string{"test", "postgres-", "mysql-", "cockroachdb-"} + +// SnapshotFailHandler dumps cluster state when a test fails. +// It saves SpiceDBClusters, Pods, Jobs, and Secrets, along with pod logs and +// events, from the test and database namespaces. func SnapshotFailHandler(message string, callerSkip ...int) { defer Fail(message, callerSkip...) @@ -444,7 +448,7 @@ func SaveClusterState(directory string, gvrs []schema.GroupVersionResource) { GinkgoWriter.Println("error fetching namespaces", err) } for _, n := range namespaces.Items { - if !strings.HasPrefix(n.Name, "test") { + if !isDumpedNamespace(n.Name) { continue } GinkgoWriter.Println("dumping namespace", n.Name) @@ -481,5 +485,84 @@ func SaveClusterState(directory string, gvrs []schema.GroupVersionResource) { } } } + + savePodLogs(ctx, k, n.Name, filepath.Join(namespacePath, "logs")) + saveEvents(ctx, k, n.Name, filepath.Join(namespacePath, "events.yaml")) + } +} + +func isDumpedNamespace(name string) bool { + for _, prefix := range dumpedNamespacePrefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// savePodLogs writes the current log of every container in the namespace +// to `-.log`. If a container has restarted, the log of the +// previous instance is written to `-.previous.log`, which +// is where a crash-looping container's error ends up. +func savePodLogs(ctx context.Context, k kubernetes.Interface, namespace, directory string) { + pods, err := k.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + GinkgoWriter.Println("error fetching pods for logs from namespace", namespace, err) + return + } + if len(pods.Items) == 0 { + return + } + if err := os.MkdirAll(directory, 0o700); err != nil { + GinkgoWriter.Println("error making directory for logs", namespace, err) + return + } + + for _, pod := range pods.Items { + statuses := make([]corev1.ContainerStatus, 0, len(pod.Status.InitContainerStatuses)+len(pod.Status.ContainerStatuses)) + statuses = append(statuses, pod.Status.InitContainerStatuses...) + statuses = append(statuses, pod.Status.ContainerStatuses...) + for _, status := range statuses { + name := pod.Name + "-" + status.Name + savePodLog(ctx, k, namespace, pod.Name, status.Name, false, filepath.Join(directory, name+".log")) + if status.RestartCount > 0 { + savePodLog(ctx, k, namespace, pod.Name, status.Name, true, filepath.Join(directory, name+".previous.log")) + } + } + } +} + +func savePodLog(ctx context.Context, k kubernetes.Interface, namespace, pod, container string, previous bool, path string) { + logs, err := k.CoreV1().Pods(namespace).GetLogs(pod, &corev1.PodLogOptions{ + Container: container, + Previous: previous, + }).DoRaw(ctx) + if err != nil { + GinkgoWriter.Println("error fetching logs for", namespace, pod, container, "previous:", previous, err) + return + } + if err := os.WriteFile(path, logs, 0o600); err != nil { + GinkgoWriter.Println("error writing logs for", namespace, pod, container, err) + } +} + +// saveEvents writes the namespace's events, which carry scheduling and image +// pull failures that don't show up in any object's status. +func saveEvents(ctx context.Context, k kubernetes.Interface, namespace, path string) { + events, err := k.CoreV1().Events(namespace).List(ctx, metav1.ListOptions{}) + if err != nil { + GinkgoWriter.Println("error fetching events from namespace", namespace, err) + return + } + if len(events.Items) == 0 { + return + } + out, err := yaml.Marshal(events.Items) + if err != nil { + GinkgoWriter.Println("error marshalling events from namespace", namespace, err) + return + } + if err := os.WriteFile(path, out, 0o600); err != nil { + GinkgoWriter.Println("error writing events from namespace", namespace, err) } }