critest: test metric descriptors

.github/workflows/crio.yml

@@ -60,8 +60,8 @@ jobs:
       - name: Configure CRI-O
         run: |
           sudo mkdir -p /etc/crio/crio.conf.d
-          printf '[crio.runtime]\nlog_level = "debug"\n[crio.image]\nshort_name_mode = "disabled"\n' | sudo tee /etc/crio/crio.conf.d/01-log-level.conf
-
+          printf '[crio.runtime]\nlog_level = "debug"\n[crio.image]\nshort_name_mode = "disabled"\n' | sudo tee -a /etc/crio/crio.conf.d/01-base.conf
+          printf '[crio.stats]\nincluded_pod_metrics = [\n"all",\n]\n' | sudo tee -a /etc/crio/crio.conf.d/01-base.conf
       - name: Configure CRI-O to use conmon-rs intead of the default conmon
         if: ${{matrix.monitor == 'conmon-rs'}}
         run: |

pkg/validate/pod.go

@@ -20,16 +20,96 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"slices"
+	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"google.golang.org/grpc/codes"
+	grpcstatus "google.golang.org/grpc/status"
 	internalapi "k8s.io/cri-api/pkg/apis"
 	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
 
 	"sigs.k8s.io/cri-tools/pkg/common"
 	"sigs.k8s.io/cri-tools/pkg/framework"
 )
 
+// expectedMetricDescriptorNames contains all expected metric descriptor names
+// based on metrics returned by kubelet with CRI-O and cadvisor on the legacy cadvisor stats provider
+// on kubernetes 1.37.
+var expectedMetricDescriptorNames = []string{
+	"container_blkio_device_usage_total",
+	"container_cpu_cfs_periods_total",
+	"container_cpu_cfs_throttled_periods_total",
+	"container_cpu_cfs_throttled_seconds_total",
+	"container_cpu_system_seconds_total",
+	"container_cpu_usage_seconds_total",
+	"container_cpu_user_seconds_total",
+	"container_file_descriptors",
+	"container_fs_inodes_free",
+	"container_fs_inodes_total",
+	"container_fs_limit_bytes",
+	"container_fs_reads_bytes_total",
+	"container_fs_reads_total",
+	"container_fs_usage_bytes",
+	"container_fs_writes_bytes_total",
+	"container_fs_writes_total",
+	"container_hugetlb_max_usage_bytes",
+	"container_hugetlb_usage_bytes",
+	"container_last_seen",
+	"container_memory_cache",
+	"container_memory_failcnt",
+	"container_memory_failures_total",
+	"container_memory_kernel_usage",
+	"container_memory_mapped_file",
+	"container_memory_max_usage_bytes",
+	"container_memory_rss",
+	"container_memory_swap",
+	"container_memory_usage_bytes",
+	"container_memory_working_set_bytes",
+	"container_network_receive_bytes_total",
+	"container_network_receive_errors_total",
+	"container_network_receive_packets_dropped_total",
+	"container_network_receive_packets_total",
+	"container_network_transmit_bytes_total",
+	"container_network_transmit_errors_total",
+	"container_network_transmit_packets_dropped_total",
+	"container_network_transmit_packets_total",
+	"container_oom_events_total",
+	"container_pressure_cpu_stalled_seconds_total",
+	"container_pressure_cpu_waiting_seconds_total",
+	"container_pressure_io_stalled_seconds_total",
+	"container_pressure_io_waiting_seconds_total",
+	"container_pressure_memory_stalled_seconds_total",
+	"container_pressure_memory_waiting_seconds_total",
+	"container_processes",
+	"container_sockets",
+	"container_spec_cpu_period",
+	"container_spec_cpu_quota",
+	"container_spec_cpu_shares",
+	"container_spec_memory_limit_bytes",
+	"container_spec_memory_reservation_limit_bytes",
+	"container_spec_memory_swap_limit_bytes",
+	"container_start_time_seconds",
+	"container_threads",
+	"container_threads_max",
+	"container_ulimits_soft",
+}
+
+// optionalValuesForMetricDescriptors contains the metric descriptors that have
+// optional values due to test environment limitations.
+var optionalValuesForMetricDescriptors = []string{
+	// All of these depend on blkio cgroup accounting, which doesn't work on
+	// GitHub actions runners because the overlay filesystem is typically backed
+	// by a virtual disk that doesn't report per-device I/O stats.
+	"container_blkio_device_usage_total",
+	"container_fs_reads_bytes_total",
+	"container_fs_reads_total",
+	"container_fs_writes_bytes_total",
+	"container_fs_writes_total",
+}
+
 var _ = framework.KubeDescribe("PodSandbox", func() {
 	f := framework.NewDefaultCRIFramework()
 
@@ -84,6 +164,74 @@ var _ = framework.KubeDescribe("PodSandbox", func() {
 			podID = "" // no need to cleanup pod
 		})
 	})
+	Context("runtime should support metrics operations", func() {
+		var (
+			podID     string
+			podConfig *runtimeapi.PodSandboxConfig
+		)
+
+		BeforeEach(func() {
+			_, err := rc.ListMetricDescriptors(context.TODO())
+			if err != nil {
+				s, ok := grpcstatus.FromError(err)
+				Expect(ok && s.Code() == codes.Unimplemented).To(BeTrue(), "Expected CRI metric descriptors call to either be not supported, or not error")
+
+				if s.Code() == codes.Unimplemented {
+					Skip("CRI Metrics endpoints not supported by this runtime version")
+				}
+			}
+		})
+
+		AfterEach(func() {
+			if podID != "" {
+				By("stop PodSandbox")
+				Expect(rc.StopPodSandbox(context.TODO(), podID)).NotTo(HaveOccurred())
+				By("delete PodSandbox")
+				Expect(rc.RemovePodSandbox(context.TODO(), podID)).NotTo(HaveOccurred())
+			}
+		})
+
+		It("runtime should support returning metrics descriptors [Conformance]", func() {
+			By("list metric descriptors")
+
+			descs := listMetricDescriptors(rc)
+
+			By("verify expected metric descriptors are present")
+			testMetricDescriptors(descs)
+		})
+
+		It("runtime should support listing pod sandbox metrics [Conformance]", func() {
+			By("create pod sandbox")
+
+			podID, podConfig = framework.CreatePodSandboxForContainer(context.TODO(), rc)
+
+			By("create container in pod")
+
+			ic := f.CRIClient.CRIImageClient
+			containerID := createContainerForMetrics(context.TODO(), rc, ic, podID, podConfig)
+
+			By("start container")
+			startContainer(context.TODO(), rc, containerID)
+
+			_, _, err := rc.ExecSync(
+				context.TODO(), containerID, []string{"/bin/sh", "-c", "for i in $(seq 1 10); do echo hi >> /var/lib/mydisktest/inode_test_file_$i; done; sync"},
+				time.Duration(defaultExecSyncTimeout)*time.Second,
+			)
+
+			Expect(err).ToNot(HaveOccurred())
+
+			By("list metric descriptors")
+
+			descs := listMetricDescriptors(rc)
+
+			By("list pod sandbox metrics")
+
+			metrics := listPodSandboxMetrics(rc)
+
+			By("verify pod metrics are present")
+			testPodSandboxMetrics(metrics, descs, podID)
+		})
+	})
 })
 
 // podSandboxFound returns whether PodSandbox is found.
@@ -170,6 +318,17 @@ func listPodSandbox(ctx context.Context, c internalapi.RuntimeService, filter *r
 	return pods
 }
 
+// listMetricDescriptors lists MetricDescriptors.
+func listMetricDescriptors(c internalapi.RuntimeService) []*runtimeapi.MetricDescriptor {
+	By("List MetricDescriptors.")
+
+	descs, err := c.ListMetricDescriptors(context.TODO())
+	framework.ExpectNoError(err, "failed to list MetricDescriptors status: %v", err)
+	framework.Logf("List MetricDescriptors succeed")
+
+	return descs
+}
+
 // createLogTempDir creates the log temp directory for podSandbox.
 func createLogTempDir(podSandboxName string) (hostPath, podLogPath string) {
 	hostPath, err := os.MkdirTemp("", "podLogTest")
@@ -200,3 +359,116 @@ func createPodSandboxWithLogDirectory(ctx context.Context, c internalapi.Runtime
 
 	return framework.RunPodSandbox(ctx, c, podConfig), podConfig, hostPath
 }
+
+// testMetricDescriptors verifies that all expected metric descriptors are present.
+func testMetricDescriptors(descs []*runtimeapi.MetricDescriptor) {
+	returnedDescriptors := make(map[string]*runtimeapi.MetricDescriptor)
+
+	for _, desc := range descs {
+		returnedDescriptors[desc.GetName()] = desc
+		Expect(desc.GetHelp()).NotTo(BeEmpty(), "Metric descriptor %q should have help text", desc.GetName())
+		Expect(desc.GetLabelKeys()).NotTo(BeEmpty(), "Metric descriptor %q should have label keys", desc.GetName())
+	}
+
+	missingMetrics := []string{}
+
+	for _, expectedName := range expectedMetricDescriptorNames {
+		_, found := returnedDescriptors[expectedName]
+		if !found {
+			missingMetrics = append(missingMetrics, expectedName)
+		}
+	}
+
+	Expect(missingMetrics).To(BeEmpty(), "Expected %s metrics to be present and they were not", strings.Join(missingMetrics, " "))
+}
+
+// listPodSandboxMetrics lists PodSandboxMetrics.
+func listPodSandboxMetrics(c internalapi.RuntimeService) []*runtimeapi.PodSandboxMetrics {
+	By("List PodSandboxMetrics.")
+
+	metrics, err := c.ListPodSandboxMetrics(context.TODO())
+	framework.ExpectNoError(err, "failed to list PodSandboxMetrics: %v", err)
+	framework.Logf("List PodSandboxMetrics succeed")
+
+	return metrics
+}
+
+// testPodSandboxMetrics verifies that metrics are present for the specified pod.
+func testPodSandboxMetrics(allMetrics []*runtimeapi.PodSandboxMetrics, descs []*runtimeapi.MetricDescriptor, podID string) {
+	var podMetrics *runtimeapi.PodSandboxMetrics
+
+	for _, m := range allMetrics {
+		if m.GetPodSandboxId() == podID {
+			podMetrics = m
+
+			break
+		}
+	}
+
+	Expect(podMetrics).NotTo(BeNil(), "Metrics for pod %q should be present", podID)
+
+	metricNamesFound := make(map[string][]string)
+
+	for _, metric := range podMetrics.GetMetrics() {
+		if len(metricNamesFound[metric.GetName()]) == 0 {
+			metricNamesFound[metric.GetName()] = metric.GetLabelValues()
+		}
+	}
+
+	for _, containerMetric := range podMetrics.GetContainerMetrics() {
+		for _, metric := range containerMetric.GetMetrics() {
+			if len(metricNamesFound[metric.GetName()]) == 0 {
+				metricNamesFound[metric.GetName()] = metric.GetLabelValues()
+			}
+		}
+	}
+
+	missingMetrics := []string{}
+
+	for _, expectedName := range expectedMetricDescriptorNames {
+		if len(metricNamesFound[expectedName]) == 0 {
+			if slices.Contains(optionalValuesForMetricDescriptors, expectedName) {
+				continue // skip optional values
+			}
+
+			missingMetrics = append(missingMetrics, expectedName)
+		}
+	}
+
+	Expect(missingMetrics).To(BeEmpty(), "Expected %s metrics to be present and they were not", strings.Join(missingMetrics, " "))
+
+	mismatchedLabels := []string{}
+
+	for _, desc := range descs {
+		if len(metricNamesFound[desc.GetName()]) != len(desc.GetLabelKeys()) {
+			if slices.Contains(optionalValuesForMetricDescriptors, desc.GetName()) {
+				continue // skip optional values
+			}
+
+			mismatchedLabels = append(mismatchedLabels, desc.GetName())
+		}
+	}
+
+	Expect(mismatchedLabels).To(BeEmpty(), "Expected %s metrics to have same set of labels in ListMetricDescriptors and ListPodSandboxMetrics", strings.Join(mismatchedLabels, ","))
+}
+
+// createContainerForMetrics creates a container for metrics.
+func createContainerForMetrics(ctx context.Context, rc internalapi.RuntimeService, ic internalapi.ImageManagerService, podID string, podConfig *runtimeapi.PodSandboxConfig) string {
+	containerName := "container-for-metrics-" + framework.NewUUID()
+	containerConfig := &runtimeapi.ContainerConfig{
+		Metadata: framework.BuildContainerMetadata(containerName, framework.DefaultAttempt),
+		Image:    &runtimeapi.ImageSpec{Image: framework.TestContext.TestImageList.DefaultTestContainerImage},
+		Command:  framework.DefaultContainerCommand,
+		Linux: &runtimeapi.LinuxContainerConfig{
+			Resources: &runtimeapi.LinuxContainerResources{
+				CpuPeriod:              100000,
+				CpuQuota:               20000,
+				CpuShares:              1024,
+				MemoryLimitInBytes:     64 * 1024 * 1024,
+				MemorySwapLimitInBytes: 64 * 1024 * 1024,
+			},
+		},
+	}
+
+	return framework.CreateContainer(ctx, rc, ic, containerConfig, podID, podConfig)
+}