feat(observability): measure gateway provision duration

README.md

@@ -181,6 +181,8 @@ The API server exposes Prometheus metrics on its metrics port (default `:8080/me
 |---|---|---|
 | `hypershell_gateways_total{phase="Running"|"Provisioning"|"Degraded"|"Failed"}` | Gauge | Number of gateways by phase. Queried live from the database on each scrape. |
 
+The control plane also exports `gateway.provision.duration` through OTLP. This histogram measures the time in seconds from Gateway creation to its first successful `Running` phase. A standard Prometheus conversion exposes it as `gateway_provision_duration_seconds`.
+
 ### Grafana dashboard
 
 A pre-built Grafana dashboard is provided at `dashboards/hypershell-dashboard.yml`. It is packaged as a Kubernetes ConfigMap with the `grafana_dashboard: "true"` label so it is picked up automatically by the Grafana sidecar.

components/control-plane/internal/otel/metrics.go

@@ -10,9 +10,10 @@ import (
 )
 
 var (
-	reconcileDuration metric.Int64Histogram
-	reconcileErrors   metric.Int64Counter
-	watchReconnects   metric.Int64Counter
+	reconcileDuration        metric.Int64Histogram
+	gatewayProvisionDuration metric.Float64Histogram
+	reconcileErrors          metric.Int64Counter
+	watchReconnects          metric.Int64Counter
 )
 
 func registerMetrics() error {
@@ -28,6 +29,16 @@ func registerMetrics() error {
 		return err
 	}
 
+	gatewayProvisionDuration, err = meter.Float64Histogram(
+		"gateway.provision.duration",
+		metric.WithUnit("s"),
+		metric.WithDescription("Time from Gateway creation until its first successful transition to Running"),
+		metric.WithExplicitBucketBoundaries(1, 5, 10, 15, 30, 45, 60, 90, 120, 180, 300, 600, 900),
+	)
+	if err != nil {
+		return err
+	}
+
 	reconcileErrors, err = meter.Int64Counter(
 		"reconcile.errors",
 		metric.WithUnit("{error}"),
@@ -56,6 +67,15 @@ func RecordReconcileDuration(ctx context.Context, kind, eventType string, start
 	))
 }
 
+// RecordGatewayProvisionDuration records one successful create-to-Running
+// duration. The caller owns the one-observation rule for each Gateway.
+func RecordGatewayProvisionDuration(ctx context.Context, duration time.Duration) {
+	if gatewayProvisionDuration == nil || duration < 0 {
+		return
+	}
+	gatewayProvisionDuration.Record(ctx, duration.Seconds())
+}
+
 // RecordReconcileError increments the reconcile error counter.
 func RecordReconcileError(ctx context.Context, kind string) {
 	if reconcileErrors == nil {

components/control-plane/internal/otel/metrics_test.go

@@ -0,0 +1,83 @@
+package otel
+
+import (
+	"context"
+	"reflect"
+	"testing"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/metric/metricdata"
+)
+
+func TestRecordGatewayProvisionDuration(t *testing.T) {
+	previousProvider := otel.GetMeterProvider()
+	previousReconcileDuration := reconcileDuration
+	previousGatewayProvisionDuration := gatewayProvisionDuration
+	previousReconcileErrors := reconcileErrors
+	previousWatchReconnects := watchReconnects
+
+	reader := sdkmetric.NewManualReader()
+	provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader))
+	otel.SetMeterProvider(provider)
+	reconcileDuration = nil
+	gatewayProvisionDuration = nil
+	reconcileErrors = nil
+	watchReconnects = nil
+	t.Cleanup(func() {
+		otel.SetMeterProvider(previousProvider)
+		reconcileDuration = previousReconcileDuration
+		gatewayProvisionDuration = previousGatewayProvisionDuration
+		reconcileErrors = previousReconcileErrors
+		watchReconnects = previousWatchReconnects
+		_ = provider.Shutdown(context.Background())
+	})
+
+	if err := registerMetrics(); err != nil {
+		t.Fatalf("registerMetrics() returned an error: %v", err)
+	}
+
+	RecordGatewayProvisionDuration(context.Background(), 37_500*time.Millisecond)
+	RecordGatewayProvisionDuration(context.Background(), -1*time.Second)
+
+	var collected metricdata.ResourceMetrics
+	if err := reader.Collect(context.Background(), &collected); err != nil {
+		t.Fatalf("Collect() returned an error: %v", err)
+	}
+
+	for _, scope := range collected.ScopeMetrics {
+		for _, gotMetric := range scope.Metrics {
+			if gotMetric.Name != "gateway.provision.duration" {
+				continue
+			}
+			if gotMetric.Unit != "s" {
+				t.Fatalf("metric unit = %q, want s", gotMetric.Unit)
+			}
+			histogram, ok := gotMetric.Data.(metricdata.Histogram[float64])
+			if !ok {
+				t.Fatalf("metric data type = %T, want float64 histogram", gotMetric.Data)
+			}
+			if len(histogram.DataPoints) != 1 {
+				t.Fatalf("data point count = %d, want 1", len(histogram.DataPoints))
+			}
+			point := histogram.DataPoints[0]
+			if point.Count != 1 {
+				t.Fatalf("sample count = %d, want 1", point.Count)
+			}
+			if point.Sum != 37.5 {
+				t.Fatalf("sample sum = %v, want 37.5", point.Sum)
+			}
+			if point.Attributes.Len() != 0 {
+				t.Fatalf("metric attributes = %v, want none", point.Attributes)
+			}
+			wantBounds := []float64{1, 5, 10, 15, 30, 45, 60, 90, 120, 180, 300, 600, 900}
+			if !reflect.DeepEqual(point.Bounds, wantBounds) {
+				t.Fatalf("bucket bounds = %v, want %v", point.Bounds, wantBounds)
+			}
+			return
+		}
+	}
+
+	t.Fatal("gateway.provision.duration metric was not collected")
+}

components/control-plane/internal/reconciler/gateway_vocabulary.go

@@ -0,0 +1,11 @@
+package reconciler
+
+// These values are the shared Gateway health vocabulary for the reconciler and
+// health paths. Keep phase comparisons and phase updates on this vocabulary.
+const (
+	gatewayPhaseProvisioning = "Provisioning"
+	gatewayPhaseRunning      = "Running"
+	gatewayPhaseDegraded     = "Degraded"
+	gatewayPhaseFailed       = "Failed"
+	gatewayStatusHealthy     = "Healthy"
+)

components/control-plane/internal/reconciler/health.go

@@ -223,7 +223,7 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl
 	// observable workload. Leave Pending gateways to the provisioning path and
 	// Failed gateways to a subsequent spec change.
 	switch phase {
-	case "Running", "Degraded", "Provisioning":
+	case gatewayPhaseRunning, gatewayPhaseDegraded, gatewayPhaseProvisioning:
 	default:
 		return
 	}
@@ -273,7 +273,7 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl
 			return
 		}
 		h.clearRouteTimer(gatewayID)
-		desiredPhase, desiredStatus = "Degraded", reason
+		desiredPhase, desiredStatus = gatewayPhaseDegraded, reason
 	case h.exposure != nil && isRoutedGateway(gw):
 		// Deployment is Ready; a routed gateway additionally requires its external
 		// exposure to be observed Ready before it can be Running.
@@ -285,7 +285,7 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl
 		}
 	default:
 		h.clearRouteTimer(gatewayID)
-		desiredPhase, desiredStatus = "Running", "Healthy"
+		desiredPhase, desiredStatus = gatewayPhaseRunning, gatewayStatusHealthy
 	}
 
 	// active_sandbox_count is maintained independently by the event-driven
@@ -297,10 +297,14 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl
 		return
 	}
 
-	if _, err := client.UpdateGateway(ctx, update); err != nil {
+	response, err := client.UpdateGateway(ctx, update)
+	if err != nil {
 		log.Printf("WARN gateway health: update %s to %s: %v", gatewayID, desiredPhase, err)
 		return
 	}
+	if isGatewayProvisionCompletion(phase, desiredPhase) {
+		observeGatewayProvisionDuration(ctx, response.GetGateway())
+	}
 
 	log.Printf("INFO gateway health: %s %s -> %s (%s)", gatewayID, phase, desiredPhase, desiredStatus)
 }
@@ -314,7 +318,7 @@ func (h *GatewayHealthReconciler) reconcileGatewayHealth(ctx context.Context, cl
 // observations retain normal ownership of phase and status so operational
 // failures remain visible.
 func observedGatewayHealthUpdate(gatewayID, currentPhase, currentStatus, desiredPhase, desiredStatus string, keycloakConfigured bool) *pb.UpdateGatewayRequest {
-	if keycloakConfigured && isGatewayKeycloakClientStatus(currentStatus) && desiredPhase == "Running" && desiredStatus == "Healthy" {
+	if keycloakConfigured && isGatewayKeycloakClientStatus(currentStatus) && desiredPhase == gatewayPhaseRunning && desiredStatus == gatewayStatusHealthy {
 		if currentPhase == desiredPhase {
 			return nil
 		}
@@ -560,21 +564,21 @@ func (h *GatewayHealthReconciler) evaluateRouteReadiness(ctx context.Context, ga
 	}
 	if rr.Ready {
 		h.clearRouteTimer(gatewayID)
-		return "Running", "Healthy"
+		return gatewayPhaseRunning, gatewayStatusHealthy
 	}
 
-	if currentPhase == "Provisioning" {
+	if currentPhase == gatewayPhaseProvisioning {
 		since := h.markRouteNotReady(gatewayID)
 		if h.now().Sub(since) >= h.routeReadyTimeout {
 			h.clearRouteTimer(gatewayID)
-			return "Degraded", fmt.Sprintf("route not ready after %s: %s", h.routeReadyTimeout, rr.Reason)
+			return gatewayPhaseDegraded, fmt.Sprintf("route not ready after %s: %s", h.routeReadyTimeout, rr.Reason)
 		}
-		return "Provisioning", rr.Reason
+		return gatewayPhaseProvisioning, rr.Reason
 	}
 
 	// currentPhase is Running (lost readiness) or Degraded (still unhealthy).
 	h.clearRouteTimer(gatewayID)
-	return "Degraded", rr.Reason
+	return gatewayPhaseDegraded, rr.Reason
 }
 
 // markRouteNotReady records the first time the gateway's Deployment was observed

components/control-plane/internal/reconciler/metrics.go

@@ -0,0 +1,82 @@
+package reconciler
+
+import (
+	"context"
+	"sync"
+	"time"
+
+	pb "github.com/openshift-online/hypershell/components/api-server/pkg/api/grpc/hypershell/v1"
+	cpotel "github.com/openshift-online/hypershell/components/control-plane/internal/otel"
+)
+
+// observedGatewayProvisions coordinates the event-driven and health reconcile
+// paths. These paths can observe the same first Running transition at the same
+// time. The Gateway identifier stays in process memory and is not an OTel
+// attribute.
+var observedGatewayProvisions sync.Map
+
+// observeGatewayProvisionDuration records the time from API creation until the
+// first successful transition to Running. It ignores incomplete or invalid
+// timestamps so telemetry cannot change reconciliation behavior.
+func observeGatewayProvisionDuration(ctx context.Context, gw *pb.Gateway) {
+	duration, ok := gatewayProvisionDuration(gw)
+	if !ok {
+		return
+	}
+	gatewayID := gw.GetMetadata().GetId()
+	if !claimGatewayProvisionObservation(gatewayID) {
+		return
+	}
+	cpotel.RecordGatewayProvisionDuration(ctx, duration)
+}
+
+func claimGatewayProvisionObservation(gatewayID string) bool {
+	if gatewayID == "" {
+		return false
+	}
+	_, loaded := observedGatewayProvisions.LoadOrStore(gatewayID, struct{}{})
+	return !loaded
+}
+
+func forgetGatewayProvisionObservation(gatewayID string) {
+	observedGatewayProvisions.Delete(gatewayID)
+}
+
+// suppressGatewayProvisionObservation prevents work on a previously Running or
+// Degraded Gateway from producing a new provision observation. The caller uses
+// the stored phase for a normal event. The retry adapter supplies the phase that
+// it cleared when it bypasses the phase gate.
+func suppressGatewayProvisionObservation(gatewayID, previousPhase string) {
+	if previousPhase == gatewayPhaseRunning || previousPhase == gatewayPhaseDegraded {
+		claimGatewayProvisionObservation(gatewayID)
+	}
+}
+
+func gatewayProvisionDuration(gw *pb.Gateway) (time.Duration, bool) {
+	if gw == nil || gw.GetMetadata() == nil {
+		return 0, false
+	}
+	createdAt := gw.GetMetadata().GetCreatedAt()
+	runningAt := gw.GetMetadata().GetUpdatedAt()
+	if createdAt == nil || runningAt == nil {
+		return 0, false
+	}
+	if err := createdAt.CheckValid(); err != nil {
+		return 0, false
+	}
+	if err := runningAt.CheckValid(); err != nil {
+		return 0, false
+	}
+	duration := runningAt.AsTime().Sub(createdAt.AsTime())
+	if duration < 0 {
+		return 0, false
+	}
+	return duration, true
+}
+
+// isGatewayProvisionCompletion excludes later recovery transitions. A new
+// gateway can stay in Provisioning after the first reconcile while its route
+// becomes ready. A Running gateway that fails moves through Degraded instead.
+func isGatewayProvisionCompletion(currentPhase, desiredPhase string) bool {
+	return currentPhase == gatewayPhaseProvisioning && desiredPhase == gatewayPhaseRunning
+}