gantry acr artifact streaming

.github/workflows/gantry-e2e.yaml

@@ -10,6 +10,7 @@ on:
       - internal/gantry/**
       - deploy/gantry/**
       - images/gantry/**
+      - images/gantry-node-config/**
       - e2e/gantry/**
       - hack/cmd/render-manifests/**
       - go.mod
@@ -25,6 +26,7 @@ on:
       - internal/gantry/**
       - deploy/gantry/**
       - images/gantry/**
+      - images/gantry-node-config/**
       - e2e/gantry/**
       - hack/cmd/render-manifests/**
       - go.mod
@@ -62,7 +64,9 @@ jobs:
           cache-dependency-path: go.sum
 
       - name: Install kind
-        run: go install sigs.k8s.io/kind@v0.30.0
+        run: |
+          make install-kind
+          echo "$PWD/bin" >> "$GITHUB_PATH"
 
       # The suite skips when prerequisites are absent. Verify them first so CI
       # cannot report success without creating a cluster.

.github/workflows/nightly.yaml

@@ -266,6 +266,7 @@ jobs:
   #   - netboot                       : PXE artifact referenced by metalman
   #   - unbounded-storage-supervisor  : per-Site storage supervisor DaemonSet
   #   - gantry                        : cluster-wide gantry DaemonSet
+  #   - gantry-node-config            : optional AKS OverlayBD configurator
   #   - orca                          : deployed separately by deploy-orca
   #
   # machine-ops-controller is omitted on purpose: the operator synthesizes
@@ -294,6 +295,8 @@ jobs:
             file: images/unbounded-operator/Containerfile
           - name: gantry
             file: images/gantry/Containerfile
+          - name: gantry-node-config
+            file: images/gantry-node-config/Containerfile
           - name: orca
             file: images/orca/Containerfile
           - name: inventory-aggregator

.github/workflows/release.yaml

@@ -821,7 +821,7 @@ jobs:
     needs: version
     runs-on: ubuntu-latest
     strategy:
-      # One bad component must not cancel the other ten. Runner minutes are the
+      # One bad component must not cancel the other eleven. Runner minutes are the
       # wrong thing to optimize on a release: knowing the full blast radius in
       # one pass beats discovering it one re-run at a time, and it keeps the
       # re-run down to the job that actually failed. When v0.5.0's
@@ -854,6 +854,10 @@ jobs:
             file: images/gantry/Containerfile
             platforms: linux/amd64,linux/arm64
             trivy: true
+          - name: gantry-node-config
+            file: images/gantry-node-config/Containerfile
+            platforms: linux/amd64,linux/arm64
+            trivy: true
           - name: unbounded-storage-supervisor
             file: images/unbounded-storage-supervisor/Containerfile
             platforms: linux/amd64,linux/arm64

Makefile

@@ -148,6 +148,7 @@ GANTRY_CHART_APP_VERSION ?= $(VERSION_TAG)
 GANTRY_CHART_PACKAGE_DIR := build/charts
 GANTRY_CHART_STAGE_DIR := tmp/gantry-chart-package
 GANTRY_CHART_IMAGE_REPOSITORY ?= $(CONTAINER_REGISTRY)/gantry
+GANTRY_NODE_CONFIG_IMAGE ?= $(CONTAINER_REGISTRY)/gantry-node-config:$(VERSION_TAG)
 
 # unbounded-storage-supervisor (Go binary; distinct from the Rust crate below)
 UNBOUNDED_STORAGE_SUPERVISOR_BIN=bin/unbounded-storage-supervisor
@@ -291,7 +292,7 @@ NET_FRONTEND_CACHE_FILE    := $(NET_FRONTEND_DIST_DIR)/.frontend-build-key
 # Frontend build toggle (dev builds produce unminified output with sourcemaps).
 REACT_DEV ?= false
 
-.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests
+.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm install-kind generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests
 .PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest
 .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local
 .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push
@@ -312,6 +313,7 @@ help: ## Show this help
 	@echo "  install-tools                    Install gofumpt, golangci-lint, protoc-gen-go, protoc-gen-go-grpc, controller-gen, actionlint"
 	@echo "  install-protoc                   Download pinned protoc into bin/protoc/"
 	@echo "  install-helm                     Download pinned Helm into bin/"
+	@echo "  install-kind                     Install pinned kind into bin/"
 	@echo ""
 	@echo "Development:"
 	@echo "  fmt                              Format Go source (gofumpt + wsl_v5)"
@@ -464,6 +466,10 @@ ACTIONLINT_VERSION ?= v1.7.12
 HELM_VERSION ?= 3.21.3
 HELM ?= $(CURDIR)/bin/helm
 HELM_STAMP := $(CURDIR)/bin/.helm-v$(HELM_VERSION)
+KIND_VERSION ?= v0.30.0
+KIND ?= $(CURDIR)/bin/kind
+KIND_STAMP := $(CURDIR)/bin/.kind-$(KIND_VERSION)
+GANTRY_E2E_RUN ?= .
 
 HELM_UNAME_S := $(shell uname -s)
 HELM_UNAME_M := $(shell uname -m)
@@ -529,6 +535,17 @@ install-protoc: $(PROTOC) ## Download pinned protoc into bin/protoc/
 
 install-helm: $(HELM) ## Download pinned Helm into bin/
 
+install-kind: $(KIND) ## Install pinned kind into bin/
+
+$(KIND_STAMP):
+	@mkdir -p $(dir $(KIND))
+	GOBIN=$(CURDIR)/bin $(GOCMD) install sigs.k8s.io/kind@$(KIND_VERSION)
+	@$(KIND) version
+	@touch $(KIND_STAMP)
+
+$(KIND): $(KIND_STAMP)
+	@test -x $(KIND) || { rm -f $(KIND_STAMP); $(MAKE) $(KIND_STAMP); }
+
 $(HELM_STAMP):
 	@test -n "$(HELM_SHA256)" || { echo "unsupported Helm platform $(HELM_OS)-$(HELM_ARCH)" >&2; exit 1; }
 	@mkdir -p $(dir $(HELM)) tmp
@@ -618,9 +635,16 @@ test: lint machina-manifests token-refresher-manifests machine-ops-manifests pla
 
 endif
 
-e2e-gantry: $(HELM) ## Run the kind-based Gantry e2e suite
-	CONTAINER_ENGINE="$(CONTAINER_ENGINE)" KIND_EXPERIMENTAL_PROVIDER="$(CONTAINER_ENGINE)" PATH="$(CURDIR)/bin:$$PATH" \
-		$(GOTEST) -tags=e2e -count=1 -timeout=120m -v ./e2e/gantry
+e2e-gantry: $(HELM) $(KIND) ## Run the kind-based Gantry e2e suite
+	@set -eu; \
+	if [ "$(CONTAINER_ENGINE)" = "podman" ] && [ "$$($(CONTAINER_ENGINE) info --format '{{.Host.Security.Rootless}}')" = "true" ]; then \
+		command -v systemd-run >/dev/null 2>&1 || { echo "rootless Podman requires systemd-run" >&2; exit 1; }; \
+		exec systemd-run --user --scope --quiet -p Delegate=yes \
+			env CONTAINER_ENGINE="$(CONTAINER_ENGINE)" KIND_EXPERIMENTAL_PROVIDER="$(CONTAINER_ENGINE)" PATH="$(CURDIR)/bin:$$PATH" \
+			$(GOTEST) -tags=e2e -count=1 -timeout=120m -run '$(GANTRY_E2E_RUN)' -v ./e2e/gantry; \
+	fi; \
+	exec env CONTAINER_ENGINE="$(CONTAINER_ENGINE)" KIND_EXPERIMENTAL_PROVIDER="$(CONTAINER_ENGINE)" PATH="$(CURDIR)/bin:$$PATH" \
+		$(GOTEST) -tags=e2e -count=1 -timeout=120m -run '$(GANTRY_E2E_RUN)' -v ./e2e/gantry
 
 e2e-playpen: ## Run the kind-based playpen e2e suite
 	$(GOTEST) -tags=e2e ./e2e/playpen -v -timeout=10m
@@ -1433,6 +1457,16 @@ image-gantry-local: ## Build the gantry container image locally (single-arch)
 image-gantry-push: image-gantry-local ## Build and push the gantry container image
 	$(CONTAINER_ENGINE) push $(GANTRY_IMAGE)
 
+.PHONY: image-gantry-node-config-local image-gantry-node-config-push
+image-gantry-node-config-local: ## Build the Gantry OverlayBD node configurator image locally
+	$(CONTAINER_ENGINE) build \
+		-t gantry-node-config:$(VERSION_TAG) -t $(GANTRY_NODE_CONFIG_IMAGE) \
+		-f ./images/gantry-node-config/Containerfile .
+	$(call trivy-maybe,$(GANTRY_NODE_CONFIG_IMAGE))
+
+image-gantry-node-config-push: image-gantry-node-config-local ## Build and push the Gantry OverlayBD node configurator image
+	$(CONTAINER_ENGINE) push $(GANTRY_NODE_CONFIG_IMAGE)
+
 ##@ Orca
 
 .PHONY: orca orca-build orca-manifests orca-oci orca-oci-push \
@@ -1583,7 +1617,7 @@ images-net-all: image-net-controller-local image-net-node-local ## Build all unb
 
 images-net-all-push: image-net-controller-push image-net-node-push ## Build and push all unbounded-net container images
 
-images-local: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-storage-supervisor-local image-unbounded-operator-local image-net-controller-local image-net-node-local image-gantry-local ## Build all container images locally
+images-local: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-storage-supervisor-local image-unbounded-operator-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-node-config-local ## Build all container images locally
 
 ##@ Net Frontend
 

api/machina/v1alpha3/site_types.go

@@ -196,6 +196,27 @@ type StorageComponentSpec struct {
 // sets enabled to false.
 type GantryComponentSpec struct {
 	SiteComponentSpec `json:",inline"`
+
+	// ArtifactStreaming configures Gantry as the node-local range proxy for
+	// AKS ACR Artifact Streaming. It is disabled when omitted. Because Gantry is
+	// a cluster singleton, every Site that enables it must use the same node
+	// selector.
+	// +optional
+	ArtifactStreaming *GantryArtifactStreamingSpec `json:"artifactStreaming,omitempty"`
+}
+
+// GantryArtifactStreamingSpec selects the AKS nodes whose Azure-managed
+// OverlayBD installation should be configured to use Gantry.
+type GantryArtifactStreamingSpec struct {
+	// Enabled controls the Gantry range endpoint and OverlayBD configurator.
+	// +optional
+	Enabled bool `json:"enabled,omitempty"`
+
+	// NodeSelector must select only AKS node pools with Artifact Streaming
+	// enabled. It is required when Enabled is true.
+	// +kubebuilder:validation:MinProperties=1
+	// +optional
+	NodeSelector map[string]string `json:"nodeSelector,omitempty"`
 }
 
 // TokenRefresherComponentSpec configures bootstrap token refresh for a Site.

cmd/gantry/agent_metrics.go

@@ -234,6 +234,63 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics {
 	return p
 }
 
+type artifactStreamingMetrics struct {
+	requests  *prometheus.CounterVec
+	bytes     *prometheus.CounterVec
+	duration  *prometheus.HistogramVec
+	firstByte *prometheus.HistogramVec
+	rejected  *prometheus.CounterVec
+	inflight  *prometheus.GaugeVec
+}
+
+func newArtifactStreamingMetrics(reg *metrics.Registry) *artifactStreamingMetrics {
+	m := &artifactStreamingMetrics{
+		requests: reg.NewCounterVec("streaming", prometheus.CounterOpts{
+			Name: "gantry_streaming_requests_total",
+			Help: "OverlayBD range requests completed, labeled by bounded source and outcome.",
+		}, []string{"source", "outcome"}),
+		bytes: reg.NewCounterVec("streaming", prometheus.CounterOpts{
+			Name: "gantry_streaming_bytes_total",
+			Help: "OverlayBD range bytes served from complete local blobs, complete peers, or signed origin.",
+		}, []string{"source"}),
+		duration: reg.NewHistogramVec("streaming", prometheus.HistogramOpts{
+			Name:    "gantry_streaming_request_duration_seconds",
+			Help:    "End-to-end OverlayBD range request duration by source and outcome.",
+			Buckets: prometheus.ExponentialBuckets(0.001, 2, 16),
+		}, []string{"source", "outcome"}),
+		firstByte: reg.NewHistogramVec("streaming", prometheus.HistogramOpts{
+			Name:    "gantry_streaming_time_to_first_byte_seconds",
+			Help:    "Time until Gantry commits response headers for an OverlayBD range.",
+			Buckets: prometheus.ExponentialBuckets(0.001, 2, 16),
+		}, []string{"source"}),
+		rejected: reg.NewCounterVec("streaming", prometheus.CounterOpts{
+			Name: "gantry_streaming_rejected_total",
+			Help: "OverlayBD requests rejected before source selection, by bounded reason.",
+		}, []string{"reason"}),
+		inflight: reg.NewGaugeVec("streaming", prometheus.GaugeOpts{
+			Name: "gantry_streaming_inflight",
+			Help: "OverlayBD response bodies currently being served by source.",
+		}, []string{"source"}),
+	}
+
+	for _, source := range []string{"local", "peer", "origin"} {
+		m.bytes.WithLabelValues(source).Add(0)
+		m.firstByte.WithLabelValues(source)
+		m.inflight.WithLabelValues(source).Set(0)
+
+		for _, outcome := range []string{"success", "error"} {
+			m.requests.WithLabelValues(source, outcome).Add(0)
+			m.duration.WithLabelValues(source, outcome)
+		}
+	}
+
+	for _, reason := range []string{"range", "origin_url"} {
+		m.rejected.WithLabelValues(reason).Add(0)
+	}
+
+	return m
+}
+
 type layerProgressTracker struct {
 	mu              sync.Mutex
 	gauge           *prometheus.GaugeVec

cmd/gantry/agent_shutdown.go

@@ -20,6 +20,7 @@ import (
 type shutdownDeps struct {
 	logger         *slog.Logger
 	mirrorSrv      *mirror.Server
+	streamingSrv   interface{ Drain() }
 	transferStop   func(context.Context) error
 	mirrorStop     func(context.Context) error
 	cdsubSrc       cdsub.ImageSource
@@ -54,6 +55,10 @@ func gracefulShutdown(d shutdownDeps) {
 
 	d.mirrorSrv.Drain()
 
+	if d.streamingSrv != nil {
+		d.streamingSrv.Drain()
+	}
+
 	if err := d.transferStop(shutdownCtx); err != nil {
 		d.logger.Warn("transfer shutdown error", slog.Any("err", err))
 	}

cmd/gantry/agent_streaming.go

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"net/http"
+	"strings"
+
+	streamingapi "github.com/Azure/unbounded/internal/gantry/streaming"
+)
+
+// routeNodeLocalHandlers dispatches /blobs/ before the mirror's ServeMux can
+// clean repeated slashes in the embedded origin URL.
+func routeNodeLocalHandlers(streamingHandler, mirrorHandler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.HasPrefix(r.RequestURI, streamingapi.HandlerPrefix) || r.URL.Path == streamingapi.ReadinessPath {
+			streamingHandler.ServeHTTP(w, r)
+
+			return
+		}
+
+		mirrorHandler.ServeHTTP(w, r)
+	})
+}

cmd/gantry/agent_streaming_test.go

@@ -0,0 +1,58 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestRouteNodeLocalHandlersPreservesRawStreamingTarget(t *testing.T) {
+	t.Parallel()
+
+	rawTarget := "/blobs/https://data.example/account//docker/registry/v2/blobs/sha256/ab/value/data?sig=a%2Bb%2Fc%3D"
+	streamingCalled := false
+	streamingHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		streamingCalled = true
+
+		if r.RequestURI != rawTarget {
+			t.Fatalf("RequestURI = %q, want %q", r.RequestURI, rawTarget)
+		}
+
+		w.WriteHeader(http.StatusNoContent)
+	})
+	mirrorHandler := http.HandlerFunc(func(http.ResponseWriter, *http.Request) {
+		t.Fatal("mirror handler called for streaming request")
+	})
+
+	req := httptest.NewRequest(http.MethodGet, rawTarget, nil)
+	response := httptest.NewRecorder()
+	routeNodeLocalHandlers(streamingHandler, mirrorHandler).ServeHTTP(response, req)
+
+	if !streamingCalled || response.Code != http.StatusNoContent {
+		t.Fatalf("called/status = %v/%d, want true/204", streamingCalled, response.Code)
+	}
+}
+
+func TestRouteNodeLocalHandlersFallsBackToMirror(t *testing.T) {
+	t.Parallel()
+
+	mirrorCalled := false
+	handler := routeNodeLocalHandlers(
+		http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("streaming handler called") }),
+		http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+			mirrorCalled = true
+
+			w.WriteHeader(http.StatusOK)
+		}),
+	)
+
+	response := httptest.NewRecorder()
+	handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/v2/", nil))
+
+	if !mirrorCalled || response.Code != http.StatusOK {
+		t.Fatalf("called/status = %v/%d, want true/200", mirrorCalled, response.Code)
+	}
+}