diff --git a/.github/workflows/gantry-e2e.yaml b/.github/workflows/gantry-e2e.yaml index 80b511c52..cfdcc1cc4 100644 --- a/.github/workflows/gantry-e2e.yaml +++ b/.github/workflows/gantry-e2e.yaml @@ -10,6 +10,7 @@ on: - internal/gantry/** - deploy/gantry/** - images/gantry/** + - images/gantry-node-config/** - e2e/gantry/** - hack/cmd/render-manifests/** - go.mod @@ -25,6 +26,7 @@ on: - internal/gantry/** - deploy/gantry/** - images/gantry/** + - images/gantry-node-config/** - e2e/gantry/** - hack/cmd/render-manifests/** - go.mod @@ -62,7 +64,9 @@ jobs: cache-dependency-path: go.sum - name: Install kind - run: go install sigs.k8s.io/kind@v0.30.0 + run: | + make install-kind + echo "$PWD/bin" >> "$GITHUB_PATH" # The suite skips when prerequisites are absent. Verify them first so CI # cannot report success without creating a cluster. diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 0c7cfa32f..c441526ba 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -266,6 +266,7 @@ jobs: # - netboot : PXE artifact referenced by metalman # - unbounded-storage-supervisor : per-Site storage supervisor DaemonSet # - gantry : cluster-wide gantry DaemonSet + # - gantry-node-config : optional AKS OverlayBD configurator # - orca : deployed separately by deploy-orca # # machine-ops-controller is omitted on purpose: the operator synthesizes @@ -294,6 +295,8 @@ jobs: file: images/unbounded-operator/Containerfile - name: gantry file: images/gantry/Containerfile + - name: gantry-node-config + file: images/gantry-node-config/Containerfile - name: orca file: images/orca/Containerfile - name: inventory-aggregator diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index cc5afd8be..3c6193cd7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -821,7 +821,7 @@ jobs: needs: version runs-on: ubuntu-latest strategy: - # One bad component must not cancel the other ten. Runner minutes are the + # One bad component must not cancel the other eleven. Runner minutes are the # wrong thing to optimize on a release: knowing the full blast radius in # one pass beats discovering it one re-run at a time, and it keeps the # re-run down to the job that actually failed. When v0.5.0's @@ -854,6 +854,10 @@ jobs: file: images/gantry/Containerfile platforms: linux/amd64,linux/arm64 trivy: true + - name: gantry-node-config + file: images/gantry-node-config/Containerfile + platforms: linux/amd64,linux/arm64 + trivy: true - name: unbounded-storage-supervisor file: images/unbounded-storage-supervisor/Containerfile platforms: linux/amd64,linux/arm64 diff --git a/Makefile b/Makefile index 8dcfe162f..214195dd8 100644 --- a/Makefile +++ b/Makefile @@ -148,6 +148,7 @@ GANTRY_CHART_APP_VERSION ?= $(VERSION_TAG) GANTRY_CHART_PACKAGE_DIR := build/charts GANTRY_CHART_STAGE_DIR := tmp/gantry-chart-package GANTRY_CHART_IMAGE_REPOSITORY ?= $(CONTAINER_REGISTRY)/gantry +GANTRY_NODE_CONFIG_IMAGE ?= $(CONTAINER_REGISTRY)/gantry-node-config:$(VERSION_TAG) # unbounded-storage-supervisor (Go binary; distinct from the Rust crate below) UNBOUNDED_STORAGE_SUPERVISOR_BIN=bin/unbounded-storage-supervisor @@ -291,7 +292,7 @@ NET_FRONTEND_CACHE_FILE := $(NET_FRONTEND_DIST_DIR)/.frontend-build-key # Frontend build toggle (dev builds produce unminified output with sourcemaps). REACT_DEV ?= false -.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests +.PHONY: all help fmt lint lint-actions test build vulncheck check-deps kubectl-unbounded kubectl-unbounded-build install-tools install-protoc install-helm install-kind generate kubectl-unbounded forge relctl relctl-build agent-artifacts-builder agent-artifacts-builder-build orcadev unbounded-agent machina machina-build machina-oci machina-oci-push machina-manifests machine-ops-controller machine-ops-controller-build machine-ops-controller-oci machine-ops-controller-oci-push machine-ops-manifests metalman metalman-build metalman-oci metalman-oci-push unbounded-operator unbounded-operator-build unbounded-operator-manifests playpen-manifests e2e-gantry e2e-playpen gomod docs-serve unbounded-net-controller unbounded-net-controller-build unbounded-net-node unbounded-net-node-build unbounded-net-routeplan-debug unping unping-build unroute unroute-build license-check notice notice-check gantry gantry-build gantry-manifests inventory-manifests .PHONY: net-frontend net-frontend-clean net-ebpf-build net-ebpf-generate net-ebpf-verify net-manifests gantry-chart-lint gantry-chart-package release-bom release-manifests unbounded-operator-release-manifest .PHONY: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-operator-local image-unbounded-operator-push image-playpen-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-push images-local .PHONY: image-net-controller-push image-net-node-push images-net-all images-net-all-push @@ -312,6 +313,7 @@ help: ## Show this help @echo " install-tools Install gofumpt, golangci-lint, protoc-gen-go, protoc-gen-go-grpc, controller-gen, actionlint" @echo " install-protoc Download pinned protoc into bin/protoc/" @echo " install-helm Download pinned Helm into bin/" + @echo " install-kind Install pinned kind into bin/" @echo "" @echo "Development:" @echo " fmt Format Go source (gofumpt + wsl_v5)" @@ -464,6 +466,10 @@ ACTIONLINT_VERSION ?= v1.7.12 HELM_VERSION ?= 3.21.3 HELM ?= $(CURDIR)/bin/helm HELM_STAMP := $(CURDIR)/bin/.helm-v$(HELM_VERSION) +KIND_VERSION ?= v0.30.0 +KIND ?= $(CURDIR)/bin/kind +KIND_STAMP := $(CURDIR)/bin/.kind-$(KIND_VERSION) +GANTRY_E2E_RUN ?= . HELM_UNAME_S := $(shell uname -s) HELM_UNAME_M := $(shell uname -m) @@ -529,6 +535,17 @@ install-protoc: $(PROTOC) ## Download pinned protoc into bin/protoc/ install-helm: $(HELM) ## Download pinned Helm into bin/ +install-kind: $(KIND) ## Install pinned kind into bin/ + +$(KIND_STAMP): + @mkdir -p $(dir $(KIND)) + GOBIN=$(CURDIR)/bin $(GOCMD) install sigs.k8s.io/kind@$(KIND_VERSION) + @$(KIND) version + @touch $(KIND_STAMP) + +$(KIND): $(KIND_STAMP) + @test -x $(KIND) || { rm -f $(KIND_STAMP); $(MAKE) $(KIND_STAMP); } + $(HELM_STAMP): @test -n "$(HELM_SHA256)" || { echo "unsupported Helm platform $(HELM_OS)-$(HELM_ARCH)" >&2; exit 1; } @mkdir -p $(dir $(HELM)) tmp @@ -618,9 +635,16 @@ test: lint machina-manifests token-refresher-manifests machine-ops-manifests pla endif -e2e-gantry: $(HELM) ## Run the kind-based Gantry e2e suite - CONTAINER_ENGINE="$(CONTAINER_ENGINE)" KIND_EXPERIMENTAL_PROVIDER="$(CONTAINER_ENGINE)" PATH="$(CURDIR)/bin:$$PATH" \ - $(GOTEST) -tags=e2e -count=1 -timeout=120m -v ./e2e/gantry +e2e-gantry: $(HELM) $(KIND) ## Run the kind-based Gantry e2e suite + @set -eu; \ + if [ "$(CONTAINER_ENGINE)" = "podman" ] && [ "$$($(CONTAINER_ENGINE) info --format '{{.Host.Security.Rootless}}')" = "true" ]; then \ + command -v systemd-run >/dev/null 2>&1 || { echo "rootless Podman requires systemd-run" >&2; exit 1; }; \ + exec systemd-run --user --scope --quiet -p Delegate=yes \ + env CONTAINER_ENGINE="$(CONTAINER_ENGINE)" KIND_EXPERIMENTAL_PROVIDER="$(CONTAINER_ENGINE)" PATH="$(CURDIR)/bin:$$PATH" \ + $(GOTEST) -tags=e2e -count=1 -timeout=120m -run '$(GANTRY_E2E_RUN)' -v ./e2e/gantry; \ + fi; \ + exec env CONTAINER_ENGINE="$(CONTAINER_ENGINE)" KIND_EXPERIMENTAL_PROVIDER="$(CONTAINER_ENGINE)" PATH="$(CURDIR)/bin:$$PATH" \ + $(GOTEST) -tags=e2e -count=1 -timeout=120m -run '$(GANTRY_E2E_RUN)' -v ./e2e/gantry e2e-playpen: ## Run the kind-based playpen e2e suite $(GOTEST) -tags=e2e ./e2e/playpen -v -timeout=10m @@ -1433,6 +1457,16 @@ image-gantry-local: ## Build the gantry container image locally (single-arch) image-gantry-push: image-gantry-local ## Build and push the gantry container image $(CONTAINER_ENGINE) push $(GANTRY_IMAGE) +.PHONY: image-gantry-node-config-local image-gantry-node-config-push +image-gantry-node-config-local: ## Build the Gantry OverlayBD node configurator image locally + $(CONTAINER_ENGINE) build \ + -t gantry-node-config:$(VERSION_TAG) -t $(GANTRY_NODE_CONFIG_IMAGE) \ + -f ./images/gantry-node-config/Containerfile . + $(call trivy-maybe,$(GANTRY_NODE_CONFIG_IMAGE)) + +image-gantry-node-config-push: image-gantry-node-config-local ## Build and push the Gantry OverlayBD node configurator image + $(CONTAINER_ENGINE) push $(GANTRY_NODE_CONFIG_IMAGE) + ##@ Orca .PHONY: orca orca-build orca-manifests orca-oci orca-oci-push \ @@ -1583,7 +1617,7 @@ images-net-all: image-net-controller-local image-net-node-local ## Build all unb images-net-all-push: image-net-controller-push image-net-node-push ## Build and push all unbounded-net container images -images-local: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-storage-supervisor-local image-unbounded-operator-local image-net-controller-local image-net-node-local image-gantry-local ## Build all container images locally +images-local: image-machina-local image-token-refresher-local image-machine-ops-controller-local image-metalman-local image-unbounded-storage-supervisor-local image-unbounded-operator-local image-net-controller-local image-net-node-local image-gantry-local image-gantry-node-config-local ## Build all container images locally ##@ Net Frontend diff --git a/api/machina/v1alpha3/site_types.go b/api/machina/v1alpha3/site_types.go index 1a8174c61..fcf38c88d 100644 --- a/api/machina/v1alpha3/site_types.go +++ b/api/machina/v1alpha3/site_types.go @@ -196,6 +196,27 @@ type StorageComponentSpec struct { // sets enabled to false. type GantryComponentSpec struct { SiteComponentSpec `json:",inline"` + + // ArtifactStreaming configures Gantry as the node-local range proxy for + // AKS ACR Artifact Streaming. It is disabled when omitted. Because Gantry is + // a cluster singleton, every Site that enables it must use the same node + // selector. + // +optional + ArtifactStreaming *GantryArtifactStreamingSpec `json:"artifactStreaming,omitempty"` +} + +// GantryArtifactStreamingSpec selects the AKS nodes whose Azure-managed +// OverlayBD installation should be configured to use Gantry. +type GantryArtifactStreamingSpec struct { + // Enabled controls the Gantry range endpoint and OverlayBD configurator. + // +optional + Enabled bool `json:"enabled,omitempty"` + + // NodeSelector must select only AKS node pools with Artifact Streaming + // enabled. It is required when Enabled is true. + // +kubebuilder:validation:MinProperties=1 + // +optional + NodeSelector map[string]string `json:"nodeSelector,omitempty"` } // TokenRefresherComponentSpec configures bootstrap token refresh for a Site. diff --git a/api/machina/v1alpha3/zz_generated.deepcopy.go b/api/machina/v1alpha3/zz_generated.deepcopy.go index a4ca42681..214c85010 100644 --- a/api/machina/v1alpha3/zz_generated.deepcopy.go +++ b/api/machina/v1alpha3/zz_generated.deepcopy.go @@ -224,10 +224,37 @@ func (in *ExternalHostSpec) DeepCopy() *ExternalHostSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GantryArtifactStreamingSpec) DeepCopyInto(out *GantryArtifactStreamingSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GantryArtifactStreamingSpec. +func (in *GantryArtifactStreamingSpec) DeepCopy() *GantryArtifactStreamingSpec { + if in == nil { + return nil + } + out := new(GantryArtifactStreamingSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GantryComponentSpec) DeepCopyInto(out *GantryComponentSpec) { *out = *in in.SiteComponentSpec.DeepCopyInto(&out.SiteComponentSpec) + if in.ArtifactStreaming != nil { + in, out := &in.ArtifactStreaming, &out.ArtifactStreaming + *out = new(GantryArtifactStreamingSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GantryComponentSpec. diff --git a/cmd/gantry/agent_metrics.go b/cmd/gantry/agent_metrics.go index 4aa36da96..f8a252c59 100644 --- a/cmd/gantry/agent_metrics.go +++ b/cmd/gantry/agent_metrics.go @@ -234,6 +234,63 @@ func newPhase2Metrics(reg *metrics.Registry) *phase2Metrics { return p } +type artifactStreamingMetrics struct { + requests *prometheus.CounterVec + bytes *prometheus.CounterVec + duration *prometheus.HistogramVec + firstByte *prometheus.HistogramVec + rejected *prometheus.CounterVec + inflight *prometheus.GaugeVec +} + +func newArtifactStreamingMetrics(reg *metrics.Registry) *artifactStreamingMetrics { + m := &artifactStreamingMetrics{ + requests: reg.NewCounterVec("streaming", prometheus.CounterOpts{ + Name: "gantry_streaming_requests_total", + Help: "OverlayBD range requests completed, labeled by bounded source and outcome.", + }, []string{"source", "outcome"}), + bytes: reg.NewCounterVec("streaming", prometheus.CounterOpts{ + Name: "gantry_streaming_bytes_total", + Help: "OverlayBD range bytes served from complete local blobs, complete peers, or signed origin.", + }, []string{"source"}), + duration: reg.NewHistogramVec("streaming", prometheus.HistogramOpts{ + Name: "gantry_streaming_request_duration_seconds", + Help: "End-to-end OverlayBD range request duration by source and outcome.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 16), + }, []string{"source", "outcome"}), + firstByte: reg.NewHistogramVec("streaming", prometheus.HistogramOpts{ + Name: "gantry_streaming_time_to_first_byte_seconds", + Help: "Time until Gantry commits response headers for an OverlayBD range.", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 16), + }, []string{"source"}), + rejected: reg.NewCounterVec("streaming", prometheus.CounterOpts{ + Name: "gantry_streaming_rejected_total", + Help: "OverlayBD requests rejected before source selection, by bounded reason.", + }, []string{"reason"}), + inflight: reg.NewGaugeVec("streaming", prometheus.GaugeOpts{ + Name: "gantry_streaming_inflight", + Help: "OverlayBD response bodies currently being served by source.", + }, []string{"source"}), + } + + for _, source := range []string{"local", "peer", "origin"} { + m.bytes.WithLabelValues(source).Add(0) + m.firstByte.WithLabelValues(source) + m.inflight.WithLabelValues(source).Set(0) + + for _, outcome := range []string{"success", "error"} { + m.requests.WithLabelValues(source, outcome).Add(0) + m.duration.WithLabelValues(source, outcome) + } + } + + for _, reason := range []string{"range", "origin_url"} { + m.rejected.WithLabelValues(reason).Add(0) + } + + return m +} + type layerProgressTracker struct { mu sync.Mutex gauge *prometheus.GaugeVec diff --git a/cmd/gantry/agent_shutdown.go b/cmd/gantry/agent_shutdown.go index be9b23709..df51053df 100644 --- a/cmd/gantry/agent_shutdown.go +++ b/cmd/gantry/agent_shutdown.go @@ -20,6 +20,7 @@ import ( type shutdownDeps struct { logger *slog.Logger mirrorSrv *mirror.Server + streamingSrv interface{ Drain() } transferStop func(context.Context) error mirrorStop func(context.Context) error cdsubSrc cdsub.ImageSource @@ -54,6 +55,10 @@ func gracefulShutdown(d shutdownDeps) { d.mirrorSrv.Drain() + if d.streamingSrv != nil { + d.streamingSrv.Drain() + } + if err := d.transferStop(shutdownCtx); err != nil { d.logger.Warn("transfer shutdown error", slog.Any("err", err)) } diff --git a/cmd/gantry/agent_streaming.go b/cmd/gantry/agent_streaming.go new file mode 100644 index 000000000..5c11c9d15 --- /dev/null +++ b/cmd/gantry/agent_streaming.go @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net/http" + "strings" + + streamingapi "github.com/Azure/unbounded/internal/gantry/streaming" +) + +// routeNodeLocalHandlers dispatches /blobs/ before the mirror's ServeMux can +// clean repeated slashes in the embedded origin URL. +func routeNodeLocalHandlers(streamingHandler, mirrorHandler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasPrefix(r.RequestURI, streamingapi.HandlerPrefix) || r.URL.Path == streamingapi.ReadinessPath { + streamingHandler.ServeHTTP(w, r) + + return + } + + mirrorHandler.ServeHTTP(w, r) + }) +} diff --git a/cmd/gantry/agent_streaming_test.go b/cmd/gantry/agent_streaming_test.go new file mode 100644 index 000000000..294edd4dc --- /dev/null +++ b/cmd/gantry/agent_streaming_test.go @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestRouteNodeLocalHandlersPreservesRawStreamingTarget(t *testing.T) { + t.Parallel() + + rawTarget := "/blobs/https://data.example/account//docker/registry/v2/blobs/sha256/ab/value/data?sig=a%2Bb%2Fc%3D" + streamingCalled := false + streamingHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + streamingCalled = true + + if r.RequestURI != rawTarget { + t.Fatalf("RequestURI = %q, want %q", r.RequestURI, rawTarget) + } + + w.WriteHeader(http.StatusNoContent) + }) + mirrorHandler := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("mirror handler called for streaming request") + }) + + req := httptest.NewRequest(http.MethodGet, rawTarget, nil) + response := httptest.NewRecorder() + routeNodeLocalHandlers(streamingHandler, mirrorHandler).ServeHTTP(response, req) + + if !streamingCalled || response.Code != http.StatusNoContent { + t.Fatalf("called/status = %v/%d, want true/204", streamingCalled, response.Code) + } +} + +func TestRouteNodeLocalHandlersFallsBackToMirror(t *testing.T) { + t.Parallel() + + mirrorCalled := false + handler := routeNodeLocalHandlers( + http.HandlerFunc(func(http.ResponseWriter, *http.Request) { t.Fatal("streaming handler called") }), + http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mirrorCalled = true + + w.WriteHeader(http.StatusOK) + }), + ) + + response := httptest.NewRecorder() + handler.ServeHTTP(response, httptest.NewRequest(http.MethodGet, "/v2/", nil)) + + if !mirrorCalled || response.Code != http.StatusOK { + t.Fatalf("called/status = %v/%d, want true/200", mirrorCalled, response.Code) + } +} diff --git a/cmd/gantry/artifact_streaming_integration_test.go b/cmd/gantry/artifact_streaming_integration_test.go new file mode 100644 index 000000000..0f3a4d389 --- /dev/null +++ b/cmd/gantry/artifact_streaming_integration_test.go @@ -0,0 +1,328 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" //nolint:staticcheck // h2c is Gantry's peer protocol + + "github.com/Azure/unbounded/internal/gantry/advertise" + "github.com/Azure/unbounded/internal/gantry/chairs" + "github.com/Azure/unbounded/internal/gantry/coldstart" + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/httprange" + "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/ifaces/fakes" + "github.com/Azure/unbounded/internal/gantry/inflight" + "github.com/Azure/unbounded/internal/gantry/streaming" + "github.com/Azure/unbounded/internal/gantry/transfer" +) + +func TestArtifactStreamingManifestPullTransitionsOriginToPeer(t *testing.T) { + t.Parallel() + + layerBody := []byte("0123456789") + layerDigest := trackerDigestOf(layerBody) + configBody := []byte(`{"architecture":"amd64","os":"linux"}`) + configDigest := trackerDigestOf(configBody) + manifestBody := []byte(fmt.Sprintf(`{ + "schemaVersion":2, + "mediaType":"application/vnd.oci.image.manifest.v1+json", + "config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":%q,"size":%d}, + "layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":%q,"size":%d, + "annotations":{"containerd.io/snapshot/overlaybd/blob-digest":%q,"containerd.io/snapshot/overlaybd/blob-size":%q}}] + }`, configDigest.String(), len(configBody), layerDigest.String(), len(layerBody), layerDigest.String(), fmt.Sprint(len(layerBody)))) + manifestDigest := trackerDigestOf(manifestBody) + + var originRangeRequests atomic.Int32 + + originServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + originRangeRequests.Add(1) + + if got := r.Header.Get("Range"); got != "bytes=2-5" { + t.Errorf("origin Range = %q, want bytes=2-5", got) + } + + w.Header().Set("Content-Range", "bytes 2-5/10") + w.Header().Set("Content-Length", "4") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(layerBody[2:6]) //nolint:errcheck // best-effort test response + })) + t.Cleanup(originServer.Close) + + peerCache := fakes.NewCache() + + var peerRangeRequests atomic.Int32 + + peerAddr := startArtifactStreamingPeer(t, transfer.New(peerCache, + transfer.WithMetrics(func() { peerRangeRequests.Add(1) }, nil), + ).Handler()) + + dht := newAdvertisingDHT(ifaces.Provider{NodeID: "chair-peer", Addr: peerAddr}) + advertiser := advertise.New(&artifactStreamingInventory{Cache: peerCache}, dht, + advertise.WithProvideTimeout(time.Second)) + + originPuller := fakes.NewOriginPuller() + originPuller.Put(configDigest, configBody) + originPuller.Put(layerDigest, layerBody) + coordinator := &artifactStreamingChairCoordinator{ + origin: originPuller, + cache: peerCache, + advertiser: advertiser, + inflight: inflight.New(inflight.DefaultStalls(), nil), + logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } + + const epoch = int64(7) + + chair := chairs.Chair{ + ID: 0, + Holder: ifaces.PeerEndpoint{ + PeerID: "chair-peer", + P2PAddrs: []string{"/ip4/127.0.0.1/tcp/1"}, + TransferAddr: peerAddr, + }, + Generation: 3, + AssignmentEpoch: epoch, + } + resolver := coldstart.NewChairResolver(coldstart.ChairOptions{ + Chairs: &artifactStreamingChairSnapshot{snapshot: chairs.Snapshot{Epoch: epoch, Chairs: []chairs.Chair{chair}}}, + Discovery: dht, + Coord: coordinator, + Inflight: inflight.New(inflight.DefaultStalls(), nil), + SelfPeerID: "requester", + CurrentEpoch: func() int64 { return epoch }, + QueryTimeout: time.Second, + APITimeout: time.Second, + SeedCount: 1, + PollManifest: time.Millisecond, + PollLayer: time.Millisecond, + }) + + manifestCache := fakes.NewCache() + manifestCache.Put(manifestDigest, manifestBody) + prefetcher := &layerPrefetchAdapter{cache: manifestCache, resolver: resolver, logger: slog.Default()} + + policy := streaming.URLPolicy{AllowedHostSuffixes: []string{"localhost"}, AllowHTTP: true} + + originClient := streaming.NewOriginClient(policy) + + streamServer, err := streaming.NewServer( + artifactStreamingMissingRangeStore{}, + dht, + transfer.NewClient(transfer.WithRequestTimeout(5*time.Second)), + originClient, + streaming.Options{URLPolicy: policy, PeerLookupTimeout: time.Second, MaxPeerAttempts: 1}, + ) + if err != nil { + t.Fatal(err) + } + + originURL := strings.Replace(originServer.URL, "127.0.0.1", "localhost", 1) + + "?d=" + layerDigest.String() + "&sig=redacted" + requestRange := func() *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, streaming.HandlerPrefix+originURL, nil) + req.Header.Set("Range", "bytes=2-5") + + response := httptest.NewRecorder() + streamServer.ServeHTTP(response, req) + + return response + } + + first := requestRange() + if first.Code != http.StatusPartialContent || first.Body.String() != "2345" { + t.Fatalf("origin response = %d %q, want 206 2345", first.Code, first.Body.String()) + } + + prefetcher.OnManifestServed(t.Context(), "app.azurecr.io", "team/app", manifestDigest) + + if !coordinator.pulled(layerDigest) { + t.Fatalf("streaming layer %s was not dispatched to the chair", layerDigest) + } + + if present, hasErr := peerCache.Has(t.Context(), layerDigest); hasErr != nil || !present { + t.Fatalf("chair cache Has(%s) = %t, %v; want true, nil", layerDigest, present, hasErr) + } + + if got := dht.provideCount(layerDigest); got != 1 { + t.Fatalf("DHT Provide(%s) calls = %d, want 1", layerDigest, got) + } + + second := requestRange() + if second.Code != http.StatusPartialContent || second.Body.String() != "2345" { + t.Fatalf("peer response = %d %q, want 206 2345", second.Code, second.Body.String()) + } + + if got := originRangeRequests.Load(); got != 1 { + t.Fatalf("origin range requests = %d, want 1 after peer advertisement", got) + } + + if got := peerRangeRequests.Load(); got != 1 { + t.Fatalf("peer range requests = %d, want 1", got) + } +} + +type artifactStreamingMissingRangeStore struct{} + +func (artifactStreamingMissingRangeStore) OpenRange(_ context.Context, d digest.Digest, _ httprange.Range) (io.ReadCloser, int64, error) { + return nil, 0, &ifaces.ErrNotFound{Digest: d} +} + +type artifactStreamingInventory struct { + *fakes.Cache +} + +func (*artifactStreamingInventory) Inventory(context.Context) ([]digest.Digest, error) { + return nil, nil +} + +type artifactStreamingChairSnapshot struct { + snapshot chairs.Snapshot +} + +func (s *artifactStreamingChairSnapshot) Snapshot(context.Context, int64) (chairs.Snapshot, error) { + return s.snapshot, nil +} + +func (s *artifactStreamingChairSnapshot) RefreshChair(_ context.Context, id chairs.ID) (chairs.Chair, error) { + for _, chair := range s.snapshot.Chairs { + if chair.ID == id { + return chair, nil + } + } + + return chairs.Chair{}, fmt.Errorf("chair %s not found", id.Name()) +} + +type artifactStreamingChairCoordinator struct { + mu sync.Mutex + origin ifaces.OriginPuller + cache ifaces.LocalContentStore + advertiser *advertise.Advertiser + inflight *inflight.Map + logger *slog.Logger + pulls []digest.Digest +} + +func (c *artifactStreamingChairCoordinator) PleasePullChair(ctx context.Context, _ ifaces.PeerEndpoint, registry, repository string, kind ifaces.OriginRefKind, digests []digest.Digest, _ ifaces.ChairAssignment) ([]ifaces.PleasePullOutcome, error) { + outcomes := make([]ifaces.PleasePullOutcome, 0, len(digests)) + for _, child := range digests { + c.mu.Lock() + c.pulls = append(c.pulls, child) + c.mu.Unlock() + + handle, existing, already := c.inflight.Start(child, kind, 0) + if already { + outcomes = append(outcomes, ifaces.PleasePullOutcome{Digest: child, Outcome: ifaces.PleasePullAlreadyPulling, StartedAt: existing.StartedAt}) + continue + } + + runOriginPull(ctx, c.origin, c.cache, nil, c.logger, handle, registry, repository, child, kind, time.Second, + func(markCtx context.Context, marked digest.Digest) bool { + return c.advertiser.Notify(markCtx, marked, true) + }, nil, nil, leaseMetricHooks{}) + outcomes = append(outcomes, ifaces.PleasePullOutcome{Digest: child, Outcome: ifaces.PleasePullStarted}) + } + + return outcomes, nil +} + +func (c *artifactStreamingChairCoordinator) pulled(want digest.Digest) bool { + c.mu.Lock() + defer c.mu.Unlock() + + for _, pulled := range c.pulls { + if pulled == want { + return true + } + } + + return false +} + +type advertisingDHT struct { + mu sync.Mutex + provider ifaces.Provider + provided map[digest.Digest]int +} + +func newAdvertisingDHT(provider ifaces.Provider) *advertisingDHT { + return &advertisingDHT{provider: provider, provided: map[digest.Digest]int{}} +} + +func (d *advertisingDHT) FindProviders(_ context.Context, child digest.Digest) ([]ifaces.Provider, error) { + d.mu.Lock() + defer d.mu.Unlock() + + if d.provided[child] == 0 { + return nil, nil + } + + return []ifaces.Provider{d.provider}, nil +} + +func (d *advertisingDHT) Provide(_ context.Context, child digest.Digest) error { + d.mu.Lock() + defer d.mu.Unlock() + + d.provided[child]++ + + return nil +} + +func (d *advertisingDHT) Withdraw(_ context.Context, child digest.Digest) error { + d.mu.Lock() + defer d.mu.Unlock() + + delete(d.provided, child) + + return nil +} + +func (*advertisingDHT) Health() float64 { return 1 } + +func (d *advertisingDHT) provideCount(child digest.Digest) int { + d.mu.Lock() + defer d.mu.Unlock() + + return d.provided[child] +} + +func startArtifactStreamingPeer(t *testing.T, handler http.Handler) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + httpServer := &http.Server{ + Handler: h2c.NewHandler(handler, &http2.Server{}), //nolint:staticcheck // h2c is Gantry's peer protocol + ReadHeaderTimeout: time.Second, + } + go func() { _ = httpServer.Serve(listener) }() //nolint:errcheck // test cleanup owns shutdown + + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _ = httpServer.Shutdown(ctx) //nolint:errcheck // best-effort test cleanup + }) + + return listener.Addr().String() +} diff --git a/cmd/gantry/main.go b/cmd/gantry/main.go index b6aa80176..e13bb3a25 100644 --- a/cmd/gantry/main.go +++ b/cmd/gantry/main.go @@ -49,6 +49,7 @@ import ( "github.com/Azure/unbounded/internal/gantry/mirror" "github.com/Azure/unbounded/internal/gantry/negcache" "github.com/Azure/unbounded/internal/gantry/registryauth" + streamingapi "github.com/Azure/unbounded/internal/gantry/streaming" "github.com/Azure/unbounded/internal/gantry/transfer" "github.com/Azure/unbounded/internal/version" ) @@ -148,6 +149,7 @@ func runAgent(args []string) error { reg.RegisterDefaultCollectors() inst := newPhase1Metrics(reg) p2 := newPhase2Metrics(reg) + streamingMetrics := newArtifactStreamingMetrics(reg) layerProgress := newLayerProgressTracker(p2.layerCompletedAt, c.NodeName, time.Now) p9 := newPhase9Metrics(reg) // Storage mode info: emit a single time-series at 1 for the @@ -457,12 +459,15 @@ func runAgent(args []string) error { InstallHolder: func(holder chairs.Holder) error { return installChairHolder(disco.LibP2P().Peerstore(), holder) }, - Claimer: chairManager, - Logger: logger, - APITimeout: c.ChairAPITimeout, - HolderCount: c.ChairHolderCount, - SeedCount: c.ChairSeedCount, - TrustedFailureClasses: configuredFailureClasses(c.OriginFailureClassesTrustedClusterWide), + Claimer: chairManager, + Logger: logger, + APITimeout: c.ChairAPITimeout, + HolderCount: c.ChairHolderCount, + SeedCount: c.ChairSeedCount, + PrefetchCoordinatorReplicas: c.PrefetchCoordinatorReplicas, + PrefetchMaxConcurrentGroups: c.PrefetchMaxConcurrentGroups, + PrefetchDispatchJitter: c.PrefetchDispatchJitter, + TrustedFailureClasses: configuredFailureClasses(c.OriginFailureClassesTrustedClusterWide), OnSeedRecruit: func(kind string, selectable, contacted, accepted int) { p3.coldStartSeedSelectable.WithLabelValues(kind).Observe(float64(selectable)) p3.coldStartSeedContacted.WithLabelValues(kind).Observe(float64(contacted)) @@ -474,6 +479,14 @@ func runAgent(args []string) error { OnChairCall: func(kind, outcome string, seconds float64) { p3.coldStartChairCallDur.WithLabelValues(kind, outcome).Observe(seconds) }, + OnPrefetchBatch: func(pullers, digests int) { + p3.prefetchBatchesTotal.Inc() + p3.prefetchDigestsTotal.Add(float64(digests)) + p3.prefetchPullersPerBatch.Observe(float64(pullers)) + }, + OnPrefetchGroup: func(target, outcome string) { + p3.prefetchGroupsTotal.WithLabelValues(target, outcome).Inc() + }, }) coldStartResolver = coldStartAdapter{r: realResolver} layerPrefetcher = newLayerPrefetcher(realResolver, cstore, logger, layerProgress.observeManifest) @@ -660,12 +673,54 @@ func runAgent(args []string) error { mirror.WithStartupReadinessGate(), ) - mirrorStop, err := mirrorSrv.ListenAndServe(c.MirrorListen) + var artifactStreamingSrv *streamingapi.Server + + mirrorHandler := mirrorSrv.Handler() + + if c.ArtifactStreamingEnabled { + urlPolicy := streamingapi.URLPolicy{AllowedHostSuffixes: c.ArtifactStreamingAllowedHostSuffixes} + + artifactStreamingSrv, err = streamingapi.NewServer(cdstore, disco, peerClient, streamingapi.NewOriginClient(urlPolicy), streamingapi.Options{ + URLPolicy: urlPolicy, + SelfPeerID: ifaces.NodeID(disco.PeerID().String()), + StartupGated: true, + Logger: logger, + Metrics: streamingapi.MetricsHooks{ + OnRequest: func(source, outcome string, duration time.Duration, bytes int64) { + streamingMetrics.requests.WithLabelValues(source, outcome).Inc() + streamingMetrics.duration.WithLabelValues(source, outcome).Observe(duration.Seconds()) + + if bytes > 0 { + streamingMetrics.bytes.WithLabelValues(source).Add(float64(bytes)) + } + }, + OnFirstByte: func(source string, duration time.Duration) { + streamingMetrics.firstByte.WithLabelValues(source).Observe(duration.Seconds()) + }, + OnInflight: func(source string, delta int) { + streamingMetrics.inflight.WithLabelValues(source).Add(float64(delta)) + }, + OnReject: func(reason string) { + streamingMetrics.rejected.WithLabelValues(reason).Inc() + }, + }, + }) + if err != nil { + return fmt.Errorf("artifact streaming server: %w", err) + } + + mirrorHandler = routeNodeLocalHandlers(artifactStreamingSrv, mirrorHandler) + } + + mirrorStop, err := mirrorSrv.ListenAndServeHandler(c.MirrorListen, mirrorHandler) if err != nil { return fmt.Errorf("mirror listen: %w", err) } - logger.Info("mirror endpoint listening", slog.String("addr", c.MirrorListen)) + logger.Info("mirror endpoint listening", + slog.String("addr", c.MirrorListen), + slog.Bool("artifact_streaming", artifactStreamingSrv != nil), + ) // /4 - cdsub event loop. cdsub no longer calls DHT.Provide // directly in containerd mode; every event is routed through the @@ -878,6 +933,11 @@ func runAgent(args []string) error { case <-t.C: if _, ok := readyCheck(); ok { mirrorSrv.MarkReady() + + if artifactStreamingSrv != nil { + artifactStreamingSrv.MarkReady() + } + logger.Info("mirror: startup gate released; /v2/ now serving") return @@ -923,6 +983,7 @@ func runAgent(args []string) error { gracefulShutdown(shutdownDeps{ logger: logger, mirrorSrv: mirrorSrv, + streamingSrv: artifactStreamingSrv, transferStop: transferStop, mirrorStop: mirrorStop, cdsubSrc: cdsubSrc, diff --git a/cmd/gantry/prefetch_manifest_test.go b/cmd/gantry/prefetch_manifest_test.go index 9f9fd8cd7..0ae368022 100644 --- a/cmd/gantry/prefetch_manifest_test.go +++ b/cmd/gantry/prefetch_manifest_test.go @@ -8,16 +8,33 @@ import ( "errors" "io" "log/slog" + "os" "strings" "sync" "testing" "time" + "github.com/Azure/unbounded/internal/gantry/coldstart" "github.com/Azure/unbounded/internal/gantry/digest" "github.com/Azure/unbounded/internal/gantry/ifaces" "github.com/Azure/unbounded/internal/gantry/manifest" ) +type manifestPrefetchResolverStub struct { + registry string + repository string + children []coldstart.ChildDigest +} + +func (s *manifestPrefetchResolverStub) PrefetchManifestChildren(_ context.Context, _ digest.Digest, children []coldstart.ChildDigest, registry, repository string) error { + s.registry = registry + s.repository = repository + + s.children = append([]coldstart.ChildDigest(nil), children...) + + return nil +} + // delayedManifestStore returns ErrNotFound until availableAfter opens, which // models containerd committing the streamed manifest a moment after the mirror // finishes serving it. @@ -146,3 +163,43 @@ func TestLayerPrefetchAdapterReportsManifestChildrenWithoutResolver(t *testing.T t.Fatalf("manifest callback = %s with %d children, want %s with 3", gotManifest, gotChildren, manifestDigest) } } + +func TestLayerPrefetchAdapterDispatchesStreamingManifestChildren(t *testing.T) { + manifestDigest := testDigest(t, "a") + configDigest := testDigest(t, "b") + layer0 := testDigest(t, "c") + layer1 := testDigest(t, "d") + + body, err := os.ReadFile("testdata/acr-streaming-manifest.json") + if err != nil { + t.Fatal(err) + } + + resolver := &manifestPrefetchResolverStub{} + adapter := &layerPrefetchAdapter{ + cache: &delayedManifestStore{body: string(body), ready: true}, + logger: slog.Default(), + resolver: resolver, + } + + adapter.OnManifestServed(context.Background(), "app.azurecr.io", "team/app", manifestDigest) + + if resolver.registry != "app.azurecr.io" || resolver.repository != "team/app" { + t.Fatalf("origin = %s/%s, want app.azurecr.io/team/app", resolver.registry, resolver.repository) + } + + want := []coldstart.ChildDigest{ + {Digest: configDigest, Kind: ifaces.KindConfig}, + {Digest: layer0, Kind: ifaces.KindBlob}, + {Digest: layer1, Kind: ifaces.KindBlob}, + } + if len(resolver.children) != len(want) { + t.Fatalf("children = %+v, want %+v", resolver.children, want) + } + + for index := range want { + if resolver.children[index] != want[index] { + t.Fatalf("child %d = %+v, want %+v", index, resolver.children[index], want[index]) + } + } +} diff --git a/cmd/gantry/testdata/acr-streaming-manifest.json b/cmd/gantry/testdata/acr-streaming-manifest.json new file mode 100644 index 000000000..2183a7b98 --- /dev/null +++ b/cmd/gantry/testdata/acr-streaming-manifest.json @@ -0,0 +1,29 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "size": 1024 + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "size": 1048576, + "annotations": { + "containerd.io/snapshot/overlaybd/blob-digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "containerd.io/snapshot/overlaybd/blob-size": "1048576" + } + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "size": 2097152, + "annotations": { + "containerd.io/snapshot/overlaybd/blob-digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "containerd.io/snapshot/overlaybd/blob-size": "2097152" + } + } + ] +} \ No newline at end of file diff --git a/deploy/gantry/README.md b/deploy/gantry/README.md index 9fbf3ef32..63e20b945 100644 --- a/deploy/gantry/README.md +++ b/deploy/gantry/README.md @@ -18,7 +18,7 @@ and examples used by development and benchmark tooling. | `chart/templates/serviceaccount.yaml` | `rendered/serviceaccount.yaml` | Namespace + ServiceAccount + Role + PriorityClass. | | `chart/templates/configmap.yaml` | `rendered/configmap.yaml` | Default `config.yaml` (mirrors `config.NewDefault()`). | | `chart/templates/rendezvous-leases.yaml` | `rendered/rendezvous-leases.yaml` | Fixed chair Lease set. | -| `chart/templates/node-config.yaml` | Standalone chart only | Continuously reconciles containerd's default Gantry mirror route. | +| `chart/templates/node-config.yaml` | Standalone chart only | Continuously reconciles containerd's Gantry mirror route. | | `examples/registry-secret.example.yaml.tmpl` | `rendered/examples/registry-secret.example.yaml` | Template Secret for upstream-registry credentials. | | `examples/networkpolicy.yaml.tmpl` | `rendered/examples/networkpolicy.yaml` | **Hardening overlay (NOT applied by default).** See [Hardening overlays](#hardening-overlays) below. | | `hosts.toml.template` | (not rendered) | containerd registry mirror config; one file per upstream registry under `/etc/containerd/certs.d//hosts.toml`. | @@ -28,9 +28,9 @@ and examples used by development and benchmark tooling. - Operator-managed clusters use the manifests embedded in the `unbounded-operator` binary. The operator never runs Helm. - Clusters without the operator install the released OCI chart. The chart - continuously reconciles `/etc/containerd/certs.d/_default/hosts.toml` on - every selected node. Containerd must already be configured to read - `/etc/containerd/certs.d`; the chart does not edit or restart containerd. + continuously reconciles its route under `/etc/containerd/certs.d` on every + selected node. Containerd must already be configured to read that directory; + the chart does not edit or restart containerd. The paths are mutually exclusive. `PriorityClass/gantry-low` records the active manager, and both installers reject ownership by the other path. @@ -81,11 +81,15 @@ the Gantry DaemonSet normally; the mirror activates when the pod starts listening on `127.0.0.1:5000`. Standalone Helm installations run `DaemonSet/gantry-containerd-config`. Its -resident reconciler checks the default `hosts.toml` every five seconds and -atomically restores the chart-owned payload when the file is missing or -different, including after a node upgrade resets host configuration. Graceful -shutdown removes the file only when it still matches the chart payload. Set -`nodeConfig.enabled=false` when another node-management system owns this file. +resident reconciler checks its `hosts.toml` files every five seconds and +atomically restores chart-owned payloads when they are missing or different, +including after a node upgrade resets host configuration. Graceful shutdown +removes a file only when it still matches the chart payload. Set +`nodeConfig.enabled=false` when another node-management system owns these files. + +The default profile owns `_default/hosts.toml`. Artifact Streaming instead owns +one registry-specific `/hosts.toml` for each configured upstream, +leaving AKS's `_default/hosts.toml` unchanged. Externally managed installations can instead drop a registry-specific `hosts.toml` at: @@ -98,6 +102,192 @@ derived from `hosts.toml.template` (substitute `${REGISTRY_SERVER}` with the registry's `https://...` URL). containerd reloads `certs.d` on its own; no restart needed. +## ACR Artifact Streaming + +This integration configures an existing AKS Artifact Streaming node pool. It +does not install OverlayBD or register its snapshotter with containerd. Enable +it only on nodes where AKS already provides +`/opt/acr/tools/overlaybd/config.sh`, `overlaybd-tcmu`, and +`overlaybd-snapshotter`. + +For a standalone Helm installation, enable the Gantry range endpoint and its +host configurator together. Point Gantry's OCI upstream at the node-local AKS +Artifact Streaming mirror and select only the streaming node pool: + +```yaml +nodeSelector: + gantry-streaming: "true" + +gantry: + artifactStreaming: + enabled: true + upstreamRegistries: + - name: .azurecr.io + endpoint: http://127.0.0.1:8578?ns=.azurecr.io + +overlaybdConfig: + enabled: true + nodeSelector: + gantry-streaming: "true" +``` + +In this mode Gantry uses host networking so it can reach the loopback-only AKS +mirror. Containerd resolves through Gantry on port 5000; Gantry forwards OCI +requests to the AKS mirror on port 8578, preserving its `ns` query. The AKS +mirror returns the OverlayBD manifest, which triggers Gantry's complete-layer +chair pulls. OverlayBD sends range reads to the same Gantry listener under +`/blobs/`, where the source order remains local, peer, then signed ACR origin. + +The registry-specific containerd file lists the AKS mirror after Gantry. If +Gantry is unavailable, containerd continues through port 8578 with Artifact +Streaming but without Gantry peer reuse. + +The configurator waits for `/artifact-streaming/readyz`, snapshots the current +host configuration under `/var/lib/gantry/overlaybd-config`, then uses the +AKS-provided configuration tool to set OverlayBD's P2P address to +`http://localhost:5000/blobs`. It restarts the OverlayBD services only when the +effective configuration changes. A second writer is never overwritten: if the +host file differs from Gantry's managed snapshot, apply or rollback preserves +that file and reports the conflict. + +For an operator-managed installation, opt in through every participating +`Site` using the same selector. This path assumes `unbounded-agent` or another +node manager already routes containerd through Gantry. Use the standalone Helm +chart on stock AKS nodes so the registry-specific route is installed alongside +the OverlayBD configuration. + +```yaml +apiVersion: unbounded-cloud.io/v1alpha3 +kind: Site +metadata: + name: +spec: + components: + gantry: + enabled: true + artifactStreaming: + enabled: true + nodeSelector: + kubernetes.azure.com/agentpool: +``` + +The operator rejects an empty selector, a Site that enables Artifact Streaming +while disabling Gantry, or conflicting selectors across Sites. + +### Artifact Streaming rollback + +Drain workloads using active OverlayBD devices before disabling the operator +setting. The operator issues deletion of +`DaemonSet/gantry-overlaybd-config` before applying the non-streaming Gantry +DaemonSet, and the configurator's `preStop` restores the original host +configuration when it still owns the current value. Kubernetes deletion and +pod termination are asynchronous, which is why workloads must be drained +before this transition. + +For standalone Helm, keep the Gantry endpoint available during restoration: + +```sh +helm upgrade gantry oci://ghcr.io/azure/charts/gantry \ + --reuse-values \ + --set overlaybdConfig.enabled=false \ + --set gantry.artifactStreaming.enabled=true \ + --wait + +kubectl -n gantry-system wait --for=delete \ + daemonset/gantry-overlaybd-config --timeout=5m +``` + +After the configurator is gone and streaming workloads are drained, disable +`gantry.artifactStreaming.enabled`. If the host OverlayBD file was changed by +another owner after Gantry configured it, rollback deliberately leaves that +new value in place; inspect the configurator logs and resolve ownership before +removing the saved state. + +### Artifact Streaming metrics + +Labels are bounded: `source` is `local`, `peer`, or `origin`; `outcome` is +`success` or `error`; `reason` is `range` or `origin_url`. + +| Metric | Use | +| --- | --- | +| `gantry_streaming_requests_total{source,outcome}` | Which source served each range, and whether it succeeded. | +| `gantry_streaming_bytes_total{source}` | Bytes per source; `origin` is ACR data-plane egress. | +| `gantry_streaming_time_to_first_byte_seconds{source}` | Latency before the range body starts. | +| `gantry_streaming_request_duration_seconds{source,outcome}` | End-to-end range duration. | +| `gantry_streaming_rejected_total{reason}` | Requests refused before a source was chosen. | +| `gantry_streaming_inflight{source}` | Range bodies currently streaming. | + +`/artifact-streaming/readyz` on the node-local mirror port reports endpoint +readiness. It is loopback-only, so probe it from the node rather than Prometheus. + +### Artifact Streaming alerts + +These separate the failure domains that need different responses. Tune the +thresholds against a real workload before paging on them. + +```yaml +- alert: GantryStreamingErrors + expr: | + sum by (instance) (rate(gantry_streaming_requests_total{outcome="error"}[5m])) + / sum by (instance) (rate(gantry_streaming_requests_total[5m])) > 0.05 + for: 10m + annotations: + summary: Gantry is failing OverlayBD range requests on {{ $labels.instance }}. + +- alert: GantryStreamingOriginFailing + expr: sum(rate(gantry_streaming_requests_total{source="origin",outcome="error"}[5m])) > 0 + for: 10m + annotations: + summary: Signed-origin range reads are failing; suspect ACR or the host allowlist. + +- alert: GantryStreamingNoPeerReuse + expr: | + sum(rate(gantry_streaming_requests_total{source="peer",outcome="success"}[30m])) == 0 + and sum(rate(gantry_streaming_requests_total{source="origin",outcome="success"}[30m])) > 0 + for: 30m + annotations: + summary: Every range is coming from ACR; peer reuse is not happening. + +- alert: GantryStreamingRejectingOriginURLs + expr: sum(rate(gantry_streaming_rejected_total{reason="origin_url"}[5m])) > 0 + for: 5m + annotations: + summary: Gantry refuses the OverlayBD origin URL; check the host suffix allowlist. +``` + +Origin reads are capped at 32 in flight per node. A sustained +`gantry_streaming_inflight{source="origin"}` at that ceiling means ranges are +queueing behind ACR rather than failing. + +### Artifact Streaming incident response + +| Symptom | Likely cause | Action | +| --- | --- | --- | +| Pods on streaming nodes stall on first read | Gantry agent unavailable on that node | OverlayBD has no second path while `p2pConfig` targets Gantry. Cordon the node, drain streaming workloads, then set `overlaybdConfig.enabled=false` so new devices use ACR directly. | +| `GantryStreamingOriginFailing` while peers still serve | ACR data plane, or an expired SAS | Ranges backed by a complete peer keep working, so do not disable Gantry; it is the only thing still serving. Escalate to ACR. | +| `GantryStreamingRejectingOriginURLs` | Allowlist does not cover the registry data endpoint | Add it to `gantry.artifactStreaming.allowedHostSuffixes` and roll the agent. | +| Ranges return to `source="origin"` after a peer had served them | Provider record outlived the blob (containerd GC or eviction) | Expected. Gantry fails over within the same request and suppresses that provider for 3m. Investigate only if it persists, which points at containerd GC pressure. | +| `/artifact-streaming/readyz` returns 503 | Agent still starting, or draining | The gate is sticky until startup completes. If it persists, check the containerd socket; containerd is Gantry's only content store. | +| OverlayBD device errors while Gantry is healthy | OverlayBD daemon or TCMU backend | Outside Gantry. Restore direct origin with `overlaybdConfig.enabled=false` and engage AKS support. | + +### Artifact Streaming compatibility + +Gantry configures an existing AKS Artifact Streaming installation. It does not +install, version, or upgrade OverlayBD. + +| Component | Requirement | Owner | +| --- | --- | --- | +| Node pool | Created with Artifact Streaming enabled | AKS | +| Registry | Premium ACR serving streaming artifacts | ACR | +| OverlayBD runtime | `overlaybd-tcmu` and `overlaybd-snapshotter` units present and active | AKS | +| OverlayBD config tool | `/opt/acr/tools/overlaybd/config.sh` | AKS | +| Kernel backend | TCMU, as provisioned by the node image | AKS | +| containerd | Reads `/etc/containerd/certs.d`; Unbounded-managed nodes pin 2.1.8 | Unbounded / AKS | +| Host config keys written | `p2pConfig.enable` and `p2pConfig.address` only | Gantry | + +Every other key in `/etc/overlaybd/overlaybd.json` is left exactly as the node +image shipped it, and rollback restores the original file. + ## What to verify after rollout | Check | How | @@ -111,6 +301,8 @@ on its own; no restart needed. | Advertiser reconciling | `gantry_advertise_reconcile_total` increases at the configured cadence | | Leases are being created on `please_pull` | `gantry_containerd_lease_created_total` increments during cold-start rollouts | | Origin fallback is rare | `p2p_origin_fallback_total` stays at ~0 | +| Streaming endpoint is ready | `/artifact-streaming/readyz` returns 200 on port 5000 for enabled target nodes | +| Streaming source transition | `gantry_streaming_requests_total{source="origin",outcome="success"}` serves cold ranges, then `source="peer"` increases after complete providers advertise | See `docs/detailed-design.md` ยง7.6 for the full metric catalog. diff --git a/deploy/gantry/chart/templates/NOTES.txt b/deploy/gantry/chart/templates/NOTES.txt index 25e7bbd0f..be74ee53f 100644 --- a/deploy/gantry/chart/templates/NOTES.txt +++ b/deploy/gantry/chart/templates/NOTES.txt @@ -4,7 +4,7 @@ Verify the agent rollout: kubectl -n {{ .Release.Namespace }} rollout status daemonset/gantry {{- if .Values.nodeConfig.enabled }} -The chart continuously reconciles containerd's default Gantry mirror route: +The chart continuously reconciles containerd's Gantry mirror route: kubectl -n {{ .Release.Namespace }} rollout status daemonset/gantry-containerd-config {{- else }} Containerd mirror routing is externally managed because nodeConfig.enabled=false. diff --git a/deploy/gantry/chart/templates/_helpers.tpl b/deploy/gantry/chart/templates/_helpers.tpl index 04aed261f..4836917db 100644 --- a/deploy/gantry/chart/templates/_helpers.tpl +++ b/deploy/gantry/chart/templates/_helpers.tpl @@ -11,12 +11,31 @@ {{- end -}} {{- end }} +{{- define "gantry.overlaybdConfigImage" -}} +{{- if and .Values.overlaybdConfig.image.reference .Values.overlaybdConfig.image.digest -}} +{{- fail "overlaybdConfig.image.reference and overlaybdConfig.image.digest are mutually exclusive" -}} +{{- end -}} +{{- if .Values.overlaybdConfig.image.reference -}} +{{- .Values.overlaybdConfig.image.reference -}} +{{- else if .Values.overlaybdConfig.image.digest -}} +{{- printf "%s@%s" .Values.overlaybdConfig.image.repository .Values.overlaybdConfig.image.digest -}} +{{- else -}} +{{- printf "%s:%s" .Values.overlaybdConfig.image.repository (default .Chart.AppVersion .Values.overlaybdConfig.image.tag) -}} +{{- end -}} +{{- end }} + {{- define "gantry.validate" -}} {{- $mountPath := trimSuffix "/" .Values.containerd.mountPath -}} {{- $socketPrefix := printf "%s/" $mountPath -}} {{- if not (hasPrefix $socketPrefix .Values.containerd.socketPath) -}} {{- fail "containerd.socketPath must be located under containerd.mountPath" -}} {{- end -}} +{{- if and .Values.overlaybdConfig.enabled (not .Values.gantry.artifactStreaming.enabled) -}} +{{- fail "overlaybdConfig.enabled requires gantry.artifactStreaming.enabled" -}} +{{- end -}} +{{- if and .Values.overlaybdConfig.enabled (empty .Values.overlaybdConfig.nodeSelector) -}} +{{- fail "overlaybdConfig.enabled requires an explicit overlaybdConfig.nodeSelector" -}} +{{- end -}} {{- end }} {{- define "gantry.managerLabel" -}} @@ -32,9 +51,34 @@ app.kubernetes.io/name: gantry {{ include "gantry.managerLabel" . }} {{- end }} -{{- define "gantry.nodeConfigHosts" -}} +{{- define "gantry.nodeConfigDefaultHosts" -}} # Managed by the Gantry Helm chart. [host."http://127.0.0.1:5000"] capabilities = ["pull", "resolve"] dial_timeout = "200ms" +{{- end }} + +{{- define "gantry.nodeConfigRegistryHosts" -}} +# Managed by the Gantry Helm chart. +server = "https://{{ .name }}" + +[host."http://127.0.0.1:5000"] + capabilities = ["pull", "resolve"] + dial_timeout = "200ms" + +# Preserve AKS Artifact Streaming when Gantry is unavailable. +[host."http://127.0.0.1:8578"] + capabilities = ["pull", "resolve"] +{{- end }} + +{{- define "gantry.nodeConfigPayload" -}} +{{- if .Values.gantry.artifactStreaming.enabled -}} +{{- range $index, $registry := .Values.gantry.upstreamRegistries }} +hosts-{{ $index }}.toml: | + {{- include "gantry.nodeConfigRegistryHosts" $registry | nindent 2 }} +{{- end }} +{{- else }} +hosts.toml: | + {{- include "gantry.nodeConfigDefaultHosts" . | nindent 2 }} +{{- end }} {{- end }} \ No newline at end of file diff --git a/deploy/gantry/chart/templates/configmap.yaml b/deploy/gantry/chart/templates/configmap.yaml index e4a3c237f..0ab0a25e5 100644 --- a/deploy/gantry/chart/templates/configmap.yaml +++ b/deploy/gantry/chart/templates/configmap.yaml @@ -10,8 +10,16 @@ data: {{- if .Values.gantry.config }} {{- tpl .Values.gantry.config . | nindent 4 }} {{- else }} + {{- if .Values.gantry.artifactStreaming.enabled }} + mirror_listen: "127.0.0.1:5000" + mirror_bind_allow_non_loopback: false + {{- else }} mirror_listen: "0.0.0.0:5000" mirror_bind_allow_non_loopback: true + {{- end }} + artifact_streaming_enabled: {{ .Values.gantry.artifactStreaming.enabled }} + artifact_streaming_allowed_host_suffixes: + {{- toYaml .Values.gantry.artifactStreaming.allowedHostSuffixes | nindent 6 }} transfer_listen: "0.0.0.0:5001" chair_listen: "0.0.0.0:5002" metrics_listen: "0.0.0.0:9095" diff --git a/deploy/gantry/chart/templates/daemonset.yaml b/deploy/gantry/chart/templates/daemonset.yaml index 5446a856e..b12fbe68c 100644 --- a/deploy/gantry/chart/templates/daemonset.yaml +++ b/deploy/gantry/chart/templates/daemonset.yaml @@ -26,8 +26,8 @@ spec: {{- end }} spec: serviceAccountName: gantry - hostNetwork: false - dnsPolicy: ClusterFirst + hostNetwork: {{ .Values.gantry.artifactStreaming.enabled }} + dnsPolicy: {{ ternary "ClusterFirstWithHostNet" "ClusterFirst" .Values.gantry.artifactStreaming.enabled }} priorityClassName: gantry-low nodeSelector: {{- toYaml .Values.nodeSelector | nindent 8 }} @@ -80,8 +80,10 @@ spec: ports: - name: mirror containerPort: 5000 + {{- if not .Values.gantry.artifactStreaming.enabled }} hostPort: 5000 hostIP: 127.0.0.1 + {{- end }} protocol: TCP - name: transfer containerPort: 5001 diff --git a/deploy/gantry/chart/templates/node-config.yaml b/deploy/gantry/chart/templates/node-config.yaml index d33ef4e3d..ddae7883f 100644 --- a/deploy/gantry/chart/templates/node-config.yaml +++ b/deploy/gantry/chart/templates/node-config.yaml @@ -10,8 +10,7 @@ metadata: {{- include "gantry.managerLabel" . | nindent 4 }} app.kubernetes.io/component: node-config data: - hosts.toml: | - {{- include "gantry.nodeConfigHosts" . | nindent 4 }} + {{- include "gantry.nodeConfigPayload" . | nindent 2 }} --- apiVersion: apps/v1 kind: DaemonSet @@ -38,7 +37,7 @@ spec: {{- include "gantry.managerLabel" . | nindent 8 }} app.kubernetes.io/component: node-config annotations: - checksum/hosts-config: {{ include "gantry.nodeConfigHosts" . | sha256sum }} + checksum/hosts-config: {{ include "gantry.nodeConfigPayload" . | sha256sum }} spec: automountServiceAccountToken: false priorityClassName: gantry-low @@ -58,38 +57,46 @@ spec: - sh - -c - | - target_file=/host-certs/_default/hosts.toml - if cmp -s /config/hosts.toml "$target_file"; then - rm -f "$target_file" - fi + for source_file in /config/*/hosts.toml; do + relative_path=${source_file#/config/} + target_file=/host-certs/$relative_path + if cmp -s "$source_file" "$target_file"; then + rm -f "$target_file" + fi + done command: - sh - -c - | set -eu - target_dir=/host-certs/_default - target_file="$target_dir/hosts.toml" - source_file=/config/hosts.toml - temp_file="$target_dir/.hosts.toml.gantry.tmp" - while true; do - mkdir -p "$target_dir" - if ! cmp -s "$source_file" "$target_file"; then - cp "$source_file" "$temp_file" - chmod 0644 "$temp_file" - mv "$temp_file" "$target_file" - echo "reconciled $target_file" - fi + for source_file in /config/*/hosts.toml; do + relative_path=${source_file#/config/} + target_file=/host-certs/$relative_path + target_dir=$(dirname "$target_file") + temp_file="$target_dir/.hosts.toml.gantry.tmp" + + mkdir -p "$target_dir" + if ! cmp -s "$source_file" "$target_file"; then + cp "$source_file" "$temp_file" + chmod 0644 "$temp_file" + mv "$temp_file" "$target_file" + echo "reconciled $target_file" + fi + done sleep {{ .Values.nodeConfig.reconcileIntervalSeconds }} done readinessProbe: exec: command: - - cmp - - -s - - /config/hosts.toml - - /host-certs/_default/hosts.toml + - sh + - -c + - | + for source_file in /config/*/hosts.toml; do + relative_path=${source_file#/config/} + cmp -s "$source_file" "/host-certs/$relative_path" || exit 1 + done initialDelaySeconds: 1 periodSeconds: 5 resources: @@ -118,4 +125,14 @@ spec: - name: config configMap: name: gantry-containerd-hosts + items: + {{- if .Values.gantry.artifactStreaming.enabled }} + {{- range $index, $registry := .Values.gantry.upstreamRegistries }} + - key: hosts-{{ $index }}.toml + path: {{ $registry.name }}/hosts.toml + {{- end }} + {{- else }} + - key: hosts.toml + path: _default/hosts.toml + {{- end }} {{- end }} \ No newline at end of file diff --git a/deploy/gantry/chart/templates/overlaybd-config.yaml b/deploy/gantry/chart/templates/overlaybd-config.yaml new file mode 100644 index 000000000..9baa0ce92 --- /dev/null +++ b/deploy/gantry/chart/templates/overlaybd-config.yaml @@ -0,0 +1,76 @@ +{{- if or .Values.overlaybdConfig.enabled .Values.overlaybdConfig.renderOnly }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: gantry-overlaybd-config + namespace: {{ .Release.Namespace }} + labels: + {{- include "gantry.labels" . | nindent 4 }} + app.kubernetes.io/component: overlaybd-config +spec: + selector: + matchLabels: + app.kubernetes.io/name: gantry + app.kubernetes.io/component: overlaybd-config + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + template: + metadata: + labels: + {{- include "gantry.labels" . | nindent 8 }} + app.kubernetes.io/component: overlaybd-config + annotations: + gantry.unbounded-cloud.io/overlaybd-address: {{ .Values.overlaybdConfig.address | quote }} + spec: + automountServiceAccountToken: false + hostNetwork: true + hostPID: true + dnsPolicy: ClusterFirstWithHostNet + priorityClassName: gantry-low + nodeSelector: + {{- toYaml .Values.overlaybdConfig.nodeSelector | nindent 8 }} + tolerations: + {{- toYaml .Values.overlaybdConfig.tolerations | nindent 8 }} + terminationGracePeriodSeconds: 60 + containers: + - name: configure + image: {{ include "gantry.overlaybdConfigImage" . }} + imagePullPolicy: {{ .Values.overlaybdConfig.image.pullPolicy }} + env: + - name: OVERLAYBD_P2P_ADDRESS + value: {{ .Values.overlaybdConfig.address | quote }} + resources: + {{- toYaml .Values.resources.overlaybdConfig | nindent 12 }} + lifecycle: + preStop: + exec: + command: ["/usr/local/bin/configure-overlaybd.sh", "restore"] + securityContext: + privileged: true + runAsNonRoot: false + runAsUser: 0 + runAsGroup: 0 + readOnlyRootFilesystem: true + seccompProfile: + type: RuntimeDefault + readinessProbe: + exec: + command: ["test", "-f", "/run/gantry-overlaybd/configured"] + initialDelaySeconds: 1 + periodSeconds: 5 + volumeMounts: + - name: host-root + mountPath: /host + - name: run + mountPath: /run/gantry-overlaybd + volumes: + - name: host-root + hostPath: + path: / + type: Directory + - name: run + emptyDir: {} +{{- end }} \ No newline at end of file diff --git a/deploy/gantry/chart/values-operator.yaml b/deploy/gantry/chart/values-operator.yaml index 8d47c3f0e..3fddc8c1a 100644 --- a/deploy/gantry/chart/values-operator.yaml +++ b/deploy/gantry/chart/values-operator.yaml @@ -11,3 +11,8 @@ manager: unbounded-operator nodeConfig: enabled: false + +overlaybdConfig: + # Internal render switch. The operator mutator removes this workload unless + # a Site explicitly enables Artifact Streaming. + renderOnly: true diff --git a/deploy/gantry/chart/values.schema.json b/deploy/gantry/chart/values.schema.json index 66540ba39..eb61b6d09 100644 --- a/deploy/gantry/chart/values.schema.json +++ b/deploy/gantry/chart/values.schema.json @@ -10,6 +10,7 @@ "tolerations", "containerd", "nodeConfig", + "overlaybdConfig", "registryCredentials", "resources", "gantry" @@ -143,6 +144,73 @@ } } }, + "overlaybdConfig": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "address", + "image", + "nodeSelector", + "tolerations" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "address": { + "type": "string", + "pattern": "^http://localhost:[0-9]+/blobs$" + }, + "image": { + "type": "object", + "additionalProperties": false, + "required": [ + "reference", + "repository", + "tag", + "digest", + "pullPolicy" + ], + "properties": { + "reference": { + "type": "string" + }, + "repository": { + "type": "string", + "minLength": 1 + }, + "tag": { + "type": "string" + }, + "digest": { + "type": "string", + "pattern": "^(|sha256:[0-9a-f]{64})$" + }, + "pullPolicy": { + "type": "string", + "enum": [ + "Always", + "IfNotPresent", + "Never" + ] + } + } + }, + "nodeSelector": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "tolerations": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, "registryCredentials": { "type": "object", "additionalProperties": false, @@ -162,7 +230,8 @@ "required": [ "agent", "init", - "nodeConfig" + "nodeConfig", + "overlaybdConfig" ], "properties": { "agent": { @@ -173,6 +242,9 @@ }, "nodeConfig": { "type": "object" + }, + "overlaybdConfig": { + "type": "object" } } }, @@ -182,6 +254,7 @@ "required": [ "config", "pprofListen", + "artifactStreaming", "upstreamRegistries" ], "properties": { @@ -191,6 +264,27 @@ "pprofListen": { "type": "string" }, + "artifactStreaming": { + "type": "object", + "additionalProperties": false, + "required": [ + "enabled", + "allowedHostSuffixes" + ], + "properties": { + "enabled": { + "type": "boolean" + }, + "allowedHostSuffixes": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + } + }, "upstreamRegistries": { "type": "array", "items": { diff --git a/deploy/gantry/chart/values.yaml b/deploy/gantry/chart/values.yaml index 41664fb7a..2924f3f6f 100644 --- a/deploy/gantry/chart/values.yaml +++ b/deploy/gantry/chart/values.yaml @@ -28,6 +28,19 @@ nodeConfig: hostPath: /etc/containerd/certs.d reconcileIntervalSeconds: 5 +overlaybdConfig: + enabled: false + address: http://localhost:5000/blobs + image: + reference: "" + repository: ghcr.io/azure/gantry-node-config + tag: "" + digest: "" + pullPolicy: IfNotPresent + nodeSelector: {} + tolerations: + - operator: Exists + registryCredentials: secretName: gantry-registry-credentials @@ -53,10 +66,23 @@ resources: limits: cpu: 50m memory: 16Mi + overlaybdConfig: + requests: + cpu: 5m + memory: 16Mi + limits: + cpu: 100m + memory: 64Mi gantry: config: "" pprofListen: "" + artifactStreaming: + enabled: false + allowedHostSuffixes: + - .azurecr.io + - .data.mcr.microsoft.com + - .blob.core.windows.net upstreamRegistries: - name: registry.example.com endpoint: https://registry.example.com diff --git a/deploy/gantry/render_test.go b/deploy/gantry/render_test.go index 31aea7a3b..da2ac88da 100644 --- a/deploy/gantry/render_test.go +++ b/deploy/gantry/render_test.go @@ -101,6 +101,94 @@ func TestDaemonSetMountsContainerdRuntimeDirectory(t *testing.T) { } } +func TestArtifactStreamingDefaultsRenderDisabled(t *testing.T) { + t.Parallel() + + outputDir := renderTemplates(t) + + raw, err := os.ReadFile(filepath.Join(outputDir, "configmap.yaml")) + if err != nil { + t.Fatalf("read rendered configmap: %v", err) + } + + var configMap struct { + Data map[string]string `yaml:"data"` + } + if err := yaml.Unmarshal(raw, &configMap); err != nil { + t.Fatalf("unmarshal rendered configmap: %v", err) + } + + config := configMap.Data["config.yaml"] + for _, expected := range []string{ + "artifact_streaming_enabled: false", + ".data.mcr.microsoft.com", + ".blob.core.windows.net", + } { + if !strings.Contains(config, expected) { + t.Errorf("rendered config missing %q:\n%s", expected, config) + } + } +} + +func TestOverlayBDConfiguratorIsOptIn(t *testing.T) { + t.Parallel() + + outputDir := renderStandaloneTemplates(t) + if _, err := os.Stat(filepath.Join(outputDir, "overlaybd-config.yaml")); !os.IsNotExist(err) { + t.Fatalf("overlaybd-config.yaml exists by default; err=%v", err) + } +} + +func TestOverlayBDConfiguratorEnabled(t *testing.T) { + t.Parallel() + + outputDir := renderChart(t, false, + "--set", "overlaybdConfig.enabled=true", + "--set", "gantry.artifactStreaming.enabled=true", + "--set-string", "overlaybdConfig.image.reference=gantry-node-config:test", + "--set-string", "overlaybdConfig.nodeSelector.kubernetes\\.azure\\.com/host-os=AzureLinux", + ) + + raw, err := os.ReadFile(filepath.Join(outputDir, "overlaybd-config.yaml")) + if err != nil { + t.Fatalf("read rendered OverlayBD configurator: %v", err) + } + + var daemonSet struct { + Spec struct { + Template struct { + Spec struct { + HostPID bool `yaml:"hostPID"` + NodeSelector map[string]string `yaml:"nodeSelector"` + Containers []struct { + Image string `yaml:"image"` + SecurityContext struct { + Privileged bool `yaml:"privileged"` + } `yaml:"securityContext"` + } `yaml:"containers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"spec"` + } + if err := yaml.Unmarshal(raw, &daemonSet); err != nil { + t.Fatalf("unmarshal OverlayBD configurator: %v", err) + } + + if !daemonSet.Spec.Template.Spec.HostPID { + t.Error("hostPID = false, want true") + } + + if got := daemonSet.Spec.Template.Spec.NodeSelector["kubernetes.azure.com/host-os"]; got != "AzureLinux" { + t.Errorf("node selector = %q, want AzureLinux", got) + } + + if len(daemonSet.Spec.Template.Spec.Containers) != 1 || + daemonSet.Spec.Template.Spec.Containers[0].Image != "gantry-node-config:test" || + !daemonSet.Spec.Template.Spec.Containers[0].SecurityContext.Privileged { + t.Fatalf("unexpected configurator container: %+v", daemonSet.Spec.Template.Spec.Containers) + } +} + func TestRendersFixedChairLeaseSet(t *testing.T) { t.Parallel() @@ -296,7 +384,9 @@ func TestStandaloneProfileContinuouslyReconcilesNodeConfig(t *testing.T) { manifest := string(raw) for _, fragment := range []string{ "path: /etc/containerd/certs.d", - "target_file=\"$target_dir/hosts.toml\"", + "path: _default/hosts.toml", + "for source_file in /config/*/hosts.toml; do", + "relative_path=${source_file#/config/}", "while true; do", "if ! cmp -s \"$source_file\" \"$target_file\"; then", "mv \"$temp_file\" \"$target_file\"", @@ -359,15 +449,20 @@ func TestStandaloneProfileContinuouslyReconcilesNodeConfig(t *testing.T) { } hostRoot := t.TempDir() + sourceRoot := t.TempDir() targetDir := filepath.Join(hostRoot, "_default") + sourceDir := filepath.Join(sourceRoot, "_default") + if err := os.MkdirAll(sourceDir, 0o755); err != nil { + t.Fatalf("create source directory: %v", err) + } - sourceFile := filepath.Join(t.TempDir(), "hosts.toml") + sourceFile := filepath.Join(sourceDir, "hosts.toml") if err := os.WriteFile(sourceFile, []byte(hostsConfig), 0o644); err != nil { t.Fatalf("write source hosts config: %v", err) } - reconcileScript = strings.Replace(reconcileScript, "target_dir=/host-certs/_default", fmt.Sprintf("target_dir=%q", targetDir), 1) - reconcileScript = strings.Replace(reconcileScript, "source_file=/config/hosts.toml", fmt.Sprintf("source_file=%q", sourceFile), 1) + reconcileScript = strings.ReplaceAll(reconcileScript, "/host-certs", hostRoot) + reconcileScript = strings.ReplaceAll(reconcileScript, "/config", sourceRoot) reconcileScript = strings.Replace(reconcileScript, "sleep 5", "sleep 0.05", 1) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) @@ -394,6 +489,99 @@ func TestStandaloneProfileContinuouslyReconcilesNodeConfig(t *testing.T) { waitForFileContent(t, targetFile, hostsConfig) } +func TestArtifactStreamingUsesAKSMirrorChain(t *testing.T) { + t.Parallel() + + const registry = "gantrystreamacr1.azurecr.io" + + outputDir := renderChart(t, false, + "--set", "gantry.artifactStreaming.enabled=true", + "--set-string", "gantry.upstreamRegistries[0].name="+registry, + "--set-string", "gantry.upstreamRegistries[0].endpoint=http://127.0.0.1:8578?ns="+registry, + ) + + daemonSetRaw, err := os.ReadFile(filepath.Join(outputDir, "daemonset.yaml")) + if err != nil { + t.Fatalf("read rendered daemonset: %v", err) + } + + var daemonSet struct { + Spec struct { + Template struct { + Spec struct { + HostNetwork bool `yaml:"hostNetwork"` + DNSPolicy string `yaml:"dnsPolicy"` + Containers []struct { + Name string `yaml:"name"` + Ports []struct { + Name string `yaml:"name"` + HostPort int `yaml:"hostPort"` + } `yaml:"ports"` + } `yaml:"containers"` + } `yaml:"spec"` + } `yaml:"template"` + } `yaml:"spec"` + } + if err := yaml.Unmarshal(daemonSetRaw, &daemonSet); err != nil { + t.Fatalf("unmarshal rendered daemonset: %v", err) + } + + if !daemonSet.Spec.Template.Spec.HostNetwork { + t.Fatal("artifact-streaming Gantry hostNetwork = false, want true") + } + + if got := daemonSet.Spec.Template.Spec.DNSPolicy; got != "ClusterFirstWithHostNet" { + t.Fatalf("artifact-streaming Gantry dnsPolicy = %q", got) + } + + for _, container := range daemonSet.Spec.Template.Spec.Containers { + if container.Name != "gantry" { + continue + } + + for _, port := range container.Ports { + if port.Name == "mirror" && port.HostPort != 0 { + t.Fatalf("host-networked mirror hostPort = %d, want omitted", port.HostPort) + } + } + } + + configRaw, err := os.ReadFile(filepath.Join(outputDir, "configmap.yaml")) + if err != nil { + t.Fatalf("read rendered configmap: %v", err) + } + + for _, fragment := range []string{ + `mirror_listen: "127.0.0.1:5000"`, + `endpoint: "http://127.0.0.1:8578?ns=` + registry + `"`, + } { + if !strings.Contains(string(configRaw), fragment) { + t.Fatalf("artifact-streaming config missing %q:\n%s", fragment, configRaw) + } + } + + nodeConfigRaw, err := os.ReadFile(filepath.Join(outputDir, "node-config.yaml")) + if err != nil { + t.Fatalf("read rendered node config: %v", err) + } + + manifest := string(nodeConfigRaw) + for _, fragment := range []string{ + `server = "https://` + registry + `"`, + `[host."http://127.0.0.1:5000"]`, + `[host."http://127.0.0.1:8578"]`, + "path: " + registry + "/hosts.toml", + } { + if !strings.Contains(manifest, fragment) { + t.Fatalf("AKS node config missing %q:\n%s", fragment, manifest) + } + } + + if strings.Contains(manifest, "path: _default/hosts.toml") { + t.Fatalf("AKS node config overwrites _default:\n%s", manifest) + } +} + func waitForFileContent(t *testing.T, path, want string) { t.Helper() @@ -417,6 +605,13 @@ func TestStandaloneAndOperatorProfilesShareCoreResources(t *testing.T) { standaloneObjects := renderedObjects(t, renderStandaloneTemplates(t)) delete(operatorObjects, "Namespace//unbounded-system") + + const overlayBDConfigKey = "DaemonSet/unbounded-system/gantry-overlaybd-config" + if _, ok := operatorObjects[overlayBDConfigKey]; !ok { + t.Fatalf("operator profile is missing %s", overlayBDConfigKey) + } + + delete(operatorObjects, overlayBDConfigKey) delete(standaloneObjects, "ConfigMap/unbounded-system/gantry-containerd-hosts") delete(standaloneObjects, "DaemonSet/unbounded-system/gantry-containerd-config") @@ -467,7 +662,7 @@ func renderStandaloneTemplates(t *testing.T) string { return renderChart(t, false) } -func renderChart(t *testing.T, operatorProfile bool) string { +func renderChart(t *testing.T, operatorProfile bool, extraArgs ...string) string { t.Helper() deployDir := filepath.Dir(sourceFile(t)) @@ -499,6 +694,8 @@ func renderChart(t *testing.T, operatorProfile bool) string { ) } + args = append(args, extraArgs...) + cmd := exec.Command(helm, args...) output, err := cmd.CombinedOutput() diff --git a/deploy/machina/crd/unbounded-cloud.io_sites.yaml b/deploy/machina/crd/unbounded-cloud.io_sites.yaml index 4161a7b45..0da339416 100644 --- a/deploy/machina/crd/unbounded-cloud.io_sites.yaml +++ b/deploy/machina/crd/unbounded-cloud.io_sites.yaml @@ -100,6 +100,26 @@ spec: (for example the Gantry print column) whenever a components block is present. A site that omits the components block entirely is not defaulted. properties: + artifactStreaming: + description: |- + ArtifactStreaming configures Gantry as the node-local range proxy for + AKS ACR Artifact Streaming. It is disabled when omitted. Because Gantry is + a cluster singleton, every Site that enables it must use the same node + selector. + properties: + enabled: + description: Enabled controls the Gantry range endpoint + and OverlayBD configurator. + type: boolean + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector must select only AKS node pools with Artifact Streaming + enabled. It is required when Enabled is true. + minProperties: 1 + type: object + type: object enabled: description: Enabled controls whether the component is reconciled. type: boolean diff --git a/designs/gantry-acr-artifact-streaming-implementation-plan.md b/designs/gantry-acr-artifact-streaming-implementation-plan.md new file mode 100644 index 000000000..006c86d43 --- /dev/null +++ b/designs/gantry-acr-artifact-streaming-implementation-plan.md @@ -0,0 +1,761 @@ +# Gantry ACR Artifact Streaming Implementation Plan + +**Status:** Draft for review + +This plan implements the behavior described in +[Gantry with ACR Artifact Streaming](gantry-acr-artifact-streaming.md). The +design document is the source of truth for system behavior. This document maps +that behavior to code changes, tests, deployment work, rollout order, and +production acceptance criteria. + +## Target behavior + +Two independent paths run after Gantry serves a streaming manifest: + +1. **Complete-layer seeding:** the existing manifest prefetch path sends + layer-specific `please_pull` requests to Gantry chairs. Chairs download, + verify, commit, and advertise complete layers. +2. **Range reads:** OverlayBD sends `Range: bytes=N-M` to a node-local Gantry + endpoint. Gantry serves the range from a complete local layer, a complete + peer layer, or the signed Azure origin URL. + +The range path does not start, cancel, or own chair pulls. Manifest processing +remains the only owner of proactive complete-layer seeding. + +```text +manifest -> existing Gantry prefetch -> chairs pull complete layers + +OverlayBD range -> local complete blob + -> complete peer provider + -> signed origin URL +``` + +## Baseline implementation decisions + +The plan uses the following baseline so the first implementation has one clear +contract: + +- Keep existing manifest-triggered complete-layer prefetch unchanged. +- Add `GET /blobs/` to the existing node-local Gantry + listener on port 5000. +- Accept one exact bounded range, `bytes=N-M`. Reject suffix, open-ended, and + multipart ranges at the OverlayBD endpoint. +- Query only complete digest providers already advertised in the existing DHT. +- Use a short, bounded provider lookup. Do not wait for an in-progress chair + pull before using the signed origin URL. +- Do not add range-keyed DHT records or range-specific chair RPCs. +- Do not cache partial origin ranges in the first implementation. +- Do not send signed URLs or OverlayBD registry authorization to peers. +- Keep complete content in containerd; do not add another complete-object + store. +- Ship the feature disabled until the node's OverlayBD configuration is ready. + +A transient range cache and cluster-wide range ownership remain possible later +optimizations. Neither is required for the design's local/peer/origin source +transition. + +## Package boundaries + +### New packages + +| Package | Responsibility | +|---|---| +| `internal/gantry/httprange` | Parse exact request ranges and validate `Content-Range` responses. | +| `internal/gantry/streaming` | Handle OverlayBD requests, parse Azure data URLs, choose a source, and stream the response. | + +`httprange` is shared by the streaming handler and peer transfer client. It +must not import Gantry networking or storage packages. + +`streaming` depends on narrow interfaces for local range reads, DHT discovery, +peer range fetches, and signed-origin range fetches. It must not import the OCI +mirror package or the chair coordinator. + +### Existing packages changed + +| Package/file | Change | +|---|---| +| `internal/gantry/containerdstore` | Add an efficient complete-object range reader backed by containerd `ReaderAt`. | +| `internal/gantry/transfer/client.go` | Add exact bounded peer range requests. | +| `internal/gantry/transfer/transfer.go` | Keep the existing server wire contract; add conformance tests for exact ranges. | +| `internal/gantry/config/config.go` | Add feature, origin-policy, lookup, and concurrency configuration. | +| `internal/gantry/mirror/mirror.go` | Permit composition of the mirror handler with the streaming handler on one listener. Do not add streaming source logic here. | +| `cmd/gantry/main.go` | Construct and wire the streaming server, shared listener, metrics, and shutdown. | +| `cmd/gantry/agent_metrics.go` | Register bounded streaming metrics. | +| `deploy/gantry/chart` | Render configuration and optional OverlayBD node configuration. | + +## Phase 0: Lock and test upstream contracts + +Before production code changes, create redacted fixtures and tests for the +assumptions that join the four systems. + +### ACR fixture capture + +Using a test ACR streaming artifact, capture and redact: + +- the tag-resolution response; +- the resolved streaming manifest and config; +- layer descriptor annotations, media types, digests, and sizes; +- the registry blob redirect shape; +- the signed data URL host/path/query shape; and +- representative OverlayBD ranges during attach and initial process startup. + +Remove registry names, account identifiers, signatures, tokens, and expiry +values before committing fixtures. + +### Contract tests + +Add tests proving: + +1. Containerd's remote snapshot annotations carry the original image + reference and layer digest. +2. The OverlayBD snapshotter writes the expected `repoBlobUrl`, digest, and + size without downloading the complete layer. +3. Gantry's existing `manifest.TypedChildren` classifies the streaming config + and layers correctly. +4. `layerPrefetchAdapter.OnManifestServed` sends the streaming children through + `PrefetchManifestChildren` with registry and repository intact. +5. OverlayBD sends the post-redirect URL to `p2pConfig.address` with an exact + bounded range. +6. ACR Artifact Streaming uses redirect mode in supported environments. Direct + registry `200` self mode is unsupported by this integration. + +### Decision gates + +The fixture run must settle these values before their defaults merge: + +- peer lookup latency budget; +- maximum peer attempts per range; +- origin concurrency limit; and +- whether an OverlayBD request-size limit is necessary when responses are + streamed without buffering. + +**Phase exit:** the test fixtures reproduce the control flow in the design, and +the supported Azure URL/authentication contract is written down without live +credentials. + +## Phase 1: Shared exact-range primitives + +Add `internal/gantry/httprange` with a small immutable value type: + +```go +type Range struct { + Start int64 + End int64 // inclusive +} +``` + +Required operations: + +- parse exactly `bytes=N-M`; +- reject negative, reversed, overflowing, suffix, open-ended, and multipart + ranges; +- compute length with overflow checks; +- validate a range against a known complete-object size; +- format the outbound `Range` header; +- parse and validate `Content-Range: bytes N-M/TOTAL`; and +- format local `206` and `416` headers. + +Do not replace the existing OCI resume parser. Gantry's complete-object path +uses `bytes=N-` to resume interrupted transfers; OverlayBD uses `bytes=N-M`. +Keeping the types separate prevents bounded reads from accidentally becoming +full suffix downloads. + +### Tests + +Cover boundary values, integer overflow, zero-length objects, one-byte reads, +end-of-object reads, malformed whitespace, multiple ranges, and mismatched +`Content-Length`/`Content-Range`. + +**Phase exit:** all range validation is centralized and neither streaming nor +transfer code parses range strings itself. + +## Phase 2: Complete-object range access + +### Local containerd ranges + +Add a narrow production capability to `containerdstore.Store`, for example: + +```go +OpenRange(ctx, digest, range) (body io.ReadCloser, totalSize int64, err error) +``` + +Implementation requirements: + +- apply the configured containerd namespace; +- open the committed descriptor through `content.Store.ReaderAt`; +- validate the requested range against `ReaderAt.Size()`; +- return an `io.SectionReader` over only the requested bytes; +- preserve `ErrNotFound` versus `ErrUnavailable`; and +- count a local range hit independently from ordinary mirror cache hits. + +Do not seek by reading and discarding a potentially large prefix. The +containerd backend already provides random access. + +### Peer ranges + +Add a separate exact-range method to `transfer.Client`, rather than changing +the semantics of `FetchFromPeer`: + +```go +FetchRangeFromPeer(ctx, peerAddr, digest, range) +``` + +The method: + +- builds the existing digest-addressed peer URL using the valid placeholder + repository when no repository is needed; +- sets `Gantry-Mirrored: 1`; +- sends `Range: bytes=N-M`; +- does not send registry `Authorization`; +- requires `206`; +- validates start, end, total size, and content length; and +- returns existing typed not-found, unavailable, busy, and protocol errors so + provider failure handling remains consistent. + +The peer server already serves exact ranges from committed objects. Add tests +that pin this as a backward-compatible wire contract. A new Gantry requester +must be able to issue an exact range to an older peer that already has the +current transfer server behavior. + +### Narrow interfaces + +Define the peer and local range interfaces in `streaming`, or in a neutral +interface file if fakes are reused broadly. Do not add exact-range methods to +the existing `ifaces.PeerDialer` unless the ordinary mirror path needs them; +that would force unrelated fakes and call sites to implement unused behavior. + +**Phase exit:** a unit/integration test reads the same byte range from a real +containerd store and from a Gantry transfer server and receives identical +bytes and metadata. + +## Phase 3: Azure URL and signed-origin client + +### Raw request preservation + +OverlayBD constructs: + +```text +http://localhost:5000/blobs/ +``` + +The embedded URL may contain repeated slashes, percent escapes, and a SAS query. +The handler must recover it from the raw request target. It must not call +`path.Clean`, round-trip through `url.Values.Encode`, or use a router that +redirects repeated slashes. + +Add a test whose embedded URL contains: + +- `https://`; +- a `//docker/registry/...` path segment; +- percent-encoded query values; +- `+`, `/`, and `=` inside a signature; and +- several query parameters in their original order. + +The reconstructed origin URL must be byte-for-byte equivalent. + +### Structured Azure URL parser + +Implement structured parsing for the URL forms observed in Phase 0. At minimum, +the parser returns: + +```text +origin URL +source digest +redacted host identity +origin class (ACR data, MCR data, Azure Blob) +``` + +Validation requirements: + +- `https` only in production; +- no userinfo or fragment; +- no IP-literal host; +- host must match an explicitly configured exact host or suffix; +- digest must be SHA-256 with 64 lowercase hexadecimal characters; +- if the digest appears in more than one location, all values must match; and +- unsupported URL forms fail before any network request. + +Do not copy Peerd's regular expressions as the validation boundary. Use +`net/url` plus host- and path-specific parsing. + +### Signed-origin HTTP client + +Add a streaming-specific HTTP client with bounded connect, TLS handshake, and +response-header timeouts. It should follow Gantry's existing origin transport +policy where applicable, but use a redirect callback that revalidates every +target against the artifact-streaming allowlist. + +For each read: + +- send the exact range; +- do not add Gantry registry credentials to a signed data URL; +- require a valid `206` response; +- validate `Content-Range` and `Content-Length` before committing response + headers to OverlayBD; +- stream the body without buffering the full range; +- close the body on every error path; and +- preserve request cancellation when OverlayBD disconnects. + +Direct registry `200` self mode is unsupported. The endpoint accepts only the +structured signed redirect URL forms covered by the fixtures and never forwards +inbound registry authorization to an origin or peer. + +### Secret handling + +No log, metric, trace attribute, error string, or panic may contain: + +- the raw request URI; +- the full origin URL; +- a SAS query or signature; +- an Authorization header; or +- a redirect Location value. + +Tests should install a log capture hook and assert a sentinel signature never +appears on success or failure. + +**Phase exit:** a local Azure-origin fixture can serve exact ranges through the +client, while SSRF, redirect, malformed response, and secret-leak tests fail +closed. + +## Phase 4: Gantry streaming server + +Add `internal/gantry/streaming.Server` with these dependencies: + +- complete local range store; +- digest-keyed DHT discovery; +- exact peer range client; +- signed-origin range client; +- bounded configuration; and +- metrics/logger hooks. + +### Request flow + +For each valid `GET /blobs/...` request: + +1. Parse the exact range and Azure origin URL. +2. Try the complete local content store. +3. If local content is absent, perform one bounded complete-provider lookup. +4. Filter self and providers currently suppressed as stale, unavailable, busy, + or suspicious. +5. Try providers within the configured attempt and wall-clock budgets. +6. If no provider serves the exact range, fetch it from the signed origin URL. +7. Return `206` with exact `Content-Range`, `Content-Length`, + `Accept-Ranges: bytes`, and `application/octet-stream`. + +Important ownership rules: + +- The range handler does not call `ColdStartResolver.Resolve`. +- It does not issue `please_pull`. +- It does not wait through the mirror's multi-minute peer rediscovery budget. +- It does not write partial bytes into containerd. +- It does not advertise partial bytes. +- It checks local/DHT state again on every request, which provides the automatic + transition to a completed chair provider. + +### Error behavior + +- Malformed request or unsupported URL: `400`. +- Syntactically valid but unsatisfiable range with known size: `416` and + `Content-Range: bytes */TOTAL`. +- Local containerd unavailable: continue to peer/origin; record the storage + failure separately. +- Peer not found, busy, stale, or unavailable: try the next provider within the + request budget. +- Signed origin `401`/`403`: return an authorization failure without logging the + URL. +- Signed origin `404`: return not found. +- Signed origin `429`: preserve a bounded `Retry-After` when valid. +- Origin protocol or transport failure: `502` or `503` according to whether the + response was malformed or unavailable. +- Failure after response bytes started: terminate the body; do not append an + HTTP error body to binary data. + +### Resource controls + +- Separate semaphore for signed-origin range requests so range traffic cannot + consume every connection while chairs are performing complete pulls. +- Reuse the existing transfer serve cap for peer responses unless measurements + show range reads require a distinct cap. +- Do not add node-local range singleflight until there is a buffering or fan-out + design that preserves streaming and bounds memory. Measure duplicate ranges + first. + +### Tests + +Use table-driven source-selection tests for: + +- local hit; +- local miss and peer hit; +- first peer stale, second peer hit; +- peer busy then origin; +- no providers then origin; +- local storage unavailable then origin; +- chair provider appearing between two range requests; and +- origin failure after all complete providers fail. + +Assert that peer calls receive only digest and range, never the signed URL. + +**Phase exit:** the streaming server passes the source-transition test: request +one range from signed origin, mark a complete provider available, then request a +second range from the peer without restarting any component. + +## Phase 5: Listener, lifecycle, configuration, and metrics + +### HTTP listener composition + +Keep the existing OCI mirror handler unchanged for `/v2` behavior. Compose the +shared port at the top level: + +```text +/blobs/ -> streaming.Server +all else -> mirror.Server.Handler() +``` + +Do not put `/blobs/` behind `http.ServeMux` path cleaning. Use a small raw-prefix +dispatcher before the existing mirror mux so embedded `https://` and repeated +slashes remain intact. + +Refactor `mirror.ListenAndServe` so the existing listener/shutdown helper can +serve a supplied composite handler. Retain the current wrapper for mirror unit +tests and callers that do not enable Artifact Streaming. + +### Startup and shutdown + +- Feature disabled: `/blobs/` is not registered and returns `404`. +- Feature enabled but Gantry not ready: return a retryable status without + attempting origin. +- On drain, stop accepting new streaming requests and allow active bodies to + finish within the existing HTTP shutdown budget. +- Add `streaming.Server.Drain` to the graceful shutdown sequence before closing + the shared listener. +- Keep the peer transfer endpoint available while active streaming requests + drain. + +### Gantry configuration + +Expose only the two decisions an operator actually owns: + +```text +artifact_streaming_enabled +artifact_streaming_allowed_host_suffixes +``` + +The peer lookup budget, peer attempt count, origin read concurrency, and origin +response-header timeout are tuning values rather than operator policy: a wrong +value degrades cold-start latency in ways an operator cannot observe from the +outside. Keep them as defaults in `internal/gantry/streaming` beside the +behavior they govern and change them with a measurement, not a knob. + +Validation must reject an enabled configuration with an empty allowlist or a +malformed host entry. + +### Metrics + +Add bounded metrics with no registry, repository, or URL labels: + +```text +gantry_streaming_requests_total{source,outcome} +gantry_streaming_bytes_total{source} +gantry_streaming_request_duration_seconds{source,outcome} +gantry_streaming_time_to_first_byte_seconds{source} +gantry_streaming_rejected_total{reason} +gantry_streaming_inflight{source} +``` + +Use `source=local|peer|origin`. Materialize bounded label combinations at zero +like existing Gantry metrics. Track provider lookup outcomes with the existing +DHT metrics only if doing so does not mix OCI-mirror and streaming semantics; +otherwise add a streaming-specific bounded lookup counter. + +### Chart changes + +- Render the new settings in `deploy/gantry/chart/templates/configmap.yaml`. +- Add values to `deploy/gantry/chart/values.yaml` and the operator profile. +- Keep port 5000 and the current loopback `hostPort`; no new port is needed. +- Update the hardening NetworkPolicy example only if the source address seen + after hostPort translation differs for OverlayBD and containerd. +- Add render tests proving the feature defaults off and enabled values survive + Helm rendering. + +**Phase exit:** `make gantry-manifests`, Helm lint, config tests, metrics tests, +and graceful shutdown tests pass with the feature disabled and enabled. + +## Phase 6: Preserve and prove complete-layer seeding + +No new chair protocol is planned. Add coverage around the existing path so +future streaming changes cannot accidentally disable Gantry's primary value. + +### Tests + +Using the redacted streaming manifest fixture, prove: + +1. Gantry serves the manifest by digest. +2. `OnManifestServed` opens and parses it from containerd. +3. Every config/layer child has the expected kind. +4. Missing children are grouped by chair and dispatched through existing + `please_pull` calls. +5. A completed chair pull commits to containerd and triggers advertisement. +6. The streaming handler's next request discovers that provider and stops using + signed origin for that digest. + +Do not add a persistent digest-to-repository catalog unless a demonstrated +failure requires range requests to restart complete pulls. In the current +design, manifest processing already owns complete pull dispatch and the range +request always has a signed fallback capability. + +**Phase exit:** one integration test exercises both concurrent paths from a +single manifest: first range from origin, complete chair commit, second range +from peer. + +## Phase 7: AKS OverlayBD configuration + +AKS Artifact Streaming node pools already install the OverlayBD daemon and +snapshotter. Gantry must configure the existing daemon to use the local endpoint: + +```json +{ + "p2pConfig": { + "enable": true, + "address": "http://localhost:5000/blobs" + } +} +``` + +### Helm/operator resources + +Add an explicitly enabled OverlayBD node-configuration workload modeled on the +upstream Peerd configurator: + +- privileged access only where required to enter the host namespaces; +- no service-account token when Kubernetes API access is unnecessary; +- node selector/affinity limiting it to Artifact Streaming node pools; +- use the AKS-provided `/opt/acr/tools/overlaybd/config.sh` rather than editing + JSON with string replacement; +- set only `p2pConfig` and leave every other OverlayBD setting exactly as the + host had it; +- verify Gantry readiness before enabling the proxy; +- record the previous values for rollback; +- avoid repeated service restarts when the desired config is already present; + and +- report readiness only after the host config and running service agree. + +Expose values such as: + +```text +gantry.artifactStreaming.enabled +overlaybdConfig.enabled +overlaybdConfig.nodeSelector +overlaybdConfig.address +``` + +The operator-managed profile renders the resource into its explicit manifest +allowlist, but reconciliation applies it only when +`spec.components.gantry.artifactStreaming.enabled` is true with a non-empty +`nodeSelector`. Disabled reconciliation explicitly deletes an older +configurator before applying Gantry without the streaming endpoint. Do not +silently deploy privileged host mutation merely because Gantry is enabled. + +### Rollout order + +1. Deploy Gantry with the streaming endpoint enabled and ready. +2. Configure OverlayBD node by node to call Gantry. +3. Validate a new streaming device on each updated node. +4. Roll back in reverse order: disable OverlayBD P2P first, then roll back + Gantry. + +**Phase exit:** a fresh AKS Artifact Streaming node can be configured and rolled +back declaratively, and reapplying identical desired state does not restart +OverlayBD services. + +## Phase 8: End-to-end validation and operational release + +### Kind contract tests + +Kind does not need a real OverlayBD block device to validate Gantry's HTTP and +peer behavior. The `e2e/gantry` contract uses a strict range client, a +cluster-local TLS origin, and a complete blob inserted on a peer node to prove: + +- raw signed URL preservation; +- origin range on a cold miss; +- exact peer range after a complete provider appears; +- stale/busy provider fallback; +- metrics; and +- secret-free logs and artifacts. + +### AKS E2E + +Use a Premium ACR, an Artifact Streaming-enabled node pool, and a real streaming +artifact. Add repository Make targets for all setup and benchmark operations. +The scenario must cover: + +- first cold Pod with no complete provider; +- manifest-triggered complete chair pulls; +- transition from signed origin to peer ranges; +- multi-node scale-out; +- private ACR/private endpoint; +- chair failure; +- signed URL expiry; +- stale provider; +- Gantry rollout while a workload reads files continuously; and +- node reboot/replacement. + +### Measurements + +Report direct measurements separately from derived values: + +- Pod readiness latency; +- first range time to first byte; +- streaming requests and bytes by local/peer/origin source; +- complete origin copies per digest; +- time from manifest observation to first complete provider; +- ACR data-plane bytes; +- peer bytes; and +- errors/retries during component restart. + +Do not claim causal improvement unless the scenario holds image, node type, +cluster size, registry topology, and workload access pattern constant. + +### Documentation and operations + +Update: + +- Gantry user guide and chart values; +- AKS installation and rollback instructions; +- Unbounded agent configuration reference; +- metrics and alerting reference; +- compatibility matrix for containerd, snapshotter, OverlayBD daemon, and + kernel backend; and +- incident procedures for Gantry unavailable, ACR unavailable, stale provider, + and OverlayBD device failures. + +**Phase exit:** all acceptance criteria below pass on the supported production +environment and the rollback procedure has been exercised. + +## Pull request sequence + +Each item should be independently reviewable and preserve existing OCI pulls. + +1. **Contract fixtures and tests**: redacted ACR manifest/URL fixtures and + existing prefetch assertions. +2. **Exact-range primitives**: `httprange` package and tests. +3. **Complete-object range readers**: containerd range reader and exact peer + client with transfer conformance tests. +4. **Azure signed-origin client**: structured URL parser, redirect policy, + response validation, and secret-leak tests. +5. **Streaming server**: local/peer/origin source resolver and handler tests. +6. **Agent wiring**: listener composition, drain, config, metrics, and chart + rendering. Feature remains disabled by default. +7. **Concurrent-path integration**: manifest `please_pull`, chair completion, + provider advertisement, and source transition test. +8. **AKS node configurator**: opt-in host configuration, readiness, rollback, + and chart/operator integration. +9. **Production E2E and operations**: AKS tests, measurements, dashboards, + alerts, compatibility matrix, and runbooks. + +PRs 2 through 7 can merge without enabling the feature. Production enablement +requires the opt-in AKS node configuration and end-to-end acceptance gates. + +## Validation commands + +Use the narrowest check after each change, then the full Gantry target before +merging. + +```text +go test ./internal/gantry/httprange +go test ./internal/gantry/streaming +go test ./internal/gantry/transfer +go test ./internal/gantry/containerdstore +go test ./internal/gantry/config +go test ./cmd/gantry +make gantry-manifests +make gantry-chart-lint +make e2e-gantry +make gantry +``` + +## Rollout and rollback + +### Rollout + +1. Upgrade Gantry everywhere with Artifact Streaming disabled. +2. Verify existing OCI pulls, DHT, chairs, and transfer behavior are unchanged. +3. Enable the Gantry streaming endpoint. +4. Verify endpoint readiness on every target node. +5. Enable OverlayBD `p2pConfig` node by node. +6. Run a streaming smoke workload on each updated node group. +7. Expand only after origin/peer metrics and active filesystem reads are healthy. + +Mixed Gantry versions are acceptable during step 1 because the existing peer +server already supports exact ranges; only upgraded requesters expose the local +OverlayBD endpoint. + +### Rollback + +1. Disable OverlayBD P2P configuration so new devices use direct origin. +2. Drain affected streaming workloads before removing the node-local endpoint. +3. Disable the Gantry streaming endpoint. +4. Roll back Gantry. + +There is no content migration. Complete layers remain ordinary containerd +content and existing DHT records remain valid. + +## Risks and mitigations + +| Risk | Mitigation | +|---|---| +| Go path cleaning corrupts embedded URLs | Raw-prefix dispatch before `ServeMux`; byte-preservation tests. | +| Signed credentials leak into logs or peers | Redacted structured logging, no URL peer field, sentinel leak tests. | +| Arbitrary URL creates SSRF | HTTPS-only structured parser, explicit host suffixes, redirect revalidation. | +| First reads amplify origin traffic | Preserve bounded chair pulls; bound origin range concurrency; measure duplicates before adding range ownership. | +| Range reads starve complete pulls | Separate concurrency pools and metrics for range origin traffic and chair origin traffic. | +| Partial response is presented as valid | Strict `206`, `Content-Range`, length, and EOF validation. | +| Snapshot annotations are absent on transfer-service pulls | Explicit containerd configuration plus contract/readiness tests. | +| Host configuration disrupts active devices | Idempotent writes, controlled node rollout, no restart on no-op, exercised rollback. | +| Partial bytes are confused with complete providers | Never write partial ranges to containerd or advertise them in the complete-object DHT. | + +## Acceptance criteria + +### Functional + +- A real ACR streaming Pod starts when no complete Gantry provider exists. +- Its first required range can be served from the signed origin URL. +- The same manifest triggers existing complete-layer chair pulls. +- After a chair advertises a layer, later ranges use the peer without ACR data + traffic for those requests. +- Existing non-streaming OCI pulls behave unchanged. + +### Correctness and security + +- Every response contains exactly the requested bytes and valid range headers. +- Complete peer layers were committed and digest-verified by containerd. +- Partial origin ranges never produce complete provider records. +- Unsupported URLs and redirects fail closed. +- No signed URL, SAS value, or authorization value appears in logs, metrics, + DHT records, peer requests, or test artifacts. + +### Reliability + +- Stale, busy, and unavailable peers fall back within the bounded request + budget. +- Existing complete peers continue serving when ACR is unavailable. +- Controlled Gantry rollout and workload drain procedures complete without + corrupting active reads. +- Rollback has been executed successfully. + +### Operations + +- Source-specific requests, bytes, latency, rejection, and in-flight metrics are + available with bounded labels. +- Alerts and runbooks distinguish Gantry failure, peer exhaustion, signed-origin + failure, and ACR failure. +- Supported component versions and kernel prerequisites are documented. + +## Out of scope for the first implementation + +- Partial-range DHT advertisements. +- Cluster-wide range ownership. +- Persistent partial-range caching. +- Changes to containerd's image pull algorithm. +- Changes to OverlayBD or the OverlayBD snapshotter. +- Provisioning OverlayBD on Unbounded-managed nodes. +- Replacing complete-layer chairs with on-demand range owners. \ No newline at end of file diff --git a/designs/gantry-acr-artifact-streaming.md b/designs/gantry-acr-artifact-streaming.md new file mode 100644 index 000000000..a05ec705c --- /dev/null +++ b/designs/gantry-acr-artifact-streaming.md @@ -0,0 +1,385 @@ +# Gantry with ACR Artifact Streaming + +**Status:** Draft for discussion + +## Summary + +ACR Artifact Streaming and Gantry solve different parts of container startup. + +ACR Artifact Streaming lets a container start without downloading every image +layer in full. OverlayBD mounts the image as a remote block device and reads +only the byte ranges required by the container. + +Gantry reduces repeated registry traffic across the cluster. After seeing an +image manifest, Gantry asks a bounded set of chairs to download the complete +layers. Those layers become verified peer seeds for every other node. + +This design combines both behaviors: + +```text +first Pod: read required ranges immediately +background: Gantry downloads and distributes complete layers +later Pods: read ranges from Gantry peers instead of ACR +``` + +The complete background downloads are intentional. They are the value Gantry +adds to Artifact Streaming: fast first startup followed by cluster-wide +convergence on complete local content. + +## The problem + +Artifact Streaming improves startup latency, but by itself each node can keep +reading image data from ACR. During a large rollout, many nodes may request the +same ranges or eventually consume most of the same layers. + +Gantry already prevents this duplication for normal OCI pulls. It coordinates +a small number of complete origin downloads and distributes the resulting +objects between peers. However, OverlayBD does not request complete OCI objects +while a container is running. It requests byte ranges from a signed ACR data +URL. + +Gantry therefore needs to support two related data paths: + +1. its existing complete-object path for proactive layer seeding; and +2. a new range path used by OverlayBD while complete layers are unavailable or + while only a subset of their bytes is needed. + +## Goals + +- Preserve Artifact Streaming's ability to start the first Pod without waiting + for complete image layers. +- Preserve Gantry's proactive, bounded complete-layer pulls. +- Serve OverlayBD ranges from local or peer content as soon as a complete layer + is available anywhere in the cluster. +- Keep signed ACR URLs on the node that received them. +- Require no Pod specification changes. +- Continue using containerd as the source of truth for complete content. +- Continue using Gantry's digest-keyed DHT for complete provider discovery. + + +## Two paths run at the same time + +The easiest way to understand the design is to separate the complete-layer path +from the range-read path. + +### Complete-layer path + +Gantry observes the resolved streaming manifest and extracts its child layer +digests. It assigns complete pulls for those digests to a bounded chair cohort. + +Each chair: + +1. downloads its assigned complete layer from ACR; +2. commits the layer to its local containerd content store; +3. verifies the layer's OCI digest; and +4. advertises itself as a provider for that digest. + +This is the same Gantry behavior used for ordinary OCI layers. It runs in the +background and does not block remote snapshot creation. + +### Range-read path + +OverlayBD reads only the byte ranges required to mount and use the image. It +sends each range to a node-local Gantry endpoint. + +Gantry serves the range from the best source currently available: + +1. the complete layer in the local containerd store; +2. a complete layer held by a Gantry peer; or +3. the signed ACR data URL supplied by OverlayBD. + +The source can change between requests. An early request may use ACR while a +chair is still downloading the layer. A later request for the same layer can use +the completed peer copy. + +## Cold Pull walkthrough + +Assume a Pod using `acr.azurecr.io/team/app:latest` is scheduled to a node that +has none of the image locally. ACR Artifact Streaming is enabled and containerd +uses the OverlayBD snapshotter. + +### 1. Kubelet requests the image + +Kubelet calls containerd's CRI `PullImage` operation with the tagged image. +Containerd selects the OverlayBD snapshotter for the workload. + +No large layer has been downloaded yet. + +### 2. ACR resolves the tag to the streaming image + +Containerd first asks Gantry to resolve the tag: + +```text +HEAD /v2/team/app/manifests/latest?ns=acr.azurecr.io +``` + +Gantry deliberately does not cache or resolve mutable tags. It returns `503`, +which makes containerd continue to ACR. ACR resolves the tag to the generated +streaming manifest and returns its immutable digest. + +### 3. Containerd fetches the streaming manifest through Gantry + +Containerd now fetches the immutable manifest: + +```text +GET /v2/team/app/manifests/sha256: +``` + +Because this request is digest-addressed, Gantry handles it through its normal +local, peer, chair, and origin logic. The manifest contains the image config and +the streaming layer descriptors. + +### 4. Gantry starts complete-layer pulls + +After serving the manifest, Gantry parses its children. For each missing layer, +Gantry selects chairs and sends a layer-specific `please_pull` request. This is an existing gantry optimization. + +The selected chairs begin downloading complete layers immediately. Different +layers can be assigned to different chairs, spreading origin and disk work +across the seed cohort. + +This work is asynchronous. Containerd does not wait for the complete downloads. + +### 5. Containerd creates remote snapshots + +Containerd passes the image reference, layer digest, layer size, and OverlayBD +annotations to the snapshotter. The snapshotter creates backing-store metadata +similar to: + +```json +{ + "repoBlobUrl": "https://acr.azurecr.io/v2/team/app/blobs", + "lowers": [ + { + "digest": "sha256:", + "size": 123456789 + } + ] +} +``` + +The snapshotter commits a remote snapshot. Under containerd's remote snapshot +protocol, this tells containerd that it does not need to download and unpack the +complete layer before continuing. + +At this point the manifest and config are local, while the large layer data may +still be remote. + +### 6. OverlayBD resolves the layer data URL + +When containerd creates the container root filesystem, the snapshotter asks the +OverlayBD daemon to attach the remote image. + +OverlayBD opens: + +```text +https://acr.azurecr.io/v2/team/app/blobs/sha256: +``` + +OverlayBD authenticates to ACR. ACR can redirect the blob request to a signed +ACR data or Azure Blob Storage URL. That signed URL grants temporary access to +the layer data. + +### 7. OverlayBD asks Gantry for a range + +OverlayBD needs small reads to load layer indexes and later to satisfy +filesystem I/O. With P2P acceleration enabled, it sends the resolved URL and the +exact range to Gantry: + +```http +GET /blobs/ +Range: bytes=N-M +``` + +The signed URL contains enough information for Gantry to identify the streaming +layer digest. + +Gantry then chooses a source: + +- **Local complete layer:** read `N-M` from the local containerd store. +- **Complete peer layer:** ask the peer for `N-M` by digest. +- **No complete provider yet:** fetch `N-M` from the signed URL. + +The first range never needs to wait for a complete chair pull. Gantry can use +the signed origin URL while complete seeding continues. + +### 8. The container starts + +After OverlayBD has loaded enough index and filesystem metadata, it exposes the +block device, the root filesystem mounts, and containerd starts the process. + +The application can continue producing range reads while the chairs download +complete layers in the background. + +### 9. Gantry takes over later reads + +When a chair finishes a layer, containerd verifies and commits it. Gantry then +advertises that chair as a provider for the layer digest. + +Subsequent range requests can use the peer's complete copy: + +```text +OverlayBD on node A + -> Gantry on node A + -> Gantry chair holding the complete layer + -> exact requested bytes +``` + +ACR is no longer involved in those reads. + +## End-to-end sequence + +```mermaid +sequenceDiagram + participant K as kubelet + participant C as containerd + participant G as Gantry requester + participant CH as Gantry chair + participant S as OverlayBD snapshotter + participant O as OverlayBD daemon + participant A as ACR + + K->>C: PullImage(image:tag) + C->>G: resolve tag + G-->>C: fall through to ACR + C->>A: resolve tag + A-->>C: streaming manifest digest + C->>G: get manifest by digest + G-->>C: streaming manifest + + par complete-layer seeding + G->>CH: please_pull(layer digest) + CH->>A: download complete layer + and remote snapshot startup + C->>S: prepare remote layer + S-->>C: remote snapshot ready + S->>O: attach remote image + O->>A: resolve layer blob + A-->>O: signed data URL + O->>G: read range N-M + G->>A: read range N-M + A-->>G: range bytes + G-->>O: range bytes + O-->>C: block device ready + end + + CH-->>G: complete provider advertised + O->>G: read later range X-Y + G->>CH: read X-Y from complete layer + CH-->>G: range bytes + G-->>O: range bytes + C-->>K: container running +``` + +The diagram shows the coldest case, where the first range reaches ACR before a +chair finishes. If a chair finishes first, Gantry can serve the first range from +that peer instead. + +## Warm and scale-out behavior + +When the same image is scheduled on more nodes, its complete layers are likely +already advertised by Gantry providers. + +Containerd still creates OverlayBD remote snapshots, so startup remains lazy. +However, the OverlayBD range requests now resolve to cluster peers rather than +ACR. New nodes receive only the ranges they need, while complete peer copies are +available if they later consume more of the image. + +The result is: + +- first-node startup remains range-based; +- origin downloads remain bounded by the chair cohort; +- scale-out traffic moves to the cluster network; and +- the cluster converges on complete, verified layers without making every node + download each layer from ACR. + +## Content and trust boundaries + +Complete layers and partial ranges have different trust properties. + +### Complete layers + +Complete layers are stored in containerd and verified against their OCI digest +before Gantry advertises them. Existing digest-keyed DHT records mean "this peer +can open and serve the complete verified object." + +### Signed origin ranges + +The signed URL is a temporary origin capability obtained by OverlayBD. Gantry +uses it only on the local node when no complete provider can serve the range. + +Gantry must not: + +- log the complete URL or SAS query; +- put it in metrics; +- advertise it in the DHT; or +- send it to a peer. + +Peer range requests contain only the layer digest and requested byte range. + +Partial origin ranges do not become normal Gantry provider records. If Gantry +later adds a transient range cache, it remains separate from containerd's +complete-object store and from complete-object DHT advertisements. + +## Failure behavior + +### A chair is still downloading + +The range request uses the signed origin URL. It does not wait indefinitely for +the chair. Later requests can use the peer after the complete layer is committed +and advertised. + +### A provider is stale or unavailable + +Gantry tries another complete provider. If no provider succeeds, it uses the +signed origin URL for that range. + +### ACR is unavailable + +Existing complete local and peer layers remain usable. A range with no complete +provider cannot be satisfied until ACR recovers or another provider appears. + +### Gantry restarts + +An active OverlayBD device depends on its node-local Gantry endpoint while P2P +configuration is enabled. This integration does not change OverlayBD to add a +second fail-open path. Production rollout therefore keeps the Gantry DaemonSet +available, uses its readiness and drain behavior for controlled replacement, +and drains active streaming workloads before disabling the OverlayBD P2P +configuration or removing the Gantry endpoint. + +## Deployment model + +Gantry exposes the OverlayBD endpoint on its existing node-local listener: + +```text +http://localhost:5000/blobs +``` + +OverlayBD is configured with: + +```json +{ + "p2pConfig": { + "enable": true, + "address": "http://localhost:5000/blobs" + } +} +``` + +The node must also: + +- run the OverlayBD daemon and snapshotter; +- register the snapshotter socket with containerd; +- select OverlayBD for Artifact Streaming workloads; and +- enable containerd's remote snapshot annotations. + +AKS Artifact Streaming node pools already provide most of this stack. For +nodes without that AKS-provided stack, provisioning and managing OverlayBD is +out of scope for this integration. + +No containerd or OverlayBD snapshotter algorithm change is currently expected. +The existing remote snapshot contracts provide the required image reference, +digest, size, backing-store configuration, and mount behavior. + + diff --git a/docs/content/guides/gantry.md b/docs/content/guides/gantry.md index e233889ff..83d7211b4 100644 --- a/docs/content/guides/gantry.md +++ b/docs/content/guides/gantry.md @@ -130,9 +130,53 @@ containerd's `certs.d` directory. Include the port when the image reference uses a non-default port. Gantry requires at least one accelerated upstream registry. +### 4. Optional: Enable ACR Artifact Streaming +ACR Artifact Streaming support requires an AKS node pool that already runs the +OverlayBD daemon and snapshotter. Gantry configures that existing installation; +it does not provision OverlayBD on other nodes. -### 4. Private Registry Authentication +For the standalone chart, set both feature gates and select only the AKS +streaming node pool: + +```yaml +gantry: + artifactStreaming: + enabled: true + +overlaybdConfig: + enabled: true + nodeSelector: + kubernetes.azure.com/agentpool: +``` + +For operator-managed Gantry, use the `Site` API instead: + +```yaml +spec: + components: + gantry: + enabled: true + artifactStreaming: + enabled: true + nodeSelector: + kubernetes.azure.com/agentpool: +``` + +Every Site that enables the cluster-wide feature must specify the same +selector. The node configurator records the previous OverlayBD configuration, +does not restart services on a no-op, and restores its previous value on +graceful removal only when no other writer changed the file. + +To roll back, drain active streaming workloads first. Operator-managed installs +then set `spec.components.gantry.artifactStreaming.enabled` to `false`; the +operator removes the configurator before rolling Gantry back without the range +endpoint. Standalone installs first set `overlaybdConfig.enabled=false` while +leaving `gantry.artifactStreaming.enabled=true`, wait for +`DaemonSet/gantry-overlaybd-config` to be deleted, and only then disable the +Gantry endpoint. + +### 5. Private Registry Authentication #### Requester-Delegated Authentication @@ -172,7 +216,7 @@ file causes the pod to fail, so do not add `credentials_path` unless the Secret is present. Prefer requester-delegated authentication when possible because it avoids distributing a shared registry credential to every node. -### 5. Verify Distribution +### 6. Verify Distribution First verify one agent directly. Select a pod and forward its health and metrics listener: diff --git a/e2e/gantry/README.md b/e2e/gantry/README.md index 3e8f252e2..3df064074 100644 --- a/e2e/gantry/README.md +++ b/e2e/gantry/README.md @@ -1,10 +1,10 @@ # gantry - end-to-end test suite This directory holds the kind-based integration suite. It boots a real -Kubernetes cluster on Docker, builds the gantry container image, and +Kubernetes cluster on Docker or rootless Podman, builds the gantry container image, and deploys the DaemonSet. The current smoke scenario asserts rollout readiness, installs containerd mirror config, pulls through Gantry on -one worker, then verifies a second worker reuses warmed content through +one node, then verifies a second node reuses warmed content through peer fetch. ## Status @@ -36,6 +36,16 @@ peer fetch. - โœ… Containerd restart recovery - restarts containerd on a worker, verifies the socket inode changes, and confirms the same Gantry pod reconnects and serves a subsequent image pull without restarting. +- โœ… ACR Artifact Streaming contract - serves the first exact range from a + cluster-local signed TLS origin, commits and advertises a complete blob on + another node, then serves the next range from that peer. The origin rejects + any request whose URI did not arrive byte-for-byte, so this also pins raw + signed-URL preservation through the real listener, and the test asserts the + SAS value never appears in any agent's logs or metrics. +- โœ… Stale provider fallback - removes the blob from the provider's containerd + while its DHT record survives, then asserts the next range fails over to the + signed origin inside one request and the peer serves nothing further. + Busy peers (429 with `Retry-After`) remain covered by unit tests only. Additional scenarios listed below should each land as their own commit. @@ -44,14 +54,20 @@ Additional scenarios listed below should each land as their own commit. The harness shells out to standard CLIs; no extra Go deps. Install: - [Docker](https://docs.docker.com/get-docker/) or [Podman](https://podman.io/) (engine running) -- [kind](https://kind.sigs.k8s.io/) โ‰ฅ 0.20 - [kubectl](https://kubernetes.io/docs/tasks/tools/) โ‰ฅ 1.28 - Go โ‰ฅ 1.26 (matching root `go.mod`) +`make e2e-gantry` installs the CI-pinned Kind version into `bin/`. Rootless +Podman additionally requires a running user systemd manager; the Make target +automatically runs the suite in a transient `Delegate=yes` scope. + ## Running ```sh make e2e-gantry # boot kind, build+load image, deploy, run tests, tear down + +# Run only the Artifact Streaming live contract. +make e2e-gantry GANTRY_E2E_RUN=TestE2E_ArtifactStreamingOriginThenPeer ``` The root Makefile defaults `CONTAINER_ENGINE` to `podman`. Set @@ -76,7 +92,7 @@ prereq CLIs: | Step | What it does | | --- | --- | -| `bootCluster()` | `kind create cluster --config kind-config.yaml` | +| `bootCluster()` | creates the Docker CI topology from `kind-config.yaml`, or the reduced rootless Podman topology from `kind-config-rootless.yaml` | | `buildAndLoadImage()` | builds `images/gantry/Containerfile` as `docker.io/library/gantry:e2e`, saves an image archive, then loads the archive into kind | | `applyManifests()` | renders `deploy/gantry/chart` with the pinned Helm binary, rewrites the DaemonSet image policy for the side-loaded image, and applies the core manifests (NetworkPolicy is intentionally not applied) | | `waitForRollout()` | polls `kubectl rollout status ds/gantry -n unbounded-system` | @@ -90,9 +106,11 @@ The smoke test also installs a `hosts.toml` mirror entry for for Gantry advertisement, then pulls the same image on a second worker and asserts peer-fetch metrics increase. -The kind config (`kind-config.yaml`) declares one control-plane + two -worker nodes - enough to exercise multi-peer coord paths in future -scenarios. +The Docker CI config (`kind-config.yaml`) declares one control-plane and seven +workers to exercise eight-chair fanout. The rootless Podman config declares one +control-plane and one worker. Both run Gantry and tolerate test workloads, so +the local topology still proves cross-node peer service while staying below the +host user's common 128 inotify-instance limit. ## Build tag @@ -130,8 +148,9 @@ The scenarios below are still gaps. Each should land as a focused commit. ## Caveats -- The kind cluster boot takes ~60โ€“120 s. The Makefile target reserves - a 10-minute test timeout to absorb that. +- Two clean local Artifact Streaming runs completed in 93 seconds and 137 + seconds on rootless Podman in the measured development environment. The Make + target retains the suite's 120-minute hard timeout for the full scenario set. - The default kind containerd uses namespace `k8s.io`, matching the gantry `containerd_namespace` default - no extra config needed. - Containerd socket access is mandatory. The default DaemonSet runs with diff --git a/e2e/gantry/artifact_streaming_e2e_test.go b/e2e/gantry/artifact_streaming_e2e_test.go new file mode 100644 index 000000000..01b64ddbd --- /dev/null +++ b/e2e/gantry/artifact_streaming_e2e_test.go @@ -0,0 +1,628 @@ +//go:build e2e + +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package e2e + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "math/big" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + utilyaml "k8s.io/apimachinery/pkg/util/yaml" +) + +const ( + streamingOriginName = "gantry-streaming-origin" + streamingOriginHost = streamingOriginName + "." + namespace + ".svc.cluster.local" +) + +// artifactStreamingOriginRequestURI builds the ACR data-path form OverlayBD +// receives after the registry redirect, doubled slash and encoded query +// included. The origin rejects anything that does not arrive byte-for-byte. +func artifactStreamingOriginRequestURI(digest, sentinel string) string { + hex := strings.TrimPrefix(digest, "sha256:") + + return "/account//docker/registry/v2/blobs/sha256/" + hex[:2] + "/" + hex + "/data" + + "?se=2030-01-01T00%3A00%3A00Z&sig=" + sentinel + "&sp=r&sv=2018-03-28" +} + +type artifactStreamingFixture struct { + t *testing.T + ctx context.Context + harness *harness + body []byte + digest string + originURL string + sentinel string + requesterPod string + providerPod string + providerNode string +} + +func setupArtifactStreaming(t *testing.T) *artifactStreamingFixture { + t.Helper() + + h := newHarness(t) + h.checkPrereqs() + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + t.Cleanup(cancel) + + h.bootCluster(ctx) + t.Cleanup(func() { + teardownCtx, teardownCancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer teardownCancel() + + h.teardown(teardownCtx) + }) + + h.buildAndLoadImage(ctx) + h.applyManifests(ctx) + h.waitForRollout(ctx) + + // A real SAS value is opaque, so use a sentinel that makes a leak unambiguous. + const sentinel = "e2eSasValueMustNotLeak" + + body := []byte(fmt.Sprintf("0123456789-artifact-streaming-%d", time.Now().UnixNano())) + digest := artifactStreamingDigest(body) + requestURI := artifactStreamingOriginRequestURI(digest, sentinel) + + caPEM, certPEM, keyPEM := generateArtifactStreamingCertificate(t, streamingOriginHost) + h.installArtifactStreamingOrigin(ctx, body, requestURI, caPEM, certPEM, keyPEM) + h.enableArtifactStreaming(ctx, caPEM) + h.waitForRollout(ctx) + h.checkReadyz(ctx) + + workers := h.workerNodes(ctx) + + return &artifactStreamingFixture{ + t: t, + ctx: ctx, + harness: h, + body: body, + digest: digest, + originURL: "https://" + streamingOriginHost + ":8443" + requestURI, + sentinel: sentinel, + requesterPod: h.gantryPodOnNode(ctx, workers[0]), + providerPod: h.gantryPodOnNode(ctx, workers[1]), + providerNode: workers[1], + } +} + +func (f *artifactStreamingFixture) request() { + f.t.Helper() + + response := f.harness.requestArtifactStreamingRange(f.ctx, f.requesterPod, f.originURL, "bytes=2-5") + assertArtifactStreamingResponse(f.t, response, len(f.body)) +} + +func (f *artifactStreamingFixture) streamingSuccesses(source string) float64 { + f.t.Helper() + + return f.harness.metricSumOnPod(f.ctx, f.requesterPod, + "gantry_streaming_requests_total", `source="`+source+`"`, `outcome="success"`) +} + +func (f *artifactStreamingFixture) waitForStreamingSuccess(source string, before float64) { + f.t.Helper() + + f.harness.waitForMetricIncreaseOnPod(f.ctx, f.requesterPod, + "gantry_streaming_requests_total", before, `source="`+source+`"`, `outcome="success"`) +} + +// serveFromPeer publishes the blob on the provider node and drives ranges until +// the requester resolves one from the advertised peer. +func (f *artifactStreamingFixture) serveFromPeer() { + f.t.Helper() + + advertiseBefore := f.harness.metricSumOnPod(f.ctx, f.providerPod, "gantry_advertise_total") + f.harness.ingestArtifactStreamingBlob(f.ctx, f.providerNode, f.digest, f.body) + f.harness.waitForMetricIncreaseOnPod(f.ctx, f.providerPod, "gantry_advertise_total", advertiseBefore) + + peerBefore := f.streamingSuccesses("peer") + deadline := time.Now().Add(2 * time.Minute) + + for { + f.request() + + if f.streamingSuccesses("peer") > peerBefore { + return + } + + if time.Now().After(deadline) { + f.harness.dumpDiagnostics(f.ctx) + f.t.Fatal("artifact streaming did not transition from signed origin to complete peer within 2m") + } + + select { + case <-f.ctx.Done(): + f.t.Fatalf("context canceled waiting for peer range service: %v", f.ctx.Err()) + case <-time.After(2 * time.Second): + } + } +} + +func TestE2E_ArtifactStreamingOriginThenPeer(t *testing.T) { + f := setupArtifactStreaming(t) + + originBefore := f.streamingSuccesses("origin") + f.request() + f.waitForStreamingSuccess("origin", originBefore) + + f.serveFromPeer() + + f.harness.assertNoSignedQueryLeak(f.ctx, f.sentinel) +} + +// TestE2E_ArtifactStreamingStaleProviderFallsBackToOrigin covers the provider +// record outliving the blob: libp2p has no protocol-level withdraw, so a peer +// that can no longer serve must be failed over inside the same request. +func TestE2E_ArtifactStreamingStaleProviderFallsBackToOrigin(t *testing.T) { + f := setupArtifactStreaming(t) + + f.serveFromPeer() + f.harness.removeArtifactStreamingBlob(f.ctx, f.providerNode, f.digest) + + originBefore := f.streamingSuccesses("origin") + peerBefore := f.streamingSuccesses("peer") + + f.request() + f.waitForStreamingSuccess("origin", originBefore) + + if peerAfter := f.streamingSuccesses("peer"); peerAfter != peerBefore { + t.Fatalf("peer served %.0f ranges after its blob was removed", peerAfter-peerBefore) + } +} + +// assertNoSignedQueryLeak pins that a signed origin credential never reaches +// anything an operator or scrape can read back. +func (h *harness) assertNoSignedQueryLeak(ctx context.Context, sentinel string) { + h.t.Helper() + + for _, pod := range h.gantryPods(ctx) { + logs, err := h.runOut(ctx, "kubectl", "-n", namespace, "logs", pod, "-c", "gantry", "--tail=-1") + if err != nil { + h.t.Fatalf("read %s logs: %v", pod, err) + } + + if strings.Contains(logs, sentinel) { + h.t.Fatalf("signed origin query leaked into %s logs", pod) + } + + if strings.Contains(h.fetchPodMetrics(ctx, pod), sentinel) { + h.t.Fatalf("signed origin query leaked into %s metrics", pod) + } + } +} + +type artifactStreamingResponse struct { + status int + body string + contentRange string +} + +func assertArtifactStreamingResponse(t *testing.T, response artifactStreamingResponse, total int) { + t.Helper() + + wantContentRange := fmt.Sprintf("bytes 2-5/%d", total) + if response.status != http.StatusPartialContent || response.body != "2345" || response.contentRange != wantContentRange { + t.Fatalf("streaming response = status %d, body %q, Content-Range %q; want 206, 2345, %s", response.status, response.body, response.contentRange, wantContentRange) + } +} + +func artifactStreamingDigest(body []byte) string { + sum := sha256.Sum256(body) + + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func generateArtifactStreamingCertificate(t *testing.T, host string) ([]byte, []byte, []byte) { + t.Helper() + + now := time.Now() + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "Gantry streaming E2E CA"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + IsCA: true, + } + + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + + serverKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + + serverTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: host}, + DNSNames: []string{host}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + + serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, caTemplate, &serverKey.PublicKey, caKey) + if err != nil { + t.Fatal(err) + } + + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}), + pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)}) +} + +func (h *harness) installArtifactStreamingOrigin(ctx context.Context, body []byte, expectedRequestURI string, caPEM, certPEM, keyPEM []byte) { + h.t.Helper() + + manifest := artifactStreamingOriginManifest(body, expectedRequestURI, caPEM, certPEM, keyPEM) + + if err := h.runWithInput(ctx, manifest, "kubectl", "apply", "-f", "-"); err != nil { + h.t.Fatalf("apply artifact streaming origin: %v", err) + } + + if err := h.run(ctx, "kubectl", "-n", namespace, "rollout", "restart", "deployment/"+streamingOriginName); err != nil { + h.t.Fatalf("restart artifact streaming origin: %v", err) + } + + if err := h.run(ctx, "kubectl", "-n", namespace, "rollout", "status", "deployment/"+streamingOriginName, "--timeout=2m"); err != nil { + h.t.Fatalf("wait for artifact streaming origin: %v", err) + } +} + +func artifactStreamingOriginManifest(body []byte, expectedRequestURI string, caPEM, certPEM, keyPEM []byte) string { + return fmt.Sprintf(`apiVersion: v1 +kind: ConfigMap +metadata: + name: %[1]s-ca + namespace: %[2]s +binaryData: + ca.crt: %[3]s +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: %[1]s-content + namespace: %[2]s +binaryData: + blob: %[4]s +data: + nginx.conf: | + worker_processes 1; + events { worker_connections 128; } + http { + access_log /dev/stdout; + error_log /dev/stderr; + server { + listen 8443 ssl; + ssl_certificate /tls/tls.crt; + ssl_certificate_key /tls/tls.key; + location / { + if ($request_uri = "%[7]s") { rewrite ^ /blob last; } + return 421; + } + location = /blob { root /usr/share/nginx/html; } + } + } +--- +apiVersion: v1 +kind: Secret +metadata: + name: %[1]s-tls + namespace: %[2]s +type: kubernetes.io/tls +data: + tls.crt: %[5]s + tls.key: %[6]s +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %[1]s + namespace: %[2]s +spec: + replicas: 1 + selector: + matchLabels: {app: %[1]s} + template: + metadata: + labels: {app: %[1]s} + spec: + containers: + - name: origin + image: registry.k8s.io/nginx-slim:0.27 + ports: + - {name: https, containerPort: 8443} + readinessProbe: + tcpSocket: {port: https} + volumeMounts: + - {name: config, mountPath: /etc/nginx/nginx.conf, subPath: nginx.conf, readOnly: true} + - {name: config, mountPath: /usr/share/nginx/html/blob, subPath: blob, readOnly: true} + - {name: tls, mountPath: /tls, readOnly: true} + volumes: + - name: config + configMap: {name: %[1]s-content} + - name: tls + secret: {secretName: %[1]s-tls} +--- +apiVersion: v1 +kind: Service +metadata: + name: %[1]s + namespace: %[2]s +spec: + selector: {app: %[1]s} + ports: + - {name: https, port: 8443, targetPort: https} +`, streamingOriginName, namespace, + base64.StdEncoding.EncodeToString(caPEM), + base64.StdEncoding.EncodeToString(body), + base64.StdEncoding.EncodeToString(certPEM), + base64.StdEncoding.EncodeToString(keyPEM), + expectedRequestURI) +} + +func TestArtifactStreamingOriginManifestIsValidYAML(t *testing.T) { + t.Parallel() + + manifest := artifactStreamingOriginManifest( + []byte("body"), + "/account//docker/registry/v2/blobs/sha256/ab/"+strings.Repeat("a", 64)+"/data?sig=x", + []byte("ca"), + []byte("certificate"), + []byte("key"), + ) + if strings.ContainsRune(manifest, '\t') { + t.Fatal("origin manifest contains a tab") + } + + decoder := utilyaml.NewYAMLOrJSONDecoder(strings.NewReader(manifest), 4096) + documents := 0 + + for { + var document map[string]any + if err := decoder.Decode(&document); err != nil { + if err == io.EOF { + break + } + + t.Fatalf("decode document %d: %v", documents+1, err) + } + + if len(document) == 0 { + continue + } + + documents++ + } + + if documents != 5 { + t.Fatalf("decoded documents = %d, want 5", documents) + } +} + +func (h *harness) enableArtifactStreaming(ctx context.Context, caPEM []byte) { + h.t.Helper() + + raw, err := os.ReadFile(filepath.Join(h.manifests, "configmap.yaml")) + if err != nil { + h.t.Fatal(err) + } + + patched, err := patchConfigMapForE2E(string(raw)) + if err != nil { + h.t.Fatal(err) + } + + patched, err = patchConfigMapForArtifactStreamingE2E(patched) + if err != nil { + h.t.Fatal(err) + } + + if err := h.runWithInput(ctx, patched, "kubectl", "apply", "-f", "-"); err != nil { + h.t.Fatalf("enable artifact streaming config: %v", err) + } + + patch := map[string]any{"spec": map[string]any{"template": map[string]any{ + "metadata": map[string]any{"annotations": map[string]any{ + "gantry.unbounded-cloud.io/streaming-origin-ca": artifactStreamingDigest(caPEM), + }}, + "spec": map[string]any{ + "containers": []any{map[string]any{ + "name": "gantry", + "env": []any{map[string]any{"name": "SSL_CERT_FILE", "value": "/etc/gantry-streaming-ca/ca.crt"}}, + "volumeMounts": []any{map[string]any{"name": "streaming-origin-ca", "mountPath": "/etc/gantry-streaming-ca", "readOnly": true}}, + }}, + "volumes": []any{map[string]any{"name": "streaming-origin-ca", "configMap": map[string]any{"name": streamingOriginName + "-ca"}}}, + }, + }}} + + patchJSON, err := json.Marshal(patch) + if err != nil { + h.t.Fatal(err) + } + + if err := h.run(ctx, "kubectl", "-n", namespace, "patch", "daemonset", dsName, "--type=strategic", "-p", string(patchJSON)); err != nil { + h.t.Fatalf("mount artifact streaming origin CA: %v", err) + } +} + +func patchConfigMapForArtifactStreamingE2E(raw string) (string, error) { + const ( + enabledFrom = " artifact_streaming_enabled: false" + enabledTo = " artifact_streaming_enabled: true" + ) + + if strings.Count(raw, enabledFrom) != 1 { + return "", fmt.Errorf("patchConfigMapForArtifactStreamingE2E: enabled anchor found %d times", strings.Count(raw, enabledFrom)) + } + + const hostsAnchor = " artifact_streaming_allowed_host_suffixes:\n" + if strings.Count(raw, hostsAnchor) != 1 { + return "", fmt.Errorf("patchConfigMapForArtifactStreamingE2E: allowed hosts anchor found %d times", strings.Count(raw, hostsAnchor)) + } + + patched := strings.Replace(raw, enabledFrom, enabledTo, 1) + patched = strings.Replace(patched, hostsAnchor, hostsAnchor+" - "+streamingOriginHost+"\n", 1) + + const ( + reconcileFrom = ` advertise_reconcile_interval: "1m"` + reconcileTo = ` advertise_reconcile_interval: "2s"` + ) + + if strings.Count(patched, reconcileFrom) != 1 { + return "", fmt.Errorf("patchConfigMapForArtifactStreamingE2E: advertise reconcile anchor found %d times", strings.Count(patched, reconcileFrom)) + } + + patched = strings.Replace(patched, reconcileFrom, reconcileTo, 1) + + return patched, nil +} + +func (h *harness) ingestArtifactStreamingBlob(ctx context.Context, node, digest string, body []byte) { + h.t.Helper() + + ref := "gantry-e2e-streaming-" + strings.TrimPrefix(digest, "sha256:")[:16] + if err := h.runWithInput(ctx, string(body), h.containerEngine, "exec", "-i", node, + "ctr", "-n", "k8s.io", "content", "ingest", + "--expected-digest", digest, "--expected-size", fmt.Sprint(len(body)), ref); err != nil { + h.t.Fatalf("ingest artifact streaming blob on %s: %v", node, err) + } + + if err := h.run(ctx, h.containerEngine, "exec", node, + "ctr", "-n", "k8s.io", "content", "label", digest, + "containerd.io/gc.root="+time.Now().UTC().Format(time.RFC3339Nano)); err != nil { + h.t.Fatalf("pin artifact streaming blob on %s: %v", node, err) + } + + if err := h.run(ctx, h.containerEngine, "exec", node, + "ctr", "-n", "k8s.io", "content", "get", digest); err != nil { + h.t.Fatalf("verify artifact streaming blob on %s: %v", node, err) + } + + h.t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := h.run(cleanupCtx, h.containerEngine, "exec", node, + "ctr", "-n", "k8s.io", "content", "rm", digest); err != nil { + h.t.Logf("remove artifact streaming blob from %s: %v", node, err) + } + }) +} + +func (h *harness) removeArtifactStreamingBlob(ctx context.Context, node, digest string) { + h.t.Helper() + + if err := h.run(ctx, h.containerEngine, "exec", node, + "ctr", "-n", "k8s.io", "content", "rm", digest); err != nil { + h.t.Fatalf("remove artifact streaming blob from %s: %v", node, err) + } +} + +func (h *harness) requestArtifactStreamingRange(ctx context.Context, pod, originURL, requestedRange string) artifactStreamingResponse { + h.t.Helper() + + port := freeLocalPort(h.t) + + forwardCtx, cancel := context.WithCancel(ctx) + defer cancel() + + cmd := exec.CommandContext(forwardCtx, "kubectl", "-n", namespace, "port-forward", "pod/"+pod, fmt.Sprintf("%d:5000", port)) + cmd.Dir = h.repoRoot + cmd.Env = os.Environ() + + var output bytes.Buffer + + cmd.Stdout = &output + + cmd.Stderr = &output + if err := cmd.Start(); err != nil { + h.t.Fatalf("start artifact streaming port-forward: %v", err) + } + + done := make(chan error, 1) + go func() { done <- cmd.Wait() }() + + defer func() { + cancel() + + select { + case <-done: + case <-time.After(5 * time.Second): + _ = cmd.Process.Kill() //nolint:errcheck // best-effort teardown + } + }() + + url := fmt.Sprintf("http://127.0.0.1:%d/blobs/%s", port, originURL) + deadline := time.Now().Add(20 * time.Second) + + var lastErr error + + for time.Now().Before(deadline) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + h.t.Fatalf("build artifact streaming request: %v", err) + } + + req.Header.Set("Range", requestedRange) + + response, requestErr := http.DefaultClient.Do(req) + if requestErr == nil { + responseBody, readErr := io.ReadAll(response.Body) + closeBody(response) + + if readErr != nil { + h.t.Fatalf("read artifact streaming response: %v", readErr) + } + + return artifactStreamingResponse{status: response.StatusCode, body: string(responseBody), contentRange: response.Header.Get("Content-Range")} + } + + lastErr = requestErr + + select { + case waitErr := <-done: + h.t.Fatalf("artifact streaming port-forward exited early (%v): %s", waitErr, output.String()) + case <-time.After(500 * time.Millisecond): + } + } + + h.t.Fatalf("artifact streaming request unavailable: %v (port-forward logs: %s)", lastErr, output.String()) + + return artifactStreamingResponse{} +} diff --git a/e2e/gantry/harness_e2e.go b/e2e/gantry/harness_e2e.go index 2c4f19a08..257dd3bfe 100644 --- a/e2e/gantry/harness_e2e.go +++ b/e2e/gantry/harness_e2e.go @@ -51,6 +51,7 @@ type harness struct { artifacts string manifests string containerEngine string + kindConfig string keepCluster bool } @@ -67,13 +68,15 @@ func newHarness(t *testing.T) *harness { } manifests := renderGantryChart(t, root) + containerEngine := resolveContainerEngine(t) return &harness{ t: t, repoRoot: root, artifacts: artifacts, manifests: manifests, - containerEngine: resolveContainerEngine(t), + containerEngine: containerEngine, + kindConfig: resolveKindConfig(root, containerEngine), keepCluster: os.Getenv("E2E_KEEP") == "1", } } @@ -125,6 +128,23 @@ func resolveContainerEngine(t *testing.T) string { return engine } +func resolveKindConfig(root, containerEngine string) string { + if configured := strings.TrimSpace(os.Getenv("GANTRY_E2E_KIND_CONFIG")); configured != "" { + if filepath.IsAbs(configured) { + return configured + } + + return filepath.Join(root, configured) + } + + name := "kind-config.yaml" + if containerEngine == "podman" { + name = "kind-config-rootless.yaml" + } + + return filepath.Join(root, "e2e", "gantry", name) +} + // checkPrereqs fails the test fast if any required CLI is missing or // the configured container engine isn't running. func (h *harness) checkPrereqs() { @@ -141,7 +161,7 @@ func (h *harness) checkPrereqs() { } } -// bootCluster creates the kind cluster declared by kind-config.yaml. +// bootCluster creates the kind cluster declared by the selected config. // Idempotent: if a cluster with the same name already exists we use it. func (h *harness) bootCluster(ctx context.Context) { h.t.Helper() @@ -151,8 +171,9 @@ func (h *harness) bootCluster(ctx context.Context) { return } - cfg := filepath.Join(h.repoRoot, "e2e", "gantry", "kind-config.yaml") - if err := h.run(ctx, "kind", "create", "cluster", "--config", cfg, "--wait", "120s"); err != nil { + h.t.Logf("creating kind cluster with %s", h.kindConfig) + + if err := h.run(ctx, "kind", "create", "cluster", "--config", h.kindConfig, "--wait", "120s"); err != nil { h.t.Fatalf("kind create cluster: %v", err) } } @@ -286,7 +307,14 @@ func (h *harness) kindNodes(ctx context.Context) []string { func (h *harness) workerNodes(ctx context.Context) []string { h.t.Helper() - out, err := h.runOut(ctx, "kubectl", "get", "nodes", "-l", "!node-role.kubernetes.io/control-plane", "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + args := []string{"get", "nodes"} + if h.containerEngine != "podman" { + args = append(args, "-l", "!node-role.kubernetes.io/control-plane") + } + + args = append(args, "-o", "jsonpath={range .items[*]}{.metadata.name}{\"\\n\"}{end}") + + out, err := h.runOut(ctx, "kubectl", args...) if err != nil { h.t.Fatalf("kubectl get nodes: %v", err) } diff --git a/e2e/gantry/harness_patch_test.go b/e2e/gantry/harness_patch_test.go index a24c9c5a5..d62c7b44d 100644 --- a/e2e/gantry/harness_patch_test.go +++ b/e2e/gantry/harness_patch_test.go @@ -31,6 +31,32 @@ func TestResolveContainerEngine(t *testing.T) { } } +func TestResolveKindConfig(t *testing.T) { + const root = "/repo" + + t.Setenv("GANTRY_E2E_KIND_CONFIG", "") + + if got := resolveKindConfig(root, "docker"); got != "/repo/e2e/gantry/kind-config.yaml" { + t.Fatalf("docker kind config = %q", got) + } + + if got := resolveKindConfig(root, "podman"); got != "/repo/e2e/gantry/kind-config-rootless.yaml" { + t.Fatalf("podman kind config = %q", got) + } + + t.Setenv("GANTRY_E2E_KIND_CONFIG", "custom/kind.yaml") + + if got := resolveKindConfig(root, "podman"); got != "/repo/custom/kind.yaml" { + t.Fatalf("relative override = %q", got) + } + + t.Setenv("GANTRY_E2E_KIND_CONFIG", "/tmp/kind.yaml") + + if got := resolveKindConfig(root, "docker"); got != "/tmp/kind.yaml" { + t.Fatalf("absolute override = %q", got) + } +} + // TestPatchDaemonSetForE2E_TargetsGantryContainerOnly is the // regression test for the twelfth-review finding: the harness's // previous strings.Replace(..., 1) on the bare "imagePullPolicy: @@ -229,3 +255,41 @@ func TestPatchConfigMapForE2E_FailsLoudWhenAnchorMissing(t *testing.T) { t.Errorf("error message = %q, want it to mention 'anchor not found' so the operator knows to update configMapUpstreamRegistriesAnchor", err.Error()) } } + +func TestPatchConfigMapForArtifactStreamingE2E(t *testing.T) { + repoRoot := repoRoot(t) + manifests := renderGantryChart(t, repoRoot) + + raw, err := os.ReadFile(filepath.Join(manifests, "configmap.yaml")) + if err != nil { + t.Fatal(err) + } + + base, err := patchConfigMapForE2E(string(raw)) + if err != nil { + t.Fatal(err) + } + + patched, err := patchConfigMapForArtifactStreamingE2E(base) + if err != nil { + t.Fatal(err) + } + + if !strings.Contains(patched, " artifact_streaming_enabled: true") { + t.Fatal("artifact streaming was not enabled") + } + + if !strings.Contains(patched, " - "+streamingOriginHost) { + t.Fatalf("artifact streaming allowed hosts do not include %s", streamingOriginHost) + } + + if !strings.Contains(patched, ` advertise_reconcile_interval: "2s"`) { + t.Fatal("artifact streaming E2E does not shorten advertiser reconciliation") + } +} + +func TestPatchConfigMapForArtifactStreamingE2EFailsWithoutAnchors(t *testing.T) { + if _, err := patchConfigMapForArtifactStreamingE2E("data: {}"); err == nil { + t.Fatal("patchConfigMapForArtifactStreamingE2E succeeded without config anchors") + } +} diff --git a/e2e/gantry/kind-config-rootless.yaml b/e2e/gantry/kind-config-rootless.yaml new file mode 100644 index 000000000..793277e96 --- /dev/null +++ b/e2e/gantry/kind-config-rootless.yaml @@ -0,0 +1,16 @@ +# Reduced local topology for rootless Podman. +# +# Rootless node containers share the host user's inotify-instance quota. The +# eight-node Docker CI topology can exhaust the common default of 128 instances +# before the control-plane CRI plugin starts. One control-plane and one worker +# retain the cross-node peer contract while keeping local resource use bounded. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +name: gantry-e2e +containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" +nodes: + - role: control-plane + - role: worker \ No newline at end of file diff --git a/hack/cmd/gantry-benchmark/artifact_streaming_test.go b/hack/cmd/gantry-benchmark/artifact_streaming_test.go new file mode 100644 index 000000000..13811413b --- /dev/null +++ b/hack/cmd/gantry-benchmark/artifact_streaming_test.go @@ -0,0 +1,333 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "io" + "reflect" + "slices" + "strings" + "testing" + "time" +) + +type artifactStreamingRunner struct { + recordingRunner +} + +func (r *artifactStreamingRunner) Run(ctx context.Context, stdin []byte, name string, args ...string) ([]byte, error) { + _, _ = r.recordingRunner.Run(ctx, stdin, name, args...) + + return []byte(`{"status":"Succeeded"}`), nil +} + +type artifactStreamingPollingRunner struct { + recordingRunner + outputs [][]byte +} + +func (r *artifactStreamingPollingRunner) Run(ctx context.Context, stdin []byte, name string, args ...string) ([]byte, error) { + _, _ = r.recordingRunner.Run(ctx, stdin, name, args...) + output := r.outputs[0] + r.outputs = r.outputs[1:] + + return output, nil +} + +type readyDaemonSetRunner struct { + recordingRunner +} + +func (r *readyDaemonSetRunner) Run(ctx context.Context, stdin []byte, name string, args ...string) ([]byte, error) { + _, _ = r.recordingRunner.Run(ctx, stdin, name, args...) + + return []byte(`{"status":{"desiredNumberScheduled":1000,"numberReady":1000}}`), nil +} + +func TestPrepareArtifactStreaming(t *testing.T) { + runner := &artifactStreamingRunner{} + benchmark := &benchmark{ + config: benchmarkConfig{GantryACRName: "benchstreamacr"}, + commands: runner, + } + state := benchmarkState{ + RunID: "run_123", + WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "benchstreamacr.azurecr.io", + } + image := "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + if err := benchmark.prepareArtifactStreaming(context.Background(), state, image); err != nil { + t.Fatalf("prepareArtifactStreaming: %v", err) + } + + want := [][]string{ + {"acr", "artifact-streaming", "update", "--name", "benchstreamacr", "--repository", "gantry-benchmark-pull", "--enable-streaming", "true", "--only-show-errors", "--output", "json"}, + {"acr", "artifact-streaming", "create", "--name", "benchstreamacr", "--image", "gantry-benchmark-pull@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "--no-wait", "--only-show-errors", "--output", "json"}, + } + if len(runner.commands) != len(want) { + t.Fatalf("commands = %d, want %d", len(runner.commands), len(want)) + } + + for index, command := range runner.commands { + if command.name != "az" || !reflect.DeepEqual(command.args, want[index]) { + t.Fatalf("command %d = %s %v, want az %v", index, command.name, command.args, want[index]) + } + } +} + +func TestRequireArtifactStreamingSucceededRejectsIncompleteOperation(t *testing.T) { + for name, output := range map[string]string{ + "pending": `{"status":"Running"}`, + "failed": `{"status":"Failed"}`, + "malformed": `{`, + } { + t.Run(name, func(t *testing.T) { + if err := requireArtifactStreamingSucceeded([]byte(output)); err == nil { + t.Fatal("operation unexpectedly succeeded") + } + }) + } +} + +func TestPrepareArtifactStreamingPollsOperation(t *testing.T) { + runner := &artifactStreamingPollingRunner{outputs: [][]byte{ + []byte(`{"status":"Succeeded"}`), + []byte(`{"id":"operation-1","status":"Running"}`), + []byte(`{"status":"Succeeded"}`), + }} + benchmark := &benchmark{ + config: benchmarkConfig{ + GantryACRName: "benchstreamacr", + ArtifactStreamingTimeout: time.Second, + ArtifactStreamingPoll: time.Millisecond, + }, + commands: runner, + stdout: io.Discard, + } + state := benchmarkState{ + WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "benchstreamacr.azurecr.io", + } + image := "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + if err := benchmark.prepareArtifactStreaming(context.Background(), state, image); err != nil { + t.Fatalf("prepareArtifactStreaming: %v", err) + } + + if len(runner.commands) != 3 { + t.Fatalf("commands = %d, want 3", len(runner.commands)) + } + status := runner.commands[2] + if status.name != "az" || !slices.Contains(status.args, "operation-1") { + t.Fatalf("status command = %s %v", status.name, status.args) + } +} + +func TestPrepareArtifactStreamingPollsByImageWhenCreateOutputIsEmpty(t *testing.T) { + runner := &artifactStreamingPollingRunner{outputs: [][]byte{ + []byte(`{"status":"Succeeded"}`), + nil, + []byte(`{"status":"Succeeded"}`), + }} + benchmark := &benchmark{ + config: benchmarkConfig{ + GantryACRName: "benchstreamacr", + ArtifactStreamingTimeout: time.Second, + ArtifactStreamingPoll: time.Millisecond, + }, + commands: runner, + stdout: io.Discard, + } + state := benchmarkState{ + WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "benchstreamacr.azurecr.io", + } + image := "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + + if err := benchmark.prepareArtifactStreaming(context.Background(), state, image); err != nil { + t.Fatalf("prepareArtifactStreaming: %v", err) + } + + status := runner.commands[2] + if !slices.Contains(status.args, "--image") || !slices.Contains(status.args, "gantry-benchmark-pull@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") { + t.Fatalf("status command = %s %v", status.name, status.args) + } + if slices.Contains(status.args, "--repository") || slices.Contains(status.args, "--id") { + t.Fatalf("status command mixes image and operation ID lookup: %v", status.args) + } +} + +func TestArtifactStreamingConfigIsOptIn(t *testing.T) { + classic, err := loadBenchmarkConfig(envFromMap(nil)) + if err != nil { + t.Fatalf("load classic config: %v", err) + } + if classic.ArtifactStreaming || classic.NodePool != "" { + t.Fatalf("classic config ArtifactStreaming=%t NodePool=%q", classic.ArtifactStreaming, classic.NodePool) + } + + streaming, err := loadBenchmarkConfig(envFromMap(map[string]string{ + "BENCHMARK_MODE": "direct", + "BENCHMARK_ARTIFACT_STREAMING": "true", + "BENCHMARK_NODE_POOL": "stream", + })) + if err != nil { + t.Fatalf("load streaming config: %v", err) + } + if !streaming.ArtifactStreaming || streaming.NodePool != "stream" { + t.Fatalf("streaming config ArtifactStreaming=%t NodePool=%q", streaming.ArtifactStreaming, streaming.NodePool) + } + + if _, err := loadBenchmarkConfig(envFromMap(map[string]string{ + "BENCHMARK_ARTIFACT_STREAMING": "true", + })); err == nil { + t.Fatal("proxy mode unexpectedly accepted Artifact Streaming") + } +} + +func TestPreparedStandaloneRequiresArtifactStreamingConversion(t *testing.T) { + state := benchmarkState{ + StandaloneGantry: true, + ArtifactStreaming: true, + GantryACRLoginServer: "benchstreamacr.azurecr.io", + GantryColdImage: "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + WorkloadPayloadSHA256: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + WorkloadRepository: "gantry-benchmark-pull", + } + + if _, _, err := state.preparedImages(); err == nil { + t.Fatal("preparedImages accepted an unconverted Artifact Streaming image") + } + + state.ArtifactStreamingPrepared = true + state.ArtifactStreamingImage = "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + if _, _, err := state.preparedImages(); err != nil { + t.Fatalf("preparedImages rejected converted image: %v", err) + } +} + +func TestGantryRuntimeImage(t *testing.T) { + const digestImage = "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + const streamingImage = "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + + classic := benchmarkState{GantryColdImage: digestImage} + if got, err := classic.gantryRuntimeImage(); err != nil || got != digestImage { + t.Fatalf("classic runtime image = %q, %v", got, err) + } + + streaming := benchmarkState{ + ArtifactStreaming: true, + ArtifactStreamingImage: streamingImage, + GantryACRLoginServer: "benchstreamacr.azurecr.io", + WorkloadRepository: "gantry-benchmark-pull", + } + if got, err := streaming.gantryRuntimeImage(); err != nil || got != streamingImage { + t.Fatalf("streaming runtime image = %q, %v", got, err) + } +} + +func TestArtifactStreamingRuntimeImageRejectsTagAndOriginalDigest(t *testing.T) { + const originalImage = "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + state := benchmarkState{ + ArtifactStreaming: true, + GantryColdImage: originalImage, + GantryACRLoginServer: "benchstreamacr.azurecr.io", + WorkloadRepository: "gantry-benchmark-pull", + ArtifactStreamingImage: "benchstreamacr.azurecr.io/gantry-benchmark-pull:converted", + } + if _, err := state.gantryRuntimeImage(); err == nil { + t.Fatal("Artifact Streaming runtime accepted a tag reference") + } + + state.ArtifactStreamingImage = originalImage + if _, err := state.gantryRuntimeImage(); err == nil { + t.Fatal("Artifact Streaming runtime accepted the original image digest") + } +} + +func TestResolveArtifactStreamingImageReturnsConvertedDigest(t *testing.T) { + runner := &artifactStreamingPollingRunner{outputs: [][]byte{ + []byte(`{"digest":"sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}`), + }} + benchmark := &benchmark{ + config: benchmarkConfig{GantryACRName: "benchstreamacr"}, + commands: runner, + } + state := benchmarkState{ + WorkloadRepository: "gantry-benchmark-pull", + GantryACRLoginServer: "benchstreamacr.azurecr.io", + } + originalImage := "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + taggedImage := "benchstreamacr.azurecr.io/gantry-benchmark-pull:converted" + + resolved, err := benchmark.resolveArtifactStreamingImage(context.Background(), state, originalImage, taggedImage) + if err != nil { + t.Fatalf("resolveArtifactStreamingImage: %v", err) + } + want := "benchstreamacr.azurecr.io/gantry-benchmark-pull@sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + if resolved != want { + t.Fatalf("resolved image = %q, want %q", resolved, want) + } + if len(runner.commands) != 1 || runner.commands[0].name != "az" || + !reflect.DeepEqual(runner.commands[0].args, []string{ + "acr", "manifest", "show-metadata", + "--registry", "benchstreamacr", + "--name", "gantry-benchmark-pull:converted", + "--only-show-errors", + "--output", "json", + }) { + t.Fatalf("resolver command = %+v", runner.commands) + } +} + +func TestValidateArtifactStreamingGantryConfig(t *testing.T) { + const registry = "benchstreamacr.azurecr.io" + raw := `artifact_streaming_enabled: true +upstream_registries: + - name: benchstreamacr.azurecr.io + endpoint: http://127.0.0.1:8578?ns=benchstreamacr.azurecr.io +` + + if err := validateArtifactStreamingGantryConfig(raw, registry); err != nil { + t.Fatalf("validateArtifactStreamingGantryConfig: %v", err) + } + + for name, invalid := range map[string]string{ + "disabled": strings.Replace(raw, "artifact_streaming_enabled: true", "artifact_streaming_enabled: false", 1), + "direct ACR": strings.Replace(raw, "http://127.0.0.1:8578?ns="+registry, "https://"+registry, 1), + } { + t.Run(name, func(t *testing.T) { + if err := validateArtifactStreamingGantryConfig(invalid, registry); err == nil { + t.Fatal("validation unexpectedly passed") + } + }) + } +} + +func TestValidateStreamingDaemonSetUsesGantryNamespace(t *testing.T) { + runner := &readyDaemonSetRunner{} + benchmark := &benchmark{ + config: benchmarkConfig{ + Namespace: "gantry-benchmark", + GantryNamespace: "gantry-system", + NodeCount: 1000, + }, + commands: runner, + } + + if err := benchmark.validateDaemonSet(context.Background(), benchmark.config.GantryNamespace, "gantry-overlaybd-config"); err != nil { + t.Fatalf("validateDaemonSet: %v", err) + } + + if len(runner.commands) != 1 { + t.Fatalf("commands = %d, want 1", len(runner.commands)) + } + command := runner.commands[0] + want := []string{"-n", "gantry-system", "get", "daemonset", "gantry-overlaybd-config", "-o", "json"} + if command.name != "kubectl" || !reflect.DeepEqual(command.args, want) { + t.Fatalf("command = %s %v, want kubectl %v", command.name, command.args, want) + } +} diff --git a/hack/cmd/gantry-benchmark/cluster.go b/hack/cmd/gantry-benchmark/cluster.go index e971f6cbd..b2faf5707 100644 --- a/hack/cmd/gantry-benchmark/cluster.go +++ b/hack/cmd/gantry-benchmark/cluster.go @@ -57,8 +57,8 @@ func (b *benchmark) validateContext(ctx context.Context) error { } func (b *benchmark) targetNodes(ctx context.Context) ([]string, error) { - platformParts := strings.SplitN(b.config.ImagePlatform, "/", 2) - if len(platformParts) != 2 || platformParts[0] == "" || platformParts[1] == "" { + selector := b.config.nodeSelector() + if selector["kubernetes.io/os"] == "" || selector["kubernetes.io/arch"] == "" { return nil, fmt.Errorf("image platform BENCHMARK_IMAGE_PLATFORM=%q must have os/architecture form", b.config.ImagePlatform) } @@ -74,9 +74,15 @@ func (b *benchmark) targetNodes(ctx context.Context) ([]string, error) { result := make([]string, 0, len(list.Items)) for _, node := range list.Items { - if node.Metadata.Labels["kubernetes.io/os"] != platformParts[0] || - node.Metadata.Labels["kubernetes.io/arch"] != platformParts[1] || - node.Spec.Unschedulable { + matches := true + for key, value := range selector { + if node.Metadata.Labels[key] != value { + matches = false + + break + } + } + if !matches || node.Spec.Unschedulable { continue } diff --git a/hack/cmd/gantry-benchmark/cluster_test.go b/hack/cmd/gantry-benchmark/cluster_test.go index 09dca9362..8a046b3d6 100644 --- a/hack/cmd/gantry-benchmark/cluster_test.go +++ b/hack/cmd/gantry-benchmark/cluster_test.go @@ -3,7 +3,45 @@ package main -import "testing" +import ( + "reflect" + "testing" +) + +func TestNodeSelector(t *testing.T) { + tests := []struct { + name string + pool string + want map[string]string + }{ + { + name: "classic default", + want: map[string]string{"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}, + }, + { + name: "dedicated pool", + pool: "stream", + want: map[string]string{"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64", "agentpool": "stream"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + config := benchmarkConfig{ImagePlatform: "linux/amd64", NodePool: test.pool} + if got := config.nodeSelector(); !reflect.DeepEqual(got, test.want) { + t.Fatalf("nodeSelector() = %v, want %v", got, test.want) + } + }) + } +} + +func TestGantryAgentSelector(t *testing.T) { + config := benchmarkConfig{GantryDaemonSet: "gantry"} + want := "app.kubernetes.io/name=gantry,app.kubernetes.io/component=agent" + if got := config.gantryAgentSelector(); got != want { + t.Fatalf("gantryAgentSelector() = %q, want %q", got, want) + } +} func TestValidateGantryStatus(t *testing.T) { status := daemonSetStatus{} diff --git a/hack/cmd/gantry-benchmark/config.go b/hack/cmd/gantry-benchmark/config.go index 3710d4268..953652726 100644 --- a/hack/cmd/gantry-benchmark/config.go +++ b/hack/cmd/gantry-benchmark/config.go @@ -50,6 +50,7 @@ type benchmarkConfig struct { BaselineACRUsername string BaselineACRPassword string GantryACRLoginServer string + GantryACRName string GantryACRUsername string GantryACRPassword string ACRLoginServer string @@ -60,6 +61,10 @@ type benchmarkConfig struct { ImagePlatform string ContainerEngine string ConfirmedContext string + NodePool string + ArtifactStreaming bool + ArtifactStreamingTimeout time.Duration + ArtifactStreamingPoll time.Duration NodeCount int ImageSizeMiB int ImageLayers int @@ -131,6 +136,21 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { return benchmarkConfig{}, err } + artifactStreaming, err := envBool(getenv, "BENCHMARK_ARTIFACT_STREAMING", false) + if err != nil { + return benchmarkConfig{}, err + } + + artifactStreamingTimeout, err := envDuration(getenv, "BENCHMARK_ARTIFACT_STREAMING_TIMEOUT", 2*time.Hour) + if err != nil { + return benchmarkConfig{}, err + } + + artifactStreamingPoll, err := envDuration(getenv, "BENCHMARK_ARTIFACT_STREAMING_POLL_INTERVAL", 30*time.Second) + if err != nil { + return benchmarkConfig{}, err + } + telemetryTimeout, err := envDuration(getenv, "BENCHMARK_TELEMETRY_TIMEOUT", 15*time.Minute) if err != nil { return benchmarkConfig{}, err @@ -183,6 +203,7 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { BaselineACRUsername: getenv("BASELINE_ACR_USERNAME"), BaselineACRPassword: getenv("BASELINE_ACR_PASSWORD"), GantryACRLoginServer: getenv("GANTRY_ACR_LOGIN_SERVER"), + GantryACRName: getenv("GANTRY_ACR_NAME"), GantryACRUsername: getenv("GANTRY_ACR_USERNAME"), GantryACRPassword: getenv("GANTRY_ACR_PASSWORD"), ACRLoginServer: getenv("ACR_LOGIN_SERVER"), @@ -193,6 +214,10 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { ImagePlatform: envDefault(getenv, "BENCHMARK_IMAGE_PLATFORM", "linux/amd64"), ContainerEngine: envDefault(getenv, "CONTAINER_ENGINE", "podman"), ConfirmedContext: getenv("BENCHMARK_CONFIRM_CONTEXT"), + NodePool: getenv("BENCHMARK_NODE_POOL"), + ArtifactStreaming: artifactStreaming, + ArtifactStreamingTimeout: artifactStreamingTimeout, + ArtifactStreamingPoll: artifactStreamingPoll, NodeCount: nodeCount, ImageSizeMiB: imageSizeMiB, ImageLayers: imageLayers, @@ -241,6 +266,13 @@ func loadBenchmarkConfig(getenv func(string) string) (benchmarkConfig, error) { return benchmarkConfig{}, errors.New("benchmark maximum latency ratio must be greater than zero") } + if config.ArtifactStreaming && config.Mode != benchmarkModeDirect { + return benchmarkConfig{}, errors.New("BENCHMARK_ARTIFACT_STREAMING requires BENCHMARK_MODE=direct") + } + if config.ArtifactStreamingTimeout <= 0 || config.ArtifactStreamingPoll <= 0 { + return benchmarkConfig{}, errors.New("Artifact Streaming timeout and poll interval must be greater than zero") + } + if config.TelemetryTimeout <= 0 { return benchmarkConfig{}, errors.New("benchmark telemetry timeout must be greater than zero") } @@ -256,6 +288,18 @@ func (c benchmarkConfig) usesProxy() bool { return c.Mode != benchmarkModeDirect } +func (c benchmarkConfig) gantryUpstreamEndpoint() string { + if c.ArtifactStreaming { + return "http://127.0.0.1:8578?ns=" + c.GantryACRLoginServer + } + + return "https://" + c.GantryACRLoginServer +} + +func (c benchmarkConfig) gantryAgentSelector() string { + return "app.kubernetes.io/name=" + c.GantryDaemonSet + ",app.kubernetes.io/component=agent" +} + func (c benchmarkConfig) registryForPhase(phase proxyPhase) (phaseRegistry, error) { if c.usesProxy() { return phaseRegistry{ @@ -303,6 +347,9 @@ func (c benchmarkConfig) validateEnable() error { } else { required["BASELINE_ACR_LOGIN_SERVER"] = c.BaselineACRLoginServer required["GANTRY_ACR_LOGIN_SERVER"] = c.GantryACRLoginServer + if c.ArtifactStreaming { + required["GANTRY_ACR_NAME"] = c.GantryACRName + } if c.BaselineACRLoginServer != "" && strings.EqualFold(c.BaselineACRLoginServer, c.GantryACRLoginServer) { return errors.New("baseline and Gantry ACR login servers must be different") @@ -349,6 +396,9 @@ func (c benchmarkConfig) nodeSelector() map[string]string { selector["kubernetes.io/os"] = parts[0] selector["kubernetes.io/arch"] = parts[1] } + if c.NodePool != "" { + selector["agentpool"] = c.NodePool + } return selector } diff --git a/hack/cmd/gantry-benchmark/enable.go b/hack/cmd/gantry-benchmark/enable.go index afee259e5..32cfbf739 100644 --- a/hack/cmd/gantry-benchmark/enable.go +++ b/hack/cmd/gantry-benchmark/enable.go @@ -101,7 +101,11 @@ func (b *benchmark) enable(ctx context.Context) (returnErr error) { } if !b.config.usesProxy() { - if err := validateDirectGantryRegistry([]byte(originalConfig), b.config.GantryACRLoginServer); err != nil { + if err := validateDirectGantryRegistry( + []byte(originalConfig), + b.config.GantryACRLoginServer, + b.config.gantryUpstreamEndpoint(), + ); err != nil { return fmt.Errorf("validate dedicated Gantry ACR configuration: %w", err) } } @@ -117,11 +121,13 @@ func (b *benchmark) enable(ctx context.Context) (returnErr error) { GantryConfigMap: b.config.GantryConfigMap, MonitoringNamespace: b.config.MonitoringNamespace, PrometheusService: b.config.PrometheusService, + NodePool: b.config.NodePool, NodeCount: b.config.NodeCount, ImagePlatform: b.config.ImagePlatform, ImageSizeMiB: b.config.ImageSizeMiB, ImageLayers: b.config.ImageLayers, WorkloadRepository: b.config.WorkloadRepository, + ArtifactStreaming: b.config.ArtifactStreaming, BaselineACRLoginServer: b.config.BaselineACRLoginServer, GantryACRLoginServer: b.config.GantryACRLoginServer, ACRLoginServer: b.config.ACRLoginServer, @@ -149,6 +155,7 @@ func (b *benchmark) enable(ctx context.Context) (returnErr error) { MonitoringLabel: b.config.KPSRelease, NodeOS: strings.SplitN(b.config.ImagePlatform, "/", 2)[0], NodeArch: strings.SplitN(b.config.ImagePlatform, "/", 2)[1], + NodePool: b.config.NodePool, ProxyImage: b.config.ProxyImage, ACRLoginServer: b.config.ACRLoginServer, RunID: runID, @@ -338,6 +345,7 @@ type proxyManifestData struct { MonitoringLabel string NodeOS string NodeArch string + NodePool string ProxyImage string ACRLoginServer string RunID string diff --git a/hack/cmd/gantry-benchmark/enable_test.go b/hack/cmd/gantry-benchmark/enable_test.go index 3cc8cee5d..72a46a15e 100644 --- a/hack/cmd/gantry-benchmark/enable_test.go +++ b/hack/cmd/gantry-benchmark/enable_test.go @@ -59,6 +59,31 @@ func TestRenderProxyManifest(t *testing.T) { } } +func TestRenderMonitoringManifestSelectsBenchmarkPool(t *testing.T) { + repoRoot, err := findRepoRoot() + if err != nil { + t.Fatalf("findRepoRoot: %v", err) + } + + benchmark := &benchmark{config: benchmarkConfig{RepoRoot: repoRoot}} + rendered, err := benchmark.renderManifest(monitoringManifestPath, proxyManifestData{ + Namespace: "gantry-benchmark", + GantryNamespace: "gantry-system", + MonitoringLabel: "kps", + NodeOS: "linux", + NodeArch: "amd64", + NodePool: "stream", + RunID: "run-1", + }) + if err != nil { + t.Fatalf("renderManifest: %v", err) + } + + if !strings.Contains(string(rendered), "agentpool: stream") { + t.Fatalf("streaming monitoring manifest does not select the benchmark pool") + } +} + // The Gantry PodMonitor lives outside the proxy template because direct mode // installs no proxy but still needs gantry_benchmark-labeled agent samples. func TestRenderMonitoringManifest(t *testing.T) { @@ -84,9 +109,13 @@ func TestRenderMonitoringManifest(t *testing.T) { if strings.Contains(string(rendered), "{{") { t.Fatalf("rendered manifest contains an unresolved template expression") } + if strings.Contains(string(rendered), "agentpool:") { + t.Fatalf("classic monitoring manifest unexpectedly selects a node pool") + } if !strings.Contains(string(rendered), `targetLabel: gantry_benchmark`) || - !strings.Contains(string(rendered), `- controller-revision-hash`) { + !strings.Contains(string(rendered), `- controller-revision-hash`) || + !strings.Contains(string(rendered), `app.kubernetes.io/component: agent`) { t.Fatalf("rendered manifest is missing benchmark scrape or Gantry revision labels") } @@ -112,6 +141,9 @@ func TestRenderMonitoringManifest(t *testing.T) { } for _, metric := range []string{ + "gantry_streaming_(requests|bytes|rejected)_total", + "gantry_streaming_(request_duration|time_to_first_byte)_seconds_(bucket|sum|count)", + "gantry_streaming_inflight", "p2p_peer_fetch_duration_seconds_(bucket|sum|count)", "p2p_dht_lookup_duration_seconds_(bucket|sum|count)", "gantry_peer_fetch_last_timestamp_seconds", @@ -361,7 +393,10 @@ func TestContainerdPullTuningManifest(t *testing.T) { } } - if !bytes.Contains(deployScript, []byte("migrate_legacy_gantry_install\n \"$repo_root/bin/helm\" upgrade --install gantry")) { + migrateIndex := bytes.Index(deployScript, []byte("migrate_legacy_gantry_install")) + helmArgsIndex := bytes.Index(deployScript, []byte("local helm_args=(")) + helmRunIndex := bytes.Index(deployScript, []byte("\"$repo_root/bin/helm\" \"${helm_args[@]}\"")) + if migrateIndex < 0 || helmArgsIndex <= migrateIndex || helmRunIndex <= helmArgsIndex { t.Fatal("deploy script does not migrate legacy Gantry resources before Helm install") } } diff --git a/hack/cmd/gantry-benchmark/gantry_config.go b/hack/cmd/gantry-benchmark/gantry_config.go index 0d1962a47..46e27f6f2 100644 --- a/hack/cmd/gantry-benchmark/gantry_config.go +++ b/hack/cmd/gantry-benchmark/gantry_config.go @@ -13,14 +13,13 @@ import ( gantryconfig "github.com/Azure/unbounded/internal/gantry/config" ) -func validateDirectGantryRegistry(raw []byte, registryName string) error { +func validateDirectGantryRegistry(raw []byte, registryName, expectedEndpoint string) error { config := gantryconfig.NewDefault() if err := config.LoadYAML(bytes.NewReader(raw)); err != nil { return fmt.Errorf("load Gantry config: %w", err) } matches := 0 - wantEndpoint := "https://" + registryName for _, registry := range config.UpstreamRegistries { if registry.Name != registryName { @@ -29,8 +28,8 @@ func validateDirectGantryRegistry(raw []byte, registryName string) error { matches++ - if strings.TrimSuffix(registry.Endpoint, "/") != wantEndpoint { - return fmt.Errorf("gantry registry %q endpoint is %q, want %q", registryName, registry.Endpoint, wantEndpoint) + if strings.TrimSuffix(registry.Endpoint, "/") != expectedEndpoint { + return fmt.Errorf("gantry registry %q endpoint is %q, want %q", registryName, registry.Endpoint, expectedEndpoint) } } diff --git a/hack/cmd/gantry-benchmark/gantry_config_test.go b/hack/cmd/gantry-benchmark/gantry_config_test.go index 71660e412..864b7cc1d 100644 --- a/hack/cmd/gantry-benchmark/gantry_config_test.go +++ b/hack/cmd/gantry-benchmark/gantry_config_test.go @@ -110,9 +110,21 @@ func TestValidateDirectGantryRegistry(t *testing.T) { endpoint: https://gantry.azurecr.io `) - if err := validateDirectGantryRegistry(raw, "gantry.azurecr.io"); err != nil { + if err := validateDirectGantryRegistry(raw, "gantry.azurecr.io", "https://gantry.azurecr.io"); err != nil { t.Fatalf("validateDirectGantryRegistry: %v", err) } + + streaming := []byte(`upstream_registries: + - name: gantry.azurecr.io + endpoint: http://127.0.0.1:8578?ns=gantry.azurecr.io +`) + if err := validateDirectGantryRegistry( + streaming, + "gantry.azurecr.io", + "http://127.0.0.1:8578?ns=gantry.azurecr.io", + ); err != nil { + t.Fatalf("validate streaming registry: %v", err) + } } func TestValidateDirectGantryRegistryRejectsBaselineEndpoint(t *testing.T) { @@ -121,7 +133,7 @@ func TestValidateDirectGantryRegistryRejectsBaselineEndpoint(t *testing.T) { endpoint: https://baseline.azurecr.io `) - if err := validateDirectGantryRegistry(raw, "gantry.azurecr.io"); err == nil || !strings.Contains(err.Error(), "want") { + if err := validateDirectGantryRegistry(raw, "gantry.azurecr.io", "https://gantry.azurecr.io"); err == nil || !strings.Contains(err.Error(), "want") { t.Fatalf("error = %v, want endpoint mismatch", err) } } diff --git a/hack/cmd/gantry-benchmark/gantry_only.go b/hack/cmd/gantry-benchmark/gantry_only.go index c8fee16e3..41b69bb3a 100644 --- a/hack/cmd/gantry-benchmark/gantry_only.go +++ b/hack/cmd/gantry-benchmark/gantry_only.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "context" "encoding/json" "errors" @@ -241,6 +242,19 @@ func (b *benchmark) prepareStandaloneGantry(ctx context.Context) error { return err } + if state.ArtifactStreaming { + if err := b.prepareArtifactStreaming(ctx, state, gantryImage); err != nil { + return err + } + streamingImage, err := b.resolveArtifactStreamingImage(ctx, state, gantryImage, standaloneGantryTaggedImage(state)) + if err != nil { + return err + } + + state.ArtifactStreamingPrepared = true + state.ArtifactStreamingImage = streamingImage + } + state.StandaloneGantry = true state.BaselineImage = "" state.GantryColdImage = gantryImage @@ -261,6 +275,284 @@ func (b *benchmark) prepareStandaloneGantry(ctx context.Context) error { return nil } +func (b *benchmark) prepareAdoptedStandaloneGantry(ctx context.Context, image, streamingImage, payloadSHA string) error { + state, err := b.loadState(ctx) + if err != nil { + return err + } + + if state.Status != "enabled" { + return fmt.Errorf("benchmark state is %q, run enable before prepare-gantry-standalone-adopt", state.Status) + } + if state.usesProxy() { + return fmt.Errorf("prepare-gantry-standalone-adopt requires direct dual-ACR mode") + } + if err := b.requireLock(ctx, state.RunID); err != nil { + return err + } + if err := b.validateContext(ctx); err != nil { + return err + } + + repository, digestValue, err := splitImageReference(image, state.GantryACRLoginServer) + if err != nil { + return fmt.Errorf("adopt standalone image: %w", err) + } + if repository != state.WorkloadRepository || !strings.HasPrefix(digestValue, "sha256:") { + return fmt.Errorf("adopted image must be a digest-pinned %s image", state.WorkloadRepository) + } + if parsed, err := digest.Parse(payloadSHA); err != nil || parsed.Algorithm() != digest.SHA256 { + return fmt.Errorf("payload fingerprint %q must be a sha256 digest", payloadSHA) + } + + if state.ArtifactStreaming { + state.ArtifactStreamingImage = streamingImage + if err := state.validateArtifactStreamingImage(); err != nil { + return err + } + if err := b.verifyArtifactStreamingImage(ctx, state, image); err != nil { + return err + } + state.ArtifactStreamingPrepared = true + } + + state.StandaloneGantry = true + state.BaselineImage = "" + state.GantryColdImage = image + state.WorkloadPayloadSHA256 = payloadSHA + state.WorkloadComparisonMode = workloadComparisonRandomShape + state.Status = "images-prepared" + + if _, _, err := state.preparedImages(); err != nil { + return err + } + if err := b.saveState(ctx, state); err != nil { + return err + } + + writeAll(b.stdout, fmt.Sprintf("adopted standalone Gantry image %s for %s\n", image, state.RunID)) + + return nil +} + +func standaloneGantryTaggedImage(state benchmarkState) string { + tag := strings.ReplaceAll(state.RunID+"-gantry-fresh", "_", "-") + + return fmt.Sprintf("%s/%s:%s", state.GantryACRLoginServer, state.WorkloadRepository, tag) +} + +func (b *benchmark) resolveArtifactStreamingImage( + ctx context.Context, + state benchmarkState, + originalImage string, + taggedImage string, +) (string, error) { + originalRepository, originalDigest, err := splitImageReference(originalImage, state.GantryACRLoginServer) + if err != nil { + return "", fmt.Errorf("resolve original image: %w", err) + } + taggedRepository, tag, err := splitImageReference(taggedImage, state.GantryACRLoginServer) + if err != nil { + return "", fmt.Errorf("resolve converted image tag: %w", err) + } + if originalRepository != state.WorkloadRepository || taggedRepository != state.WorkloadRepository { + return "", fmt.Errorf("Artifact Streaming image repositories must both be %q", state.WorkloadRepository) + } + if strings.HasPrefix(tag, "sha256:") { + return "", fmt.Errorf("converted image lookup must use a tag: %s", taggedImage) + } + + commandContext, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + output, err := b.commands.Run(commandContext, nil, + "az", "acr", "manifest", "show-metadata", + "--registry", b.config.GantryACRName, + "--name", taggedRepository+":"+tag, + "--only-show-errors", + "--output", "json", + ) + if err != nil { + return "", fmt.Errorf("resolve converted Artifact Streaming manifest %s: %w", taggedImage, err) + } + + var metadata struct { + Digest string `json:"digest"` + } + if err := json.Unmarshal(output, &metadata); err != nil { + return "", fmt.Errorf("decode converted Artifact Streaming manifest metadata: %w", err) + } + convertedDigest, err := digest.Parse(metadata.Digest) + if err != nil || convertedDigest.Algorithm() != digest.SHA256 { + return "", fmt.Errorf("converted Artifact Streaming manifest has invalid digest %q", metadata.Digest) + } + if metadata.Digest == originalDigest { + return "", fmt.Errorf("converted Artifact Streaming manifest digest still equals original image digest %s", originalDigest) + } + + return fmt.Sprintf("%s/%s@%s", state.GantryACRLoginServer, taggedRepository, convertedDigest), nil +} + +func (b *benchmark) verifyArtifactStreamingImage(ctx context.Context, state benchmarkState, imageReference string) error { + repository, digestValue, err := splitImageReference(imageReference, state.GantryACRLoginServer) + if err != nil { + return fmt.Errorf("verify Artifact Streaming image: %w", err) + } + image := repository + "@" + digestValue + + output, err := b.runArtifactStreamingCommand(ctx, + "operation", "show", + "--name", b.config.GantryACRName, + "--image", image, + "--only-show-errors", + "--output", "json", + ) + if err != nil { + return fmt.Errorf("verify Artifact Streaming image %s: %w", image, err) + } + if err := requireArtifactStreamingSucceeded(output); err != nil { + return fmt.Errorf("verify Artifact Streaming image %s: %w", image, err) + } + + return nil +} + +func (b *benchmark) prepareArtifactStreaming(ctx context.Context, state benchmarkState, imageReference string) error { + if b.config.GantryACRName == "" { + return fmt.Errorf("BENCHMARK_ARTIFACT_STREAMING requires GANTRY_ACR_NAME") + } + + repository, digest, err := splitImageReference(imageReference, state.GantryACRLoginServer) + if err != nil { + return fmt.Errorf("resolve Artifact Streaming image: %w", err) + } + if !strings.HasPrefix(digest, "sha256:") { + return fmt.Errorf("Artifact Streaming image must be digest-pinned: %s", imageReference) + } + image := repository + "@" + digest + + updateOutput, err := b.runArtifactStreamingCommand(ctx, + "update", + "--name", b.config.GantryACRName, + "--repository", state.WorkloadRepository, + "--enable-streaming", "true", + "--only-show-errors", + "--output", "json", + ) + if err != nil { + return fmt.Errorf("enable Artifact Streaming on repository %s: %w", state.WorkloadRepository, err) + } + if err := requireArtifactStreamingSucceeded(updateOutput); err != nil { + return fmt.Errorf("enable Artifact Streaming on repository %s: %w", state.WorkloadRepository, err) + } + + createOutput, err := b.runArtifactStreamingCommand(ctx, + "create", + "--name", b.config.GantryACRName, + "--image", image, + "--no-wait", + "--only-show-errors", + "--output", "json", + ) + if err != nil { + return fmt.Errorf("convert image %s for Artifact Streaming: %w", image, err) + } + operation := artifactStreamingOperation{Status: "Submitted"} + lookupArgs := []string{"--image", image} + operationLabel := image + if len(bytes.TrimSpace(createOutput)) != 0 { + operation, err = parseArtifactStreamingOperation(createOutput) + if err != nil { + return fmt.Errorf("convert image %s for Artifact Streaming: %w", image, err) + } + if operation.Status == "Succeeded" { + return nil + } + if operation.ID != "" { + lookupArgs = []string{"--repository", repository, "--id", operation.ID} + operationLabel = operation.ID + } + } + + writeAll(b.stdout, fmt.Sprintf("Artifact Streaming conversion %s status=%s\n", operationLabel, operation.Status)) + + deadline := time.NewTimer(b.config.ArtifactStreamingTimeout) + defer deadline.Stop() + ticker := time.NewTicker(b.config.ArtifactStreamingPoll) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-deadline.C: + return fmt.Errorf("Artifact Streaming conversion %s did not complete within %s", operationLabel, b.config.ArtifactStreamingTimeout) + case <-ticker.C: + statusArgs := []string{ + "operation", "show", + "--name", b.config.GantryACRName, + } + statusArgs = append(statusArgs, lookupArgs...) + statusArgs = append(statusArgs, + "--only-show-errors", + "--output", "json", + ) + statusOutput, err := b.runArtifactStreamingCommand(ctx, statusArgs...) + if err != nil { + return fmt.Errorf("read Artifact Streaming conversion %s: %w", operationLabel, err) + } + + operation, err = parseArtifactStreamingOperation(statusOutput) + if err != nil { + return fmt.Errorf("read Artifact Streaming conversion %s: %w", operationLabel, err) + } + writeAll(b.stdout, fmt.Sprintf("Artifact Streaming conversion %s status=%s\n", operationLabel, operation.Status)) + + switch operation.Status { + case "Succeeded": + return nil + case "Failed", "Canceled", "Cancelled": + return fmt.Errorf("Artifact Streaming conversion %s ended with status %s", operationLabel, operation.Status) + } + } + } +} + +func (b *benchmark) runArtifactStreamingCommand(ctx context.Context, args ...string) ([]byte, error) { + commandContext, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + commandArgs := append([]string{"acr", "artifact-streaming"}, args...) + + return b.commands.Run(commandContext, nil, "az", commandArgs...) +} + +type artifactStreamingOperation struct { + ID string `json:"id"` + Status string `json:"status"` +} + +func parseArtifactStreamingOperation(output []byte) (artifactStreamingOperation, error) { + var operation artifactStreamingOperation + if err := json.Unmarshal(output, &operation); err != nil { + return artifactStreamingOperation{}, fmt.Errorf("decode operation: %w", err) + } + + return operation, nil +} + +func requireArtifactStreamingSucceeded(output []byte) error { + operation, err := parseArtifactStreamingOperation(output) + if err != nil { + return err + } + if operation.Status != "Succeeded" { + return fmt.Errorf("operation status is %q, want Succeeded", operation.Status) + } + + return nil +} + func (b *benchmark) prepareAdoptedFreshGantryOnly(ctx context.Context, baselineRunID, image, payloadSHA string) error { state, err := b.loadState(ctx) if err != nil { @@ -605,6 +897,13 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { return fmt.Errorf("benchmark state is %q, run preflight before run-gantry", state.Status) } + if state.ArtifactStreaming != b.config.ArtifactStreaming { + return fmt.Errorf("benchmark Artifact Streaming state=%t does not match BENCHMARK_ARTIFACT_STREAMING=%t", state.ArtifactStreaming, b.config.ArtifactStreaming) + } + if state.NodePool != b.config.NodePool { + return fmt.Errorf("benchmark node pool state=%q does not match BENCHMARK_NODE_POOL=%q", state.NodePool, b.config.NodePool) + } + if err := b.requireLock(ctx, state.RunID); err != nil { return err } @@ -625,6 +924,10 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { if err != nil { return err } + runtimeImage, err := state.gantryRuntimeImage() + if err != nil { + return err + } var baselineResult phaseResult if !state.StandaloneGantry { @@ -709,7 +1012,7 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { writeAll(b.stdout, fmt.Sprintf("running Gantry-only cold pull on %d nodes\n", b.config.NodeCount)) - job, err := b.runPullJob(ctx, state, proxyPhaseGantryCold, gantryImage) + job, err := b.runPullJob(ctx, state, proxyPhaseGantryCold, runtimeImage) if err != nil { return err } @@ -751,14 +1054,11 @@ func (b *benchmark) runGantryOnly(ctx context.Context) (returnErr error) { return err } - if err := requireFinalLayerResponseTimestamps(diagnosticTimestamps, diagnosticsAfter.PodNodes); err != nil { - return err - } - diagnostics, err := subtractGantryDiagnosticSnapshots(diagnosticsBefore, diagnosticsAfter, diagnosticTimestamps) if err != nil { return err } + b.warnIncompleteFinalLayerResponseTimestamps(diagnostics) bytes, bytesSource := deriveOriginBytes(b.config, proxyPhaseGantryCold, proxyPhaseTotals{}, metrics, job) diff --git a/hack/cmd/gantry-benchmark/hosts_routing.go b/hack/cmd/gantry-benchmark/hosts_routing.go index 42c8d3979..a1e6a60d1 100644 --- a/hack/cmd/gantry-benchmark/hosts_routing.go +++ b/hack/cmd/gantry-benchmark/hosts_routing.go @@ -63,6 +63,9 @@ server = "http://%s:5002" case hostsModeGantry: if state.GantryRoutingStrategy == gantryRoutingFailOpen { fallbackServer := "https://" + registry + if state.ArtifactStreaming { + fallbackServer = "http://127.0.0.1:8578" + } if state.usesProxy() { if state.ProxyClusterIP == "" { @@ -126,6 +129,10 @@ func (s benchmarkState) registryForHosts(mode hostsMode) (string, error) { } func (b *benchmark) installHosts(ctx context.Context, state benchmarkState, mode hostsMode) error { + if state.ArtifactStreaming { + return nil + } + content, err := renderHosts(state, mode) if err != nil { return err @@ -279,6 +286,10 @@ exec sleep 2147483647 } func (b *benchmark) restoreHosts(ctx context.Context, state benchmarkState) error { + if state.ArtifactStreaming { + return nil + } + if _, err := b.commands.Run( ctx, nil, @@ -561,10 +572,14 @@ func nodeDaemonSet( } func (b *benchmark) validateBenchmarkDaemonSet(ctx context.Context, name string) error { + return b.validateDaemonSet(ctx, b.config.Namespace, name) +} + +func (b *benchmark) validateDaemonSet(ctx context.Context, namespace, name string) error { output, err := b.commands.Run( ctx, nil, - "kubectl", "-n", b.config.Namespace, + "kubectl", "-n", namespace, "get", "daemonset", name, "-o", "json", ) if err != nil { diff --git a/hack/cmd/gantry-benchmark/hosts_routing_test.go b/hack/cmd/gantry-benchmark/hosts_routing_test.go index d595dd348..e505b8acd 100644 --- a/hack/cmd/gantry-benchmark/hosts_routing_test.go +++ b/hack/cmd/gantry-benchmark/hosts_routing_test.go @@ -51,6 +51,22 @@ func TestRenderHosts(t *testing.T) { } } +func TestArtifactStreamingDoesNotMutateHostsRouting(t *testing.T) { + runner := &recordingRunner{} + benchmark := &benchmark{commands: runner} + state := benchmarkState{ArtifactStreaming: true} + + if err := benchmark.installHosts(context.Background(), state, hostsModeGantry); err != nil { + t.Fatalf("installHosts: %v", err) + } + if err := benchmark.restoreHosts(context.Background(), state); err != nil { + t.Fatalf("restoreHosts: %v", err) + } + if len(runner.commands) != 0 { + t.Fatalf("streaming routing executed %d commands", len(runner.commands)) + } +} + func TestRenderHostsGantryResolvesOnlyToLoopback(t *testing.T) { const registry = "gantry.azurecr.io" @@ -132,6 +148,45 @@ func TestRenderHostsDirectGantryFailOpenResolvesGantryThenACR(t *testing.T) { } } +func TestRenderHostsArtifactStreamingFailOpenResolvesGantryThenAKS(t *testing.T) { + const registry = "gantry.azurecr.io" + + state := benchmarkState{ + RunID: "run-1", + Mode: benchmarkModeDirect, + GantryRoutingStrategy: gantryRoutingFailOpen, + GantryACRLoginServer: registry, + ArtifactStreaming: true, + } + + hostsFile, err := renderHosts(state, hostsModeGantry) + if err != nil { + t.Fatalf("render Gantry: %v", err) + } + + hostDirectory := filepath.Join(t.TempDir(), registry) + if err := os.MkdirAll(hostDirectory, 0o750); err != nil { + t.Fatal(err) + } + + if err := os.WriteFile(filepath.Join(hostDirectory, "hosts.toml"), []byte(hostsFile), 0o640); err != nil { + t.Fatal(err) + } + + resolver := containerdconfig.ConfigureHosts(context.Background(), containerdconfig.HostOptions{ + HostDir: containerdconfig.HostDirFromRoot(filepath.Dir(hostDirectory)), + }) + + resolved, err := resolver(registry) + if err != nil { + t.Fatalf("resolve containerd hosts: %v", err) + } + + if len(resolved) != 2 || resolved[0].Host != "127.0.0.1:5000" || resolved[1].Host != "127.0.0.1:8578" { + t.Fatalf("resolved hosts = %#v, want Gantry followed by AKS Artifact Streaming", resolved) + } +} + func TestRenderHostsProxyGantryFailOpenUsesCountingProxy(t *testing.T) { state := benchmarkState{ RunID: "run-1", diff --git a/hack/cmd/gantry-benchmark/image.go b/hack/cmd/gantry-benchmark/image.go index b1428758d..19c958bbe 100644 --- a/hack/cmd/gantry-benchmark/image.go +++ b/hack/cmd/gantry-benchmark/image.go @@ -63,6 +63,10 @@ func (b *benchmark) loginRegistry(ctx context.Context, loginServer, username, pa } func (b *benchmark) prepareImages(ctx context.Context) error { + if b.config.ArtifactStreaming { + return fmt.Errorf("Artifact Streaming requires prepare-gantry-standalone") + } + state, err := b.loadState(ctx) if err != nil { return err diff --git a/hack/cmd/gantry-benchmark/job.go b/hack/cmd/gantry-benchmark/job.go index 1d474d958..cc4a1481c 100644 --- a/hack/cmd/gantry-benchmark/job.go +++ b/hack/cmd/gantry-benchmark/job.go @@ -99,11 +99,8 @@ func (b *benchmark) runPullJob(ctx context.Context, state benchmarkState, phase "metadata": map[string]any{"labels": podLabels}, "spec": map[string]any{ "restartPolicy": "Never", - "nodeSelector": map[string]string{ - "kubernetes.io/os": platformParts[0], - "kubernetes.io/arch": platformParts[1], - }, - "tolerations": []any{map[string]any{"operator": "Exists"}}, + "nodeSelector": b.config.nodeSelector(), + "tolerations": []any{map[string]any{"operator": "Exists"}}, "affinity": map[string]any{ "podAntiAffinity": map[string]any{ "requiredDuringSchedulingIgnoredDuringExecution": []any{ diff --git a/hack/cmd/gantry-benchmark/main.go b/hack/cmd/gantry-benchmark/main.go index 808d081bd..407572f5b 100644 --- a/hack/cmd/gantry-benchmark/main.go +++ b/hack/cmd/gantry-benchmark/main.go @@ -113,6 +113,12 @@ func runCLI(ctx context.Context, args []string, stdout, stderr io.Writer) error } return benchmark.prepareStandaloneGantry(ctx) + case "prepare-gantry-standalone-adopt": + if len(args) != 4 { + return fmt.Errorf("usage: gantry-benchmark prepare-gantry-standalone-adopt ") + } + + return benchmark.prepareAdoptedStandaloneGantry(ctx, args[1], args[2], args[3]) case "prepare-gantry-adopt": if len(args) != 4 { return fmt.Errorf("usage: gantry-benchmark prepare-gantry-adopt ") @@ -161,6 +167,8 @@ Subcommands: generate new random bytes and build only a fresh Gantry image prepare-gantry-standalone generate and build one Gantry image without a baseline + prepare-gantry-standalone-adopt + adopt an existing converted image for a standalone run prepare-gantry-adopt adopt an already-pushed fresh Gantry image by immutable digest prepare-gantry-pool diff --git a/hack/cmd/gantry-benchmark/peer_telemetry.go b/hack/cmd/gantry-benchmark/peer_telemetry.go index 0d7848bfa..677d8a985 100644 --- a/hack/cmd/gantry-benchmark/peer_telemetry.go +++ b/hack/cmd/gantry-benchmark/peer_telemetry.go @@ -48,9 +48,11 @@ type gantryPodDiagnosticMeasurement struct { } type gantryDiagnosticPhaseMeasurement struct { - Pods []gantryPodDiagnosticMeasurement `json:"pods"` - Source string `json:"source"` - Complete bool `json:"complete"` + Pods []gantryPodDiagnosticMeasurement `json:"pods"` + Source string `json:"source"` + Complete bool `json:"complete"` + FinalLayerResponseTimestampsComplete bool `json:"final_layer_response_timestamps_complete"` + MissingFinalLayerResponseTimestampPodNames []string `json:"missing_final_layer_response_timestamp_pod_names,omitempty"` } type gantryDiagnosticSnapshot struct { @@ -111,7 +113,7 @@ func (b *benchmark) gantryPodNodes(ctx context.Context, revision string) (map[st ctx, nil, "kubectl", "-n", b.config.GantryNamespace, - "get", "pods", "-l", "app.kubernetes.io/name="+b.config.GantryDaemonSet, + "get", "pods", "-l", b.config.gantryAgentSelector(), "-o", "json", ) if err != nil { @@ -233,7 +235,8 @@ func (b *benchmark) fetchGantryDiagnosticTimestamps( revision, ) - raw, err := b.queryPrometheusRange(ctx, query, window, performanceTelemetryStep) + queryWindow := diagnosticTimestampQueryWindow(window, time.Now().UTC()) + raw, err := b.queryPrometheusRange(ctx, query, queryWindow, performanceTelemetryStep) if err != nil { return nil, err } @@ -292,6 +295,14 @@ func (b *benchmark) fetchGantryDiagnosticTimestamps( return result, nil } +func diagnosticTimestampQueryWindow(eventWindow telemetryWindow, collectedAt time.Time) telemetryWindow { + if collectedAt.After(eventWindow.FinishedAt) { + eventWindow.FinishedAt = collectedAt + } + + return eventWindow +} + func subtractGantryDiagnosticSnapshots( before, after gantryDiagnosticSnapshot, timestamps map[string]map[string]float64, @@ -339,10 +350,14 @@ func subtractGantryDiagnosticSnapshots( }) } + missingTimestamps := missingFinalLayerResponseTimestampPods(timestamps, after.PodNodes) + return gantryDiagnosticPhaseMeasurement{ - Pods: pods, - Source: "per-pod Prometheus counter deltas and timestamp gauges", - Complete: len(pods) > 0, + Pods: pods, + Source: "per-pod Prometheus counter deltas and timestamp gauges", + Complete: len(pods) > 0, + FinalLayerResponseTimestampsComplete: len(missingTimestamps) == 0, + MissingFinalLayerResponseTimestampPodNames: missingTimestamps, }, nil } @@ -357,10 +372,10 @@ func finalLayerResponseCompletedTimestamp(timestamps map[string]float64) float64 return latest } -func requireFinalLayerResponseTimestamps( +func missingFinalLayerResponseTimestampPods( timestamps map[string]map[string]float64, podNodes map[string]string, -) error { +) []string { missing := make([]string, 0) for pod := range podNodes { @@ -369,18 +384,22 @@ func requireFinalLayerResponseTimestamps( } } - if len(missing) == 0 { - return nil - } - sort.Strings(missing) - return fmt.Errorf( - "final layer response completion timestamp missing for %d/%d Gantry pods: %s", - len(missing), - len(podNodes), - strings.Join(missing, ","), - ) + return missing +} + +func (b *benchmark) warnIncompleteFinalLayerResponseTimestamps(measurement gantryDiagnosticPhaseMeasurement) { + if measurement.FinalLayerResponseTimestampsComplete { + return + } + + writeAll(b.stderr, fmt.Sprintf( + "warning: final layer response completion timestamp missing for %d/%d Gantry pods: %s; preserving successful benchmark result with incomplete diagnostic telemetry\n", + len(measurement.MissingFinalLayerResponseTimestampPodNames), + len(measurement.Pods), + strings.Join(measurement.MissingFinalLayerResponseTimestampPodNames, ","), + )) } func (b *benchmark) fetchGantryPeerByteSnapshot(ctx context.Context, revision string) (peerByteSnapshot, error) { diff --git a/hack/cmd/gantry-benchmark/peer_telemetry_test.go b/hack/cmd/gantry-benchmark/peer_telemetry_test.go index 4bc0b343b..37d6ac34a 100644 --- a/hack/cmd/gantry-benchmark/peer_telemetry_test.go +++ b/hack/cmd/gantry-benchmark/peer_telemetry_test.go @@ -30,7 +30,7 @@ func (r *diagnosticTimestampRunner) Run(_ context.Context, _ []byte, _ string, a )), nil } -func TestFetchGantryDiagnosticTimestampsUsesExactJobWindow(t *testing.T) { +func TestFetchGantryDiagnosticTimestampsFiltersValuesToExactJobWindow(t *testing.T) { runner := &diagnosticTimestampRunner{} benchmark := &benchmark{ config: benchmarkConfig{ @@ -65,24 +65,40 @@ func TestFetchGantryDiagnosticTimestampsUsesExactJobWindow(t *testing.T) { } } -func TestRequireFinalLayerResponseTimestamps(t *testing.T) { +func TestDiagnosticTimestampQueryWindowIncludesPostJobScrapes(t *testing.T) { + eventWindow := telemetryWindow{ + StartedAt: time.Date(2026, time.August, 4, 1, 2, 0, 0, time.UTC), + FinishedAt: time.Date(2026, time.August, 4, 1, 5, 0, 0, time.UTC), + } + collectedAt := eventWindow.FinishedAt.Add(20 * time.Second) + + queryWindow := diagnosticTimestampQueryWindow(eventWindow, collectedAt) + if !queryWindow.StartedAt.Equal(eventWindow.StartedAt) || !queryWindow.FinishedAt.Equal(collectedAt) { + t.Fatalf("query window = %+v, want event start and collection end", queryWindow) + } +} + +func TestMissingFinalLayerResponseTimestampPods(t *testing.T) { timestamps := map[string]map[string]float64{ "gantry-a": { "gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}": 1234, }, } - podNodes := map[string]string{"gantry-a": "node-a", "gantry-b": "node-b"} + podNodes := map[string]string{"gantry-a": "node-a", "gantry-b": "node-b", "gantry-c": "node-c"} - err := requireFinalLayerResponseTimestamps(timestamps, podNodes) - if err == nil || !strings.Contains(err.Error(), "gantry-b") { - t.Fatalf("error = %v, want missing gantry-b completion", err) + missing := missingFinalLayerResponseTimestampPods(timestamps, podNodes) + if strings.Join(missing, ",") != "gantry-b,gantry-c" { + t.Fatalf("missing = %v, want sorted gantry-b and gantry-c", missing) } timestamps["gantry-b"] = map[string]float64{ "gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=origin}": 1235, } - if err := requireFinalLayerResponseTimestamps(timestamps, podNodes); err != nil { - t.Fatalf("requireFinalLayerResponseTimestamps: %v", err) + timestamps["gantry-c"] = map[string]float64{ + "gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=local}": 1236, + } + if missing := missingFinalLayerResponseTimestampPods(timestamps, podNodes); len(missing) != 0 { + t.Fatalf("missing = %v, want complete timestamp coverage", missing) } } @@ -205,7 +221,7 @@ func TestSubtractGantryDiagnosticSnapshots(t *testing.T) { t.Fatalf("subtractGantryDiagnosticSnapshots: %v", err) } - if !measurement.Complete || len(measurement.Pods) != 1 { + if !measurement.Complete || !measurement.FinalLayerResponseTimestampsComplete || len(measurement.Pods) != 1 { t.Fatalf("measurement = %+v, want one complete pod", measurement) } @@ -217,6 +233,30 @@ func TestSubtractGantryDiagnosticSnapshots(t *testing.T) { } } +func TestSubtractGantryDiagnosticSnapshotsRecordsMissingFinalLayerTimestamp(t *testing.T) { + before := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a", "gantry-b": "node-b"}, + Counters: map[string]map[string]float64{"gantry-a": {}, "gantry-b": {}}, + } + after := gantryDiagnosticSnapshot{ + PodNodes: map[string]string{"gantry-a": "node-a", "gantry-b": "node-b"}, + Counters: map[string]map[string]float64{"gantry-a": {}, "gantry-b": {}}, + } + timestamps := map[string]map[string]float64{ + "gantry-a": {"gantry_mirror_response_completed_timestamp_seconds{kind=layer,source=peer}": 1234}, + } + + measurement, err := subtractGantryDiagnosticSnapshots(before, after, timestamps) + if err != nil { + t.Fatalf("subtractGantryDiagnosticSnapshots: %v", err) + } + + if measurement.FinalLayerResponseTimestampsComplete || + strings.Join(measurement.MissingFinalLayerResponseTimestampPodNames, ",") != "gantry-b" { + t.Fatalf("measurement = %+v, want gantry-b recorded as missing diagnostic telemetry", measurement) + } +} + func TestSubtractGantryDiagnosticSnapshotsRejectsReset(t *testing.T) { before := gantryDiagnosticSnapshot{ PodNodes: map[string]string{"gantry-a": "node-a"}, diff --git a/hack/cmd/gantry-benchmark/preflight.go b/hack/cmd/gantry-benchmark/preflight.go index 596a4a0e9..f164b33c8 100644 --- a/hack/cmd/gantry-benchmark/preflight.go +++ b/hack/cmd/gantry-benchmark/preflight.go @@ -11,6 +11,8 @@ import ( "strconv" "strings" "time" + + "gopkg.in/yaml.v3" ) const registryManifestAccept = "application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json" @@ -42,6 +44,16 @@ func (b *benchmark) preflight(ctx context.Context) error { return fmt.Errorf("benchmark state is %q, run prepare before preflight or disable the run", state.Status) } + if state.ArtifactStreaming != b.config.ArtifactStreaming { + return fmt.Errorf("benchmark Artifact Streaming state=%t does not match BENCHMARK_ARTIFACT_STREAMING=%t", state.ArtifactStreaming, b.config.ArtifactStreaming) + } + if state.NodePool != b.config.NodePool { + return fmt.Errorf("benchmark node pool state=%q does not match BENCHMARK_NODE_POOL=%q", state.NodePool, b.config.NodePool) + } + if state.ArtifactStreaming && !state.StandaloneGantry { + return fmt.Errorf("Artifact Streaming requires prepare-gantry-standalone") + } + if _, _, err := state.preparedImages(); err != nil { return err } @@ -58,6 +70,12 @@ func (b *benchmark) preflight(ctx context.Context) error { return err } + if state.ArtifactStreaming { + if err := b.checkArtifactStreaming(ctx, state); err != nil { + return err + } + } + if state.AzureTelemetry { if err := b.checkAzureTelemetry(ctx); err != nil { return err @@ -91,6 +109,60 @@ func (b *benchmark) preflight(ctx context.Context) error { return nil } +func (b *benchmark) checkArtifactStreaming(ctx context.Context, state benchmarkState) error { + for _, daemonSet := range []string{"gantry-containerd-config", "gantry-overlaybd-config"} { + if err := b.validateDaemonSet(ctx, b.config.GantryNamespace, daemonSet); err != nil { + return fmt.Errorf("validate Artifact Streaming %s: %w", daemonSet, err) + } + } + + output, err := b.commands.Run( + ctx, + nil, + "kubectl", "-n", b.config.GantryNamespace, + "get", "configmap", b.config.GantryConfigMap, + "-o", "json", + ) + if err != nil { + return err + } + + var configMap struct { + Data map[string]string `json:"data"` + } + if err := json.Unmarshal(output, &configMap); err != nil { + return fmt.Errorf("decode Gantry ConfigMap: %w", err) + } + + return validateArtifactStreamingGantryConfig(configMap.Data["config.yaml"], state.GantryACRLoginServer) +} + +func validateArtifactStreamingGantryConfig(raw, registry string) error { + var config struct { + ArtifactStreamingEnabled bool `yaml:"artifact_streaming_enabled"` + UpstreamRegistries []struct { + Name string `yaml:"name"` + Endpoint string `yaml:"endpoint"` + } `yaml:"upstream_registries"` + } + if err := yaml.Unmarshal([]byte(raw), &config); err != nil { + return fmt.Errorf("decode Gantry configuration: %w", err) + } + + if !config.ArtifactStreamingEnabled { + return fmt.Errorf("Gantry Artifact Streaming is disabled") + } + + wantEndpoint := "http://127.0.0.1:8578?ns=" + registry + for _, upstream := range config.UpstreamRegistries { + if upstream.Name == registry && upstream.Endpoint == wantEndpoint { + return nil + } + } + + return fmt.Errorf("Gantry upstream %q is not chained through %q", registry, wantEndpoint) +} + func (b *benchmark) smokeProxy(ctx context.Context, state benchmarkState) error { repository, reference, err := splitImageReference(state.ProxyImage, state.ACRLoginServer) if err != nil { diff --git a/hack/cmd/gantry-benchmark/results.go b/hack/cmd/gantry-benchmark/results.go index 1bc510159..6bc369612 100644 --- a/hack/cmd/gantry-benchmark/results.go +++ b/hack/cmd/gantry-benchmark/results.go @@ -147,7 +147,7 @@ func (b *benchmark) gantryRevision(ctx context.Context) (string, error) { ctx, nil, "kubectl", "-n", b.config.GantryNamespace, - "get", "pods", "-l", "app.kubernetes.io/name="+b.config.GantryDaemonSet, + "get", "pods", "-l", b.config.gantryAgentSelector(), "-o", "json", ) if err != nil { diff --git a/hack/cmd/gantry-benchmark/run.go b/hack/cmd/gantry-benchmark/run.go index c487fd018..d49c6e342 100644 --- a/hack/cmd/gantry-benchmark/run.go +++ b/hack/cmd/gantry-benchmark/run.go @@ -19,6 +19,12 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { if state.Status != "preflight-passed" { return fmt.Errorf("benchmark state is %q, run preflight before run", state.Status) } + if state.ArtifactStreaming { + return fmt.Errorf("Artifact Streaming requires run-gantry") + } + if state.NodePool != b.config.NodePool { + return fmt.Errorf("benchmark node pool state=%q does not match BENCHMARK_NODE_POOL=%q", state.NodePool, b.config.NodePool) + } if err := b.requireLock(ctx, state.RunID); err != nil { return err @@ -314,10 +320,6 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { return err } - if err := requireFinalLayerResponseTimestamps(gantryDiagnosticTimestamps, gantryDiagnosticsAfter.PodNodes); err != nil { - return err - } - gantryDiagnostics, err := subtractGantryDiagnosticSnapshots( gantryDiagnosticsBefore, gantryDiagnosticsAfter, @@ -326,6 +328,7 @@ func (b *benchmark) runBenchmark(ctx context.Context) (returnErr error) { if err != nil { return err } + b.warnIncompleteFinalLayerResponseTimestamps(gantryDiagnostics) var gantryProxy proxyPhaseTotals diff --git a/hack/cmd/gantry-benchmark/state.go b/hack/cmd/gantry-benchmark/state.go index 97e4ad048..a4093df4f 100644 --- a/hack/cmd/gantry-benchmark/state.go +++ b/hack/cmd/gantry-benchmark/state.go @@ -10,6 +10,9 @@ import ( "fmt" "os" "path/filepath" + "strings" + + "github.com/opencontainers/go-digest" ) const ( @@ -28,6 +31,7 @@ type benchmarkState struct { GantryConfigMap string `json:"gantry_configmap"` MonitoringNamespace string `json:"monitoring_namespace"` PrometheusService string `json:"prometheus_service"` + NodePool string `json:"node_pool,omitempty"` NodeCount int `json:"node_count"` ImagePlatform string `json:"image_platform"` ImageSizeMiB int `json:"image_size_mib"` @@ -41,6 +45,9 @@ type benchmarkState struct { WorkloadPayloadSHA256 string `json:"workload_payload_sha256,omitempty"` WorkloadComparisonMode workloadComparisonMode `json:"workload_comparison_mode,omitempty"` StandaloneGantry bool `json:"standalone_gantry,omitempty"` + ArtifactStreaming bool `json:"artifact_streaming,omitempty"` + ArtifactStreamingPrepared bool `json:"artifact_streaming_prepared,omitempty"` + ArtifactStreamingImage string `json:"artifact_streaming_image,omitempty"` ProxyImage string `json:"proxy_image,omitempty"` ProxyClusterIP string `json:"proxy_cluster_ip,omitempty"` OriginalGantryConfig string `json:"original_gantry_config"` @@ -81,6 +88,15 @@ func (s benchmarkState) preparedImages() (string, string, error) { return "", "", fmt.Errorf("prepared standalone Gantry image has no payload fingerprint") } + if s.ArtifactStreaming && !s.ArtifactStreamingPrepared { + return "", "", fmt.Errorf("standalone Gantry image has not completed Artifact Streaming conversion") + } + if s.ArtifactStreaming { + if err := s.validateArtifactStreamingImage(); err != nil { + return "", "", err + } + } + gantryRepository, _, err := splitImageReference(s.GantryColdImage, s.GantryACRLoginServer) if err != nil { return "", "", fmt.Errorf("prepared Gantry image registry mismatch: %w", err) @@ -139,6 +155,43 @@ func (s benchmarkState) preparedImages() (string, string, error) { return s.BaselineImage, s.GantryColdImage, nil } +func (s benchmarkState) validateArtifactStreamingImage() error { + if s.ArtifactStreamingImage == "" { + return fmt.Errorf("standalone Gantry run has no Artifact Streaming digest reference") + } + if !strings.Contains(s.ArtifactStreamingImage, "@") { + return fmt.Errorf("Artifact Streaming runtime image must use a digest: %s", s.ArtifactStreamingImage) + } + + repository, digestValue, err := splitImageReference(s.ArtifactStreamingImage, s.GantryACRLoginServer) + if err != nil { + return fmt.Errorf("invalid Artifact Streaming runtime image: %w", err) + } + if repository != s.WorkloadRepository { + return fmt.Errorf("Artifact Streaming runtime repository %q, want %q", repository, s.WorkloadRepository) + } + parsedDigest, err := digest.Parse(digestValue) + if err != nil || parsedDigest.Algorithm() != digest.SHA256 { + return fmt.Errorf("Artifact Streaming runtime image must use a sha256 digest: %s", s.ArtifactStreamingImage) + } + if s.ArtifactStreamingImage == s.GantryColdImage { + return fmt.Errorf("Artifact Streaming runtime digest must differ from the original image digest: %s", s.ArtifactStreamingImage) + } + + return nil +} + +func (s benchmarkState) gantryRuntimeImage() (string, error) { + if !s.ArtifactStreaming { + return s.GantryColdImage, nil + } + if err := s.validateArtifactStreamingImage(); err != nil { + return "", err + } + + return s.ArtifactStreamingImage, nil +} + func (b *benchmark) saveState(ctx context.Context, state benchmarkState) error { stateForJSON := state stateForJSON.OriginalGantryConfig = "" diff --git a/hack/cmd/release-bom/main.go b/hack/cmd/release-bom/main.go index f9fa76b32..769d67a53 100644 --- a/hack/cmd/release-bom/main.go +++ b/hack/cmd/release-bom/main.go @@ -29,6 +29,7 @@ const bomSchemaVersion = 2 var releaseImageNames = []string{ "gantry", + "gantry-node-config", "host-ubuntu2404", "inventory-aggregator", "inventory-inspector", diff --git a/hack/cmd/release-bom/main_test.go b/hack/cmd/release-bom/main_test.go index 45b939e82..ddd22a5f6 100644 --- a/hack/cmd/release-bom/main_test.go +++ b/hack/cmd/release-bom/main_test.go @@ -61,6 +61,19 @@ func TestBuildBOM(t *testing.T) { t.Fatalf("first image reference = %q, want %q", got, want) } + foundGantryNodeConfig := false + + for _, image := range bom.Images { + if image.Name == "gantry-node-config" && image.Reference == "registry.example.com/project/gantry-node-config:v1.2.3" { + foundGantryNodeConfig = true + break + } + } + + if !foundGantryNodeConfig { + t.Fatal("gantry-node-config image is missing from release BOM") + } + if bom.NodeBootstrap.ContainerdVersion != goalstates.ContainerdVersion || bom.NodeBootstrap.RuncVersion != goalstates.RunCVersion || bom.NodeBootstrap.CNIPluginVersion != goalstates.CNIPluginVersion || diff --git a/hack/gantry-benchmark/Makefile b/hack/gantry-benchmark/Makefile index 5306ff718..8001faace 100644 --- a/hack/gantry-benchmark/Makefile +++ b/hack/gantry-benchmark/Makefile @@ -2,7 +2,7 @@ REPO_ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))/../..) ENV_FILE ?= $(CURDIR)/env.local DEPLOY_CONFIG ?= $(CURDIR)/deploy.env -.PHONY: help test monitor profile-gantry deploy deploy-plan deploy-scale deploy-status operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch operator-vm-prebuild operator-vm-run-full operator-vm-run-pool operator-vm-run-fresh operator-vm-run-standalone operator-vm-ensure-fail-open operator-vm-deploy-runtime-run operator-vm-deploy-runtime-status operator-vm-deploy-runtime-watch operator-vm-deploy-runtime-fast operator-vm-image-pool-status operator-vm-prune-images operator-vm-prune-images-status proxy-image proxy-push image-pool-status prebuild-gantry enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-standalone prepare-gantry-adopt prepare-gantry-pool preflight run run-gantry status disable recover-direct-config +.PHONY: help test monitor profile-gantry deploy deploy-plan deploy-scale deploy-recreate-pool deploy-status operator-vm-check operator-vm-provision operator-vm-status operator-vm-watch operator-vm-stop operator-vm-prebuild operator-vm-run-full operator-vm-run-pool operator-vm-run-fresh operator-vm-run-standalone operator-vm-ensure-fail-open operator-vm-deploy-runtime-run operator-vm-deploy-runtime-status operator-vm-deploy-runtime-watch operator-vm-deploy-runtime-fast operator-vm-image-pool-status operator-vm-prune-images operator-vm-prune-images-status proxy-image proxy-push image-pool-status prebuild-gantry enable prepare prepare-adopt prepare-gantry prepare-gantry-fresh prepare-gantry-standalone prepare-gantry-standalone-adopt prepare-gantry-adopt prepare-gantry-pool preflight run run-gantry status disable recover-direct-config help: ## Show benchmark targets @echo "" @@ -14,11 +14,13 @@ help: ## Show benchmark targets @echo " deploy Idempotently deploy the complete benchmark stack" @echo " deploy-plan Print the resolved deployment contract without mutation" @echo " deploy-scale Resize the existing AKS node pool and wait for Ready nodes" + @echo " deploy-recreate-pool Replace the streaming worker pool with cold nodes" @echo " deploy-status Report deployment readiness without mutation" @echo " operator-vm-check Validate operator VM scripts" @echo " operator-vm-provision Provision/bootstrap the private operator VM" @echo " operator-vm-status Print one live operator VM progress snapshot" @echo " operator-vm-watch Follow operator VM progress until completion" + @echo " operator-vm-stop Stop the active operator lifecycle and run cleanup" @echo " operator-vm-prebuild Start an asynchronous operator-VM pool build" @echo " operator-vm-run-full Start full lifecycle; optional ADOPT_* values reuse images" @echo " operator-vm-run-pool Start Gantry-only using the next compatible pool image" @@ -72,11 +74,25 @@ profile-gantry: operator-vm-check operator-vm-check: bash -n deploy.sh profile-gantry.sh operator-vm-provision.sh operator-vm-bootstrap.sh operator-vm-build-images.sh operator-vm-deploy-runtime.sh operator-vm-run.sh operator-vm-prebuild-images.sh operator-vm-image-pool.sh operator-vm-prune-images-remote.sh operator-vm-status.sh operator-vm-watch.sh - ./deploy.sh plan deploy.env.example >/dev/null + ./deploy.sh plan deploy.env.example | grep -Fq 'Artifact Streaming: false' + @stream_config=$$(mktemp); \ + trap 'rm -f "$$stream_config"' EXIT; \ + sed -e 's/BENCHMARK_ARTIFACT_STREAMING="false"/BENCHMARK_ARTIFACT_STREAMING="true"/' \ + deploy.env.example >"$$stream_config"; \ + ./deploy.sh plan "$$stream_config" | grep -Fq 'Artifact Streaming: true'; \ + ./deploy.sh plan "$$stream_config" | grep -Fq 'benchmark pool: stream (1000 x Standard_D8s_v3)'; \ + ./deploy.sh plan "$$stream_config" | grep -Fq 'system pool: system (1 x Standard_D4s_v5)' ! grep -Eq 'az acr login|podman (build|push|login|pull|tag)' deploy.sh ! grep -Eq '^[[:space:]]*az login([[:space:]]|$$)' deploy.sh ! grep -Eq '^[[:space:]]*export START_BENCHMARK=false' deploy.sh - grep -Fq '"$$repo_root/bin/helm" upgrade --install gantry "$$repo_root/deploy/gantry/chart"' deploy.sh + grep -Fq 'local helm_args=(' deploy.sh + grep -Fq '[[ "$$legacy" == true ]] || return 0' deploy.sh + grep -Fq 'upgrade --install gantry "$$repo_root/deploy/gantry/chart"' deploy.sh + grep -Fq '"$$repo_root/bin/helm" "$${helm_args[@]}"' deploy.sh + grep -Fq -- '--enable-artifact-streaming' deploy.sh + grep -Fq -- '--set gantry.artifactStreaming.enabled=true' deploy.sh + grep -Fq -- '--set overlaybdConfig.enabled=true' deploy.sh + grep -Fq 'endpoint=http://127.0.0.1:8578?ns=$$GANTRY_ACR_LOGIN_SERVER' deploy.sh ! grep -Fq '"$$repo_root/bin/helm" template gantry' deploy.sh ! grep -Fq 'ensure_chair_leases' deploy.sh grep -Fq 'git archive "$$source_revision" | tar -x -C "$$source_context"' deploy.sh @@ -91,6 +107,9 @@ deploy-plan: operator-vm-check deploy-scale: operator-vm-check cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh scale "$(DEPLOY_CONFIG)" +deploy-recreate-pool: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh recreate-pool "$(DEPLOY_CONFIG)" + deploy-status: operator-vm-check cd "$(REPO_ROOT)" && hack/gantry-benchmark/deploy.sh status "$(DEPLOY_CONFIG)" @@ -103,6 +122,9 @@ operator-vm-status: operator-vm-check operator-vm-watch: operator-vm-check cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-watch.sh --follow +operator-vm-stop: operator-vm-check + cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-image-pool.sh stop + operator-vm-prebuild: operator-vm-check @test -n "$(GANTRY_IMAGE_POOL_COUNT)" || { echo "GANTRY_IMAGE_POOL_COUNT is required" >&2; exit 2; } cd "$(REPO_ROOT)" && hack/gantry-benchmark/operator-vm-image-pool.sh start "$(GANTRY_IMAGE_POOL_COUNT)" @@ -229,6 +251,17 @@ prepare-gantry-standalone: set +a; \ cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-gantry-standalone +prepare-gantry-standalone-adopt: + @test -n "$(GANTRY_ONLY_STANDALONE_IMAGE)" || { echo "GANTRY_ONLY_STANDALONE_IMAGE is required" >&2; exit 2; } + @test -n "$(GANTRY_ONLY_STANDALONE_STREAMING_IMAGE)" || { echo "GANTRY_ONLY_STANDALONE_STREAMING_IMAGE is required" >&2; exit 2; } + @test -n "$(GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256)" || { echo "GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256 is required" >&2; exit 2; } + set -a; \ + [ ! -f "$(ENV_FILE)" ] || . "$(ENV_FILE)"; \ + set +a; \ + cd "$(REPO_ROOT)" && GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-gantry-standalone-adopt \ + "$(GANTRY_ONLY_STANDALONE_IMAGE)" "$(GANTRY_ONLY_STANDALONE_STREAMING_IMAGE)" \ + "$(GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256)" + prepare-gantry-adopt: @test -n "$(GANTRY_ONLY_BASELINE_RUN_ID)" || { echo "GANTRY_ONLY_BASELINE_RUN_ID is required" >&2; exit 2; } @test -n "$(GANTRY_ONLY_ADOPT_IMAGE)" || { echo "GANTRY_ONLY_ADOPT_IMAGE is required" >&2; exit 2; } diff --git a/hack/gantry-benchmark/README.md b/hack/gantry-benchmark/README.md index 82e73beb0..d4e458878 100644 --- a/hack/gantry-benchmark/README.md +++ b/hack/gantry-benchmark/README.md @@ -35,6 +35,50 @@ passes. The deployment config contains names and topology only. Credentials remain in Azure managed identities and short-lived ACR tokens. +### AKS Artifact Streaming + +Artifact Streaming is opt-in and preserves the classic deployment when unset: + +```bash +BENCHMARK_ARTIFACT_STREAMING="true" +``` + +With that setting, deployment creates a one-node System pool and a separate +`AKS_NODE_COUNT`-node User pool named `stream` by default. The benchmark, +Gantry, containerd tuning, private DNS guard, and pull workload select only the +User pool. The pool is created with AKS Artifact Streaming enabled; this cannot +be added to an existing pool. + +The Helm release routes containerd through Gantry on port 5000 and chains +Gantry's OCI upstream through the node-local AKS helper on port 8578. OverlayBD +range requests use the same Gantry listener under `/blobs/`. Fail-open routing +falls back to port 8578, preserving native Artifact Streaming if Gantry is +unavailable. + +The operator lifecycle automatically uses the standalone path: + +```text +enable -> prepare-gantry-standalone -> ACR conversion -> preflight -> run-gantry +``` + +Image preparation enables conversion on the workload repository and waits for +the freshly pushed 40 GiB image conversion to succeed before state becomes +preflight-ready. Preflight requires the containerd and OverlayBD configurator +DaemonSets on every benchmark node and validates Gantry's `:8578?ns=` upstream. + +Use the same Make entrypoints as the classic deployment: + +```bash +make -C hack/gantry-benchmark deploy-plan +make -C hack/gantry-benchmark deploy +make -C hack/gantry-benchmark operator-vm-run-standalone +make -C hack/gantry-benchmark operator-vm-watch +``` + +Artifact Streaming currently supports the standalone lifecycle only. The +classic two-phase `prepare`/`run` lifecycle remains unchanged and rejects the +streaming flag rather than measuring an unconverted image. + The workstation needs one valid Azure management-plane login before invoking `deploy.sh`; the script never invokes `az login`, `az acr login`, or workstation Podman. It publishes only the revision-labeled source carrier through an ACR diff --git a/hack/gantry-benchmark/RESULTS.md b/hack/gantry-benchmark/RESULTS.md index 764c8133a..50ed49fe5 100644 --- a/hack/gantry-benchmark/RESULTS.md +++ b/hack/gantry-benchmark/RESULTS.md @@ -1,4 +1,135 @@ -# Gantry Benchmark +# Gantry Benchmark Results + +## AKS ACR Artifact Streaming smoke test - 2026-09-25 + +Run `run-20260925-160553-7f811295` exercised ACR Artifact Streaming on 100 +AKS worker nodes with a 1 GiB, four-layer image. All 100 workload pods +succeeded. The Kubernetes Job ran from `16:09:01Z` to `16:09:34Z`, for a +33-second wall-clock duration. + +The benchmark lifecycle subsequently reported failure because the original +post-run validation required a final-layer response timestamp from every +Gantry pod and received 98 of 100. This happened after the workload succeeded. +The validation has since been changed to retain a successful result while +recording incomplete diagnostic timestamp coverage. + +This was a Gantry-only smoke test. It did not run a direct-ACR baseline, so no +baseline latency or traffic-reduction percentage is available. + +### Configuration + +| Item | Value | +| --- | --- | +| Azure region | Canada Central | +| Worker pool | 100 `Standard_D8s_v3` nodes | +| System pool | One `Standard_D4s_v5` node | +| Gantry agents | 100 running, zero container restarts | +| Image | 1,024 MiB, four layers | +| Original image digest | `sha256:2d646d6c719c04214fb92a063c2fbe15f32b40a7b8f28fa716309c8969fc276e` | +| Streaming manifest digest | `sha256:8d5d997858cea552b84120630cc7174c20f6686bd7bc7eb15edc460a9803c67f` | +| ACR conversion | Submitted `16:06:28Z`; succeeded `16:07:01Z` | +| Preflight | Passed `16:08:39Z` | +| Workload | 100 succeeded, zero failed | + +### Pod startup latency + +AKS audit logs contained all required create, binding, and started-status +events for 100 of 100 pods. Percentiles use the benchmark's nearest-rank +method. Startup is measured from the API-server create request to the first +status request that reports the workload container started. + +| Measurement | P50 | P95 | P100 | +| --- | ---: | ---: | ---: | +| Pod startup | 11.288s | 13.671s | 15.581s | +| Scheduling | 0.076s | 0.178s | 0.191s | +| Post-bind startup | 11.195s | 13.600s | 15.525s | + +The first pod create request was observed at `16:09:01.536040Z`; the last +started-status request was observed at `16:09:17.543502Z`. + +### ACR traffic + +The Gantry ACR Private Endpoint is the authoritative registry-boundary byte +measurement for the closed `16:09:00Z` to `16:13:00Z` window. + +| Measurement | Bytes | Binary units | +| --- | ---: | ---: | +| Gantry ACR Private Endpoint `PEBytesIn` | 9,926,015,448 | 9.244 GiB | +| Gantry mirror origin response bodies | 8,781,334,224 | 8.178 GiB | +| Artifact Streaming origin range bodies | 217,841,584 | 207.750 MiB | +| Derived sum of measured origin response bodies | 8,999,175,808 | 8.381 GiB | +| Private Endpoint bytes not represented by those body counters | 926,839,640 | 883.903 MiB | +| Baseline ACR Private Endpoint `PEBytesIn` | 0 | 0 GiB | + +The derived response-body sum is: + +$$ +8{,}781{,}334{,}224 + 217{,}841{,}584 = 8{,}999{,}175{,}808\ \text{bytes}. +$$ + +The remaining 926,839,640 bytes are measured at the Private Endpoint, but the +available counters do not establish their exact traffic class. Private +Endpoint traffic was 9.244% of the workload's 100 GiB logical image volume. + +ACR logged 116 workload repository pull events in the same window. All 116 +returned HTTP 200 and referenced the streaming manifest digest; no non-200 +repository event was present. + +### Artifact Streaming ranges + +These values are cumulative `gantry_streaming_*` counters collected directly +from all 100 unchanged Gantry agents after the run. All agents responded to the +collection, and none had restarted. + +| Source | Bytes | Share of bytes | Successful requests | Share of requests | Average duration | Average time to first byte | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Origin | 217,841,584 (207.750 MiB) | 60.835% | 5,857 | 50.098% | 311.806ms | 311.111ms | +| Peer | 128,409,118 (122.460 MiB) | 35.860% | 5,391 | 46.112% | 184.985ms | 184.605ms | +| Local | 11,835,807 (11.288 MiB) | 3.305% | 443 | 3.789% | 1.990ms | 0.529ms | +| **Total** | **358,086,509 (341.498 MiB)** | **100%** | **11,691** | **100%** | - | - | + +Peer plus local service accounted for a derived 140,244,925 bytes, or 39.165% +of all range bytes. Total range bytes were 0.333% of the 100 GiB logical image +volume: + +$$ +\frac{358{,}086{,}509}{100 \times 1{,}073{,}741{,}824} \times 100 += 0.333\%. +$$ + +All 11,691 range requests succeeded. Range errors, origin-URL rejections, +range rejections, and requests still in flight were all zero. + +### Gantry distribution counters + +| Measurement | Value | +| --- | ---: | +| Origin pulls started / succeeded | 56 / 56 | +| Origin layer pulls started / succeeded | 40 / 40 | +| Origin config pulls started / succeeded | 8 / 8 | +| Origin manifest pulls started / succeeded | 8 / 8 | +| Peer fetch hits | 190 | +| Peer bytes served, all kinds | 128,963,098 (122.989 MiB) | +| Direct-origin fallbacks | 0 | +| In-flight pulls after the run | 0 | + +Every recorded non-hit peer outcome was zero: busy, stall, unavailable, +not-found, authentication/configuration, local error, digest mismatch, +protocol error, and server error. + +### Evidence and limitations + +- Pod latency comes from 700 retained `AKSAuditAdmin` events covering all 100 + workload pods. +- Private Endpoint bytes and ACR events use the benchmark's exact closed Azure + metric window. +- Gantry counters were read after lifecycle cleanup because the strict + timestamp gate prevented `gantry-cold.json` from being written. They are + cumulative values from the unchanged deployment, not saved phase deltas. +- The removed benchmark namespace means Kubernetes pod objects and the normal + per-pod performance artifact are unavailable. + +## Complete-image benchmark Gantry delivers a 99% reduction in ACR traffic and improves pod startup latency across a 2000-node AKS cluster delivering a 40 GiB image to diff --git a/hack/gantry-benchmark/deploy.env.example b/hack/gantry-benchmark/deploy.env.example index ed23c1d8f..a7453f6e7 100644 --- a/hack/gantry-benchmark/deploy.env.example +++ b/hack/gantry-benchmark/deploy.env.example @@ -12,6 +12,12 @@ DEPLOY_CONFIRM="vapa-gantry-benchmark1" # Recovered and validated 1000-node topology. AZURE_LOCATION="canadacentral" AKS_KUBERNETES_VERSION="1.35" +BENCHMARK_ARTIFACT_STREAMING="false" +AKS_SYSTEM_NODE_POOL_NAME="system" +AKS_SYSTEM_NODE_COUNT="1" +AKS_SYSTEM_NODE_VM_SIZE="Standard_D4s_v5" +# Empty derives "system" for classic and "stream" for Artifact Streaming. +AKS_NODE_POOL_NAME="" AKS_NODE_COUNT="1000" AKS_NODE_VM_SIZE="Standard_D8s_v3" AKS_NODE_OS_DISK_GB="512" diff --git a/hack/gantry-benchmark/deploy.sh b/hack/gantry-benchmark/deploy.sh index 66360ea15..90e74e086 100755 --- a/hack/gantry-benchmark/deploy.sh +++ b/hack/gantry-benchmark/deploy.sh @@ -9,13 +9,14 @@ repo_root=$(cd -- "$script_dir/../.." && pwd) usage() { cat <<'USAGE' -Usage: deploy.sh [config-file] +Usage: deploy.sh [config-file] One idempotent entrypoint for the complete Gantry benchmark stack. The config file is a shell environment file. No credentials are stored in it. plan validate inputs and print the complete deployment contract scale resize the existing AKS node pool and wait for Ready nodes + recreate-pool delete and recreate the Artifact Streaming benchmark pool deploy create or validate every resource and leave the benchmark ready status report Azure and Kubernetes readiness without mutation USAGE @@ -25,7 +26,7 @@ action=${1:-plan} config_file=${2:-${GANTRY_BENCHMARK_DEPLOY_CONFIG:-$script_dir/deploy.env}} case "$action" in -plan | scale | deploy | status) ;; +plan | scale | recreate-pool | deploy | status) ;; -h | --help | help) usage exit 0 @@ -71,11 +72,20 @@ POD_CIDR=${POD_CIDR:-10.64.0.0/12} SERVICE_CIDR=${SERVICE_CIDR:-10.0.0.0/16} DNS_SERVICE_IP=${DNS_SERVICE_IP:-10.0.0.10} AKS_KUBERNETES_VERSION=${AKS_KUBERNETES_VERSION:-1.35} -AKS_NODE_POOL_NAME=${AKS_NODE_POOL_NAME:-system} +BENCHMARK_ARTIFACT_STREAMING=${BENCHMARK_ARTIFACT_STREAMING:-false} +if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + default_node_pool_name=stream +else + default_node_pool_name=system +fi +AKS_NODE_POOL_NAME=${AKS_NODE_POOL_NAME:-$default_node_pool_name} AKS_NODE_COUNT=${AKS_NODE_COUNT:-1000} AKS_NODE_VM_SIZE=${AKS_NODE_VM_SIZE:-Standard_D8s_v3} AKS_NODE_OS_DISK_GB=${AKS_NODE_OS_DISK_GB:-512} AKS_MAX_PODS=${AKS_MAX_PODS:-250} +AKS_SYSTEM_NODE_POOL_NAME=${AKS_SYSTEM_NODE_POOL_NAME:-system} +AKS_SYSTEM_NODE_COUNT=${AKS_SYSTEM_NODE_COUNT:-1} +AKS_SYSTEM_NODE_VM_SIZE=${AKS_SYSTEM_NODE_VM_SIZE:-Standard_D4s_v5} BENCHMARK_NODE_COUNT=${BENCHMARK_NODE_COUNT:-$AKS_NODE_COUNT} BENCHMARK_IMAGE_SIZE_MIB=${BENCHMARK_IMAGE_SIZE_MIB:-40960} @@ -120,6 +130,15 @@ BASELINE_ACR_DATA_HOST=${BASELINE_ACR_NAME}.${AZURE_LOCATION}.data.azurecr.io GANTRY_ACR_DATA_HOST=${GANTRY_ACR_NAME}.${AZURE_LOCATION}.data.azurecr.io [[ "$AKS_NODE_COUNT" =~ ^[1-9][0-9]*$ ]] || { echo "AKS_NODE_COUNT must be positive" >&2; exit 2; } +[[ "$BENCHMARK_ARTIFACT_STREAMING" == true || "$BENCHMARK_ARTIFACT_STREAMING" == false ]] || { + echo "BENCHMARK_ARTIFACT_STREAMING must be true or false" >&2 + exit 2 +} +[[ "$AKS_SYSTEM_NODE_COUNT" =~ ^[1-9][0-9]*$ ]] || { echo "AKS_SYSTEM_NODE_COUNT must be positive" >&2; exit 2; } +if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true && "$AKS_NODE_POOL_NAME" == "$AKS_SYSTEM_NODE_POOL_NAME" ]]; then + echo "AKS_NODE_POOL_NAME must differ from AKS_SYSTEM_NODE_POOL_NAME when Artifact Streaming is enabled" >&2 + exit 2 +fi [[ "$BENCHMARK_NODE_COUNT" == "$AKS_NODE_COUNT" ]] || { echo "BENCHMARK_NODE_COUNT must equal AKS_NODE_COUNT for this topology" >&2 exit 2 @@ -157,6 +176,10 @@ if ((adoption_values != 0 && adoption_values != 3)); then echo "ADOPT_BASELINE_IMAGE, ADOPT_GANTRY_IMAGE, and ADOPT_PAYLOAD_SHA256 must be set together" >&2 exit 2 fi +if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true && "$adoption_values" != 0 ]]; then + echo "Artifact Streaming standalone runs do not support ADOPT_BASELINE_IMAGE or ADOPT_GANTRY_IMAGE" >&2 + exit 2 +fi if ((adoption_values == 3)); then valid_adopted_image "$ADOPT_BASELINE_IMAGE" "$BASELINE_ACR_LOGIN_SERVER" || { echo "ADOPT_BASELINE_IMAGE must be an immutable gantry-benchmark-pull image in $BASELINE_ACR_LOGIN_SERVER" >&2 @@ -171,6 +194,16 @@ if ((adoption_values == 3)); then exit 2 } fi +if [[ "$action" == recreate-pool ]]; then + [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]] || { + echo "recreate-pool requires BENCHMARK_ARTIFACT_STREAMING=true" >&2 + exit 2 + } + [[ "${RECREATE_BENCHMARK_POOL_CONFIRM:-}" == "$AKS_NODE_POOL_NAME" ]] || { + echo "set RECREATE_BENCHMARK_POOL_CONFIRM=$AKS_NODE_POOL_NAME to authorize replacement" >&2 + exit 2 + } +fi assert_default() { local name=$1 local actual=$2 @@ -211,6 +244,9 @@ retry_command() { print_plan() { local image_preparation="build and push fresh workload images" + if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + image_preparation="build, push, and convert one standalone workload image" + fi if ((adoption_values == 3)); then image_preparation="adopt existing immutable workload images" fi @@ -229,7 +265,9 @@ Azure resource group: $AZURE_RESOURCE_GROUP AKS: $AZURE_AKS_CLUSTER_NAME node resource group: $AZURE_NODE_RESOURCE_GROUP - node pool: $AKS_NODE_POOL_NAME ($AKS_NODE_COUNT x $AKS_NODE_VM_SIZE) + benchmark pool: $AKS_NODE_POOL_NAME ($AKS_NODE_COUNT x $AKS_NODE_VM_SIZE) + system pool: $AKS_SYSTEM_NODE_POOL_NAME ($([[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]] && printf '%s x %s' "$AKS_SYSTEM_NODE_COUNT" "$AKS_SYSTEM_NODE_VM_SIZE" || printf 'same as benchmark pool')) + Artifact Streaming: $BENCHMARK_ARTIFACT_STREAMING node OS disk: ${AKS_NODE_OS_DISK_GB} GiB managed Kubernetes: $AKS_KUBERNETES_VERSION @@ -463,15 +501,23 @@ build_source_image() { } ensure_aks() { - local subnet_id + local subnet_id initial_pool_name initial_node_count initial_vm_size subnet_id=$(az network vnet subnet show -g "$AZURE_RESOURCE_GROUP" --vnet-name "$VNET_NAME" \ -n "$AKS_SUBNET_NAME" --query id -o tsv) + initial_pool_name=$AKS_NODE_POOL_NAME + initial_node_count=$AKS_NODE_COUNT + initial_vm_size=$AKS_NODE_VM_SIZE + if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + initial_pool_name=$AKS_SYSTEM_NODE_POOL_NAME + initial_node_count=$AKS_SYSTEM_NODE_COUNT + initial_vm_size=$AKS_SYSTEM_NODE_VM_SIZE + fi if ! az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" --output none 2>/dev/null; then log "creating AKS cluster $AZURE_AKS_CLUSTER_NAME" az aks create -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" -l "$AZURE_LOCATION" \ --tier standard --enable-managed-identity --node-resource-group "$AZURE_NODE_RESOURCE_GROUP" \ - --nodepool-name "$AKS_NODE_POOL_NAME" --node-count "$AKS_NODE_COUNT" \ - --node-vm-size "$AKS_NODE_VM_SIZE" --node-osdisk-type Managed \ + --nodepool-name "$initial_pool_name" --node-count "$initial_node_count" \ + --node-vm-size "$initial_vm_size" --node-osdisk-type Managed \ --node-osdisk-size "$AKS_NODE_OS_DISK_GB" --max-pods "$AKS_MAX_PODS" \ --os-sku Ubuntu --network-plugin azure --network-plugin-mode overlay \ --network-dataplane azure --pod-cidr "$POD_CIDR" --service-cidr "$SERVICE_CIDR" \ @@ -483,6 +529,28 @@ ensure_aks() { az aks wait -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" \ --created --interval 30 --timeout 7200 + if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + local system_pool_json + system_pool_json=$(az aks nodepool show -g "$AZURE_RESOURCE_GROUP" --cluster-name "$AZURE_AKS_CLUSTER_NAME" \ + -n "$AKS_SYSTEM_NODE_POOL_NAME" -o json) + assert_equal "AKS system node count" "$(jq -r .count <<<"$system_pool_json")" "$AKS_SYSTEM_NODE_COUNT" + assert_equal "AKS system node-pool mode" "$(jq -r .mode <<<"$system_pool_json")" System + + az extension add --name aks-preview --upgrade --only-show-errors >/dev/null + if ! az aks nodepool show -g "$AZURE_RESOURCE_GROUP" --cluster-name "$AZURE_AKS_CLUSTER_NAME" \ + -n "$AKS_NODE_POOL_NAME" --output none 2>/dev/null; then + log "creating Artifact Streaming benchmark pool $AKS_NODE_POOL_NAME" + az aks nodepool add -g "$AZURE_RESOURCE_GROUP" --cluster-name "$AZURE_AKS_CLUSTER_NAME" \ + -n "$AKS_NODE_POOL_NAME" --mode User \ + --node-count "$AKS_NODE_COUNT" --node-vm-size "$AKS_NODE_VM_SIZE" \ + --node-osdisk-type Managed --node-osdisk-size "$AKS_NODE_OS_DISK_GB" \ + --max-pods "$AKS_MAX_PODS" --os-sku Ubuntu --vnet-subnet-id "$subnet_id" \ + --kubernetes-version "$AKS_KUBERNETES_VERSION" --enable-artifact-streaming \ + --no-wait --only-show-errors -o none + fi + wait_for_node_pool_provisioning + fi + local cluster_json pool_json cluster_json=$(az aks show -g "$AZURE_RESOURCE_GROUP" -n "$AZURE_AKS_CLUSTER_NAME" -o json) assert_equal "AKS location" "$(jq -r .location <<<"$cluster_json")" "$AZURE_LOCATION" @@ -498,8 +566,82 @@ ensure_aks() { assert_equal "AKS max pods" "$(jq -r .maxPods <<<"$pool_json")" "$AKS_MAX_PODS" assert_equal "AKS node OS disk" "$(jq -r .osDiskSizeGb <<<"$pool_json")" "$AKS_NODE_OS_DISK_GB" assert_equal "AKS node OS SKU" "$(jq -r .osSku <<<"$pool_json")" Ubuntu - assert_equal "AKS node-pool mode" "$(jq -r .mode <<<"$pool_json")" System assert_equal "AKS node subnet" "$(jq -r .vnetSubnetId <<<"$pool_json")" "$subnet_id" + if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + assert_equal "AKS benchmark node-pool mode" "$(jq -r .mode <<<"$pool_json")" User + assert_equal "AKS benchmark pool Artifact Streaming" \ + "$(jq -r '.artifactStreamingProfile.enabled // false' <<<"$pool_json")" true + + else + assert_equal "AKS node-pool mode" "$(jq -r .mode <<<"$pool_json")" System + fi +} + +wait_for_node_pool_provisioning() { + local attempt state + for attempt in $(seq 1 240); do + state=$(timeout 30s az aks nodepool show -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" -n "$AKS_NODE_POOL_NAME" \ + --query provisioningState -o tsv 2>/dev/null || true) + case "$state" in + Succeeded) + log "AKS benchmark pool $AKS_NODE_POOL_NAME provisioning succeeded" + return + ;; + Failed | Canceled) + echo "AKS benchmark pool $AKS_NODE_POOL_NAME provisioning ended in $state" >&2 + return 1 + ;; + *) + log "waiting for benchmark pool provisioning: ${state:-not visible} ($attempt/240)" + sleep 30 + ;; + esac + done + + echo "AKS benchmark pool $AKS_NODE_POOL_NAME did not provision" >&2 + return 1 +} + +recreate_aks_node_pool() { + local provisioning_state + provisioning_state=$(timeout 30s az aks nodepool show -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" -n "$AKS_NODE_POOL_NAME" \ + --query provisioningState -o tsv 2>/dev/null || true) + case "$provisioning_state" in + "") + log "AKS benchmark pool $AKS_NODE_POOL_NAME is already absent" + ;; + Deleting) + log "AKS benchmark pool $AKS_NODE_POOL_NAME is already deleting" + ;; + *) + log "submitting deletion of benchmark pool $AKS_NODE_POOL_NAME" + az aks nodepool delete -g "$AZURE_RESOURCE_GROUP" --cluster-name "$AZURE_AKS_CLUSTER_NAME" \ + -n "$AKS_NODE_POOL_NAME" --no-wait --only-show-errors -o none + ;; + esac + + local attempt + for attempt in $(seq 1 240); do + if ! timeout 30s az aks nodepool show -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" -n "$AKS_NODE_POOL_NAME" \ + --output none 2>/dev/null; then + log "AKS benchmark pool $AKS_NODE_POOL_NAME deleted" + break + fi + log "waiting for benchmark pool deletion ($attempt/240)" + sleep 30 + done + if timeout 30s az aks nodepool show -g "$AZURE_RESOURCE_GROUP" \ + --cluster-name "$AZURE_AKS_CLUSTER_NAME" -n "$AKS_NODE_POOL_NAME" \ + --output none 2>/dev/null; then + echo "AKS benchmark pool $AKS_NODE_POOL_NAME did not delete" >&2 + return 1 + fi + + ensure_aks + wait_for_nodes } scale_aks_node_pool() { @@ -522,7 +664,13 @@ scale_aks_node_pool() { assert_equal "AKS max pods" "$(jq -r .maxPods <<<"$pool_json")" "$AKS_MAX_PODS" assert_equal "AKS node OS disk" "$(jq -r .osDiskSizeGb <<<"$pool_json")" "$AKS_NODE_OS_DISK_GB" assert_equal "AKS node OS SKU" "$(jq -r .osSku <<<"$pool_json")" Ubuntu - assert_equal "AKS node-pool mode" "$(jq -r .mode <<<"$pool_json")" System + if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + assert_equal "AKS node-pool mode" "$(jq -r .mode <<<"$pool_json")" User + assert_equal "AKS node-pool Artifact Streaming" \ + "$(jq -r '.artifactStreamingProfile.enabled // false' <<<"$pool_json")" true + else + assert_equal "AKS node-pool mode" "$(jq -r .mode <<<"$pool_json")" System + fi assert_equal "AKS node subnet" "$(jq -r .vnetSubnetId <<<"$pool_json")" "$subnet_id" current_count=$(jq -r .count <<<"$pool_json") @@ -651,7 +799,7 @@ wait_for_nodes() { local attempt total ready unschedulable for attempt in $(seq 1 120); do local nodes - nodes=$(kubectl get nodes -o json) + nodes=$(kubectl get nodes -l "agentpool=$AKS_NODE_POOL_NAME" -o json) total=$(jq '.items|length' <<<"$nodes") ready=$(jq '[.items[]|select(any(.status.conditions[];.type=="Ready" and .status=="True"))]|length' <<<"$nodes") unschedulable=$(jq '[.items[]|select(.spec.unschedulable==true)]|length' <<<"$nodes") @@ -736,6 +884,9 @@ install_containerd_pull_tuning() { kubectl -n "$GANTRY_NAMESPACE" delete configmap gantry-benchmark-containerd-config \ --ignore-not-found=true kubectl apply -f "$repo_root/hack/gantry-benchmark/manifests/containerd-pull-tuning.yaml" + kubectl -n "$GANTRY_NAMESPACE" patch daemonset gantry-benchmark-containerd-pull-tuning \ + --type merge \ + -p "{\"spec\":{\"template\":{\"spec\":{\"nodeSelector\":{\"kubernetes.io/os\":\"linux\",\"agentpool\":\"$AKS_NODE_POOL_NAME\"}}}}}" kubectl -n "$GANTRY_NAMESPACE" rollout status \ daemonset/gantry-benchmark-containerd-pull-tuning --timeout=45m } @@ -777,6 +928,7 @@ spec: dnsPolicy: Default nodeSelector: kubernetes.io/os: linux + agentpool: $AKS_NODE_POOL_NAME tolerations: - operator: Exists containers: @@ -935,6 +1087,7 @@ spec: spec: nodeSelector: kubernetes.io/os: linux + agentpool: $AKS_NODE_POOL_NAME tolerations: - operator: Exists containers: @@ -1000,7 +1153,7 @@ migrate_legacy_gantry_install() { kubectl get priorityclass gantry-low >/dev/null 2>&1; then legacy=true fi - [[ "$legacy" == true ]] || return + [[ "$legacy" == true ]] || return 0 local resource for resource in \ @@ -1025,6 +1178,7 @@ migrate_legacy_gantry_install() { log "removing legacy rendered Gantry resources before first Helm install" kubectl -n "$GANTRY_NAMESPACE" delete daemonset gantry gantry-containerd-config \ + gantry-overlaybd-config \ --ignore-not-found=true --wait=true kubectl -n "$GANTRY_NAMESPACE" delete \ configmap/gantry-config \ @@ -1042,18 +1196,36 @@ deploy_gantry() { export KUBECONFIG GOTOOLCHAIN=auto make -C "$repo_root" install-helm migrate_legacy_gantry_install - "$repo_root/bin/helm" upgrade --install gantry "$repo_root/deploy/gantry/chart" \ - --namespace "$GANTRY_NAMESPACE" \ - --create-namespace \ - --set-string "image.reference=$GANTRY_IMAGE" \ - --set-string 'gantry.pprofListen=127.0.0.1:6060' \ - --set-string "gantry.upstreamRegistries[0].name=$GANTRY_ACR_LOGIN_SERVER" \ - --set-string "gantry.upstreamRegistries[0].endpoint=https://$GANTRY_ACR_LOGIN_SERVER" \ - --wait \ - --timeout 45m + local helm_args=( + upgrade --install gantry "$repo_root/deploy/gantry/chart" + --namespace "$GANTRY_NAMESPACE" + --create-namespace + --set-string "image.reference=$GANTRY_IMAGE" + --set-string 'gantry.pprofListen=127.0.0.1:6060' + --set-string "nodeSelector.agentpool=$AKS_NODE_POOL_NAME" + --set-string "gantry.upstreamRegistries[0].name=$GANTRY_ACR_LOGIN_SERVER" + ) + if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + helm_args+=( + --set gantry.artifactStreaming.enabled=true + --set overlaybdConfig.enabled=true + --set-string "overlaybdConfig.image.reference=$GANTRY_NODE_CONFIG_IMAGE" + --set-string "overlaybdConfig.nodeSelector.agentpool=$AKS_NODE_POOL_NAME" + --set-string "gantry.upstreamRegistries[0].endpoint=http://127.0.0.1:8578?ns=$GANTRY_ACR_LOGIN_SERVER" + ) + else + helm_args+=( + --set-string "gantry.upstreamRegistries[0].endpoint=https://$GANTRY_ACR_LOGIN_SERVER" + ) + fi + helm_args+=(--wait --timeout 45m) + "$repo_root/bin/helm" "${helm_args[@]}" kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry-containerd-config --timeout=30m kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry --timeout=45m + if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + kubectl -n "$GANTRY_NAMESPACE" rollout status daemonset/gantry-overlaybd-config --timeout=45m + fi } provision_operator() { @@ -1065,6 +1237,7 @@ provision_operator() { export OPERATOR_BUILD_DISK_IOPS OPERATOR_BUILD_DISK_MBPS OPERATOR_SUBNET_NAME OPERATOR_SUBNET_CIDR export BENCHMARK_SOURCE_IMAGE=$SOURCE_IMAGE BENCHMARK_SOURCE_REVISION=$source_revision export BENCHMARK_NODE_COUNT BENCHMARK_IMAGE_SIZE_MIB BENCHMARK_IMAGE_LAYERS + export BENCHMARK_ARTIFACT_STREAMING BENCHMARK_NODE_POOL=$AKS_NODE_POOL_NAME export BENCHMARK_AZURE_TELEMETRY=true BENCHMARK_MINIMUM_BYTE_REDUCTION BENCHMARK_MAXIMUM_LATENCY_RATIO export ADOPT_BASELINE_IMAGE ADOPT_GANTRY_IMAGE ADOPT_PAYLOAD_SHA256 AZURE_BASELINE_ACR_PRIVATE_ENDPOINT_RESOURCE_ID=$(az network private-endpoint show \ @@ -1102,12 +1275,13 @@ build_operator_images() { --query 'value[0].message' -o tsv) local result_json result_json=$(tr -d '\r' <<<"$output" | sed -n 's/^DEPLOYMENT_IMAGES_JSON=//p' | tail -1) - jq -e 'type == "object" and (.gantry_image | type == "string") and (.baseline_probe_image | type == "string")' \ + jq -e 'type == "object" and (.gantry_image | type == "string") and (.gantry_node_config_image | type == "string") and (.baseline_probe_image | type == "string")' \ <<<"$result_json" >/dev/null || { echo "operator did not return valid deployment image JSON" >&2 return 1 } GANTRY_IMAGE=$(jq -r .gantry_image <<<"$result_json") + GANTRY_NODE_CONFIG_IMAGE=$(jq -r .gantry_node_config_image <<<"$result_json") BASELINE_PROBE_IMAGE=$(jq -r .baseline_probe_image <<<"$result_json") [[ "$GANTRY_IMAGE" == "$GANTRY_ACR_LOGIN_SERVER/gantry@sha256:"* ]] || { echo "operator did not return an immutable Gantry image" >&2 @@ -1117,6 +1291,11 @@ build_operator_images() { echo "operator did not return an immutable baseline probe image" >&2 return 1 } + if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true && + "$GANTRY_NODE_CONFIG_IMAGE" != "$GANTRY_ACR_LOGIN_SERVER/gantry-node-config@sha256:"* ]]; then + echo "operator did not return an immutable Gantry node-config image" >&2 + return 1 + fi } acquire_operator_run_command_lock() { @@ -1139,6 +1318,12 @@ if [[ "$action" == scale ]]; then log "AKS node pool scale complete" exit 0 fi +if [[ "$action" == recreate-pool ]]; then + recreate_aks_node_pool + release_operator_run_command_lock + log "AKS benchmark pool recreation complete" + exit 0 +fi ensure_group ensure_vnet @@ -1176,10 +1361,15 @@ assert_equal "Gantry ACR public access" \ kubectl -n "$MONITORING_NAMESPACE" get endpoints "$PROMETHEUS_SERVICE" -o json | \ jq -e '.subsets | any(.addresses | length > 0)' >/dev/null -for daemonset in gantry-benchmark-containerd-pull-tuning gantry-acr-private-dns-guard gantry-containerd-config gantry; do +daemonsets=(gantry-benchmark-containerd-pull-tuning gantry-acr-private-dns-guard gantry-containerd-config gantry) +if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + daemonsets+=(gantry-overlaybd-config) +fi +for daemonset in "${daemonsets[@]}"; do namespace=$GANTRY_NAMESPACE desired=$(kubectl -n "$namespace" get daemonset "$daemonset" -o jsonpath='{.status.desiredNumberScheduled}') ready=$(kubectl -n "$namespace" get daemonset "$daemonset" -o jsonpath='{.status.numberReady}') + assert_equal "$daemonset desired" "$desired" "$AKS_NODE_COUNT" assert_equal "$daemonset readiness" "$ready" "$desired" done diff --git a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl index 439fc8028..6157d8776 100644 --- a/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl +++ b/hack/gantry-benchmark/manifests/monitoring.yaml.tmpl @@ -19,6 +19,7 @@ spec: selector: matchLabels: app.kubernetes.io/name: gantry + app.kubernetes.io/component: agent podTargetLabels: - controller-revision-hash podMetricsEndpoints: @@ -29,7 +30,7 @@ spec: - action: keep sourceLabels: - __name__ - regex: process_cpu_seconds_total|gantry_storage_mode_info|p2p_dht_health_score|p2p_libp2p_conns|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_layer_download_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total|p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests|groups)_total|p2p_prefetch_pullers_per_manifest_(bucket|sum|count)|p2p_cold_start_seed_chairs_contacted_(bucket|sum|count)|p2p_cold_start_selectable_chairs_(bucket|sum|count)|p2p_cold_start_seed_chairs_accepted_(bucket|sum|count)|p2p_cold_start_chair_dispatch_total|p2p_cold_start_chair_call_duration_seconds_(bucket|sum|count)|go_sched_latencies_seconds_(bucket|sum|count)|go_goroutines|p2p_origin_pull_queue_wait_seconds_(bucket|sum|count)|p2p_origin_pull_duration_seconds_(bucket|sum|count) + regex: process_cpu_seconds_total|gantry_storage_mode_info|gantry_streaming_(requests|bytes|rejected)_total|gantry_streaming_(request_duration|time_to_first_byte)_seconds_(bucket|sum|count)|gantry_streaming_inflight|p2p_dht_health_score|p2p_libp2p_conns|p2p_dht_lookup_total|p2p_dht_lookup_duration_seconds_(bucket|sum|count)|gantry_peer_serve_bytes_total|gantry_origin_bytes_total|p2p_origin_pull_total|p2p_peer_fetch_total|gantry_peer_fetch_last_timestamp_seconds|p2p_peer_fetch_duration_seconds_(bucket|sum|count)|p2p_origin_pull_success_total|p2p_origin_fallback_total|p2p_in_flight_pulls|gantry_peer_fetch_bytes_total|gantry_mirror_bytes_served_total|gantry_mirror_response_completed_timestamp_seconds|gantry_layer_download_completed_timestamp_seconds|gantry_containerd_commit_observed_total|gantry_containerd_commit_observed_timestamp_seconds|gantry_containerd_commit_observation_duration_seconds_(bucket|sum|count)|gantry_containerd_commit_latest_observation_duration_seconds|gantry_containerd_commit_missing_after_stream_total|p2p_coord_please_pull_(served|started|declined)_total|p2p_coord_pull_intent_served_total|p2p_prefetch_(batches|digests|groups)_total|p2p_prefetch_pullers_per_manifest_(bucket|sum|count)|p2p_cold_start_seed_chairs_contacted_(bucket|sum|count)|p2p_cold_start_selectable_chairs_(bucket|sum|count)|p2p_cold_start_seed_chairs_accepted_(bucket|sum|count)|p2p_cold_start_chair_dispatch_total|p2p_cold_start_chair_call_duration_seconds_(bucket|sum|count)|go_sched_latencies_seconds_(bucket|sum|count)|go_goroutines|p2p_origin_pull_queue_wait_seconds_(bucket|sum|count)|p2p_origin_pull_duration_seconds_(bucket|sum|count) - action: replace targetLabel: gantry_benchmark replacement: "true" @@ -58,6 +59,9 @@ spec: nodeSelector: kubernetes.io/os: {{ .NodeOS }} kubernetes.io/arch: {{ .NodeArch }} + {{- with .NodePool }} + agentpool: {{ . }} + {{- end }} tolerations: - operator: Exists containers: diff --git a/hack/gantry-benchmark/operator-vm-bootstrap.sh b/hack/gantry-benchmark/operator-vm-bootstrap.sh index d8735d0c2..365a3760f 100755 --- a/hack/gantry-benchmark/operator-vm-bootstrap.sh +++ b/hack/gantry-benchmark/operator-vm-bootstrap.sh @@ -11,11 +11,12 @@ Usage: operator-vm-bootstrap.sh \ \ \ \ - + \ + USAGE } -[[ $# -eq 23 ]] || { usage >&2; exit 2; } +[[ $# -eq 25 ]] || { usage >&2; exit 2; } subscription_id=$1 resource_group=$2 @@ -40,9 +41,16 @@ source_revision=${20} adopt_baseline_image=${21} adopt_gantry_image=${22} adopt_payload_sha256=${23} +artifact_streaming=${24} +benchmark_node_pool=${25} [[ "$adopt_baseline_image" != - ]] || adopt_baseline_image="" [[ "$adopt_gantry_image" != - ]] || adopt_gantry_image="" [[ "$adopt_payload_sha256" != - ]] || adopt_payload_sha256="" +[[ "$benchmark_node_pool" != - ]] || benchmark_node_pool="" +[[ "$artifact_streaming" == true || "$artifact_streaming" == false ]] || { + echo "artifact-streaming must be true or false" >&2 + exit 2 +} adoption_values=0 for value in "$adopt_baseline_image" "$adopt_gantry_image" "$adopt_payload_sha256"; do @@ -155,6 +163,11 @@ if ! grep -Fq "UUID=$build_uuid " /etc/fstab; then fi mount "$build_mount" 2>/dev/null || mount -a findmnt --mountpoint "$build_mount" >/dev/null +build_block_device=$(basename "$(readlink -f "$build_device")") +if [[ -w "/sys/class/block/$build_block_device/device/rescan" ]]; then + echo 1 >"/sys/class/block/$build_block_device/device/rescan" +fi +resize2fs "$build_device" install -d -m 0711 "$build_mount/containers" install -d -m 0755 /etc/containers @@ -313,6 +326,9 @@ BENCHMARK_NAMESPACE="gantry-benchmark" BENCHMARK_NODE_COUNT="$node_count" BENCHMARK_IMAGE_SIZE_MIB="$image_size_mib" BENCHMARK_IMAGE_LAYERS="$image_layers" +BENCHMARK_ARTIFACT_STREAMING="$artifact_streaming" +BENCHMARK_NODE_POOL="$benchmark_node_pool" +GANTRY_ONLY_STANDALONE="$artifact_streaming" BENCHMARK_IMAGE_PLATFORM="linux/amd64" BENCHMARK_WORKLOAD_REPOSITORY="gantry-benchmark-pull" BENCHMARK_ROLLOUT_TIMEOUT="15m" diff --git a/hack/gantry-benchmark/operator-vm-build-images.sh b/hack/gantry-benchmark/operator-vm-build-images.sh index 9e93d19d4..15cb00bbd 100755 --- a/hack/gantry-benchmark/operator-vm-build-images.sh +++ b/hack/gantry-benchmark/operator-vm-build-images.sh @@ -32,9 +32,12 @@ if [[ -f "$image_state" ]]; then # shellcheck source=/dev/null . "$image_state" if [[ "$GANTRY_IMAGE" == "$gantry_login/gantry@sha256:"* && - "$BASELINE_PROBE_IMAGE" == "$baseline_login/gantry-deploy-probe@sha256:"* ]]; then - jq -cn --arg gantry_image "$GANTRY_IMAGE" --arg baseline_probe_image "$BASELINE_PROBE_IMAGE" \ - '{gantry_image:$gantry_image,baseline_probe_image:$baseline_probe_image}' | \ + "$BASELINE_PROBE_IMAGE" == "$baseline_login/gantry-deploy-probe@sha256:"* && + ( "$BENCHMARK_ARTIFACT_STREAMING" != true || "${GANTRY_NODE_CONFIG_IMAGE:-}" == "$gantry_login/gantry-node-config@sha256:"* ) ]]; then + jq -cn --arg gantry_image "$GANTRY_IMAGE" \ + --arg gantry_node_config_image "${GANTRY_NODE_CONFIG_IMAGE:-}" \ + --arg baseline_probe_image "$BASELINE_PROBE_IMAGE" \ + '{gantry_image:$gantry_image,gantry_node_config_image:$gantry_node_config_image,baseline_probe_image:$baseline_probe_image}' | \ sed 's/^/DEPLOYMENT_IMAGES_JSON=/' exit 0 fi @@ -71,6 +74,16 @@ podman build --isolation chroot --platform linux/amd64 \ gantry_digest_file=/var/lib/gantry-benchmark/gantry-deploy.digest podman push --digestfile "$gantry_digest_file" "$gantry_tag" >>"$log_file" 2>&1 gantry_digest=$(tr -d '[:space:]' <"$gantry_digest_file") + +gantry_node_config_digest="" +if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true ]]; then + gantry_node_config_tag="$gantry_login/gantry-node-config:benchmark-$source_short" + podman build --isolation chroot --platform linux/amd64 \ + --tag "$gantry_node_config_tag" --file images/gantry-node-config/Containerfile . >>"$log_file" 2>&1 + gantry_node_config_digest_file=/var/lib/gantry-benchmark/gantry-node-config-deploy.digest + podman push --digestfile "$gantry_node_config_digest_file" "$gantry_node_config_tag" >>"$log_file" 2>&1 + gantry_node_config_digest=$(tr -d '[:space:]' <"$gantry_node_config_digest_file") +fi podman logout "$gantry_login" >>"$log_file" 2>&1 registry_login "$baseline_acr" "$baseline_login" @@ -83,14 +96,21 @@ probe_digest=$(tr -d '[:space:]' <"$probe_digest_file") podman logout "$baseline_login" >>"$log_file" 2>&1 GANTRY_IMAGE="$gantry_login/gantry@$gantry_digest" +GANTRY_NODE_CONFIG_IMAGE="" +if [[ -n "$gantry_node_config_digest" ]]; then + GANTRY_NODE_CONFIG_IMAGE="$gantry_login/gantry-node-config@$gantry_node_config_digest" +fi BASELINE_PROBE_IMAGE="$baseline_login/gantry-deploy-probe@$probe_digest" cat >"$image_state" <&2; exit 2; } mode_config='GANTRY_ONLY_STANDALONE="true"' + if [[ -n "${GANTRY_ONLY_STANDALONE_IMAGE:-}" || -n "${GANTRY_ONLY_STANDALONE_STREAMING_IMAGE:-}" || -n "${GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256:-}" ]]; then + : "${GANTRY_ONLY_STANDALONE_IMAGE:?Set GANTRY_ONLY_STANDALONE_IMAGE with the complete standalone adoption set}" + : "${GANTRY_ONLY_STANDALONE_STREAMING_IMAGE:?Set GANTRY_ONLY_STANDALONE_STREAMING_IMAGE with the complete standalone adoption set}" + : "${GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256:?Set GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256 with the complete standalone adoption set}" + printf -v mode_config '%s\nGANTRY_ONLY_STANDALONE_IMAGE=%q\nGANTRY_ONLY_STANDALONE_STREAMING_IMAGE=%q\nGANTRY_ONLY_STANDALONE_PAYLOAD_SHA256=%q' \ + "$mode_config" "$GANTRY_ONLY_STANDALONE_IMAGE" "$GANTRY_ONLY_STANDALONE_STREAMING_IMAGE" "$GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256" + fi local_repo_root=$(git -C "$(dirname "${BASH_SOURCE[0]}")" rev-parse --show-toplevel) standalone_paths=( + hack/cmd/gantry-benchmark/config.go + hack/cmd/gantry-benchmark/enable.go hack/cmd/gantry-benchmark/gantry_only.go + hack/cmd/gantry-benchmark/hosts_routing.go hack/cmd/gantry-benchmark/main.go + hack/cmd/gantry-benchmark/peer_telemetry.go + hack/cmd/gantry-benchmark/preflight.go + hack/cmd/gantry-benchmark/results.go hack/cmd/gantry-benchmark/state.go + hack/gantry-benchmark/manifests/monitoring.yaml.tmpl hack/gantry-benchmark/operator-vm-run.sh ) standalone_base_hashes_base64=$( @@ -500,6 +519,26 @@ SCRIPT script=${script/__PRUNE_WORKER_BASE64__/$prune_worker} invoke_remote "$script" ;; + stop) + (($# == 0)) || { usage >&2; exit 2; } + script=$(cat <<'SCRIPT' +set -Eeuo pipefail +systemctl stop gantry-benchmark-operator.service || true +source /etc/gantry-benchmark/env +export HOME="${BENCHMARK_OPERATOR_HOME:-/var/lib/gantry-benchmark}" +export KUBECONFIG="${KUBECONFIG:-$HOME/kubeconfig}" +export BENCHMARK_CONFIRM_CONTEXT="$(kubectl config current-context)" +if kubectl -n "${BENCHMARK_NAMESPACE:-gantry-benchmark}" get configmap gantry-benchmark-state >/dev/null 2>&1 || + kubectl -n "${GANTRY_NAMESPACE:-gantry-system}" get configmap gantry-benchmark-lock >/dev/null 2>&1; then + cd "$BENCHMARK_REPO_ROOT" + make -C hack/gantry-benchmark disable +fi +systemctl show gantry-benchmark-operator.service \ + --property=ActiveState --property=SubState --property=Result --property=ExecMainStatus --no-pager +SCRIPT +) + invoke_remote "$script" + ;; prune-status) (($# == 0)) || { usage >&2; exit 2; } diff --git a/hack/gantry-benchmark/operator-vm-provision.sh b/hack/gantry-benchmark/operator-vm-provision.sh index 6943cb9b9..7ca3740d3 100755 --- a/hack/gantry-benchmark/operator-vm-provision.sh +++ b/hack/gantry-benchmark/operator-vm-provision.sh @@ -39,6 +39,8 @@ BENCHMARK_SOURCE_REVISION="${BENCHMARK_SOURCE_REVISION:-}" BENCHMARK_NODE_COUNT="${BENCHMARK_NODE_COUNT:-5}" BENCHMARK_IMAGE_SIZE_MIB="${BENCHMARK_IMAGE_SIZE_MIB:-128}" BENCHMARK_IMAGE_LAYERS="${BENCHMARK_IMAGE_LAYERS:-4}" +BENCHMARK_ARTIFACT_STREAMING="${BENCHMARK_ARTIFACT_STREAMING:-false}" +BENCHMARK_NODE_POOL="${BENCHMARK_NODE_POOL:-}" BENCHMARK_AZURE_TELEMETRY="${BENCHMARK_AZURE_TELEMETRY:-true}" BENCHMARK_MINIMUM_BYTE_REDUCTION="${BENCHMARK_MINIMUM_BYTE_REDUCTION:-0.70}" BENCHMARK_MAXIMUM_LATENCY_RATIO="${BENCHMARK_MAXIMUM_LATENCY_RATIO:-3.0}" @@ -47,6 +49,15 @@ ADOPT_GANTRY_IMAGE="${ADOPT_GANTRY_IMAGE:-}" ADOPT_PAYLOAD_SHA256="${ADOPT_PAYLOAD_SHA256:-}" START_BENCHMARK="${START_BENCHMARK:-false}" +[[ "$BENCHMARK_ARTIFACT_STREAMING" == true || "$BENCHMARK_ARTIFACT_STREAMING" == false ]] || { + echo "BENCHMARK_ARTIFACT_STREAMING must be true or false" >&2 + exit 2 +} +if [[ "$BENCHMARK_ARTIFACT_STREAMING" == true && -z "$BENCHMARK_NODE_POOL" ]]; then + echo "BENCHMARK_NODE_POOL is required when Artifact Streaming is enabled" >&2 + exit 2 +fi + repo_root=$(git rev-parse --show-toplevel) bootstrap_script="$repo_root/hack/gantry-benchmark/operator-vm-bootstrap.sh" key_path="$repo_root/tmp/gantry-benchmark-operator-key" @@ -143,6 +154,24 @@ if ! az disk show -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_BUILD_DISK_NAME" --ou --disk-mbps-read-write "$OPERATOR_BUILD_DISK_MBPS" \ --only-show-errors \ -o none +else + current_build_disk_gb=$(az disk show -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_BUILD_DISK_NAME" \ + --query diskSizeGB -o tsv) + if ((current_build_disk_gb > OPERATOR_BUILD_DISK_GB)); then + echo "operator build disk is ${current_build_disk_gb} GiB; shrinking to ${OPERATOR_BUILD_DISK_GB} GiB is not supported" >&2 + exit 1 + fi + if ((current_build_disk_gb < OPERATOR_BUILD_DISK_GB)); then + echo "expanding operator build disk: ${current_build_disk_gb} GiB -> ${OPERATOR_BUILD_DISK_GB} GiB" + az disk update \ + -g "$AZURE_RESOURCE_GROUP" \ + -n "$OPERATOR_BUILD_DISK_NAME" \ + --size-gb "$OPERATOR_BUILD_DISK_GB" \ + --disk-iops-read-write "$OPERATOR_BUILD_DISK_IOPS" \ + --disk-mbps-read-write "$OPERATOR_BUILD_DISK_MBPS" \ + --only-show-errors \ + -o none + fi fi build_disk_id=$(az disk show -g "$AZURE_RESOURCE_GROUP" -n "$OPERATOR_BUILD_DISK_NAME" --query id -o tsv) @@ -219,6 +248,8 @@ az vm run-command invoke \ "${ADOPT_BASELINE_IMAGE:--}" \ "${ADOPT_GANTRY_IMAGE:--}" \ "${ADOPT_PAYLOAD_SHA256:--}" \ + "$BENCHMARK_ARTIFACT_STREAMING" \ + "${BENCHMARK_NODE_POOL:--}" \ --only-show-errors \ -o json diff --git a/hack/gantry-benchmark/operator-vm-run.sh b/hack/gantry-benchmark/operator-vm-run.sh index 4e737abe2..13c393d9b 100755 --- a/hack/gantry-benchmark/operator-vm-run.sh +++ b/hack/gantry-benchmark/operator-vm-run.sh @@ -151,6 +151,14 @@ if [[ "${GANTRY_ONLY_STANDALONE:-false}" == true && -n "${GANTRY_ONLY_BASELINE_R echo "GANTRY_ONLY_STANDALONE and GANTRY_ONLY_BASELINE_RUN_ID are mutually exclusive" >&2 exit 2 fi +standalone_adoption_values=0 +for value in "${GANTRY_ONLY_STANDALONE_IMAGE:-}" "${GANTRY_ONLY_STANDALONE_STREAMING_IMAGE:-}" "${GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256:-}"; do + [[ -z "$value" ]] || standalone_adoption_values=$((standalone_adoption_values + 1)) +done +if ((standalone_adoption_values != 0 && standalone_adoption_values != 3)); then + echo "GANTRY_ONLY_STANDALONE_IMAGE, GANTRY_ONLY_STANDALONE_STREAMING_IMAGE, and GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256 must be set together" >&2 + exit 2 +fi if [[ -n "${GANTRY_ONLY_BASELINE_RUN_ID:-}" ]]; then preparation_modes=0 [[ -z "${GANTRY_ONLY_ADOPT_IMAGE:-}" ]] || ((preparation_modes += 1)) @@ -191,7 +199,11 @@ make -C hack/gantry-benchmark enable run_id=$(kubectl -n "${BENCHMARK_NAMESPACE:-gantry-benchmark}" get configmap gantry-benchmark-state -o jsonpath='{.data.state\.json}' | jq -er '.run_id') log "enabled benchmark $run_id" if [[ "${GANTRY_ONLY_STANDALONE:-false}" == true ]]; then - write_progress "prepare" "generating a standalone Gantry image with no baseline" + if ((standalone_adoption_values == 3)); then + write_progress "prepare" "adopting an existing standalone Gantry image" + else + write_progress "prepare" "generating a standalone Gantry image with no baseline" + fi elif [[ -n "${ADOPT_BASELINE_IMAGE:-}" || -n "${ADOPT_GANTRY_IMAGE:-}" || -n "${ADOPT_PAYLOAD_SHA256:-}" ]]; then : "${ADOPT_BASELINE_IMAGE:?Set ADOPT_BASELINE_IMAGE with the full adoption set}" : "${ADOPT_GANTRY_IMAGE:?Set ADOPT_GANTRY_IMAGE with the full adoption set}" @@ -215,7 +227,7 @@ fi needs_baseline_credentials=false needs_gantry_credentials=false if [[ "${GANTRY_ONLY_STANDALONE:-false}" == true ]]; then - needs_gantry_credentials=true + [[ "$standalone_adoption_values" == 3 ]] || needs_gantry_credentials=true elif [[ -z "${ADOPT_BASELINE_IMAGE:-}" ]]; then if [[ -z "${GANTRY_ONLY_BASELINE_RUN_ID:-}" ]]; then needs_baseline_credentials=true @@ -255,7 +267,13 @@ if [[ "$needs_baseline_credentials" == true || "$needs_gantry_credentials" == tr fi if [[ "${GANTRY_ONLY_STANDALONE:-false}" == true ]]; then - GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-gantry-standalone + if ((standalone_adoption_values == 3)); then + GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-gantry-standalone-adopt \ + "$GANTRY_ONLY_STANDALONE_IMAGE" "$GANTRY_ONLY_STANDALONE_STREAMING_IMAGE" \ + "$GANTRY_ONLY_STANDALONE_PAYLOAD_SHA256" + else + GOTOOLCHAIN=auto go run ./hack/cmd/gantry-benchmark prepare-gantry-standalone + fi elif [[ -n "${ADOPT_BASELINE_IMAGE:-}" ]]; then make -C hack/gantry-benchmark prepare-adopt \ ADOPT_BASELINE_IMAGE="$ADOPT_BASELINE_IMAGE" \ diff --git a/hack/scripts/gantry-streaming-aks.sh b/hack/scripts/gantry-streaming-aks.sh new file mode 100755 index 000000000..b6188c544 --- /dev/null +++ b/hack/scripts/gantry-streaming-aks.sh @@ -0,0 +1,271 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# gantry-streaming-aks.sh -- Stand up a throwaway AKS cluster and Premium ACR +# for testing Gantry against ACR Artifact Streaming, with streaming enabled +# from the start rather than staged on afterwards. +# +# Usage: +# ACR= hack/scripts/gantry-streaming-aks.sh up +# hack/scripts/gantry-streaming-aks.sh cleanup-operator +# hack/scripts/gantry-streaming-aks.sh install +# hack/scripts/gantry-streaming-aks.sh verify +# hack/scripts/gantry-streaming-aks.sh uninstall +# hack/scripts/gantry-streaming-aks.sh down +# +# Environment: +# ACR required on 'up'; must be globally unique +# RESOURCE_GROUP default gantry-streaming +# LOCATION default westus3 +# CLUSTER default gantry-streaming +# STREAM_POOL default stream +# STREAM_NODES default 2; peer reuse cannot be observed with fewer +# NODE_LABEL default gantry-streaming=true +# NODE_VM_SIZE default Standard_D4s_v5 +# IMAGE_TAG default current git describe; shared by both Gantry images +# +# Artifact streaming is an AKS/ACR preview feature and can only be enabled when +# a node pool is created, never on an existing pool. The az flags below move +# with the preview; check the current Azure docs if one is rejected. +# +# Azure operations here are long-running. Interrupting this script does not +# cancel the server-side operation: re-run 'up' to converge, or 'down' to +# remove everything. + +set -euo pipefail + +RESOURCE_GROUP="${RESOURCE_GROUP:-gantry-streaming}" +LOCATION="${LOCATION:-westus3}" +CLUSTER="${CLUSTER:-gantry-streaming}" +ACR="${ACR:-}" +STREAM_POOL="${STREAM_POOL:-stream}" +STREAM_NODES="${STREAM_NODES:-2}" +SYSTEM_NODES="${SYSTEM_NODES:-1}" +NODE_LABEL="${NODE_LABEL:-gantry-streaming=true}" +NODE_VM_SIZE="${NODE_VM_SIZE:-Standard_D4s_v5}" +NAMESPACE="${NAMESPACE:-unbounded-system}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +IMAGE_TAG="${IMAGE_TAG:-$(git -C "$REPO_ROOT" describe --tags --always --dirty 2>/dev/null || echo dev)}" +IMAGE_TAG="${IMAGE_TAG//\//-}" + +log() { printf '\n== %s\n' "$*"; } +die() { + printf 'error: %s\n' "$*" >&2 + exit 1 +} + +require_tools() { + for bin in "$@"; do + command -v "$bin" >/dev/null 2>&1 || die "missing required tool: $bin" + done +} + +require_azure() { + require_tools az + az account show >/dev/null 2>&1 || die "run 'az login' first" +} + +cmd_up() { + require_azure + require_tools kubectl make + [ -n "$ACR" ] || die "set ACR to a globally unique registry name" + + log "Enabling the aks-preview extension" + az extension add --name aks-preview --upgrade --only-show-errors >/dev/null + + log "Resource group $RESOURCE_GROUP in $LOCATION" + az group create -n "$RESOURCE_GROUP" -l "$LOCATION" --only-show-errors >/dev/null + + # Artifact streaming requires a Premium registry. + log "Premium ACR $ACR" + if ! az acr show -n "$ACR" >/dev/null 2>&1; then + az acr create -n "$ACR" -g "$RESOURCE_GROUP" --sku Premium --only-show-errors >/dev/null + fi + + log "AKS cluster $CLUSTER (long-running)" + if ! az aks show -n "$CLUSTER" -g "$RESOURCE_GROUP" >/dev/null 2>&1; then + # Overlay is pinned so the network profile carries a podCidr, which + # get-aks-cluster-cidrs.sh needs to build the 'site init' command. + az aks create -n "$CLUSTER" -g "$RESOURCE_GROUP" \ + --node-count "$SYSTEM_NODES" --node-vm-size "$NODE_VM_SIZE" \ + --network-plugin azure --network-plugin-mode overlay \ + --attach-acr "$ACR" --generate-ssh-keys --only-show-errors >/dev/null + fi + + log "Artifact streaming node pool $STREAM_POOL (${STREAM_NODES} nodes)" + if ! az aks nodepool show --cluster-name "$CLUSTER" -g "$RESOURCE_GROUP" -n "$STREAM_POOL" >/dev/null 2>&1; then + az aks nodepool add --cluster-name "$CLUSTER" -g "$RESOURCE_GROUP" -n "$STREAM_POOL" \ + --node-count "$STREAM_NODES" --node-vm-size "$NODE_VM_SIZE" \ + --enable-artifact-streaming --labels "$NODE_LABEL" --only-show-errors >/dev/null + fi + + log "Fetching kubeconfig" + az aks get-credentials -n "$CLUSTER" -g "$RESOURCE_GROUP" --overwrite-existing --only-show-errors + + cmd_images + + cat < --enable true + +EOF +} + +cmd_images() { + require_azure + require_tools make + [ -n "$ACR" ] || die "set ACR to the registry holding the test images" + + log "Building and pushing images to $ACR" + az acr login -n "$ACR" --only-show-errors >/dev/null + make -C "$REPO_ROOT" image-gantry-push image-gantry-node-config-push \ + CONTAINER_REGISTRY="${ACR}.azurecr.io" VERSION="$IMAGE_TAG" +} + +cmd_cleanup_operator() { + require_tools kubectl + + local manager + manager="$(kubectl get priorityclass gantry-low \ + -o jsonpath='{.metadata.annotations.gantry\.unbounded-cloud\.io/manager}' 2>/dev/null || true)" + if [ "$manager" = "helm" ]; then + die "refusing operator cleanup: Gantry is managed by Helm" + fi + + if [ "$manager" != "unbounded-operator" ] && \ + ! kubectl -n "$NAMESPACE" get deployment/unbounded-operator >/dev/null 2>&1; then + die "no unbounded-operator installation found" + fi + + log "Stopping unbounded-operator" + kubectl -n "$NAMESPACE" scale deployment/unbounded-operator --replicas=0 2>/dev/null || true + + log "Restoring OverlayBD before removing operator-managed Gantry" + kubectl -n "$NAMESPACE" delete daemonset/gantry-overlaybd-config --ignore-not-found --wait=true --timeout=2m + + log "Removing operator namespace and cluster-scoped resources" + kubectl delete namespace "$NAMESPACE" --ignore-not-found --wait=true --timeout=3m + kubectl delete clusterrole \ + unbounded-operator unbounded-net-controller unbounded-net-node \ + unbounded-net-status-viewer token-refresher --ignore-not-found + kubectl delete clusterrolebinding \ + unbounded-operator unbounded-net-controller unbounded-net-node \ + unbounded-net-kube-proxy token-refresher --ignore-not-found + kubectl delete apiservice \ + v1alpha1.net.unbounded-cloud.io v1alpha3.unbounded-cloud.io --ignore-not-found + kubectl delete priorityclass gantry-low --ignore-not-found + + local crds + crds="$(kubectl get crd -o name 2>/dev/null | grep -E '(\.unbounded-cloud\.io$)' || true)" + if [ -n "$crds" ]; then + printf '%s\n' "$crds" | xargs kubectl delete + fi +} + +cmd_install() { + require_tools kubectl make + [ -n "$ACR" ] || die "set ACR to the registry holding the Gantry images" + + local helm="$REPO_ROOT/bin/helm" + [ -x "$helm" ] || make -C "$REPO_ROOT" install-helm + + local key="${NODE_LABEL%%=*}" + local value="${NODE_LABEL#*=}" + local registry="${ACR}.azurecr.io" + + kubectl create namespace "$NAMESPACE" --dry-run=client -o yaml | kubectl apply -f - + + log "Installing Gantry with AKS Artifact Streaming" + "$helm" upgrade --install gantry "$REPO_ROOT/deploy/gantry/chart" \ + --namespace "$NAMESPACE" \ + --set-string "image.reference=${registry}/gantry:${IMAGE_TAG}" \ + --set image.pullPolicy=Always \ + --set-string "overlaybdConfig.image.reference=${registry}/gantry-node-config:${IMAGE_TAG}" \ + --set overlaybdConfig.image.pullPolicy=Always \ + --set-string "nodeSelector.${key}=${value}" \ + --set overlaybdConfig.enabled=true \ + --set-string "overlaybdConfig.nodeSelector.${key}=${value}" \ + --set gantry.artifactStreaming.enabled=true \ + --set-string "gantry.upstreamRegistries[0].name=${registry}" \ + --set-string "gantry.upstreamRegistries[0].endpoint=http://127.0.0.1:8578?ns=${registry}" \ + --wait --timeout 5m +} + +cmd_uninstall() { + require_tools kubectl + + local helm="$REPO_ROOT/bin/helm" + [ -x "$helm" ] || die "missing Helm binary: run 'make install-helm'" + + log "Uninstalling Gantry" + "$helm" uninstall gantry --namespace "$NAMESPACE" --wait --timeout 3m +} + +# gantry-overlaybd-config reports ready only once the host config and the +# running OverlayBD services agree, so its rollout is the host-side check. +cmd_verify() { + require_tools kubectl + + log "Gantry agent rollout" + kubectl -n "$NAMESPACE" rollout status ds/gantry --timeout=5m + + log "Containerd mirror configurator rollout" + kubectl -n "$NAMESPACE" rollout status ds/gantry-containerd-config --timeout=5m + + log "OverlayBD configurator rollout" + kubectl -n "$NAMESPACE" rollout status ds/gantry-overlaybd-config --timeout=5m + + log "Streaming metrics" + local pod + pod="$(kubectl -n "$NAMESPACE" get pods \ + -l 'app.kubernetes.io/name=gantry,app.kubernetes.io/component=agent' \ + -o jsonpath='{.items[0].metadata.name}')" + [ -n "$pod" ] || die "no gantry pod found in $NAMESPACE" + + kubectl -n "$NAMESPACE" port-forward "pod/$pod" 19095:9095 >/dev/null 2>&1 & + local forward=$! + trap "kill '$forward' 2>/dev/null || true" EXIT + + local attempt metrics + for attempt in $(seq 1 20); do + if metrics="$(curl -fsS --max-time 2 http://127.0.0.1:19095/metrics 2>/dev/null)" && + printf '%s\n' "$metrics" | grep -E '^gantry_streaming_(requests|rejected)_total'; then + return 0 + fi + + [ "$attempt" -lt 20 ] || die "streaming metrics unavailable on $pod" + sleep 1 + done +} + +cmd_down() { + require_azure + + log "Deleting resource group $RESOURCE_GROUP" + az group delete -n "$RESOURCE_GROUP" --yes --no-wait --only-show-errors + echo "Deletion runs in the background; 'az group show -n $RESOURCE_GROUP' reports progress." +} + +case "${1:-}" in +up) cmd_up ;; +images) cmd_images ;; +cleanup-operator) cmd_cleanup_operator ;; +install) cmd_install ;; +verify) cmd_verify ;; +uninstall) cmd_uninstall ;; +down) cmd_down ;; +*) + echo "usage: $0 {up|images|cleanup-operator|install|verify|uninstall|down}" >&2 + exit 2 + ;; +esac diff --git a/images/gantry-node-config/Containerfile b/images/gantry-node-config/Containerfile new file mode 100644 index 000000000..b75a2f016 --- /dev/null +++ b/images/gantry-node-config/Containerfile @@ -0,0 +1,14 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +FROM docker.io/library/ubuntu:noble@sha256:84e77dee7d1bc93fb029a45e3c6cb9d8aa4831ccfcc7103d36e876938d28895b + +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + curl jq util-linux \ + && rm -rf /var/lib/apt/lists/* + +COPY images/gantry-node-config/configure-overlaybd.sh /usr/local/bin/configure-overlaybd.sh +RUN chmod 0755 /usr/local/bin/configure-overlaybd.sh + +ENTRYPOINT ["/usr/local/bin/configure-overlaybd.sh"] \ No newline at end of file diff --git a/images/gantry-node-config/configure-overlaybd.sh b/images/gantry-node-config/configure-overlaybd.sh new file mode 100755 index 000000000..1000fd58d --- /dev/null +++ b/images/gantry-node-config/configure-overlaybd.sh @@ -0,0 +1,139 @@ +#!/bin/sh +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +set -eu + +: "${OVERLAYBD_P2P_ADDRESS:?OVERLAYBD_P2P_ADDRESS is required}" +: "${HOST_ROOT:=/host}" +: "${OVERLAYBD_CONFIG_TOOL:=/opt/acr/tools/overlaybd/config.sh}" +: "${GANTRY_STREAMING_READYZ:=http://localhost:5000/artifact-streaming/readyz}" +: "${GANTRY_READY_TIMEOUT_SECONDS:=60}" +: "${NSENTER_BIN:=nsenter}" +: "${CURL_BIN:=curl}" +: "${JQ_BIN:=jq}" +: "${READY_MARKER:=/run/gantry-overlaybd/configured}" +: "${HOST_STATE_DIR:=/var/lib/gantry/overlaybd-config}" + +HOST_CONFIG="$HOST_ROOT/etc/overlaybd/overlaybd.json" +STATE_DIR="$HOST_ROOT/var/lib/gantry/overlaybd-config" +ORIGINAL_CONFIG="$STATE_DIR/original.json" +MANAGED_CONFIG="$STATE_DIR/managed.json" + +umask 077 +mkdir -p "$STATE_DIR" "$(dirname "$READY_MARKER")" + +acquire_lock() { + exec 9>"$STATE_DIR/lock" + flock -x 9 +} + +release_lock() { + flock -u 9 + exec 9>&- +} + +host_exec() { + "$NSENTER_BIN" -t 1 -m -u -i -n -p --root=/proc/1/root --wd=/ -- "$@" +} + +# Change directory after nsenter because its --wd resolves inside this container. +host_exec_writable() { + host_exec sh -c 'cd "$1" || exit 1; shift; exec "$@"' sh "$HOST_STATE_DIR" "$@" +} + +restart_services() { + host_exec systemctl restart overlaybd-tcmu + host_exec systemctl restart overlaybd-snapshotter + host_exec systemctl is-active --quiet overlaybd-tcmu + host_exec systemctl is-active --quiet overlaybd-snapshotter +} + +wait_for_gantry() { + elapsed=0 + while ! "$CURL_BIN" --fail --silent --show-error --max-time 2 "$GANTRY_STREAMING_READYZ" >/dev/null; do + if [ "$elapsed" -ge "$GANTRY_READY_TIMEOUT_SECONDS" ]; then + echo "Gantry artifact streaming endpoint did not become ready" >&2 + return 1 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done +} + +read_value() { + "$JQ_BIN" -r "$1" "$HOST_CONFIG" +} + +verify_desired() { + [ "$(read_value '.p2pConfig.enable // false')" = true ] && + [ "$(read_value '.p2pConfig.address // ""')" = "$OVERLAYBD_P2P_ADDRESS" ] +} + +apply_config() { + wait_for_gantry + [ -f "$HOST_CONFIG" ] || { echo "missing host OverlayBD config: $HOST_CONFIG" >&2; return 1; } + host_exec test -x "$OVERLAYBD_CONFIG_TOOL" + + if [ -f "$MANAGED_CONFIG" ] && ! cmp -s "$HOST_CONFIG" "$MANAGED_CONFIG"; then + echo "refusing to overwrite concurrently changed OverlayBD config" >&2 + return 1 + fi + + if [ ! -f "$ORIGINAL_CONFIG" ]; then + cp "$HOST_CONFIG" "$ORIGINAL_CONFIG" + fi + + if verify_desired; then + cp "$HOST_CONFIG" "$MANAGED_CONFIG" + else + host_exec_writable "$OVERLAYBD_CONFIG_TOOL" p2pConfig.enable true + host_exec_writable "$OVERLAYBD_CONFIG_TOOL" p2pConfig.address "\"$OVERLAYBD_P2P_ADDRESS\"" + verify_desired || { echo "OverlayBD config did not converge" >&2; return 1; } + cp "$HOST_CONFIG" "$MANAGED_CONFIG" + restart_services + fi + + touch "$READY_MARKER" +} + +restore_config() { + rm -f "$READY_MARKER" + [ -f "$ORIGINAL_CONFIG" ] || return 0 + [ -f "$MANAGED_CONFIG" ] || { echo "missing managed OverlayBD config snapshot; preserving host config" >&2; return 0; } + + if ! cmp -s "$HOST_CONFIG" "$MANAGED_CONFIG"; then + echo "OverlayBD config changed after Gantry configuration; preserving current host config" >&2 + return 0 + fi + + if ! cmp -s "$HOST_CONFIG" "$ORIGINAL_CONFIG"; then + temporary="$HOST_CONFIG.gantry-restore" + cp "$ORIGINAL_CONFIG" "$temporary" + chmod --reference="$HOST_CONFIG" "$temporary" + mv "$temporary" "$HOST_CONFIG" + restart_services + fi + + rm -f "$ORIGINAL_CONFIG" "$MANAGED_CONFIG" +} + +case "${1:-apply}" in +apply) + acquire_lock + apply_config + release_lock + if [ "${GANTRY_OVERLAYBD_ONESHOT:-false}" != true ]; then + exec tail -f /dev/null + fi + ;; +restore) + acquire_lock + restore_config + release_lock + ;; +*) + echo "usage: $0 [apply|restore]" >&2 + exit 2 + ;; +esac \ No newline at end of file diff --git a/images/gantry-node-config/configure_overlaybd_test.go b/images/gantry-node-config/configure_overlaybd_test.go new file mode 100644 index 000000000..c14aa32ad --- /dev/null +++ b/images/gantry-node-config/configure_overlaybd_test.go @@ -0,0 +1,271 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package gantrynodeconfig_test + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +const originalConfig = `{"logConfig":{"logLevel":1},"p2pConfig":{"enable":false,"address":""},"other":{"preserve":true}}` + +type configuratorFixture struct { + hostRoot string + config string + stateDir string + log string + env []string +} + +func newConfiguratorFixture(t *testing.T) configuratorFixture { + t.Helper() + + root := t.TempDir() + hostRoot := filepath.Join(root, "host") + config := filepath.Join(hostRoot, "etc", "overlaybd", "overlaybd.json") + stateDir := filepath.Join(hostRoot, "var", "lib", "gantry", "overlaybd-config") + binDir := filepath.Join(root, "bin") + logPath := filepath.Join(root, "systemctl.log") + + for _, directory := range []string{filepath.Dir(config), binDir} { + if err := os.MkdirAll(directory, 0o755); err != nil { + t.Fatal(err) + } + } + + if err := os.WriteFile(config, []byte(originalConfig), 0o644); err != nil { + t.Fatal(err) + } + + nsenter := writeExecutable(t, binDir, "nsenter", `#!/bin/sh +wd=/ +while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do + case "$1" in + --wd=*) wd="${1#--wd=}" ;; + esac + shift +done +shift +cd "$wd" || exit 1 +exec "$@" +`) + curl := writeExecutable(t, binDir, "curl", "#!/bin/sh\nexit 0\n") + writeExecutable(t, binDir, "systemctl", `#!/bin/sh +printf '%s\n' "$*" >> "$TEST_SYSTEMCTL_LOG" +exit 0 +`) + // Match the AKS tool's jq interpolation and relative temporary file. + configTool := writeExecutable(t, binDir, "config.sh", `#!/bin/sh +key=$1 +value=$2 +jq ".$key = $value" "$TEST_HOST_CONFIG" > tmp.json +mv tmp.json "$TEST_HOST_CONFIG" +`) + + return configuratorFixture{ + hostRoot: hostRoot, + config: config, + stateDir: stateDir, + log: logPath, + env: append(os.Environ(), + "HOST_ROOT="+hostRoot, + "OVERLAYBD_P2P_ADDRESS=http://localhost:5000/blobs", + "OVERLAYBD_CONFIG_TOOL="+configTool, + "GANTRY_OVERLAYBD_ONESHOT=true", + "HOST_STATE_DIR="+stateDir, + "NSENTER_BIN="+nsenter, + "CURL_BIN="+curl, + "TEST_HOST_CONFIG="+config, + "TEST_SYSTEMCTL_LOG="+logPath, + "READY_MARKER="+filepath.Join(root, "run", "configured"), + "PATH="+binDir+":"+os.Getenv("PATH"), + ), + } +} + +func (f configuratorFixture) run(t *testing.T, action string) string { + t.Helper() + + command := exec.Command("sh", "configure-overlaybd.sh", action) + command.Env = f.env + output, err := command.CombinedOutput() + if err != nil { + t.Fatalf("configure-overlaybd.sh %s: %v\n%s", action, err, output) + } + + return string(output) +} + +func TestConfiguratorApplyIsIdempotentAndRestoreReverts(t *testing.T) { + fixture := newConfiguratorFixture(t) + fixture.run(t, "apply") + + assertDesiredConfig(t, fixture.config) + if got := countLines(t, fixture.log); got != 4 { + t.Fatalf("systemctl calls after apply = %d, want 4", got) + } + + fixture.run(t, "apply") + if got := countLines(t, fixture.log); got != 4 { + t.Fatalf("systemctl calls after no-op apply = %d, want 4", got) + } + + fixture.run(t, "restore") + restored, err := os.ReadFile(fixture.config) + if err != nil { + t.Fatal(err) + } + + if string(restored) != originalConfig { + t.Fatalf("restored config = %s, want %s", restored, originalConfig) + } + + if got := countLines(t, fixture.log); got != 8 { + t.Fatalf("systemctl calls after restore = %d, want 8", got) + } + + for _, name := range []string{"original.json", "managed.json"} { + if _, err := os.Stat(filepath.Join(fixture.stateDir, name)); !os.IsNotExist(err) { + t.Fatalf("state file %s remains after restore; err=%v", name, err) + } + } +} + +func TestConfiguratorRestorePreservesConcurrentChange(t *testing.T) { + fixture := newConfiguratorFixture(t) + fixture.run(t, "apply") + + changedPath := fixture.config + ".changed" + command := exec.Command("jq", ".other.preserve=false", fixture.config) + changed, err := command.Output() + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(changedPath, changed, 0o644); err != nil { + t.Fatal(err) + } + if err := os.Rename(changedPath, fixture.config); err != nil { + t.Fatal(err) + } + + output := fixture.run(t, "restore") + if !strings.Contains(output, "preserving current host config") { + t.Fatalf("restore output = %q, want preservation warning", output) + } + + got, err := os.ReadFile(fixture.config) + if err != nil { + t.Fatal(err) + } + if string(got) != string(changed) { + t.Fatalf("concurrent config was overwritten: %s", got) + } + + if calls := countLines(t, fixture.log); calls != 4 { + t.Fatalf("systemctl calls after refused restore = %d, want 4", calls) + } +} + +func TestConfiguratorResidentProcessReleasesRestoreLock(t *testing.T) { + fixture := newConfiguratorFixture(t) + resident := exec.Command("sh", "configure-overlaybd.sh", "apply") + resident.Env = replaceEnv(fixture.env, "GANTRY_OVERLAYBD_ONESHOT", "false") + if err := resident.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = resident.Process.Kill() + _ = resident.Wait() + }) + + deadline := time.Now().Add(2 * time.Second) + for { + if _, err := os.Stat(filepath.Join(fixture.stateDir, "managed.json")); err == nil { + break + } else if !os.IsNotExist(err) { + t.Fatal(err) + } + if time.Now().After(deadline) { + t.Fatal("resident configurator did not finish apply") + } + time.Sleep(10 * time.Millisecond) + } + + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Second) + defer cancel() + + restore := exec.CommandContext(ctx, "sh", "configure-overlaybd.sh", "restore") + restore.Env = fixture.env + if output, err := restore.CombinedOutput(); err != nil { + t.Fatalf("restore while resident configurator runs: %v\n%s", err, output) + } + + restored, err := os.ReadFile(fixture.config) + if err != nil { + t.Fatal(err) + } + if string(restored) != originalConfig { + t.Fatalf("restored config = %s, want %s", restored, originalConfig) + } +} + +func replaceEnv(environment []string, key, value string) []string { + prefix := key + "=" + replaced := append([]string(nil), environment...) + for index, entry := range replaced { + if strings.HasPrefix(entry, prefix) { + replaced[index] = prefix + value + + return replaced + } + } + + return append(replaced, prefix+value) +} + +func writeExecutable(t *testing.T, directory, name, content string) string { + t.Helper() + + path := filepath.Join(directory, name) + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatal(err) + } + + return path +} + +// assertDesiredConfig also pins that settings Gantry does not own are left +// exactly as the host had them. +func assertDesiredConfig(t *testing.T, path string) { + t.Helper() + + command := exec.Command("jq", "-e", `.p2pConfig.enable == true and .p2pConfig.address == "http://localhost:5000/blobs" and .logConfig.logLevel == 1 and .other.preserve == true`, path) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("desired config check: %v\n%s", err, output) + } +} + +func countLines(t *testing.T, path string) int { + t.Helper() + + content, err := os.ReadFile(path) + if os.IsNotExist(err) { + return 0 + } + if err != nil { + t.Fatal(err) + } + + trimmed := strings.TrimSpace(string(content)) + if trimmed == "" { + return 0 + } + + return strings.Count(trimmed, "\n") + 1 +} diff --git a/internal/gantry/coldstart/chair.go b/internal/gantry/coldstart/chair.go index 8648a94de..8650f4a57 100644 --- a/internal/gantry/coldstart/chair.go +++ b/internal/gantry/coldstart/chair.go @@ -7,10 +7,12 @@ import ( "context" "errors" "fmt" + "hash/fnv" "log/slog" "sort" "strings" "sync" + "sync/atomic" "time" "github.com/Azure/unbounded/internal/gantry/chairs" @@ -35,23 +37,26 @@ type ChairClaimer interface { } type ChairOptions struct { - Chairs ChairSnapshotCache - Discovery Discovery - Coord ifaces.ChairCoordinator - LocalPull ifaces.LocalChairPullStarter - Inflight *inflight.Map - SelfPeerID ifaces.NodeID - CurrentEpoch func() int64 - InstallHolder func(chairs.Holder) error - Claimer ChairClaimer - Logger *slog.Logger - QueryTimeout time.Duration - PollManifest time.Duration - PollLayer time.Duration - APITimeout time.Duration - HolderCount int - SeedCount int - TrustedFailureClasses []ifaces.FailureClass + Chairs ChairSnapshotCache + Discovery Discovery + Coord ifaces.ChairCoordinator + LocalPull ifaces.LocalChairPullStarter + Inflight *inflight.Map + SelfPeerID ifaces.NodeID + CurrentEpoch func() int64 + InstallHolder func(chairs.Holder) error + Claimer ChairClaimer + Logger *slog.Logger + QueryTimeout time.Duration + PollManifest time.Duration + PollLayer time.Duration + APITimeout time.Duration + HolderCount int + SeedCount int + PrefetchCoordinatorReplicas int + PrefetchMaxConcurrentGroups int + PrefetchDispatchJitter time.Duration + TrustedFailureClasses []ifaces.FailureClass // OnSeedRecruit reports one completed seed-recruitment pass: how many chairs // were selectable, how many were contacted, and how many accepted. contacted // above the effective scaled cohort means the resolver moved down the ranking, @@ -66,7 +71,9 @@ type ChairOptions struct { // outcome separates a clean reply from "deadline" (QueryTimeout expired), // so a run can tell a binding deadline from slow transport: deadline // outcomes pile up at QueryTimeout, transport shows a long ok tail. - OnChairCall func(kind, outcome string, seconds float64) + OnChairCall func(kind, outcome string, seconds float64) + OnPrefetchBatch func(pullers, digests int) + OnPrefetchGroup func(target, outcome string) } type ChairResolver struct { @@ -122,6 +129,10 @@ func NewChairResolver(opts ChairOptions) *ChairResolver { opts.APITimeout = 5 * time.Second } + if opts.PrefetchMaxConcurrentGroups <= 0 { + opts.PrefetchMaxConcurrentGroups = 64 + } + if len(opts.TrustedFailureClasses) == 0 { opts.TrustedFailureClasses = []ifaces.FailureClass{ ifaces.FailureAuth, @@ -384,7 +395,7 @@ func (r *ChairResolver) reportDispatch(kind ifaces.OriginRefKind, reason string) } } -func (r *ChairResolver) PrefetchManifestChildren(ctx context.Context, _ digest.Digest, children []ChildDigest, registry, repository string) error { +func (r *ChairResolver) PrefetchManifestChildren(ctx context.Context, manifestDigest digest.Digest, children []ChildDigest, registry, repository string) error { if registry == "" || repository == "" { return fmt.Errorf("%w: registry=%q repository=%q", ErrPrefetchInvalid, registry, repository) } @@ -402,6 +413,11 @@ func (r *ChairResolver) PrefetchManifestChildren(ctx context.Context, _ digest.D return err } + coordinators := chairs.Rank(snapshot, manifestDigest) + if !coordinatesChairPrefetch(r.opts.SelfPeerID, coordinators, r.opts.PrefetchCoordinatorReplicas) { + return nil + } + type groupKey struct { peer ifaces.NodeID chair chairs.ID @@ -454,13 +470,41 @@ func (r *ChairResolver) PrefetchManifestChildren(ctx context.Context, _ digest.D return keys[i].kind < keys[j].kind }) + dispatchOffset, dispatchDelay := prefetchDispatchPlan(r.opts.SelfPeerID, manifestDigest, len(keys), r.opts.PrefetchDispatchJitter) + if len(keys) > 1 { + rotated := append([]groupKey(nil), keys[dispatchOffset:]...) + keys = append(rotated, keys[:dispatchOffset]...) + } + + distinctPullers := make(map[ifaces.NodeID]struct{}, len(groups)) + totalDigests := 0 + for key, digests := range groups { + distinctPullers[key.peer] = struct{}{} + totalDigests += len(digests) + } + if r.opts.OnPrefetchBatch != nil { + r.opts.OnPrefetchBatch(len(distinctPullers), totalDigests) + } + + if dispatchDelay > 0 { + timer := time.NewTimer(dispatchDelay) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + } + } + var ( wg sync.WaitGroup - failures int - mu sync.Mutex + failures atomic.Int32 ) + remoteSlots := make(chan struct{}, r.opts.PrefetchMaxConcurrentGroups) - for _, key := range keys { +dispatchRemote: + for index, key := range keys { key := key chair := chairs.Chair{ ID: key.chair, @@ -470,29 +514,83 @@ func (r *ChairResolver) PrefetchManifestChildren(ctx context.Context, _ digest.D } digests := append([]digest.Digest(nil), groups[key]...) + select { + case remoteSlots <- struct{}{}: + case <-ctx.Done(): + failures.Add(int32(len(keys) - index)) + + break dispatchRemote + } + wg.Add(1) go func() { defer wg.Done() + defer func() { <-remoteSlots }() outcomes, err := r.pullChair(ctx, chair, registry, repository, key.kind, digests) if err != nil || !allChairOutcomesAccepted(outcomes) { - mu.Lock() - failures++ - mu.Unlock() + failures.Add(1) + if r.opts.OnPrefetchGroup != nil { + r.opts.OnPrefetchGroup(prefetchTarget(r.opts.SelfPeerID, chair), "error") + } + } else if r.opts.OnPrefetchGroup != nil { + r.opts.OnPrefetchGroup(prefetchTarget(r.opts.SelfPeerID, chair), "success") } }() } wg.Wait() - if failures > 0 { - return fmt.Errorf("%w: %d/%d groups errored", ErrPrefetchPartial, failures, len(keys)) + if failures.Load() > 0 { + return fmt.Errorf("%w: %d/%d groups errored", ErrPrefetchPartial, failures.Load(), len(keys)) } return nil } +func coordinatesChairPrefetch(self ifaces.NodeID, ranked []chairs.Chair, replicas int) bool { + if replicas <= 0 || replicas >= len(ranked) { + return true + } + + for _, chair := range ranked[:replicas] { + if chair.Holder.PeerID == self { + return true + } + } + + return false +} + +func prefetchDispatchPlan(self ifaces.NodeID, manifestDigest digest.Digest, groups int, maxJitter time.Duration) (int, time.Duration) { + hasher := fnv.New64a() + _, _ = hasher.Write([]byte(self)) + _, _ = hasher.Write([]byte{0}) + _, _ = hasher.Write([]byte(manifestDigest.String())) + dispatchHash := hasher.Sum64() + + offset := 0 + if groups > 1 { + offset = int(dispatchHash % uint64(groups)) + } + + delay := time.Duration(0) + if maxJitter > 0 { + delay = time.Duration(dispatchHash % uint64(maxJitter)) + } + + return offset, delay +} + +func prefetchTarget(self ifaces.NodeID, chair chairs.Chair) string { + if chair.Holder.PeerID == self { + return "local" + } + + return "remote" +} + func (r *ChairResolver) seedCount(selectable int) int { if selectable <= 0 { return 0 diff --git a/internal/gantry/coldstart/chair_test.go b/internal/gantry/coldstart/chair_test.go index 0dbd3b51e..db74fca66 100644 --- a/internal/gantry/coldstart/chair_test.go +++ b/internal/gantry/coldstart/chair_test.go @@ -54,6 +54,35 @@ type chairCoordStub struct { onCall func(ifaces.ChairAssignment) } +type boundedChairCoordStub struct { + current atomic.Int32 + maximum atomic.Int32 + release chan struct{} + started chan struct{} +} + +func (s *boundedChairCoordStub) PleasePullChair(_ context.Context, _ ifaces.PeerEndpoint, _, _ string, _ ifaces.OriginRefKind, digests []digest.Digest, _ ifaces.ChairAssignment) ([]ifaces.PleasePullOutcome, error) { + current := s.current.Add(1) + defer s.current.Add(-1) + + for { + maximum := s.maximum.Load() + if current <= maximum || s.maximum.CompareAndSwap(maximum, current) { + break + } + } + + s.started <- struct{}{} + <-s.release + + outcomes := make([]ifaces.PleasePullOutcome, 0, len(digests)) + for _, d := range digests { + outcomes = append(outcomes, ifaces.PleasePullOutcome{Digest: d, Outcome: ifaces.PleasePullStarted}) + } + + return outcomes, nil +} + func (s *chairCoordStub) PleasePullChair(_ context.Context, _ ifaces.PeerEndpoint, _, _ string, _ ifaces.OriginRefKind, digests []digest.Digest, assignment ifaces.ChairAssignment) ([]ifaces.PleasePullOutcome, error) { s.mu.Lock() defer s.mu.Unlock() @@ -252,6 +281,106 @@ func TestChairPrefetchPreservesHolderToSeedRatio(t *testing.T) { } } +func TestChairPrefetchUsesDeterministicCoordinators(t *testing.T) { + manifestDigest := digest.MustParse("sha256:edededededededededededededededededededededededededededededededed") + childDigest := digest.MustParse("sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd") + snapshot := fullChairSnapshot(5) + coordinators := chairs.Rank(snapshot, manifestDigest)[:3] + coordinatorSet := make(map[ifaces.NodeID]struct{}, len(coordinators)) + for _, chair := range coordinators { + coordinatorSet[chair.Holder.PeerID] = struct{}{} + } + + callers := 0 + for _, chair := range snapshot.Chairs { + coord := &chairCoordStub{} + resolver := coldstart.NewChairResolver(coldstart.ChairOptions{ + Chairs: &chairSnapshotStub{snapshot: snapshot}, + Discovery: &stubDisco{}, + Coord: coord, + Inflight: inflight.New(inflight.DefaultStalls(), nil), + SelfPeerID: chair.Holder.PeerID, + CurrentEpoch: func() int64 { return 5 }, + HolderCount: chairs.Count, + SeedCount: chairs.SeedCount, + PrefetchCoordinatorReplicas: 3, + }) + + if err := resolver.PrefetchManifestChildren(context.Background(), manifestDigest, []coldstart.ChildDigest{{ + Digest: childDigest, + Kind: ifaces.KindBlob, + }}, "registry.example.com", "repo/image"); err != nil { + t.Fatalf("PrefetchManifestChildren for %s: %v", chair.Holder.PeerID, err) + } + + coord.mu.Lock() + calls := len(coord.calls) + coord.mu.Unlock() + + _, coordinator := coordinatorSet[chair.Holder.PeerID] + if coordinator { + callers++ + if calls != chairs.SeedCount { + t.Fatalf("coordinator %s calls = %d, want %d", chair.Holder.PeerID, calls, chairs.SeedCount) + } + } else if calls != 0 { + t.Fatalf("non-coordinator %s made %d calls", chair.Holder.PeerID, calls) + } + } + + if callers != 3 { + t.Fatalf("coordinator callers = %d, want 3", callers) + } +} + +func TestChairPrefetchBoundsConcurrentGroups(t *testing.T) { + manifestDigest := digest.MustParse("sha256:edededededededededededededededededededededededededededededededed") + childDigest := digest.MustParse("sha256:cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd") + snapshot := fullChairSnapshot(5) + coord := &boundedChairCoordStub{ + release: make(chan struct{}), + started: make(chan struct{}, chairs.SeedCount), + } + resolver := coldstart.NewChairResolver(coldstart.ChairOptions{ + Chairs: &chairSnapshotStub{snapshot: snapshot}, + Discovery: &stubDisco{}, + Coord: coord, + Inflight: inflight.New(inflight.DefaultStalls(), nil), + SelfPeerID: "self", + CurrentEpoch: func() int64 { return 5 }, + HolderCount: chairs.Count, + SeedCount: chairs.SeedCount, + PrefetchMaxConcurrentGroups: 2, + }) + + done := make(chan error, 1) + go func() { + done <- resolver.PrefetchManifestChildren(context.Background(), manifestDigest, []coldstart.ChildDigest{{ + Digest: childDigest, + Kind: ifaces.KindBlob, + }}, "registry.example.com", "repo/image") + }() + + for range 2 { + select { + case <-coord.started: + case <-time.After(time.Second): + t.Fatal("timed out waiting for bounded prefetch groups") + } + } + if got := coord.maximum.Load(); got != 2 { + t.Fatalf("maximum concurrent groups = %d, want 2", got) + } + + close(coord.release) + if err := <-done; err != nil { + t.Fatalf("PrefetchManifestChildren: %v", err) + } + if got := coord.maximum.Load(); got > 2 { + t.Fatalf("maximum concurrent groups = %d, want <= 2", got) + } +} + func TestChairResolverRefreshesStaleChairBeforeUsingBackup(t *testing.T) { d := digest.MustParse("sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd") snapshot := fullChairSnapshot(8) diff --git a/internal/gantry/config/config.go b/internal/gantry/config/config.go index 8a5b9fe47..2dd0ee8a5 100644 --- a/internal/gantry/config/config.go +++ b/internal/gantry/config/config.go @@ -78,6 +78,15 @@ type Config struct { // for the full opt-in checklist. MirrorBindAllowNonLoopback bool `yaml:"mirror_bind_allow_non_loopback"` + // ArtifactStreamingEnabled enables the node-local OverlayBD range endpoint + // on /blobs/ under MirrorListen. + ArtifactStreamingEnabled bool `yaml:"artifact_streaming_enabled"` + + // ArtifactStreamingAllowedHostSuffixes limits signed-origin requests to + // approved Azure data endpoints. A leading dot denotes a DNS suffix; values + // without a leading dot are exact hosts. + ArtifactStreamingAllowedHostSuffixes []string `yaml:"artifact_streaming_allowed_host_suffixes"` + // TransferListen is the peer-facing HTTP/2 endpoint (the design doc). The bind is // typically 0.0.0.0; cluster-internal isolation comes from // NetworkPolicy + the `Gantry-Mirrored: 1` request-header gate + @@ -473,15 +482,21 @@ func NewDefault() *Config { return &Config{ MirrorListen: "127.0.0.1:5000", MirrorBindAllowNonLoopback: false, - TransferListen: "0.0.0.0:5001", - MetricsListen: "0.0.0.0:9095", - PprofListen: "", - Libp2pListen: nil, - Libp2pIdentityPath: "/var/lib/gantry/libp2p.key", - Libp2pConnManagerHigh: 900, - Libp2pConnManagerLow: 600, - Libp2pConnManagerGrace: time.Minute, - ChairListen: "0.0.0.0:5002", + ArtifactStreamingEnabled: false, + ArtifactStreamingAllowedHostSuffixes: []string{ + ".azurecr.io", + ".data.mcr.microsoft.com", + ".blob.core.windows.net", + }, + TransferListen: "0.0.0.0:5001", + MetricsListen: "0.0.0.0:9095", + PprofListen: "", + Libp2pListen: nil, + Libp2pIdentityPath: "/var/lib/gantry/libp2p.key", + Libp2pConnManagerHigh: 900, + Libp2pConnManagerLow: 600, + Libp2pConnManagerGrace: time.Minute, + ChairListen: "0.0.0.0:5002", NodeName: "", MembersKubeconfig: "", @@ -620,6 +635,7 @@ func (c *Config) LoadEnv(env func(string) string) error { setStr("MIRROR_LISTEN", &c.MirrorListen) setBool("MIRROR_BIND_ALLOW_NON_LOOPBACK", &c.MirrorBindAllowNonLoopback) + setBool("ARTIFACT_STREAMING_ENABLED", &c.ArtifactStreamingEnabled) setStr("TRANSFER_LISTEN", &c.TransferListen) setStr("METRICS_LISTEN", &c.MetricsListen) setStr("PPROF_LISTEN", &c.PprofListen) @@ -704,6 +720,7 @@ func (c *Config) LoadEnv(env func(string) string) error { func (c *Config) BindFlags(fs *flag.FlagSet) { fs.StringVar(&c.MirrorListen, "mirror-listen", c.MirrorListen, "address for the containerd-facing mirror endpoint (loopback)") fs.BoolVar(&c.MirrorBindAllowNonLoopback, "mirror-bind-allow-non-loopback", c.MirrorBindAllowNonLoopback, "opt in to a non-loopback mirror bind (e.g. when using hostPort + hostIP=127.0.0.1 in Kubernetes)") + fs.BoolVar(&c.ArtifactStreamingEnabled, "artifact-streaming-enabled", c.ArtifactStreamingEnabled, "enable the node-local OverlayBD /blobs/ range endpoint") fs.StringVar(&c.TransferListen, "transfer-listen", c.TransferListen, "address for the peer-facing transfer endpoint") fs.StringVar(&c.MetricsListen, "metrics-listen", c.MetricsListen, "address for the Prometheus metrics endpoint") fs.StringVar(&c.PprofListen, "pprof-listen", c.PprofListen, "optional loopback address for Go runtime profiles (empty disables pprof)") @@ -749,9 +766,9 @@ func (c *Config) BindFlags(fs *flag.FlagSet) { fs.IntVar(&c.HRWK, "hrw-k", c.HRWK, "legacy no-op membership HRW size") fs.IntVar(&c.PrefetchPullerReplicas, "prefetch-puller-replicas", c.PrefetchPullerReplicas, "legacy no-op prefetch replica count") fs.Float64Var(&c.PrefetchPullerFraction, "prefetch-puller-fraction", c.PrefetchPullerFraction, "legacy no-op prefetch fraction") - fs.IntVar(&c.PrefetchCoordinatorReplicas, "prefetch-coordinator-replicas", c.PrefetchCoordinatorReplicas, "legacy no-op prefetch coordinator count") - fs.IntVar(&c.PrefetchMaxConcurrentGroups, "prefetch-max-concurrent-groups", c.PrefetchMaxConcurrentGroups, "legacy no-op prefetch concurrency") - fs.DurationVar(&c.PrefetchDispatchJitter, "prefetch-dispatch-jitter", c.PrefetchDispatchJitter, "legacy no-op prefetch dispatch jitter") + fs.IntVar(&c.PrefetchCoordinatorReplicas, "prefetch-coordinator-replicas", c.PrefetchCoordinatorReplicas, "number of deterministic chair holders allowed to dispatch manifest prefetch") + fs.IntVar(&c.PrefetchMaxConcurrentGroups, "prefetch-max-concurrent-groups", c.PrefetchMaxConcurrentGroups, "maximum simultaneous outbound manifest prefetch groups") + fs.DurationVar(&c.PrefetchDispatchJitter, "prefetch-dispatch-jitter", c.PrefetchDispatchJitter, "maximum deterministic delay before manifest prefetch dispatch") fs.StringVar(&c.HRWTopologyScope, "hrw-topology-scope", c.HRWTopologyScope, "legacy no-op membership HRW scope") fs.StringVar(&c.ZoneLabelKey, "zone-label-key", c.ZoneLabelKey, "legacy no-op zone label key") fs.BoolVar(&c.CoordPeerAuthzEnforce, "coord-peer-authz-enforce", c.CoordPeerAuthzEnforce, "unsupported in Lease-chair mode; validation requires false") @@ -837,6 +854,19 @@ func (c *Config) Validate() error { mustAddr("transfer_listen", c.TransferListen) mustAddr("metrics_listen", c.MetricsListen) + if c.ArtifactStreamingEnabled && len(c.ArtifactStreamingAllowedHostSuffixes) == 0 { + errs = append(errs, errors.New("artifact_streaming_enabled requires artifact_streaming_allowed_host_suffixes")) + } + + for _, allowed := range c.ArtifactStreamingAllowedHostSuffixes { + value := strings.TrimSpace(strings.TrimSuffix(allowed, ".")) + + base := strings.TrimPrefix(value, ".") + if !validArtifactStreamingHostRule(value, base) { + errs = append(errs, fmt.Errorf("artifact_streaming_allowed_host_suffixes contains invalid host %q", allowed)) + } + } + if c.PprofListen != "" { mustAddr("pprof_listen", c.PprofListen) @@ -1150,6 +1180,30 @@ func (c *Config) Validate() error { return errors.Join(errs...) } +func validArtifactStreamingHostRule(value, base string) bool { + if value == "" || base == "" || net.ParseIP(base) != nil || strings.ContainsAny(base, "*/:?#@") || len(base) > 253 { + return false + } + + for _, label := range strings.Split(base, ".") { + if len(label) == 0 || len(label) > 63 || !isDNSAlphaNumeric(label[0]) || !isDNSAlphaNumeric(label[len(label)-1]) { + return false + } + + for index := 1; index < len(label)-1; index++ { + if !isDNSAlphaNumeric(label[index]) && label[index] != '-' { + return false + } + } + } + + return true +} + +func isDNSAlphaNumeric(value byte) bool { + return value >= 'a' && value <= 'z' || value >= 'A' && value <= 'Z' || value >= '0' && value <= '9' +} + // ResolveUpstream returns the UpstreamRegistry whose Name (or NSAlias) // equals ns. Returns false if ns does not match any configured registry. func (c *Config) ResolveUpstream(ns string) (UpstreamRegistry, bool) { @@ -1169,6 +1223,7 @@ func (c *Config) ResolveUpstream(ns string) (UpstreamRegistry, bool) { func (c *Config) Redacted() *Config { cp := *c cp.UpstreamRegistries = append([]UpstreamRegistry(nil), c.UpstreamRegistries...) + cp.ArtifactStreamingAllowedHostSuffixes = append([]string(nil), c.ArtifactStreamingAllowedHostSuffixes...) // CredentialsPath is a path, not the secret; safe to log as-is. return &cp } diff --git a/internal/gantry/config/config_test.go b/internal/gantry/config/config_test.go index c88a06b4b..2acfe06eb 100644 --- a/internal/gantry/config/config_test.go +++ b/internal/gantry/config/config_test.go @@ -14,6 +14,10 @@ import ( func TestDefaultsValidateAfterMinimalUpstream(t *testing.T) { c := NewDefault() + if c.ArtifactStreamingEnabled { + t.Fatal("ArtifactStreamingEnabled = true, want disabled by default") + } + if c.OriginPullProgressTimeout != 5*time.Minute { t.Fatalf("OriginPullProgressTimeout = %v, want 5m", c.OriginPullProgressTimeout) } @@ -76,6 +80,85 @@ func TestDefaultsValidateAfterMinimalUpstream(t *testing.T) { } } +func TestArtifactStreamingConfig(t *testing.T) { + t.Run("YAML", func(t *testing.T) { + c := NewDefault() + + err := c.LoadYAML(strings.NewReader(` +artifact_streaming_enabled: true +artifact_streaming_allowed_host_suffixes: + - .data.azurecr.io +`)) + if err != nil { + t.Fatal(err) + } + + if !c.ArtifactStreamingEnabled || len(c.ArtifactStreamingAllowedHostSuffixes) != 1 { + t.Fatalf("unexpected artifact streaming config: %+v", c) + } + }) + + t.Run("environment", func(t *testing.T) { + c := NewDefault() + + err := c.LoadEnv(func(key string) string { + if key == "GANTRY_ARTIFACT_STREAMING_ENABLED" { + return "true" + } + + return "" + }) + if err != nil { + t.Fatal(err) + } + + if !c.ArtifactStreamingEnabled { + t.Fatal("artifact streaming was not enabled from the environment") + } + }) + + t.Run("flags", func(t *testing.T) { + c := NewDefault() + flags := flag.NewFlagSet("test", flag.ContinueOnError) + c.BindFlags(flags) + + if err := flags.Parse([]string{"--artifact-streaming-enabled"}); err != nil { + t.Fatal(err) + } + + if !c.ArtifactStreamingEnabled { + t.Fatal("artifact streaming was not enabled from flags") + } + }) +} + +func TestValidateArtifactStreaming(t *testing.T) { + tests := []struct { + name string + mutate func(*Config) + field string + }{ + {name: "empty hosts", mutate: func(c *Config) { c.ArtifactStreamingEnabled = true; c.ArtifactStreamingAllowedHostSuffixes = nil }, field: "artifact_streaming_allowed_host_suffixes"}, + {name: "invalid host", mutate: func(c *Config) { c.ArtifactStreamingAllowedHostSuffixes = []string{"https://example.com"} }, field: "artifact_streaming_allowed_host_suffixes"}, + {name: "wildcard host", mutate: func(c *Config) { c.ArtifactStreamingAllowedHostSuffixes = []string{"*.example.com"} }, field: "artifact_streaming_allowed_host_suffixes"}, + {name: "empty label", mutate: func(c *Config) { c.ArtifactStreamingAllowedHostSuffixes = []string{"example..com"} }, field: "artifact_streaming_allowed_host_suffixes"}, + {name: "leading hyphen", mutate: func(c *Config) { c.ArtifactStreamingAllowedHostSuffixes = []string{"-bad.example"} }, field: "artifact_streaming_allowed_host_suffixes"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + c := NewDefault() + c.UpstreamRegistries = []UpstreamRegistry{{Name: "r", Endpoint: "https://r"}} + test.mutate(c) + + err := c.Validate() + if err == nil || !strings.Contains(err.Error(), test.field) { + t.Fatalf("Validate error = %v, want %s", err, test.field) + } + }) + } +} + func TestChairCapacityConfig(t *testing.T) { t.Run("environment", func(t *testing.T) { c := NewDefault() diff --git a/internal/gantry/containerdstore/store.go b/internal/gantry/containerdstore/store.go index d3d06154c..95d6af9b7 100644 --- a/internal/gantry/containerdstore/store.go +++ b/internal/gantry/containerdstore/store.go @@ -47,6 +47,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" gdigest "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/httprange" "github.com/Azure/unbounded/internal/gantry/ifaces" ) @@ -305,6 +306,53 @@ func (s *Store) Open(ctx context.Context, d gdigest.Digest) (io.ReadCloser, int6 }, size, nil } +// OpenRange returns a reader over an exact range of a committed digest. The +// total object size is returned separately for Content-Range construction. +func (s *Store) OpenRange(ctx context.Context, d gdigest.Digest, requested httprange.Range) (io.ReadCloser, int64, error) { + desc := ocispec.Descriptor{Digest: godigest.Digest(d.String())} + + ra, err := s.cs.ReaderAt(s.withNS(ctx), desc) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, 0, ctxErr + } + + if errors.Is(err, cerrdefs.ErrNotFound) { + if s.metrics.OnMiss != nil { + s.metrics.OnMiss() + } + + return nil, 0, &ifaces.ErrNotFound{Digest: d} + } + + if s.metrics.OnUnavailable != nil { + s.metrics.OnUnavailable() + } + + if s.metrics.OnOpenError != nil { + s.metrics.OnOpenError() + } + + return nil, 0, &ifaces.ErrUnavailable{Op: "ReaderAt(range)", Cause: err} + } + + size := ra.Size() + if err := requested.ValidateSize(size); err != nil { + _ = ra.Close() //nolint:errcheck // best-effort close + + return nil, size, err + } + + if s.metrics.OnHit != nil { + s.metrics.OnHit() + } + + return &readerAtCloser{ + SectionReader: io.NewSectionReader(ra, requested.Start, requested.Length()), + closer: ra, + }, size, nil +} + // Descriptor returns the OCI descriptor (mediatype unknown; size from // the store) for d. Used by callers that need to construct an // ocispec.Descriptor without opening a reader handle. Returns diff --git a/internal/gantry/containerdstore/store_test.go b/internal/gantry/containerdstore/store_test.go index 17eae8435..592255a5e 100644 --- a/internal/gantry/containerdstore/store_test.go +++ b/internal/gantry/containerdstore/store_test.go @@ -23,6 +23,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" gdigest "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/httprange" "github.com/Azure/unbounded/internal/gantry/ifaces" ) @@ -298,6 +299,54 @@ func TestStore_HasAndOpen(t *testing.T) { } } +func TestStore_OpenRange(t *testing.T) { + cs := newFake() + payload := []byte("0123456789") + cs.put(godigest.FromBytes(payload), payload) + s := New(cs) + d := mustDigest(t, payload) + + r, size, err := s.OpenRange(context.Background(), d, httprange.Range{Start: 3, End: 6}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() //nolint:errcheck // best-effort close + + if size != int64(len(payload)) { + t.Fatalf("size = %d, want %d", size, len(payload)) + } + + got, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + + if string(got) != "3456" { + t.Fatalf("body = %q, want %q", got, "3456") + } +} + +func TestStore_OpenRangeUnsatisfiable(t *testing.T) { + cs := newFake() + payload := []byte("small") + cs.put(godigest.FromBytes(payload), payload) + s := New(cs) + d := mustDigest(t, payload) + + r, size, err := s.OpenRange(context.Background(), d, httprange.Range{Start: 4, End: 5}) + if r != nil { + _ = r.Close() //nolint:errcheck // best-effort close + } + + if !errors.Is(err, httprange.ErrUnsatisfiable) { + t.Fatalf("error = %v, want ErrUnsatisfiable", err) + } + + if size != int64(len(payload)) { + t.Fatalf("size = %d, want %d", size, len(payload)) + } +} + // TestStore_HasPropagatesBackendError covers the plan-mandated // "containerd unreachable" contract: a non-NotFound error from the // backend MUST surface so callers do not advertise stale availability. diff --git a/internal/gantry/httprange/range.go b/internal/gantry/httprange/range.go new file mode 100644 index 000000000..0990eefaf --- /dev/null +++ b/internal/gantry/httprange/range.go @@ -0,0 +1,144 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +// Package httprange parses and validates exact HTTP byte ranges. +package httprange + +import ( + "errors" + "fmt" + "math" + "strconv" + "strings" +) + +var ( + ErrInvalid = errors.New("invalid byte range") + ErrUnsatisfiable = errors.New("unsatisfiable byte range") +) + +// Range is an exact inclusive byte range. +type Range struct { + Start int64 + End int64 +} + +// New constructs a validated exact range. +func New(start, end int64) (Range, error) { + if start < 0 || end < start || end-start == math.MaxInt64 { + return Range{}, fmt.Errorf("%w: %d-%d", ErrInvalid, start, end) + } + + return Range{Start: start, End: end}, nil +} + +// ParseExact parses a single bytes=N-M range. Open-ended, suffix, and +// multipart ranges are deliberately rejected. +func ParseExact(value string) (Range, error) { + value = strings.TrimSpace(value) + if !strings.HasPrefix(value, "bytes=") { + return Range{}, fmt.Errorf("%w: expected bytes=N-M", ErrInvalid) + } + + spec := strings.TrimPrefix(value, "bytes=") + if strings.Contains(spec, ",") { + return Range{}, fmt.Errorf("%w: multipart ranges are unsupported", ErrInvalid) + } + + startText, endText, ok := strings.Cut(spec, "-") + if !ok || startText == "" || endText == "" || strings.Contains(endText, "-") { + return Range{}, fmt.Errorf("%w: expected bytes=N-M", ErrInvalid) + } + + start, err := strconv.ParseInt(startText, 10, 64) + if err != nil { + return Range{}, fmt.Errorf("%w: start: %v", ErrInvalid, err) + } + + end, err := strconv.ParseInt(endText, 10, 64) + if err != nil { + return Range{}, fmt.Errorf("%w: end: %v", ErrInvalid, err) + } + + return New(start, end) +} + +// Length returns the number of bytes in the range. +func (r Range) Length() int64 { return r.End - r.Start + 1 } + +// HeaderValue returns the HTTP Range header value. +func (r Range) HeaderValue() string { + return fmt.Sprintf("bytes=%d-%d", r.Start, r.End) +} + +// ValidateSize verifies that the entire range is within an object of size. +func (r Range) ValidateSize(size int64) error { + if size < 0 { + return fmt.Errorf("%w: negative object size %d", ErrInvalid, size) + } + + if r.Start >= size || r.End >= size { + return fmt.Errorf("%w: range %d-%d, size %d", ErrUnsatisfiable, r.Start, r.End, size) + } + + return nil +} + +// ContentRange is a satisfied HTTP Content-Range value. +type ContentRange struct { + Range Range + Size int64 +} + +// ParseContentRange parses Content-Range: bytes N-M/TOTAL. +func ParseContentRange(value string) (ContentRange, error) { + value = strings.TrimSpace(value) + if !strings.HasPrefix(value, "bytes ") { + return ContentRange{}, fmt.Errorf("%w: expected bytes N-M/TOTAL", ErrInvalid) + } + + boundsText, sizeText, ok := strings.Cut(strings.TrimPrefix(value, "bytes "), "/") + if !ok || sizeText == "" || sizeText == "*" || strings.Contains(sizeText, "/") { + return ContentRange{}, fmt.Errorf("%w: expected known total size", ErrInvalid) + } + + r, err := ParseExact("bytes=" + boundsText) + if err != nil { + return ContentRange{}, err + } + + size, err := strconv.ParseInt(sizeText, 10, 64) + if err != nil || size <= 0 { + return ContentRange{}, fmt.Errorf("%w: invalid total size %q", ErrInvalid, sizeText) + } + + if err := r.ValidateSize(size); err != nil { + return ContentRange{}, err + } + + return ContentRange{Range: r, Size: size}, nil +} + +// ValidateResponse verifies that a partial response exactly matches the +// requested range and body length. +func ValidateResponse(requested Range, value string, contentLength int64) (int64, error) { + got, err := ParseContentRange(value) + if err != nil { + return 0, err + } + + if got.Range != requested { + return 0, fmt.Errorf("%w: got %d-%d, want %d-%d", ErrInvalid, + got.Range.Start, got.Range.End, requested.Start, requested.End) + } + + if contentLength >= 0 && contentLength != requested.Length() { + return 0, fmt.Errorf("%w: content length %d, want %d", ErrInvalid, + contentLength, requested.Length()) + } + + return got.Size, nil +} + +// UnsatisfiedContentRange returns the Content-Range value for a 416 response. +func UnsatisfiedContentRange(size int64) string { return fmt.Sprintf("bytes */%d", size) } diff --git a/internal/gantry/httprange/range_test.go b/internal/gantry/httprange/range_test.go new file mode 100644 index 000000000..625d80b94 --- /dev/null +++ b/internal/gantry/httprange/range_test.go @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package httprange_test + +import ( + "errors" + "math" + "testing" + + "github.com/Azure/unbounded/internal/gantry/httprange" +) + +func TestParseExact(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + value string + want httprange.Range + }{ + {name: "one byte", value: "bytes=0-0", want: httprange.Range{Start: 0, End: 0}}, + {name: "bounded", value: "bytes=456-990", want: httprange.Range{Start: 456, End: 990}}, + {name: "outer whitespace", value: " bytes=1-2 ", want: httprange.Range{Start: 1, End: 2}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + got, err := httprange.ParseExact(test.value) + if err != nil { + t.Fatal(err) + } + + if got != test.want { + t.Fatalf("range = %+v, want %+v", got, test.want) + } + }) + } +} + +func TestParseExactRejectsUnsupportedRanges(t *testing.T) { + t.Parallel() + + for _, value := range []string{ + "", + "items=0-1", + "bytes=-10", + "bytes=10-", + "bytes=0-1,4-5", + "bytes=-1-2", + "bytes=2-1", + "bytes=0-9223372036854775807", + "bytes=abc-2", + } { + t.Run(value, func(t *testing.T) { + t.Parallel() + + if _, err := httprange.ParseExact(value); !errors.Is(err, httprange.ErrInvalid) { + t.Fatalf("error = %v, want ErrInvalid", err) + } + }) + } +} + +func TestRangeValidateSize(t *testing.T) { + t.Parallel() + + r := httprange.Range{Start: 9, End: 9} + if err := r.ValidateSize(10); err != nil { + t.Fatal(err) + } + + if err := r.ValidateSize(9); !errors.Is(err, httprange.ErrUnsatisfiable) { + t.Fatalf("error = %v, want ErrUnsatisfiable", err) + } +} + +func TestValidateResponse(t *testing.T) { + t.Parallel() + + r := httprange.Range{Start: 10, End: 19} + + size, err := httprange.ValidateResponse(r, "bytes 10-19/100", 10) + if err != nil { + t.Fatal(err) + } + + if size != 100 { + t.Fatalf("size = %d, want 100", size) + } + + for _, test := range []struct { + name string + contentRange string + contentLength int64 + }{ + {name: "wrong start", contentRange: "bytes 11-19/100", contentLength: 9}, + {name: "wrong end", contentRange: "bytes 10-20/100", contentLength: 11}, + {name: "unknown size", contentRange: "bytes 10-19/*", contentLength: 10}, + {name: "range beyond size", contentRange: "bytes 10-19/19", contentLength: 10}, + {name: "wrong length", contentRange: "bytes 10-19/100", contentLength: 9}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + if _, err := httprange.ValidateResponse(r, test.contentRange, test.contentLength); err == nil { + t.Fatal("expected validation error") + } + }) + } +} + +func TestNewRejectsOverflowLength(t *testing.T) { + t.Parallel() + + if _, err := httprange.New(0, math.MaxInt64); !errors.Is(err, httprange.ErrInvalid) { + t.Fatalf("error = %v, want ErrInvalid", err) + } +} diff --git a/internal/gantry/ifaces/ifaces.go b/internal/gantry/ifaces/ifaces.go index 8d797b5ed..df50f7b40 100644 --- a/internal/gantry/ifaces/ifaces.go +++ b/internal/gantry/ifaces/ifaces.go @@ -512,3 +512,18 @@ type ErrPeerHTTPStatus struct { func (e *ErrPeerHTTPStatus) Error() string { return fmt.Sprintf("peer %s returned %d", e.PeerAddr, e.StatusCode) } + +// ErrPeerProtocol reports a peer response that violated the transfer wire +// contract. Callers should temporarily suppress the provider for this digest; +// unlike an availability error, retrying the same response immediately cannot +// succeed safely. +type ErrPeerProtocol struct { + PeerAddr string + Err error +} + +func (e *ErrPeerProtocol) Error() string { + return fmt.Sprintf("peer %s protocol error: %v", e.PeerAddr, e.Err) +} + +func (e *ErrPeerProtocol) Unwrap() error { return e.Err } diff --git a/internal/gantry/mirror/mirror.go b/internal/gantry/mirror/mirror.go index 6bd21a52c..5e149931f 100644 --- a/internal/gantry/mirror/mirror.go +++ b/internal/gantry/mirror/mirror.go @@ -3159,13 +3159,20 @@ func isDigestRef(ref string) bool { return strings.HasPrefix(ref, "sha256:") } // ListenAndServe runs the mirror on the configured loopback address. The // returned function stops the server gracefully. func (s *Server) ListenAndServe(addr string) (func(context.Context) error, error) { + return s.ListenAndServeHandler(addr, s.Handler()) +} + +// ListenAndServeHandler runs a caller-composed handler on the mirror listener. +// It allows other node-local protocols to share the loopback hostPort without +// placing their request paths behind the OCI mirror's ServeMux. +func (s *Server) ListenAndServeHandler(addr string, handler http.Handler) (func(context.Context) error, error) { ln, err := net.Listen("tcp", addr) if err != nil { return nil, fmt.Errorf("mirror: listen %s: %w", addr, err) } srv := &http.Server{ - Handler: s.Handler(), + Handler: handler, ReadHeaderTimeout: 5 * time.Second, } diff --git a/internal/gantry/origin/origin.go b/internal/gantry/origin/origin.go index 8205b8bb8..03f4a0e6e 100644 --- a/internal/gantry/origin/origin.go +++ b/internal/gantry/origin/origin.go @@ -662,12 +662,17 @@ func (r *registry) head(ctx context.Context, ref ifaces.OriginRef) (int64, strin // urlFor returns the full URL for an OriginRef. func (r *registry) urlFor(ref ifaces.OriginRef) string { + u := *r.base + basePath := strings.TrimRight(u.Path, "/") + switch ref.Kind { case ifaces.KindManifest: - return r.base.String() + "/v2/" + ref.Repository + "/manifests/" + ref.Digest.String() + u.Path = basePath + "/v2/" + ref.Repository + "/manifests/" + ref.Digest.String() default: - return r.base.String() + "/v2/" + ref.Repository + "/blobs/" + ref.Digest.String() + u.Path = basePath + "/v2/" + ref.Repository + "/blobs/" + ref.Digest.String() } + + return u.String() } // do issues a request, preferring request-scoped delegated Basic/Bearer auth. A diff --git a/internal/gantry/origin/origin_test.go b/internal/gantry/origin/origin_test.go index bea6e5433..cfbe74539 100644 --- a/internal/gantry/origin/origin_test.go +++ b/internal/gantry/origin/origin_test.go @@ -1068,6 +1068,38 @@ func TestNSAliasResolves(t *testing.T) { rc.Close() } +func TestEndpointQueryIsPreserved(t *testing.T) { + body := []byte("proxied") + d := digestOf(body) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Path; got != "/proxy/v2/owner/repo/blobs/"+d.String() { + t.Errorf("path = %q", got) + } + + if got := r.URL.Query().Get("ns"); got != "registry.example.com" { + t.Errorf("ns = %q", got) + } + + _, _ = w.Write(body) //nolint:errcheck // best-effort write + })) + defer srv.Close() + + c := newClient(t, config.UpstreamRegistry{ + Name: "registry.example.com", + Endpoint: srv.URL + "/proxy?ns=registry.example.com", + }) + + rc, _, err := c.Pull(context.Background(), ifaces.OriginRef{ + Registry: "registry.example.com", Repository: "owner/repo", Digest: d, + }) + if err != nil { + t.Fatalf("Pull: %v", err) + } + + rc.Close() +} + func TestNewRejectsBadCredentialsFile(t *testing.T) { dir := t.TempDir() diff --git a/internal/gantry/streaming/azureurl.go b/internal/gantry/streaming/azureurl.go new file mode 100644 index 000000000..68d36523e --- /dev/null +++ b/internal/gantry/streaming/azureurl.go @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package streaming + +import ( + "fmt" + "net" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/Azure/unbounded/internal/gantry/digest" +) + +const ( + HandlerPrefix = "/blobs/" + ReadinessPath = "/artifact-streaming/readyz" +) + +var digestHexPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) + +// URLPolicy limits signed-origin requests to operator-approved HTTPS hosts. +type URLPolicy struct { + AllowedHostSuffixes []string + AllowHTTP bool +} + +// OriginURL is a validated origin capability extracted from an OverlayBD +// request. Raw is retained for the outbound request and must never be logged. +type OriginURL struct { + Raw string + Digest digest.Digest +} + +// OriginURLFromRequest reconstructs and validates the URL embedded by +// OverlayBD after /blobs/. RequestURI is used so signed query bytes are not +// decoded and re-encoded. +func OriginURLFromRequest(r *http.Request, policy URLPolicy) (OriginURL, error) { + requestTarget := r.RequestURI + if requestTarget == "" { + requestTarget = r.URL.RequestURI() + } + + if !strings.HasPrefix(requestTarget, HandlerPrefix) { + return OriginURL{}, fmt.Errorf("streaming origin: request target does not start with %q", HandlerPrefix) + } + + return ParseOriginURL(strings.TrimPrefix(requestTarget, HandlerPrefix), policy) +} + +// ParseOriginURL validates a raw absolute origin URL and extracts its blob +// digest without rewriting the signed query. +func ParseOriginURL(raw string, policy URLPolicy) (OriginURL, error) { + if raw == "" { + return OriginURL{}, fmt.Errorf("streaming origin: empty URL") + } + + u, err := url.Parse(raw) + if err != nil { + return OriginURL{}, fmt.Errorf("streaming origin: invalid URL") + } + + if err := policy.ValidateURL(u); err != nil { + return OriginURL{}, err + } + + d, err := digestFromURL(u) + if err != nil { + return OriginURL{}, err + } + + return OriginURL{ + Raw: raw, + Digest: d, + }, nil +} + +// ValidateURL validates scheme and host without requiring the URL to carry a +// digest. Redirect targets use this narrower check. +func (p URLPolicy) ValidateURL(u *url.URL) error { + if u == nil || !u.IsAbs() || u.Host == "" || u.Opaque != "" { + return fmt.Errorf("streaming origin: absolute hierarchical URL required") + } + + if u.User != nil || u.Fragment != "" { + return fmt.Errorf("streaming origin: userinfo and fragments are forbidden") + } + + if u.Scheme != "https" && (!p.AllowHTTP || u.Scheme != "http") { + return fmt.Errorf("streaming origin: scheme %q is not allowed", u.Scheme) + } + + host := strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) + if host == "" || net.ParseIP(host) != nil { + return fmt.Errorf("streaming origin: DNS host required") + } + + for _, allowed := range p.AllowedHostSuffixes { + allowed = strings.ToLower(strings.TrimSpace(strings.TrimSuffix(allowed, "."))) + if allowed == "" { + continue + } + + if strings.HasPrefix(allowed, ".") { + base := strings.TrimPrefix(allowed, ".") + if host == base || strings.HasSuffix(host, allowed) { + return nil + } + + continue + } + + if host == allowed { + return nil + } + } + + return fmt.Errorf("streaming origin: host %q is not allowed", host) +} + +func digestFromURL(u *url.URL) (digest.Digest, error) { + var zero digest.Digest + + var found []string + + query, err := url.ParseQuery(u.RawQuery) + if err != nil { + return zero, fmt.Errorf("streaming origin: invalid query") + } + + if values := query["d"]; len(values) > 1 { + return zero, fmt.Errorf("streaming origin: duplicate d digest") + } else if len(values) == 1 && values[0] != "" { + value := values[0] + + hex, ok := strings.CutPrefix(value, "sha256:") + if !ok || !digestHexPattern.MatchString(hex) { + return zero, fmt.Errorf("streaming origin: invalid d digest") + } + + found = append(found, hex) + } + + segments := strings.Split(strings.Trim(u.EscapedPath(), "/"), "/") + for index, segment := range segments { + decoded, err := url.PathUnescape(segment) + if err != nil { + return zero, fmt.Errorf("streaming origin: invalid path escape") + } + + if decoded == "sha256" && index >= 4 && index+3 < len(segments) { + pathPrefix := make([]string, 4) + validPrefix := true + + for prefixIndex := range pathPrefix { + value, decodeErr := url.PathUnescape(segments[index-4+prefixIndex]) + if decodeErr != nil { + return zero, fmt.Errorf("streaming origin: invalid path escape") + } + + pathPrefix[prefixIndex] = value + } + + for prefixIndex, want := range []string{"docker", "registry", "v2", "blobs"} { + if pathPrefix[prefixIndex] != want { + validPrefix = false + break + } + } + + if !validPrefix { + continue + } + + prefix, prefixErr := url.PathUnescape(segments[index+1]) + + hex, hexErr := url.PathUnescape(segments[index+2]) + + data, dataErr := url.PathUnescape(segments[index+3]) + if prefixErr != nil || hexErr != nil || dataErr != nil || data != "data" || + len(prefix) != 2 || !digestHexPattern.MatchString(hex) || prefix != hex[:2] { + return zero, fmt.Errorf("streaming origin: invalid registry data path digest") + } + + found = append(found, hex) + } + } + + if len(found) == 0 { + return zero, fmt.Errorf("streaming origin: SHA-256 digest not found") + } + + for _, value := range found[1:] { + if value != found[0] { + return zero, fmt.Errorf("streaming origin: conflicting digests") + } + } + + return digest.Parse("sha256:" + found[0]) +} diff --git a/internal/gantry/streaming/azureurl_test.go b/internal/gantry/streaming/azureurl_test.go new file mode 100644 index 000000000..2799ad33d --- /dev/null +++ b/internal/gantry/streaming/azureurl_test.go @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package streaming_test + +import ( + "encoding/json" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/Azure/unbounded/internal/gantry/streaming" +) + +func TestRedactedAzureOriginURLFixtures(t *testing.T) { + t.Parallel() + + raw, err := os.ReadFile("testdata/azure-origin-urls.json") + if err != nil { + t.Fatal(err) + } + + var fixtures []struct { + Name string `json:"name"` + URL string `json:"url"` + Digest string `json:"digest"` + AllowedHostSuffix string `json:"allowed_host_suffix"` + } + if err := json.Unmarshal(raw, &fixtures); err != nil { + t.Fatal(err) + } + + for _, fixture := range fixtures { + t.Run(fixture.Name, func(t *testing.T) { + t.Parallel() + + got, err := streaming.ParseOriginURL(fixture.URL, streaming.URLPolicy{ + AllowedHostSuffixes: []string{fixture.AllowedHostSuffix}, + }) + if err != nil { + t.Fatal(err) + } + + if got.Digest.String() != fixture.Digest { + t.Fatalf("digest = %s, want %s", got.Digest, fixture.Digest) + } + }) + } +} + +const testDigest = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func TestOriginURLFromRequestPreservesRawSignedURL(t *testing.T) { + t.Parallel() + + raw := "https://westus.data.azurecr.io/account//docker/registry/v2/blobs/sha256/01/" + testDigest + "/data?sig=a%2Bb%2Fc%3D&se=2030-01-01T00%3A00%3A00Z" + req := httptest.NewRequest("GET", streaming.HandlerPrefix+raw, nil) + + got, err := streaming.OriginURLFromRequest(req, streaming.URLPolicy{ + AllowedHostSuffixes: []string{".data.azurecr.io"}, + }) + if err != nil { + t.Fatal(err) + } + + if got.Raw != raw { + t.Fatalf("raw URL changed:\n got: %s\nwant: %s", got.Raw, raw) + } + + if got.Digest.String() != "sha256:"+testDigest { + t.Fatalf("digest = %s, want sha256:%s", got.Digest, testDigest) + } +} + +func TestParseOriginURLForms(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + raw string + policy streaming.URLPolicy + }{ + { + name: "ACR query digest", + raw: "https://app.eastus.data.azurecr.io/?sv=1&d=sha256:" + testDigest + "&sig=redacted", + policy: streaming.URLPolicy{AllowedHostSuffixes: []string{".data.azurecr.io"}}, + }, + { + name: "MCR data path", + raw: "https://westus.data.mcr.microsoft.com/account//docker/registry/v2/blobs/sha256/01/" + testDigest + "/data?sig=redacted", + policy: streaming.URLPolicy{AllowedHostSuffixes: []string{".data.mcr.microsoft.com"}}, + }, + { + name: "Azure Blob path", + raw: "https://account.blob.core.windows.net/container//docker/registry/v2/blobs/sha256/01/" + testDigest + "/data?sig=redacted", + policy: streaming.URLPolicy{AllowedHostSuffixes: []string{".blob.core.windows.net"}}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + got, err := streaming.ParseOriginURL(test.raw, test.policy) + if err != nil { + t.Fatal(err) + } + + if got.Digest.String() != "sha256:"+testDigest { + t.Fatalf("digest = %s, want sha256:%s", got.Digest, testDigest) + } + }) + } +} + +func TestParseOriginURLRejectsUnsafeOrConflictingURL(t *testing.T) { + t.Parallel() + + policy := streaming.URLPolicy{AllowedHostSuffixes: []string{".data.azurecr.io"}} + differentDigest := strings.Repeat("a", 64) + + for _, raw := range []string{ + "http://westus.data.azurecr.io/?d=sha256:" + testDigest, + "https://127.0.0.1/?d=sha256:" + testDigest, + "https://user@westus.data.azurecr.io/?d=sha256:" + testDigest, + "https://attacker.example/?d=sha256:" + testDigest, + "https://westus.data.azurecr.io/no-digest?sig=redacted", + "https://app.azurecr.io/v2/team/app/blobs/sha256:" + testDigest, + "https://westus.data.azurecr.io/account//docker/registry/v2/blobs/sha256/aa/" + differentDigest + "/data?d=sha256:" + testDigest, + "https://westus.data.azurecr.io/docker/registry/v2/blobs/sha256/ff/" + testDigest + "/data", + "https://westus.data.azurecr.io/docker/not-registry/v2/blobs/sha256/01/" + testDigest + "/data", + } { + t.Run(raw, func(t *testing.T) { + t.Parallel() + + if _, err := streaming.ParseOriginURL(raw, policy); err == nil { + t.Fatal("expected URL validation error") + } + }) + } +} + +func TestURLPolicyAllowsHostOnLabelBoundaryOnly(t *testing.T) { + t.Parallel() + + policy := streaming.URLPolicy{AllowedHostSuffixes: []string{".blob.core.windows.net"}} + if _, err := streaming.ParseOriginURL( + "https://evilblob.core.windows.net/?d=sha256:"+testDigest, + policy, + ); err == nil { + t.Fatal("expected host boundary validation error") + } +} + +func TestParseOriginURLRejectsDuplicateDigestQuery(t *testing.T) { + t.Parallel() + + raw := "https://westus.data.azurecr.io/?d=sha256:" + testDigest + + "&d=sha256:" + strings.Repeat("a", 64) + "&sig=redacted" + if _, err := streaming.ParseOriginURL(raw, streaming.URLPolicy{ + AllowedHostSuffixes: []string{".data.azurecr.io"}, + }); err == nil { + t.Fatal("expected duplicate digest validation error") + } +} + +func TestParseOriginURLErrorDoesNotExposeSignedQuery(t *testing.T) { + t.Parallel() + + const secret = "secret-that-must-not-appear" + + raw := "https://westus.data.azurecr.io/?d=sha256:" + testDigest + "&sig=" + secret + "%zz" + + _, err := streaming.ParseOriginURL(raw, streaming.URLPolicy{ + AllowedHostSuffixes: []string{".data.azurecr.io"}, + }) + if err == nil || strings.Contains(err.Error(), secret) { + t.Fatalf("error = %q, want redacted parse failure", err) + } +} diff --git a/internal/gantry/streaming/integration_test.go b/internal/gantry/streaming/integration_test.go new file mode 100644 index 000000000..7456e0f0c --- /dev/null +++ b/internal/gantry/streaming/integration_test.go @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package streaming_test + +import ( + "context" + "crypto/sha256" + "fmt" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "golang.org/x/net/http2" + "golang.org/x/net/http2/h2c" //nolint:staticcheck // h2c is Gantry's peer protocol + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/ifaces/fakes" + "github.com/Azure/unbounded/internal/gantry/streaming" + "github.com/Azure/unbounded/internal/gantry/transfer" +) + +func TestIntegrationTransitionsFromSignedOriginToCompletePeer(t *testing.T) { + t.Parallel() + + complete := []byte("0123456789") + d := mustStreamingDigest(t, complete) + + originServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Range"); got != "bytes=2-5" { + t.Errorf("origin Range = %q, want bytes=2-5", got) + } + + w.Header().Set("Content-Range", "bytes 2-5/10") + w.Header().Set("Content-Length", "4") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(complete[2:6]) //nolint:errcheck // best-effort test response + })) + t.Cleanup(originServer.Close) + + peerCache := fakes.NewCache() + peerCache.Put(d, complete) + peerAddr := startStreamingTransferServer(t, transfer.New(peerCache).Handler()) + + policy := streaming.URLPolicy{AllowedHostSuffixes: []string{"localhost"}, AllowHTTP: true} + + originClient := streaming.NewOriginClient(policy) + + discovery := &discoveryStub{} + + server, err := streaming.NewServer( + &rangeStoreStub{err: &ifaces.ErrNotFound{Digest: d}}, + discovery, + transfer.NewClient(transfer.WithRequestTimeout(5*time.Second)), + originClient, + streaming.Options{ + URLPolicy: policy, + PeerLookupTimeout: time.Second, + MaxPeerAttempts: 2, + }, + ) + if err != nil { + t.Fatal(err) + } + + originURL := strings.Replace(originServer.URL, "127.0.0.1", "localhost", 1) + + "?d=" + d.String() + "&sig=redacted" + request := func() *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, streaming.HandlerPrefix+originURL, nil) + req.Header.Set("Range", "bytes=2-5") + + response := httptest.NewRecorder() + server.ServeHTTP(response, req) + + return response + } + + first := request() + if first.Code != http.StatusPartialContent || first.Body.String() != "2345" { + t.Fatalf("origin response = %d %q, want 206 2345", first.Code, first.Body.String()) + } + + discovery.set(ifaces.Provider{NodeID: "peer-a", Addr: peerAddr}) + + second := request() + if second.Code != http.StatusPartialContent || second.Body.String() != "2345" { + t.Fatalf("peer response = %d %q, want 206 2345", second.Code, second.Body.String()) + } + + if got := second.Header().Get("Content-Range"); got != "bytes 2-5/10" { + t.Fatalf("peer Content-Range = %q, want bytes 2-5/10", got) + } +} + +func startStreamingTransferServer(t *testing.T, handler http.Handler) string { + t.Helper() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + + httpServer := &http.Server{ + Handler: h2c.NewHandler(handler, &http2.Server{}), //nolint:staticcheck // h2c is Gantry's peer protocol + ReadHeaderTimeout: time.Second, + } + + go func() { _ = httpServer.Serve(listener) }() //nolint:errcheck // test cleanup owns shutdown + + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _ = httpServer.Shutdown(ctx) //nolint:errcheck // best-effort test cleanup + }) + + return listener.Addr().String() +} + +func mustStreamingDigest(t *testing.T, payload []byte) digest.Digest { + t.Helper() + + d, err := digest.Parse("sha256:" + fmt.Sprintf("%x", sha256.Sum256(payload))) + if err != nil { + t.Fatal(err) + } + + return d +} diff --git a/internal/gantry/streaming/origin.go b/internal/gantry/streaming/origin.go new file mode 100644 index 000000000..9c85b64a0 --- /dev/null +++ b/internal/gantry/streaming/origin.go @@ -0,0 +1,193 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package streaming + +import ( + "context" + "crypto/tls" + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/httprange" +) + +const ( + defaultOriginDialTimeout = 30 * time.Second + defaultOriginTLSHandshakeTimeout = 10 * time.Second + defaultOriginIdleConnTimeout = 90 * time.Second + + // Origin range reads are bounded separately from complete chair pulls so a + // slow registry cannot starve them. + maxConcurrentOriginReads = 32 + originResponseHeaderTimeout = 30 * time.Second +) + +// OriginStatusError reports a non-206 response from the signed origin. +type OriginStatusError struct { + StatusCode int + RetryAfter time.Duration +} + +func (e *OriginStatusError) Error() string { + return fmt.Sprintf("streaming origin returned HTTP %d", e.StatusCode) +} + +// OriginClient fetches exact ranges from validated signed origin URLs. +type OriginClient struct { + hc *http.Client + sem chan struct{} +} + +type redirectDigestContextKey struct{} + +// NewOriginClient constructs a client with independent origin concurrency. +func NewOriginClient(policy URLPolicy) *OriginClient { + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: defaultOriginDialTimeout, KeepAlive: 30 * time.Second}).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: defaultOriginIdleConnTimeout, + TLSHandshakeTimeout: defaultOriginTLSHandshakeTimeout, + ExpectContinueTimeout: time.Second, + ResponseHeaderTimeout: originResponseHeaderTimeout, + TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, + } + + hc := &http.Client{ + Transport: transport, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return fmt.Errorf("streaming origin: too many redirects") + } + + redirected, err := ParseOriginURL(req.URL.String(), policy) + if err != nil { + return err + } + + expected, ok := req.Context().Value(redirectDigestContextKey{}).(digest.Digest) + if !ok || redirected.Digest != expected { + return fmt.Errorf("streaming origin: redirect changed digest") + } + + req.Header.Del("Authorization") + req.Header.Del("Cookie") + req.Header.Del("Proxy-Authorization") + + return nil + }, + } + + return &OriginClient{hc: hc, sem: make(chan struct{}, maxConcurrentOriginReads)} +} + +// FetchRange fetches exactly requested from origin and returns the complete +// object size reported by Content-Range. +func (c *OriginClient) FetchRange(ctx context.Context, origin OriginURL, requested httprange.Range) (io.ReadCloser, int64, string, error) { + if _, err := httprange.New(requested.Start, requested.End); err != nil { + return nil, 0, "", err + } + + select { + case c.sem <- struct{}{}: + case <-ctx.Done(): + return nil, 0, "", ctx.Err() + } + + release := func() { <-c.sem } + + ctx = context.WithValue(ctx, redirectDigestContextKey{}, origin.Digest) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, origin.Raw, nil) + if err != nil { + release() + + return nil, 0, "", fmt.Errorf("streaming origin request is invalid") + } + + req.Header.Set("Accept", "application/octet-stream") + req.Header.Set("Range", requested.HeaderValue()) + + resp, err := c.hc.Do(req) + if err != nil { + release() + + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, 0, "", ctxErr + } + + return nil, 0, "", fmt.Errorf("streaming origin request failed") + } + + if resp.StatusCode != http.StatusPartialContent { + retryAfter := parseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + _ = resp.Body.Close() //nolint:errcheck // best-effort close + + release() + + return nil, 0, "", &OriginStatusError{StatusCode: resp.StatusCode, RetryAfter: retryAfter} + } + + if resp.ContentLength < 0 { + _ = resp.Body.Close() //nolint:errcheck // best-effort close + + release() + + return nil, 0, "", fmt.Errorf("streaming origin response omitted Content-Length") + } + + total, err := httprange.ValidateResponse(requested, resp.Header.Get("Content-Range"), resp.ContentLength) + if err != nil { + _ = resp.Body.Close() //nolint:errcheck // best-effort close + + release() + + return nil, 0, "", fmt.Errorf("streaming origin range response: %w", err) + } + + return &releaseReadCloser{ReadCloser: resp.Body, release: release}, total, resp.Header.Get("Content-Type"), nil +} + +func parseRetryAfter(value string, now time.Time) time.Duration { + if seconds, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64); err == nil && seconds >= 0 { + return time.Duration(seconds) * time.Second + } + + when, err := http.ParseTime(value) + if err != nil || !when.After(now) { + return 0 + } + + return when.Sub(now) +} + +type releaseReadCloser struct { + io.ReadCloser + release func() + once sync.Once +} + +func (r *releaseReadCloser) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + if err != nil { + r.once.Do(r.release) + } + + return n, err +} + +func (r *releaseReadCloser) Close() error { + err := r.ReadCloser.Close() + r.once.Do(r.release) + + return err +} diff --git a/internal/gantry/streaming/origin_test.go b/internal/gantry/streaming/origin_test.go new file mode 100644 index 000000000..a91233e25 --- /dev/null +++ b/internal/gantry/streaming/origin_test.go @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package streaming_test + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/httprange" + "github.com/Azure/unbounded/internal/gantry/streaming" +) + +func TestOriginClientFetchRange(t *testing.T) { + t.Parallel() + + body := []byte("0123456789") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Range"); got != "bytes=2-5" { + t.Errorf("Range = %q, want bytes=2-5", got) + } + + if got := r.URL.RawQuery; got != "sig=a%2Bb%2Fc%3D&d=sha256:"+testDigest { + t.Errorf("query = %q", got) + } + + w.Header().Set("Content-Range", "bytes 2-5/10") + w.Header().Set("Content-Length", "4") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write(body[2:6]) //nolint:errcheck // best-effort write + })) + t.Cleanup(server.Close) + + raw := strings.Replace(server.URL, "127.0.0.1", "localhost", 1) + "?sig=a%2Bb%2Fc%3D&d=sha256:" + testDigest + policy := streaming.URLPolicy{AllowedHostSuffixes: []string{"localhost"}, AllowHTTP: true} + + origin, err := streaming.ParseOriginURL(raw, policy) + if err != nil { + t.Fatal(err) + } + + client := streaming.NewOriginClient(policy) + + rc, total, _, err := client.FetchRange(context.Background(), origin, httprange.Range{Start: 2, End: 5}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = rc.Close() }() //nolint:errcheck // best-effort close + + if total != int64(len(body)) { + t.Fatalf("total = %d, want %d", total, len(body)) + } + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + + if string(got) != "2345" { + t.Fatalf("body = %q, want 2345", got) + } +} + +func TestOriginClientRejectsInvalidRangeResponse(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Range", "bytes 3-6/10") + w.Header().Set("Content-Length", "4") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("3456")) //nolint:errcheck // best-effort write + })) + t.Cleanup(server.Close) + + raw := strings.Replace(server.URL, "127.0.0.1", "localhost", 1) + "?d=sha256:" + testDigest + policy := streaming.URLPolicy{AllowedHostSuffixes: []string{"localhost"}, AllowHTTP: true} + + origin, err := streaming.ParseOriginURL(raw, policy) + if err != nil { + t.Fatal(err) + } + + client := streaming.NewOriginClient(policy) + + rc, _, _, err := client.FetchRange(context.Background(), origin, httprange.Range{Start: 2, End: 5}) + if rc != nil { + _ = rc.Close() //nolint:errcheck // best-effort close + } + + if err == nil { + t.Fatal("expected invalid range response error") + } +} + +func TestOriginClientRedirectRequiresSameDigest(t *testing.T) { + t.Parallel() + + differentDigest := strings.Repeat("a", 64) + final := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Fatal("redirect target should not be requested") + })) + t.Cleanup(final.Close) + + redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + target := strings.Replace(final.URL, "127.0.0.1", "localhost", 1) + "?d=sha256:" + differentDigest + http.Redirect(w, &http.Request{}, target, http.StatusTemporaryRedirect) + })) + t.Cleanup(redirect.Close) + + policy := streaming.URLPolicy{AllowedHostSuffixes: []string{"localhost"}, AllowHTTP: true} + raw := strings.Replace(redirect.URL, "127.0.0.1", "localhost", 1) + "?d=sha256:" + testDigest + + origin, err := streaming.ParseOriginURL(raw, policy) + if err != nil { + t.Fatal(err) + } + + client := streaming.NewOriginClient(policy) + + _, _, _, err = client.FetchRange(context.Background(), origin, httprange.Range{Start: 0, End: 0}) + if err == nil || strings.Contains(err.Error(), "d=sha256") { + t.Fatalf("error = %q, want redacted redirect failure", err) + } +} + +func TestOriginClientInvalidRequestErrorDoesNotExposeSignedQuery(t *testing.T) { + t.Parallel() + + const secret = "secret-that-must-not-appear" + + d, err := digest.Parse("sha256:" + testDigest) + if err != nil { + t.Fatal(err) + } + + client := streaming.NewOriginClient(streaming.URLPolicy{AllowedHostSuffixes: []string{"localhost"}, AllowHTTP: true}) + + _, _, _, err = client.FetchRange(context.Background(), streaming.OriginURL{ + Raw: "://invalid?sig=" + secret, + Digest: d, + }, httprange.Range{Start: 0, End: 0}) + if err == nil || strings.Contains(err.Error(), secret) { + t.Fatalf("error = %q, want redacted request failure", err) + } +} diff --git a/internal/gantry/streaming/providers.go b/internal/gantry/streaming/providers.go new file mode 100644 index 000000000..e95d149a5 --- /dev/null +++ b/internal/gantry/streaming/providers.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package streaming + +import ( + "errors" + "math/rand/v2" + "net/http" + "sync" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/ifaces" +) + +const ( + defaultStaleProviderTTL = 3 * time.Minute + defaultUnavailableProviderTTL = 30 * time.Second + defaultSuspiciousProviderTTL = 5 * time.Minute +) + +type providerDigestKey struct { + digest digest.Digest + nodeID ifaces.NodeID + addr string +} + +type providerFailures struct { + mu sync.Mutex + stale map[providerDigestKey]time.Time + suspicious map[providerDigestKey]time.Time + unavailable map[string]time.Time + now func() time.Time +} + +func newProviderFailures() *providerFailures { + return &providerFailures{ + stale: map[providerDigestKey]time.Time{}, + suspicious: map[providerDigestKey]time.Time{}, + unavailable: map[string]time.Time{}, + now: time.Now, + } +} + +func (f *providerFailures) filter(d digest.Digest, providers []ifaces.Provider, self ifaces.NodeID) []ifaces.Provider { + f.mu.Lock() + defer f.mu.Unlock() + + now := f.now() + f.sweep(now) + + filtered := make([]ifaces.Provider, 0, len(providers)) + for _, provider := range providers { + if provider.NodeID == self || provider.Addr == "" { + continue + } + + key := providerDigestKey{digest: d, nodeID: provider.NodeID, addr: provider.Addr} + if now.Before(f.stale[key]) || now.Before(f.suspicious[key]) || now.Before(f.unavailable[provider.Addr]) { + continue + } + + filtered = append(filtered, provider) + } + + rand.Shuffle(len(filtered), func(left, right int) { + filtered[left], filtered[right] = filtered[right], filtered[left] + }) + + return filtered +} + +func (f *providerFailures) record(d digest.Digest, provider ifaces.Provider, err error) { + f.mu.Lock() + defer f.mu.Unlock() + + now := f.now() + f.sweep(now) + + key := providerDigestKey{digest: d, nodeID: provider.NodeID, addr: provider.Addr} + + var notFound *ifaces.ErrNotFound + if errors.As(err, ¬Found) { + f.stale[key] = now.Add(defaultStaleProviderTTL) + + return + } + + var protocol *ifaces.ErrPeerProtocol + if errors.As(err, &protocol) { + f.suspicious[key] = now.Add(defaultSuspiciousProviderTTL) + + return + } + + var status *ifaces.ErrPeerHTTPStatus + if errors.As(err, &status) { + switch { + case status.StatusCode == http.StatusNotFound: + f.stale[key] = now.Add(defaultStaleProviderTTL) + case status.StatusCode == http.StatusTooManyRequests && status.RetryAfter > 0: + f.unavailable[provider.Addr] = now.Add(status.RetryAfter) + case status.StatusCode == http.StatusTooManyRequests || status.StatusCode >= http.StatusInternalServerError: + f.unavailable[provider.Addr] = now.Add(defaultUnavailableProviderTTL) + default: + f.suspicious[key] = now.Add(defaultSuspiciousProviderTTL) + } + + return + } + + f.unavailable[provider.Addr] = now.Add(defaultUnavailableProviderTTL) +} + +func (f *providerFailures) sweep(now time.Time) { + for key, until := range f.stale { + if !now.Before(until) { + delete(f.stale, key) + } + } + + for key, until := range f.suspicious { + if !now.Before(until) { + delete(f.suspicious, key) + } + } + + for addr, until := range f.unavailable { + if !now.Before(until) { + delete(f.unavailable, addr) + } + } +} diff --git a/internal/gantry/streaming/providers_test.go b/internal/gantry/streaming/providers_test.go new file mode 100644 index 000000000..7a913db7d --- /dev/null +++ b/internal/gantry/streaming/providers_test.go @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package streaming + +import ( + "errors" + "testing" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/ifaces" +) + +func TestProviderFailuresSuppressAndExpire(t *testing.T) { + d, err := digest.Parse("sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef") + if err != nil { + t.Fatal(err) + } + + now := time.Unix(100, 0) + failures := newProviderFailures() + failures.now = func() time.Time { return now } + + stale := ifaces.Provider{NodeID: "stale", Addr: "10.0.0.1:5001"} + unavailable := ifaces.Provider{NodeID: "down", Addr: "10.0.0.2:5001"} + suspicious := ifaces.Provider{NodeID: "bad", Addr: "10.0.0.3:5001"} + healthy := ifaces.Provider{NodeID: "healthy", Addr: "10.0.0.4:5001"} + + failures.record(d, stale, &ifaces.ErrNotFound{Digest: d}) + failures.record(d, unavailable, errors.New("connection refused")) + failures.record(d, suspicious, &ifaces.ErrPeerProtocol{PeerAddr: suspicious.Addr, Err: errors.New("invalid range")}) + + filtered := failures.filter(d, []ifaces.Provider{stale, unavailable, suspicious, healthy}, "self") + if len(filtered) != 1 || filtered[0] != healthy { + t.Fatalf("filtered = %+v, want only healthy", filtered) + } + + now = now.Add(defaultSuspiciousProviderTTL + time.Second) + + filtered = failures.filter(d, []ifaces.Provider{stale, unavailable, suspicious, healthy}, "self") + if len(filtered) != 4 { + t.Fatalf("after expiry filtered = %+v, want all providers", filtered) + } +} diff --git a/internal/gantry/streaming/server.go b/internal/gantry/streaming/server.go new file mode 100644 index 000000000..1b6981b91 --- /dev/null +++ b/internal/gantry/streaming/server.go @@ -0,0 +1,298 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package streaming + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "sync/atomic" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/httprange" + "github.com/Azure/unbounded/internal/gantry/ifaces" +) + +type LocalRangeStore interface { + OpenRange(ctx context.Context, d digest.Digest, requested httprange.Range) (io.ReadCloser, int64, error) +} + +type ProviderDiscovery interface { + FindProviders(ctx context.Context, d digest.Digest) ([]ifaces.Provider, error) +} + +type PeerRangeFetcher interface { + FetchRangeFromPeer(ctx context.Context, peerAddr string, d digest.Digest, requested httprange.Range) (io.ReadCloser, int64, string, error) +} + +type OriginRangeFetcher interface { + FetchRange(ctx context.Context, origin OriginURL, requested httprange.Range) (io.ReadCloser, int64, string, error) +} + +// Tuning values rather than operator configuration: changing one requires a +// latency measurement, so they live in code with the behavior they affect. +const ( + defaultPeerLookupTimeout = 250 * time.Millisecond + defaultMaxPeerAttempts = 3 +) + +type Options struct { + URLPolicy URLPolicy + + // PeerLookupTimeout and MaxPeerAttempts fall back to defaults when zero. + PeerLookupTimeout time.Duration + MaxPeerAttempts int + + SelfPeerID ifaces.NodeID + StartupGated bool + Logger *slog.Logger + Metrics MetricsHooks +} + +type MetricsHooks struct { + OnRequest func(source, outcome string, duration time.Duration, bytes int64) + OnFirstByte func(source string, duration time.Duration) + OnInflight func(source string, delta int) + OnReject func(reason string) +} + +// Server serves OverlayBD's node-local range proxy contract. +type Server struct { + local LocalRangeStore + discovery ProviderDiscovery + peer PeerRangeFetcher + origin OriginRangeFetcher + opts Options + failures *providerFailures + draining atomic.Bool + ready atomic.Bool +} + +func NewServer(local LocalRangeStore, discovery ProviderDiscovery, peer PeerRangeFetcher, origin OriginRangeFetcher, opts Options) (*Server, error) { + if local == nil || discovery == nil || peer == nil || origin == nil { + return nil, fmt.Errorf("streaming server: all dependencies are required") + } + + if opts.PeerLookupTimeout <= 0 { + opts.PeerLookupTimeout = defaultPeerLookupTimeout + } + + if opts.MaxPeerAttempts < 1 { + opts.MaxPeerAttempts = defaultMaxPeerAttempts + } + + if opts.Logger == nil { + opts.Logger = slog.Default() + } + + opts.Logger = opts.Logger.With(slog.String("subsystem", "streaming")) + + s := &Server{local: local, discovery: discovery, peer: peer, origin: origin, opts: opts, failures: newProviderFailures()} + if !opts.StartupGated { + s.ready.Store(true) + } + + return s, nil +} + +// Drain prevents new reads while allowing the shared HTTP server to finish +// active handlers. +func (s *Server) Drain() { s.draining.Store(true) } + +// MarkReady releases the sticky startup gate. +func (s *Server) MarkReady() { s.ready.Store(true) } + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + started := time.Now() + + if r.URL.Path == ReadinessPath { + if s.draining.Load() || !s.ready.Load() { + http.Error(w, "not ready", http.StatusServiceUnavailable) + + return + } + + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "ok") //nolint:errcheck // best-effort readiness body + + return + } + + if s.draining.Load() { + http.Error(w, "agent shutting down", http.StatusServiceUnavailable) + + return + } + + if !s.ready.Load() { + w.Header().Set("Retry-After", "5") + http.Error(w, "agent starting up", http.StatusServiceUnavailable) + + return + } + + if r.Method != http.MethodGet { + w.Header().Set("Allow", http.MethodGet) + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + + return + } + + requested, err := httprange.ParseExact(r.Header.Get("Range")) + if err != nil { + s.reject("range") + http.Error(w, "invalid Range", http.StatusBadRequest) + + return + } + + origin, err := OriginURLFromRequest(r, s.opts.URLPolicy) + if err != nil { + s.reject("origin_url") + http.Error(w, "invalid origin URL", http.StatusBadRequest) + + return + } + + body, total, err := s.local.OpenRange(r.Context(), origin.Digest, requested) + if err == nil { + s.writeResponse(w, body, total, "application/octet-stream", requested, "local", started) + + return + } + + if errors.Is(err, httprange.ErrUnsatisfiable) && total >= 0 { + w.Header().Set("Content-Range", httprange.UnsatisfiedContentRange(total)) + http.Error(w, "range not satisfiable", http.StatusRequestedRangeNotSatisfiable) + + return + } + + var ( + notFound *ifaces.ErrNotFound + unavailable *ifaces.ErrUnavailable + ) + if !errors.As(err, ¬Found) && !errors.As(err, &unavailable) { + s.opts.Logger.Warn("local range open failed", + slog.String("digest", origin.Digest.String()), + slog.Any("err", err), + ) + } + + lookupCtx, cancel := context.WithTimeout(r.Context(), s.opts.PeerLookupTimeout) + providers, lookupErr := s.discovery.FindProviders(lookupCtx, origin.Digest) + + cancel() + + if lookupErr == nil { + attempts := 0 + for _, provider := range s.failures.filter(origin.Digest, providers, s.opts.SelfPeerID) { + if attempts >= s.opts.MaxPeerAttempts { + break + } + + attempts++ + + peerBody, peerTotal, contentType, peerErr := s.peer.FetchRangeFromPeer(r.Context(), provider.Addr, origin.Digest, requested) + if peerErr != nil { + s.failures.record(origin.Digest, provider, peerErr) + continue + } + + s.writeResponse(w, peerBody, peerTotal, contentType, requested, "peer", started) + + return + } + } + + body, total, contentType, err := s.origin.FetchRange(r.Context(), origin, requested) + if err != nil { + s.writeOriginError(w, err) + s.observe("origin", "error", time.Since(started), 0) + + return + } + + s.writeResponse(w, body, total, contentType, requested, "origin", started) +} + +func (s *Server) writeResponse(w http.ResponseWriter, body io.ReadCloser, total int64, contentType string, requested httprange.Range, source string, started time.Time) { + defer func() { _ = body.Close() }() //nolint:errcheck // best-effort close + + if s.opts.Metrics.OnInflight != nil { + s.opts.Metrics.OnInflight(source, 1) + defer s.opts.Metrics.OnInflight(source, -1) + } + + if contentType == "" { + contentType = "application/octet-stream" + } + + w.Header().Set("Accept-Ranges", "bytes") + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", requested.Start, requested.End, total)) + w.Header().Set("Content-Length", strconv.FormatInt(requested.Length(), 10)) + + if s.opts.Metrics.OnFirstByte != nil { + s.opts.Metrics.OnFirstByte(source, time.Since(started)) + } + + w.WriteHeader(http.StatusPartialContent) + + written, err := io.CopyN(w, body, requested.Length()) + if err != nil { + s.opts.Logger.Debug("range response interrupted", + slog.String("source", source), + slog.String("digest", "redacted"), + slog.Int64("written", written), + slog.Any("err", err), + ) + s.observe(source, "error", time.Since(started), written) + + return + } + + s.observe(source, "success", time.Since(started), written) +} + +func (s *Server) writeOriginError(w http.ResponseWriter, err error) { + var statusErr *OriginStatusError + if errors.As(err, &statusErr) { + if statusErr.RetryAfter > 0 { + w.Header().Set("Retry-After", strconv.FormatInt(int64(statusErr.RetryAfter/time.Second), 10)) + } + + switch statusErr.StatusCode { + case http.StatusUnauthorized, http.StatusForbidden: + http.Error(w, "origin authorization failed", statusErr.StatusCode) + case http.StatusNotFound: + http.Error(w, "origin content not found", http.StatusNotFound) + case http.StatusTooManyRequests: + http.Error(w, "origin rate limited", http.StatusTooManyRequests) + default: + http.Error(w, "origin unavailable", http.StatusBadGateway) + } + + return + } + + http.Error(w, "origin unavailable", http.StatusServiceUnavailable) +} + +func (s *Server) reject(reason string) { + if s.opts.Metrics.OnReject != nil { + s.opts.Metrics.OnReject(reason) + } +} + +func (s *Server) observe(source, outcome string, duration time.Duration, bytes int64) { + if s.opts.Metrics.OnRequest != nil { + s.opts.Metrics.OnRequest(source, outcome, duration, bytes) + } +} diff --git a/internal/gantry/streaming/server_test.go b/internal/gantry/streaming/server_test.go new file mode 100644 index 000000000..7fb225202 --- /dev/null +++ b/internal/gantry/streaming/server_test.go @@ -0,0 +1,254 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 + +package streaming_test + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/httprange" + "github.com/Azure/unbounded/internal/gantry/ifaces" + "github.com/Azure/unbounded/internal/gantry/streaming" +) + +type rangeStoreStub struct { + body []byte + total int64 + err error +} + +func (s *rangeStoreStub) OpenRange(context.Context, digest.Digest, httprange.Range) (io.ReadCloser, int64, error) { + if s.err != nil { + return nil, s.total, s.err + } + + return io.NopCloser(bytes.NewReader(s.body)), s.total, nil +} + +type discoveryStub struct { + mu sync.Mutex + providers []ifaces.Provider +} + +func (s *discoveryStub) FindProviders(context.Context, digest.Digest) ([]ifaces.Provider, error) { + s.mu.Lock() + defer s.mu.Unlock() + + return append([]ifaces.Provider(nil), s.providers...), nil +} + +func (s *discoveryStub) set(providers ...ifaces.Provider) { + s.mu.Lock() + + s.providers = append([]ifaces.Provider(nil), providers...) + s.mu.Unlock() +} + +type peerStub struct { + body []byte + total int64 + calls int +} + +func (s *peerStub) FetchRangeFromPeer(context.Context, string, digest.Digest, httprange.Range) (io.ReadCloser, int64, string, error) { + s.calls++ + + return io.NopCloser(bytes.NewReader(s.body)), s.total, "application/octet-stream", nil +} + +type originStub struct { + body []byte + total int64 + calls int + originRaw string + requested httprange.Range +} + +func (s *originStub) FetchRange(_ context.Context, origin streaming.OriginURL, requested httprange.Range) (io.ReadCloser, int64, string, error) { + s.calls++ + s.originRaw = origin.Raw + s.requested = requested + + return io.NopCloser(bytes.NewReader(s.body)), s.total, "application/octet-stream", nil +} + +func TestServerTransitionsFromOriginToPeer(t *testing.T) { + t.Parallel() + + local := &rangeStoreStub{err: &ifaces.ErrNotFound{}} + discovery := &discoveryStub{} + peer := &peerStub{body: []byte("peer"), total: 10} + origin := &originStub{body: []byte("orig"), total: 10} + + server, err := streaming.NewServer(local, discovery, peer, origin, streaming.Options{ + URLPolicy: streaming.URLPolicy{ + AllowedHostSuffixes: []string{".data.azurecr.io"}, + }, + PeerLookupTimeout: time.Second, + MaxPeerAttempts: 2, + }) + if err != nil { + t.Fatal(err) + } + + rawOrigin := "https://westus.data.azurecr.io/?d=sha256:" + testDigest + "&sig=secret" + request := func() *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, streaming.HandlerPrefix+rawOrigin, nil) + req.Header.Set("Range", "bytes=2-5") + + response := httptest.NewRecorder() + server.ServeHTTP(response, req) + + return response + } + + first := request() + if first.Code != http.StatusPartialContent || first.Body.String() != "orig" { + t.Fatalf("first response = %d %q, want origin 206", first.Code, first.Body.String()) + } + + if origin.calls != 1 || peer.calls != 0 { + t.Fatalf("first calls: origin=%d peer=%d", origin.calls, peer.calls) + } + + discovery.set(ifaces.Provider{NodeID: "peer-a", Addr: "10.0.0.2:5001"}) + + second := request() + if second.Code != http.StatusPartialContent || second.Body.String() != "peer" { + t.Fatalf("second response = %d %q, want peer 206", second.Code, second.Body.String()) + } + + if origin.calls != 1 || peer.calls != 1 { + t.Fatalf("second calls: origin=%d peer=%d", origin.calls, peer.calls) + } + + if origin.originRaw != rawOrigin || origin.requested != (httprange.Range{Start: 2, End: 5}) { + t.Fatalf("origin request changed: raw=%q range=%+v", origin.originRaw, origin.requested) + } +} + +func TestServerPrefersLocalCompleteBlob(t *testing.T) { + t.Parallel() + + local := &rangeStoreStub{body: []byte("locl"), total: 10} + discovery := &discoveryStub{} + peer := &peerStub{} + origin := &originStub{} + + server, err := streaming.NewServer(local, discovery, peer, origin, streaming.Options{ + URLPolicy: streaming.URLPolicy{AllowedHostSuffixes: []string{".data.azurecr.io"}}, + PeerLookupTimeout: time.Second, + MaxPeerAttempts: 1, + }) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, streaming.HandlerPrefix+"https://westus.data.azurecr.io/?d=sha256:"+testDigest, nil) + req.Header.Set("Range", "bytes=2-5") + + response := httptest.NewRecorder() + server.ServeHTTP(response, req) + + if response.Code != http.StatusPartialContent || response.Body.String() != "locl" { + t.Fatalf("response = %d %q", response.Code, response.Body.String()) + } + + if peer.calls != 0 || origin.calls != 0 { + t.Fatalf("peer=%d origin=%d, want zero", peer.calls, origin.calls) + } +} + +func TestServerRejectsInvalidOriginBeforeDependencies(t *testing.T) { + t.Parallel() + + local := &rangeStoreStub{} + discovery := &discoveryStub{} + peer := &peerStub{} + origin := &originStub{} + + server, err := streaming.NewServer(local, discovery, peer, origin, streaming.Options{ + URLPolicy: streaming.URLPolicy{AllowedHostSuffixes: []string{".data.azurecr.io"}}, + PeerLookupTimeout: time.Second, + MaxPeerAttempts: 1, + }) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodGet, streaming.HandlerPrefix+"https://attacker.example/?d=sha256:"+testDigest, nil) + req.Header.Set("Range", "bytes=0-0") + + response := httptest.NewRecorder() + server.ServeHTTP(response, req) + + if response.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", response.Code) + } + + if peer.calls != 0 || origin.calls != 0 { + t.Fatalf("peer=%d origin=%d, want zero", peer.calls, origin.calls) + } +} + +func TestServerStartupGate(t *testing.T) { + t.Parallel() + + local := &rangeStoreStub{body: []byte("x"), total: 1} + + server, err := streaming.NewServer(local, &discoveryStub{}, &peerStub{}, &originStub{}, streaming.Options{ + URLPolicy: streaming.URLPolicy{AllowedHostSuffixes: []string{".data.azurecr.io"}}, + PeerLookupTimeout: time.Second, + MaxPeerAttempts: 1, + StartupGated: true, + }) + if err != nil { + t.Fatal(err) + } + + request := func() *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, streaming.HandlerPrefix+"https://westus.data.azurecr.io/?d=sha256:"+testDigest, nil) + req.Header.Set("Range", "bytes=0-0") + + response := httptest.NewRecorder() + server.ServeHTTP(response, req) + + return response + } + + if got := request().Code; got != http.StatusServiceUnavailable { + t.Fatalf("before MarkReady status = %d, want 503", got) + } + + server.MarkReady() + + if got := request().Code; got != http.StatusPartialContent { + t.Fatalf("after MarkReady status = %d, want 206", got) + } + + readyRequest := func() int { + request := httptest.NewRequest(http.MethodGet, streaming.ReadinessPath, nil) + response := httptest.NewRecorder() + server.ServeHTTP(response, request) + + return response.Code + } + + if got := readyRequest(); got != http.StatusOK { + t.Fatalf("ready endpoint status = %d, want 200", got) + } + + server.Drain() + + if got := readyRequest(); got != http.StatusServiceUnavailable { + t.Fatalf("draining endpoint status = %d, want 503", got) + } +} diff --git a/internal/gantry/streaming/testdata/azure-origin-urls.json b/internal/gantry/streaming/testdata/azure-origin-urls.json new file mode 100644 index 000000000..3bbfd91f4 --- /dev/null +++ b/internal/gantry/streaming/testdata/azure-origin-urls.json @@ -0,0 +1,20 @@ +[ + { + "name": "acr-data-query", + "url": "https://example.eastus.data.azurecr.io/?t=REDACTED&h=example.azurecr.io&c=REDACTED&r=team/app&d=sha256:dd5ad9c9c29f04b41a0155c720cf5ccab28ef6d353f1fe17a06c579c70054f0a&p=REDACTED&s=REDACTED&v=1", + "digest": "sha256:dd5ad9c9c29f04b41a0155c720cf5ccab28ef6d353f1fe17a06c579c70054f0a", + "allowed_host_suffix": ".data.azurecr.io" + }, + { + "name": "mcr-data-path", + "url": "https://westus2.data.mcr.microsoft.com/REDACTED//docker/registry/v2/blobs/sha256/1b/1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced/data?se=REDACTED&sig=REDACTED&sp=r&spr=https&sr=b&sv=2018-03-28®id=REDACTED", + "digest": "sha256:1b930d010525941c1d56ec53b97bd057a67ae1865eebf042686d2a2d18271ced", + "allowed_host_suffix": ".data.mcr.microsoft.com" + }, + { + "name": "azure-blob-data-path", + "url": "https://example.blob.core.windows.net/REDACTED//docker/registry/v2/blobs/sha256/dd/dd5ad9c9c29f04b41a0155c720cf5ccab28ef6d353f1fe17a06c579c70054f0a/data?se=REDACTED&sig=REDACTED&sp=r&spr=https&sr=b&sv=2018-03-28®id=REDACTED", + "digest": "sha256:dd5ad9c9c29f04b41a0155c720cf5ccab28ef6d353f1fe17a06c579c70054f0a", + "allowed_host_suffix": ".blob.core.windows.net" + } +] \ No newline at end of file diff --git a/internal/gantry/transfer/client.go b/internal/gantry/transfer/client.go index a30a42599..bc0dbe016 100644 --- a/internal/gantry/transfer/client.go +++ b/internal/gantry/transfer/client.go @@ -19,6 +19,7 @@ import ( "golang.org/x/net/http2" "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/httprange" "github.com/Azure/unbounded/internal/gantry/ifaces" "github.com/Azure/unbounded/internal/gantry/oci" "github.com/Azure/unbounded/internal/gantry/registryauth" @@ -202,6 +203,56 @@ func (c *Client) FetchFromPeer(ctx context.Context, peerAddr string, ref ifaces. } } +// FetchRangeFromPeer fetches one exact byte range from a peer holding a +// complete, committed blob. Registry authorization is deliberately not +// forwarded: digest and range are sufficient at the peer boundary. +func (c *Client) FetchRangeFromPeer(ctx context.Context, peerAddr string, d digest.Digest, requested httprange.Range) (io.ReadCloser, int64, string, error) { + if _, err := httprange.New(requested.Start, requested.End); err != nil { + return nil, 0, "", err + } + + url, err := buildPeerURL(peerAddr, ifaces.OriginRef{Digest: d, Kind: ifaces.KindBlob}) + if err != nil { + return nil, 0, "", err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, 0, "", err + } + + req.Header.Set(MirroredHeader, "1") + req.Header.Set("Accept", "application/octet-stream") + req.Header.Set("Range", requested.HeaderValue()) + req.URL.Scheme = "http" + + resp, err := c.hc.Do(req) + if err != nil { + return nil, 0, "", fmt.Errorf("peer dial %s: %w", peerAddr, err) + } + + switch resp.StatusCode { + case http.StatusPartialContent: + size, err := httprange.ValidateResponse(requested, resp.Header.Get("Content-Range"), resp.ContentLength) + if err != nil { + _ = resp.Body.Close() //nolint:errcheck // best-effort body close + + return nil, 0, "", &ifaces.ErrPeerProtocol{PeerAddr: peerAddr, Err: err} + } + + return c.responseBody(resp, ifaces.KindBlob), size, resp.Header.Get("Content-Type"), nil + case http.StatusNotFound: + _ = resp.Body.Close() //nolint:errcheck // best-effort body close + + return nil, 0, "", &ifaces.ErrNotFound{Digest: d} + default: + retryAfter := parseRetryAfter(resp.Header.Get("Retry-After"), time.Now()) + _ = resp.Body.Close() //nolint:errcheck // best-effort body close + + return nil, 0, "", &ifaces.ErrPeerHTTPStatus{PeerAddr: peerAddr, StatusCode: resp.StatusCode, RetryAfter: retryAfter} + } +} + func parseRetryAfter(value string, now time.Time) time.Duration { if seconds, err := strconv.ParseInt(strings.TrimSpace(value), 10, 64); err == nil && seconds >= 0 { return time.Duration(seconds) * time.Second diff --git a/internal/gantry/transfer/client_test.go b/internal/gantry/transfer/client_test.go index ccfafc764..e938da1d4 100644 --- a/internal/gantry/transfer/client_test.go +++ b/internal/gantry/transfer/client_test.go @@ -17,6 +17,7 @@ import ( "golang.org/x/net/http2/h2c" //nolint:staticcheck // h2c deliberate "github.com/Azure/unbounded/internal/gantry/digest" + "github.com/Azure/unbounded/internal/gantry/httprange" "github.com/Azure/unbounded/internal/gantry/ifaces" "github.com/Azure/unbounded/internal/gantry/ifaces/fakes" "github.com/Azure/unbounded/internal/gantry/registryauth" @@ -248,6 +249,88 @@ func TestClientFetchRange(t *testing.T) { } } +func TestClientFetchExactRange(t *testing.T) { + cache := fakes.NewCache() + body := []byte("peer-served bytes") + d := mustDigest(body) + cache.Put(d, body) + + addr := startTransferOnEphemeral(t, cache) + + rc, size, contentType, err := NewClient().FetchRangeFromPeer( + registryauth.WithAuthorization(context.Background(), "Bearer must-not-be-forwarded"), + addr, + d, + httprange.Range{Start: 5, End: 10}, + ) + if err != nil { + t.Fatalf("FetchRangeFromPeer: %v", err) + } + defer func() { _ = rc.Close() }() //nolint:errcheck // best-effort close + + if size != int64(len(body)) { + t.Fatalf("size = %d, want %d", size, len(body)) + } + + if contentType != "application/octet-stream" { + t.Fatalf("content type = %q, want application/octet-stream", contentType) + } + + got, err := io.ReadAll(rc) + if err != nil { + t.Fatal(err) + } + + if string(got) != string(body[5:11]) { + t.Fatalf("body = %q, want %q", got, body[5:11]) + } +} + +func TestClientExactRangeDoesNotForwardAuthorization(t *testing.T) { + d := mustDigest([]byte("range")) + addr := startHandlerOnEphemeral(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "" { + t.Errorf("Authorization = %q, want empty", got) + } + + w.Header().Set("Content-Range", "bytes 0-0/1") + w.Header().Set("Content-Length", "1") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("x")) //nolint:errcheck // best-effort write + })) + + ctx := registryauth.WithAuthorization(context.Background(), "Bearer requester-token") + + rc, _, _, err := NewClient().FetchRangeFromPeer(ctx, addr, d, httprange.Range{Start: 0, End: 0}) + if err != nil { + t.Fatal(err) + } + + _ = rc.Close() //nolint:errcheck // best-effort close +} + +func TestClientRejectsInvalidExactRangeResponse(t *testing.T) { + d := mustDigest([]byte("range")) + addr := startHandlerOnEphemeral(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Range", "bytes 1-2/10") + w.Header().Set("Content-Length", "2") + w.WriteHeader(http.StatusPartialContent) + _, _ = w.Write([]byte("xx")) //nolint:errcheck // best-effort write + })) + + rc, _, _, err := NewClient().FetchRangeFromPeer( + context.Background(), addr, d, httprange.Range{Start: 2, End: 3}, + ) + if rc != nil { + _ = rc.Close() //nolint:errcheck // best-effort close + } + + var protocolErr *ifaces.ErrPeerProtocol + if !errors.As(err, &protocolErr) { + t.Fatalf("error = %T %v, want ErrPeerProtocol", err, err) + } +} + func TestClientRejectsInvalidRangeResponse(t *testing.T) { body := []byte("wrong range") d := mustDigest(body) diff --git a/internal/operator/component/env.go b/internal/operator/component/env.go index 4485d06c7..7412ebdd3 100644 --- a/internal/operator/component/env.go +++ b/internal/operator/component/env.go @@ -7,7 +7,7 @@ import ( "bytes" "context" "crypto/sha256" - "encoding/base64" + "encoding/base32" "encoding/json" "errors" "fmt" @@ -416,7 +416,8 @@ func AppliedPayloadHash(obj *unstructured.Unstructured) (string, error) { sum := sha256.Sum256(data) - return base64.RawURLEncoding.EncodeToString(sum[:]), nil + // Base32 always satisfies Kubernetes label boundary rules. + return base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(sum[:]), nil } // ListSites returns every Site in the cluster. diff --git a/internal/operator/component/env_test.go b/internal/operator/component/env_test.go index 8208be59e..55a404f89 100644 --- a/internal/operator/component/env_test.go +++ b/internal/operator/component/env_test.go @@ -5,7 +5,7 @@ package component import ( "context" - "regexp" + "fmt" "testing" appsv1 "k8s.io/api/apps/v1" @@ -15,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -255,6 +256,25 @@ func TestApplyObject(t *testing.T) { } } +// Exercise enough payloads to catch invalid base64url boundary characters. +func TestAppliedPayloadHashIsAlwaysAValidLabelValue(t *testing.T) { + for i := range 2000 { + desired := &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, + ObjectMeta: metav1.ObjectMeta{Name: fmt.Sprintf("test-%d", i), Namespace: "default"}, + } + + hash, err := AppliedPayloadHash(ToUnstructured(desired)) + if err != nil { + t.Fatalf("appliedPayloadHash: %v", err) + } + + if errs := validation.IsValidLabelValue(hash); len(errs) > 0 { + t.Fatalf("hash %q for %s is not a valid label value: %v", hash, desired.Name, errs) + } + } +} + func TestApplyObjectSkipsMatchingPayload(t *testing.T) { scheme := testScheme(t) desired := &appsv1.Deployment{ @@ -271,8 +291,8 @@ func TestApplyObjectSkipsMatchingPayload(t *testing.T) { t.Fatalf("appliedPayloadHash: %v", err) } - if len(hash) > 63 || !regexp.MustCompile(`^[A-Za-z0-9_-]+$`).MatchString(hash) { - t.Fatalf("applied payload hash %q is not a valid label value", hash) + if errs := validation.IsValidLabelValue(hash); len(errs) > 0 { + t.Fatalf("applied payload hash %q is not a valid label value: %v", hash, errs) } current := desired.DeepCopy() diff --git a/internal/operator/components/gantry/gantry.go b/internal/operator/components/gantry/gantry.go index 3b051dd8d..3e781fd42 100644 --- a/internal/operator/components/gantry/gantry.go +++ b/internal/operator/components/gantry/gantry.go @@ -14,6 +14,7 @@ package gantry import ( "context" "fmt" + "maps" "strconv" "strings" @@ -54,7 +55,11 @@ const ( // agentContainerName is the gantry agent's main container. Only this // container carries the operator-managed image; the DaemonSet's busybox // init container keeps its pinned public image. - agentContainerName = "gantry" + agentContainerName = "gantry" + overlayBDConfigDaemonSetName = "gantry-overlaybd-config" + overlayBDConfigContainerName = "configure" + overlayBDConfigImageRepository = "gantry-node-config" + artifactStreamingEnabledEnv = "GANTRY_ARTIFACT_STREAMING_ENABLED" // legacyNodeConfigName and legacyNodeConfigDaemonSetName were installed by // older operator versions. The unbounded agent owns this host configuration. @@ -71,10 +76,16 @@ const ( var operatorManifestFiles = map[string]struct{}{ "configmap.yaml": {}, "daemonset.yaml": {}, + "overlaybd-config.yaml": {}, "rendezvous-leases.yaml": {}, "serviceaccount.yaml": {}, } +type artifactStreamingConfig struct { + Enabled bool + NodeSelector map[string]string +} + // Component reconciles the gantry cluster singleton. type Component struct{} @@ -112,6 +123,11 @@ func (c Component) Plan(ctx context.Context, env *component.Env, sites []unbound } } + artifactStreaming, err := resolveArtifactStreaming(sites) + if err != nil { + return nil, component.Result{}, err + } + installationManager, err := detectInstallationManager(ctx, env) if err != nil { return nil, component.Result{}, err @@ -135,6 +151,13 @@ func (c Component) Plan(ctx context.Context, env *component.Env, sites []unbound // the applies depend on those deletes, so a failure to remove the legacy // DaemonSet does not race the replacement into the cluster alongside it. legacy := legacyCleanupOperations(c.Name(), env.Namespace) + if !artifactStreaming.Enabled { + legacy = append(legacy, component.DeleteOperation(&appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "DaemonSet"}, + ObjectMeta: metav1.ObjectMeta{Namespace: env.Namespace, Name: overlayBDConfigDaemonSetName}, + }, c.Name(), "")) + } + plan.Add(legacy...) legacyRefs := make([]component.ObjectRef, 0, len(legacy)) @@ -169,7 +192,12 @@ func (c Component) Plan(ctx context.Context, env *component.Env, sites []unbound dependsOn = append(dependsOn, configOp.Ref()) } - objects, err := decodeManifests(env, applyMutator(env.Config.Image(imageRepository), configHash)) + objects, err := decodeManifests(env, applyMutator( + env.Config.Image(imageRepository), + env.Config.Image(overlayBDConfigImageRepository), + configHash, + artifactStreaming, + )) if err != nil { return nil, component.Result{}, err } @@ -187,7 +215,7 @@ func (c Component) Plan(ctx context.Context, env *component.Env, sites []unbound DependsOn: dependsOn, } - if obj.GetKind() == "DaemonSet" && obj.GetName() == daemonSetName { + if obj.GetKind() == "DaemonSet" && (obj.GetName() == daemonSetName || obj.GetName() == overlayBDConfigDaemonSetName) { op.Overridable = true } @@ -197,6 +225,40 @@ func (c Component) Plan(ctx context.Context, env *component.Env, sites []unbound return plan, component.Reconciled(), nil } +func resolveArtifactStreaming(sites []unboundedv1alpha3.Site) (artifactStreamingConfig, error) { + var resolved artifactStreamingConfig + + for index := range sites { + site := &sites[index] + + gantry := site.Spec.Components.Gantry + if gantry == nil || gantry.ArtifactStreaming == nil || !gantry.ArtifactStreaming.Enabled { + continue + } + + if !EnabledFor(site) { + return artifactStreamingConfig{}, fmt.Errorf("site/%s enables gantry artifactStreaming while gantry is disabled", site.Name) + } + + selector := gantry.ArtifactStreaming.NodeSelector + if len(selector) == 0 { + return artifactStreamingConfig{}, fmt.Errorf("site/%s gantry artifactStreaming requires nodeSelector", site.Name) + } + + if !resolved.Enabled { + resolved = artifactStreamingConfig{Enabled: true, NodeSelector: maps.Clone(selector)} + + continue + } + + if !maps.Equal(resolved.NodeSelector, selector) { + return artifactStreamingConfig{}, fmt.Errorf("site/%s gantry artifactStreaming nodeSelector conflicts with another Site", site.Name) + } + } + + return resolved, nil +} + // SetupWatches reconciles Gantry on changes to its active resources and when // legacy node-config resources appear so they can be removed. func (Component) SetupWatches(b *builder.Builder, env *component.Env) { @@ -206,7 +268,7 @@ func (Component) SetupWatches(b *builder.Builder, env *component.Env) { b.Watches(&corev1.ConfigMap{}, env.RequestSingleton(), builder.WithPredicates(env.ManagedConfigPredicate(env.InNamespaceNamed(configName, legacyNodeConfigName)))) b.Watches(&appsv1.DaemonSet{}, env.RequestSingleton(), - builder.WithPredicates(env.ManagedWorkloadPredicate(env.InNamespaceNamed(daemonSetName, legacyNodeConfigDaemonSetName)))) + builder.WithPredicates(env.ManagedWorkloadPredicate(env.InNamespaceNamed(daemonSetName, legacyNodeConfigDaemonSetName, overlayBDConfigDaemonSetName)))) b.Watches(&coordinationv1.Lease{}, env.RequestSingleton(), builder.WithPredicates(chairLeaseDeletePredicate(env.Namespace))) b.Watches(&schedulingv1.PriorityClass{}, env.RequestSingleton(), @@ -333,7 +395,7 @@ func legacyCleanupOperations(componentName, namespace string) []component.Operat // stamps the operator-derived agent image onto the agent DaemonSet, and stamps // the config payload hash on its pod template so a config change rolls the // DaemonSet. Only the agent's own container is re-imaged. -func applyMutator(agentImage, configHash string) func(*unstructured.Unstructured) error { +func applyMutator(agentImage, overlayBDConfigImage, configHash string, artifactStreaming artifactStreamingConfig) func(*unstructured.Unstructured) error { return func(obj *unstructured.Unstructured) error { if obj.GetKind() == component.CRDKind { obj.Object = nil @@ -352,13 +414,82 @@ func applyMutator(agentImage, configHash string) func(*unstructured.Unstructured return fmt.Errorf("set gantry agent image: %w", err) } + if artifactStreaming.Enabled { + if err := setNamedContainerEnv(obj, agentContainerName, artifactStreamingEnabledEnv, "true"); err != nil { + return fmt.Errorf("enable gantry artifact streaming: %w", err) + } + } + return stampConfigHash(obj, configHashAnnotation, configHash) } + if obj.GetKind() == "DaemonSet" && obj.GetName() == overlayBDConfigDaemonSetName { + if !artifactStreaming.Enabled { + obj.Object = nil + + return nil + } + + if err := component.SetNamedContainerImage(obj, overlayBDConfigContainerName, overlayBDConfigImage); err != nil { + return fmt.Errorf("set gantry OverlayBD configurator image: %w", err) + } + + selector := make(map[string]any, len(artifactStreaming.NodeSelector)) + for key, value := range artifactStreaming.NodeSelector { + selector[key] = value + } + + if err := unstructured.SetNestedMap(obj.Object, selector, "spec", "template", "spec", "nodeSelector"); err != nil { + return fmt.Errorf("set gantry OverlayBD configurator nodeSelector: %w", err) + } + } + return nil } } +func setNamedContainerEnv(obj *unstructured.Unstructured, containerName, name, value string) error { + containers, found, err := unstructured.NestedSlice(obj.Object, "spec", "template", "spec", "containers") + if err != nil { + return err + } + + if !found { + return fmt.Errorf("containers not found") + } + + for index, item := range containers { + container, ok := item.(map[string]any) + if !ok || container["name"] != containerName { + continue + } + + env, _, err := unstructured.NestedSlice(container, "env") + if err != nil { + return err + } + + for envIndex, entry := range env { + variable, ok := entry.(map[string]any) + if ok && variable["name"] == name { + variable["value"] = value + env[envIndex] = variable + container["env"] = env + containers[index] = container + + return unstructured.SetNestedSlice(obj.Object, containers, "spec", "template", "spec", "containers") + } + } + + container["env"] = append(env, map[string]any{"name": name, "value": value}) + containers[index] = container + + return unstructured.SetNestedSlice(obj.Object, containers, "spec", "template", "spec", "containers") + } + + return fmt.Errorf("container %q not found", containerName) +} + // stampConfigHash sets a config-hash annotation on a workload's pod template so a // config change rolls it. func stampConfigHash(obj *unstructured.Unstructured, annotation, hash string) error { diff --git a/internal/operator/components/gantry/gantry_test.go b/internal/operator/components/gantry/gantry_test.go index 076162184..601737494 100644 --- a/internal/operator/components/gantry/gantry_test.go +++ b/internal/operator/components/gantry/gantry_test.go @@ -188,7 +188,7 @@ func TestApplyMutatorStampsDaemonSetAndSkipsConfig(t *testing.T) { }}, }} - if err := applyMutator("ghcr.io/azure/gantry:test", "gantry-hash")(ds); err != nil { + if err := applyMutator("ghcr.io/azure/gantry:test", "ghcr.io/azure/gantry-node-config:test", "gantry-hash", artifactStreamingConfig{})(ds); err != nil { t.Fatalf("applyMutator: %v", err) } @@ -200,7 +200,7 @@ func TestApplyMutatorStampsDaemonSetAndSkipsConfig(t *testing.T) { config := &unstructured.Unstructured{Object: map[string]any{ "apiVersion": "v1", "kind": "ConfigMap", "metadata": map[string]any{"name": configName}, }} - if err := applyMutator("ghcr.io/azure/gantry:test", "gantry-hash")(config); err != nil || config.Object != nil { + if err := applyMutator("ghcr.io/azure/gantry:test", "ghcr.io/azure/gantry-node-config:test", "gantry-hash", artifactStreamingConfig{})(config); err != nil || config.Object != nil { t.Fatalf("gantry ConfigMap was not skipped: err=%v object=%#v", err, config.Object) } } @@ -209,6 +209,7 @@ func TestOperatorManifestAllowlist(t *testing.T) { want := map[string]bool{ "configmap.yaml": true, "daemonset.yaml": true, + "overlaybd-config.yaml": true, "rendezvous-leases.yaml": true, "serviceaccount.yaml": true, } @@ -253,7 +254,7 @@ func TestApplyMutatorImagesOnlyAgentContainer(t *testing.T) { }}, }} - if err := applyMutator(derived, "h")(agent); err != nil { + if err := applyMutator(derived, "ghcr.io/azure/gantry-node-config:v1.2.3", "h", artifactStreamingConfig{})(agent); err != nil { t.Fatalf("applyMutator: %v", err) } @@ -280,7 +281,7 @@ func TestApplyMutatorImagesOnlyAgentContainer(t *testing.T) { }}, }} - if err := applyMutator(derived, "h")(nodeDS); err != nil { + if err := applyMutator(derived, "ghcr.io/azure/gantry-node-config:v1.2.3", "h", artifactStreamingConfig{})(nodeDS); err != nil { t.Fatalf("applyMutator node-config: %v", err) } @@ -289,6 +290,85 @@ func TestApplyMutatorImagesOnlyAgentContainer(t *testing.T) { } } +func TestApplyMutatorEnablesArtifactStreaming(t *testing.T) { + artifact := artifactStreamingConfig{ + Enabled: true, + NodeSelector: map[string]string{"kubernetes.azure.com/agentpool": "streaming"}, + } + + agent := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "metadata": map[string]any{"name": daemonSetName}, + "spec": map[string]any{"template": map[string]any{ + "metadata": map[string]any{}, + "spec": map[string]any{"containers": []any{ + map[string]any{"name": agentContainerName, "image": "placeholder"}, + }}, + }}, + }} + overlay := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "apps/v1", + "kind": "DaemonSet", + "metadata": map[string]any{"name": overlayBDConfigDaemonSetName}, + "spec": map[string]any{"template": map[string]any{ + "spec": map[string]any{ + "nodeSelector": map[string]any{}, + "containers": []any{ + map[string]any{"name": overlayBDConfigContainerName, "image": "placeholder"}, + }, + }, + }}, + }} + + mutate := applyMutator("gantry:test", "gantry-node-config:test", "hash", artifact) + if err := mutate(agent); err != nil { + t.Fatal(err) + } + + if err := mutate(overlay); err != nil { + t.Fatal(err) + } + + containers, _, err := unstructured.NestedSlice(agent.Object, "spec", "template", "spec", "containers") + if err != nil { + t.Fatal(err) + } + + variableFound := false + + for _, entry := range containers[0].(map[string]any)["env"].([]any) { + variable := entry.(map[string]any) + if variable["name"] == artifactStreamingEnabledEnv && variable["value"] == "true" { + variableFound = true + } + } + + if !variableFound { + t.Fatal("artifact streaming environment override missing") + } + + if image := containerImage(t, overlay, "containers", overlayBDConfigContainerName); image != "gantry-node-config:test" { + t.Fatalf("configurator image = %q", image) + } + + selector, _, err := unstructured.NestedStringMap(overlay.Object, "spec", "template", "spec", "nodeSelector") + if err != nil || selector["kubernetes.azure.com/agentpool"] != "streaming" { + t.Fatalf("selector = %#v, err=%v", selector, err) + } +} + +func TestResolveArtifactStreamingRejectsConflictingSites(t *testing.T) { + sites := []unboundedv1alpha3.Site{ + {ObjectMeta: metav1.ObjectMeta{Name: "a"}, Spec: unboundedv1alpha3.SiteSpec{Components: unboundedv1alpha3.SiteComponents{Gantry: &unboundedv1alpha3.GantryComponentSpec{ArtifactStreaming: &unboundedv1alpha3.GantryArtifactStreamingSpec{Enabled: true, NodeSelector: map[string]string{"pool": "a"}}}}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "b"}, Spec: unboundedv1alpha3.SiteSpec{Components: unboundedv1alpha3.SiteComponents{Gantry: &unboundedv1alpha3.GantryComponentSpec{ArtifactStreaming: &unboundedv1alpha3.GantryArtifactStreamingSpec{Enabled: true, NodeSelector: map[string]string{"pool": "b"}}}}}}, + } + + if _, err := resolveArtifactStreaming(sites); err == nil || !strings.Contains(err.Error(), "conflicts") { + t.Fatalf("error = %v, want selector conflict", err) + } +} + func containerImage(t *testing.T, obj *unstructured.Unstructured, field, name string) string { t.Helper() @@ -356,6 +436,52 @@ func TestReconcileAppliesCoreManifestsAndSkipsExamples(t *testing.T) { } } +func TestReconcileDeletesOverlayBDConfiguratorWhenArtifactStreamingDisabled(t *testing.T) { + configurator := &appsv1.DaemonSet{ObjectMeta: metav1.ObjectMeta{ + Namespace: component.DefaultNamespace, + Name: overlayBDConfigDaemonSetName, + }} + env, applied := reconcilerEnv(t, configurator) + + res := reconcile(t, env, []unboundedv1alpha3.Site{*siteWithGantry("edge", nil)}) + if !res.Ready || res.Err != nil { + t.Fatalf("Reconcile = %+v, want ready", res) + } + + if err := env.Client.Get(t.Context(), client.ObjectKeyFromObject(configurator), configurator); !apierrors.IsNotFound(err) { + t.Fatalf("disabled artifact streaming retained configurator: %v", err) + } + + if applied["DaemonSet/"+overlayBDConfigDaemonSetName] { + t.Fatal("disabled artifact streaming reapplied configurator") + } +} + +func TestPlanAppliesOverlayBDConfiguratorWhenArtifactStreamingEnabled(t *testing.T) { + gantryEnabled := true + site := siteWithGantry("edge", &gantryEnabled) + site.Spec.Components.Gantry.ArtifactStreaming = &unboundedv1alpha3.GantryArtifactStreamingSpec{ + Enabled: true, + NodeSelector: map[string]string{"pool": "streaming"}, + } + + plan, res, err := (Component{}).Plan(t.Context(), testEnv(t), []unboundedv1alpha3.Site{*site}) + if err != nil || !res.Ready { + t.Fatalf("Plan = (%+v, %v), want ready", res, err) + } + + summary := plan.Summary() + + want := "Apply DaemonSet/unbounded-system/" + overlayBDConfigDaemonSetName + if !strings.Contains(summary, want) { + t.Fatalf("plan does not apply enabled configurator:\n%s", summary) + } + + if strings.Contains(summary, "Delete DaemonSet/unbounded-system/"+overlayBDConfigDaemonSetName) { + t.Fatalf("plan deletes enabled configurator:\n%s", summary) + } +} + func TestReconcileFailsWhenLegacyNodeConfigCleanupFails(t *testing.T) { wantErr := errors.New("delete denied") scheme := testScheme(t) @@ -572,6 +698,7 @@ func TestPlanGolden(t *testing.T) { "ConfigMap/unbounded-system/gantry-containerd-hosts " + "ClusterRoleBinding/gantry-agent " + "ClusterRole/gantry-agent " + + "DaemonSet/unbounded-system/gantry-overlaybd-config " + "ConfigMap/unbounded-system/gantry-config]" var chairPlan strings.Builder @@ -583,6 +710,7 @@ func TestPlanGolden(t *testing.T) { Delete ConfigMap/unbounded-system/gantry-containerd-hosts Delete ClusterRoleBinding/gantry-agent Delete ClusterRole/gantry-agent +Delete DaemonSet/unbounded-system/gantry-overlaybd-config CreateIfAbsent ConfigMap/unbounded-system/gantry-config Apply DaemonSet/unbounded-system/gantry [overridable]` + after + ` ` + chairPlan.String() + `Apply PriorityClass/gantry-low` + after + ` @@ -621,6 +749,7 @@ func TestExecutionOrderGolden(t *testing.T) { } want := `Delete DaemonSet/unbounded-system/gantry-containerd-config +Delete DaemonSet/unbounded-system/gantry-overlaybd-config Delete ConfigMap/unbounded-system/gantry-containerd-hosts Delete ClusterRoleBinding/gantry-agent Delete ClusterRole/gantry-agent