diff --git a/cmd/atenet/internal/router/README.md b/cmd/atenet/internal/router/README.md index d8ccf7a83..33f64bd2e 100644 --- a/cmd/atenet/internal/router/README.md +++ b/cmd/atenet/internal/router/README.md @@ -79,9 +79,8 @@ Ingress and egress are deployed separately today — `atenet-router` fronts the ingress dataplane, `atenet-egress` the egress gateway — because the two scale independently, not because they need separate binaries. -The `--atenet-router` choice only applies to the ingress dataplane. The egress -gateway is its own Deployment with a statically configured Envoy, so -`--atenet-router=agentgateway` leaves it untouched. +`--atenet-router` selects the dataplane for both Deployments. Each gateway has +its own static configuration because ingress and egress scale independently. ## status page diff --git a/cmd/atenet/internal/router/egress/egress.go b/cmd/atenet/internal/router/egress/egress.go index 0897e5455..dc3b621ad 100644 --- a/cmd/atenet/internal/router/egress/egress.go +++ b/cmd/atenet/internal/router/egress/egress.go @@ -48,6 +48,9 @@ import ( ) const ( + // agentgatewayClientCertificateAttribute is the PEM peer certificate agentgateway + // computes from the downstream TLS connection for ext_proc. + agentgatewayClientCertificateAttribute = "source.certificate" // forwardedClientCertHeader is the header Envoy fills in with details of // the mTLS peer, including the PEM chain it validated. The egress filter // chain sets forward_client_cert_details: SANITIZE_SET, so whatever a @@ -193,6 +196,13 @@ func (h *Handler) validateActor(ctx context.Context, identity *substratex509.Act // on the request into a verified ActorIdentity, or an error describing why it // cannot be trusted. func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (*substratex509.ActorIdentity, error) { + if certificate := md.Attribute(agentgatewayClientCertificateAttribute); certificate != "" { + chain, err := parseCertificateChainPEM([]byte(certificate)) + if err != nil { + return nil, err + } + return h.verifyActorCertificate(chain) + } header := md.Header(forwardedClientCertHeader) if header == "" { return nil, fmt.Errorf("request carries no %s header", forwardedClientCertHeader) @@ -296,8 +306,12 @@ func parseXFCCChain(header string) ([]*x509.Certificate, error) { return nil, fmt.Errorf("decoding the client certificate chain: %w", err) } + return parseCertificateChainPEM([]byte(chainPEM)) +} + +func parseCertificateChainPEM(chainPEM []byte) ([]*x509.Certificate, error) { var chain []*x509.Certificate - rest := []byte(chainPEM) + rest := chainPEM for { var block *pem.Block block, rest = pem.Decode(rest) @@ -314,7 +328,7 @@ func parseXFCCChain(header string) ([]*x509.Certificate, error) { chain = append(chain, cert) } if len(chain) == 0 { - return nil, fmt.Errorf("%s carries no certificate", forwardedClientCertHeader) + return nil, fmt.Errorf("client certificate value carries no certificate") } return chain, nil } diff --git a/cmd/atenet/internal/router/egress/egress_test.go b/cmd/atenet/internal/router/egress/egress_test.go index 9265ad774..7507472b5 100644 --- a/cmd/atenet/internal/router/egress/egress_test.go +++ b/cmd/atenet/internal/router/egress/egress_test.go @@ -37,6 +37,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" "github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc" "github.com/agent-substrate/substrate/internal/substratex509" @@ -230,6 +231,19 @@ func egressMetadata(xfcc string) *extproc.RequestMetadata { return extproc.NewRequestMetadata(headers, nil) } +func agentgatewayEgressMetadata(certificate string) *extproc.RequestMetadata { + return extproc.NewRequestMetadata([]*corev3.HeaderValue{ + {Key: ":method", RawValue: []byte("CONNECT")}, + {Key: ":authority", RawValue: []byte("93.184.216.34:80")}, + }, map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + agentgatewayClientCertificateAttribute: structpb.NewStringValue(certificate), + }, + }, + }) +} + func wantStatus(t *testing.T, err error, want envoy_type.StatusCode) { t.Helper() if err == nil { @@ -265,6 +279,17 @@ func TestHandleRequestHeadersAllowsVerifiedActor(t *testing.T) { } } +func TestHandleRequestHeadersAllowsAgentgatewayCertificateAttribute(t *testing.T) { + ca := newTestCA(t, "actor-identity-ca") + leaf := ca.issueActorCert(t, actorCertOptions{}) + h := egressHandler(ca.roots(), runningActor(), nil) + + certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leaf.Raw}) + if _, err := h.HandleRequestHeaders(context.Background(), agentgatewayEgressMetadata(string(certificate))); err != nil { + t.Fatalf("HandleRequestHeaders() error = %v, want nil", err) + } +} + // Every way an actor certificate can fail to prove an identity has to end in a // denial, never in a tunnel. func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) { diff --git a/cmd/atenet/internal/router/extproc/dispatch.go b/cmd/atenet/internal/router/extproc/dispatch.go index 52864d9cd..2da57f6de 100644 --- a/cmd/atenet/internal/router/extproc/dispatch.go +++ b/cmd/atenet/internal/router/extproc/dispatch.go @@ -31,6 +31,9 @@ const ( ) const ( + // directionAttribute is set from a dataplane expression, not a client + // header, by dataplanes without Envoy filter chains. + directionAttribute = "ate.extproc.direction" // EgressFilterChainName is the Envoy filter chain that terminates actor // egress CONNECTs, and so the one that selects the egress handler. It must // stay in sync with the filter chain name in @@ -63,6 +66,9 @@ const ( // an egress request misrouted to the ingress handler fails to parse as an actor // DNS name and 404s, whereas the reverse leaks control-plane state. func directionOf(req *extprocv3.ProcessingRequest) Direction { + if requestAttribute(req, directionAttribute) == string(DirectionEgress) { + return DirectionEgress + } if filterChainName(req) == EgressFilterChainName { return DirectionEgress } @@ -74,8 +80,12 @@ func directionOf(req *extprocv3.ProcessingRequest) Direction { // attributes map is keyed by the ext_proc filter's name within the HCM chain, // which we do not want to hardcode here, so scan every entry. func filterChainName(req *extprocv3.ProcessingRequest) string { + return requestAttribute(req, FilterChainNameAttribute) +} + +func requestAttribute(req *extprocv3.ProcessingRequest, name string) string { for _, attrs := range req.GetAttributes() { - if v, ok := attrs.GetFields()[FilterChainNameAttribute]; ok { + if v, ok := attrs.GetFields()[name]; ok { return v.GetStringValue() } } diff --git a/cmd/atenet/internal/router/extproc/dispatch_test.go b/cmd/atenet/internal/router/extproc/dispatch_test.go index 83252eb6d..fe43dd8f5 100644 --- a/cmd/atenet/internal/router/extproc/dispatch_test.go +++ b/cmd/atenet/internal/router/extproc/dispatch_test.go @@ -118,6 +118,20 @@ func TestDirectionOf(t *testing.T) { } } +func TestDirectionOfAgentgatewayAttribute(t *testing.T) { + req := connectRequest("", "") + req.Attributes = map[string]*structpb.Struct{ + "envoy.filters.http.ext_proc": { + Fields: map[string]*structpb.Value{ + directionAttribute: structpb.NewStringValue(string(DirectionEgress)), + }, + }, + } + if got := directionOf(req); got != DirectionEgress { + t.Errorf("directionOf() = %v, want %v", got, DirectionEgress) + } +} + // A request the client dresses up to look like egress must not be enough: only // the Envoy-asserted filter chain name selects the egress handler. func TestDirectionOfIgnoresClientSuppliedAttributeHeader(t *testing.T) { diff --git a/cmd/atenet/internal/router/extproc/metadata.go b/cmd/atenet/internal/router/extproc/metadata.go index 75779f99c..a23d41829 100644 --- a/cmd/atenet/internal/router/extproc/metadata.go +++ b/cmd/atenet/internal/router/extproc/metadata.go @@ -69,7 +69,6 @@ func NewRequestMetadata(headers []*corev3.HeaderValue, attributes map[string]*st method = val } } - return &RequestMetadata{ Headers: headersMap, Path: path, diff --git a/hack/install-ate.sh b/hack/install-ate.sh index 0eb9bae2f..febd8fae4 100755 --- a/hack/install-ate.sh +++ b/hack/install-ate.sh @@ -67,7 +67,7 @@ function usage() { echo " --delete-ate-system Delete core system" echo " --delete-all Delete core system and all registered demos" echo " --ateapi-client-auth=cert|token Select how in-cluster clients authenticate to ateapi for --deploy-ate-system (default: cert; the server always accepts both)" - echo " --atenet-router=envoy|agentgateway Select the atenet router dataplane (default: envoy)" + echo " --atenet-router=envoy|agentgateway Select the ingress and egress dataplane (default: envoy)" echo " --store-backend=redis|postgres Configure the ateapi store backend (default: redis)" echo " --otlp-endpoint URL Send all control plane telemetry to URL, not to the cluster default (see benchmarking/telemetry/README.md)" echo "" @@ -267,6 +267,15 @@ atenet_egress_manifest() { fi } +render_atenet_egress_manifest() { + if [[ "$(atenet_router)" == "agentgateway" ]]; then + kubectl kustomize manifests/ate-install/agentgateway-egress \ + --load-restrictor LoadRestrictionsNone | run_ko resolve -f - + else + run_ko resolve -f "$(atenet_egress_manifest)" + fi +} + # Apply the ate-otel-config ConfigMap that every control plane component reads # via envFrom. The full install gets it through render_ate_system_manifests, but # the targeted single-component redeploys below apply raw manifests with no @@ -452,6 +461,7 @@ create_egress_mitm_ca_pool_secret() { # Only the sdsmint egress variant mounts this pool. ensure_egress_mitm_ca_pool_secret() { + [[ "$(atenet_router)" != "agentgateway" ]] || return 0 [[ "${ATE_EXPERIMENTAL_USE_SDSMINT:-false}" == "true" ]] || return 0 run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \ || create_egress_mitm_ca_pool_secret @@ -628,7 +638,9 @@ deploy_ate_system() { # --experimental-use-sdsmint composes with every overlay instead of needing a # variant of each. ensure_egress_mitm_ca_pool_secret - run_ko apply -f "$(atenet_egress_manifest)" + local egress_manifests="" + egress_manifests="$(render_atenet_egress_manifest)" + echo "${egress_manifests}" | run_kubectl apply -f - log_step "Waiting for ATE system components to be ready..." case "$(store_backend)" in @@ -726,7 +738,9 @@ deploy_atenet() { echo "${router_manifest}" | run_kubectl apply -f - ensure_egress_mitm_ca_pool_secret - run_ko apply -f "$(atenet_egress_manifest)" + local egress_manifests="" + egress_manifests="$(render_atenet_egress_manifest)" + echo "${egress_manifests}" | run_kubectl apply -f - run_ko apply -f manifests/ate-install/atenet-dns.yaml run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout=120s diff --git a/hack/verify-egress-demo.sh b/hack/verify-egress-demo.sh index 19c69cfd9..d37eaa9b9 100755 --- a/hack/verify-egress-demo.sh +++ b/hack/verify-egress-demo.sh @@ -20,9 +20,9 @@ # # The egress demo Actor accepts {"url":"..."} and performs an HTTP GET. With # egress turned on, the Actor's outbound TCP is nftables-REDIRECTed -# into atunnel, wrapped in mTLS + HTTP CONNECT, and sent to the Envoy egress -# gateway, which terminates CONNECT and tunnels to the real destination. This -# script drives that path and shows the gateway's access log proving the actor's +# into atunnel, wrapped in mTLS + HTTP CONNECT, and sent to the egress gateway, +# which terminates CONNECT and tunnels to the real destination. This script +# drives that path and shows the ext_proc authentication log proving the actor's # client certificate + CONNECT authority were seen. set -o errexit -o nounset -o pipefail ROOT="$(git rev-parse --show-toplevel)"; cd "${ROOT}" @@ -42,10 +42,8 @@ kubectl-ate --context "${CTX}" create actor "${ACTOR}" \ --atespace "${ATESPACE}" --template ate-demo-egress/egress 2>/dev/null || true ${K} -n ate-system wait --for=condition=Ready "actor/${ACTOR}" 2>/dev/null || sleep 10 -echo "== snapshot gateway log offset ==" -# -c envoy explicitly: the gateway pod also runs the ext-proc sidecar, and the -# [egress] access log belongs to Envoy. -BEFORE=$(${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null | wc -l | tr -d ' ') +echo "== snapshot gateway authentication log offset ==" +BEFORE=$(${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=-1 2>/dev/null | wc -l | tr -d ' ') echo "== drive actor egress: GET ${TARGET_URL} via the actor ==" ${K} -n ate-system port-forward service/atenet-router 18000:80 >/tmp/pf.log 2>&1 & @@ -57,30 +55,30 @@ RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://localhost:18000/ \ -d "{\"url\":\"${TARGET_URL}\"}") || true echo "actor round-trip HTTP ${RESP} (200 = the actor fetched ${TARGET_URL} through egress)" -echo "== NEW egress gateway access log lines (proof of CONNECT+mTLS+identity) ==" -# Envoy emits the CONNECT entry asynchronously (an external dst can land seconds -# after the actor's response), so we poll. And the Actor's HTTP client keeps the +echo "== NEW egress authentication log lines (proof of CONNECT+mTLS+identity) ==" +# The CONNECT can land seconds after the actor's response, so we poll. And the +# Actor's HTTP client keeps the # tunnel alive: a repeat fetch to a host it already reached rides the open tunnel -# and produces no new access-log entry at all, so a run against a warm actor +# and produces no new authentication entry at all, so a run against a warm actor # would fail even though egress is working. Fall back to any tunnel already open -# for this actor's SAN before declaring failure. -SAN="atespace/${ATESPACE}/actor/${ACTOR}" +# for this actor before declaring failure. NEW="" for _ in $(seq 1 15); do - NEW=$(${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null \ - | tail -n +"$((BEFORE + 1))" | grep '\[egress\]' || true) + NEW=$(${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=-1 2>/dev/null \ + | tail -n +"$((BEFORE + 1))" | grep 'egress identity authenticated' || true) [ -n "${NEW}" ] && break sleep 2 done if [ -n "${NEW}" ]; then echo "${NEW}" -elif ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null \ - | grep -q "\[egress\].*${SAN}"; then +elif ${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=-1 2>/dev/null \ + | grep 'egress identity authenticated' | grep -q "${ATESPACE}.*${ACTOR}\|${ACTOR}.*${ATESPACE}"; then echo " no new CONNECT — the actor reused an already-open tunnel; its existing entries:" - ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 | grep "\[egress\].*${SAN}" | tail -3 + ${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=-1 \ + | grep 'egress identity authenticated' | grep "${ATESPACE}.*${ACTOR}\|${ACTOR}.*${ATESPACE}" | tail -3 else - echo "!! no [egress] lines for ${SAN} — dumping recent gateway logs:" - ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=20 + echo "!! no egress authentication lines for ${ATESPACE}/${ACTOR} — dumping recent ext_proc logs:" + ${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=20 exit 1 fi -echo "== PASS: actor egress traversed the Envoy egress gateway ==" +echo "== PASS: actor egress traversed the egress gateway ==" diff --git a/manifests/ate-install/agentgateway-egress/kustomization.yaml b/manifests/ate-install/agentgateway-egress/kustomization.yaml new file mode 100644 index 000000000..046293535 --- /dev/null +++ b/manifests/ate-install/agentgateway-egress/kustomization.yaml @@ -0,0 +1,22 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../atenet-egress.yaml + +components: + - ../components/agentgateway diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 9b6b735ba..157f2c33b 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -168,3 +168,45 @@ data: key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/podidentity.podcert.ate.dev/trust-bundle.pem insecureHost: true +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: atenet-egress-agentgateway-config + namespace: ate-system +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + frontendPolicies: + connect: + mode: route + + gateways: + egress: + port: 8443 + protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + root: /run/actor-id-ca-certs/ca.crt + + routes: + - name: substrate-egress + gateways: + - egress + policies: + extProc: + host: 127.0.0.1:50051 + failureMode: failClosed + requestAttributes: + ate.extproc.direction: "'egress'" + source.certificate: source.certificate + processingOptions: + requestHeaderMode: send + responseHeaderMode: skip + requestBodyMode: none + responseBodyMode: none + requestTrailerMode: skip + responseTrailerMode: skip + backends: + - dynamic: {} diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index d1cfddc96..8511f195e 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -19,7 +19,12 @@ resources: - configmap.yaml patches: - - patch: |- + - target: + version: v1 + kind: ConfigMap + name: atenet-router-envoy-config + namespace: ate-system + patch: |- apiVersion: v1 kind: ConfigMap metadata: @@ -82,7 +87,13 @@ patches: mountPath: /run/servicedns.podcert.ate.dev - name: podidentity mountPath: /run/podidentity.podcert.ate.dev - - patch: |- + - target: + group: apps + version: v1 + kind: Deployment + name: atenet-router + namespace: ate-system + patch: |- apiVersion: apps/v1 kind: Deployment metadata: @@ -95,3 +106,52 @@ patches: - name: envoy-config configMap: name: atenet-router-agentgateway-config + - target: + group: apps + version: v1 + kind: Deployment + name: atenet-egress + namespace: ate-system + patch: |- + - op: replace + path: /spec/template/spec/containers/0 + value: + name: agentgateway + image: cr.agentgateway.dev/agentgateway:v1.4.1 + args: + - -f + - /etc/agentgateway/config.yaml + ports: + - name: https + containerPort: 8443 + - name: readiness + containerPort: 15021 + - name: stats + containerPort: 15020 + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + startupProbe: + failureThreshold: 60 + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 1 + volumeMounts: + - name: config + mountPath: /etc/agentgateway + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - op: add + path: /spec/template/spec/containers/1/args/- + value: --atenet-router=agentgateway + - op: replace + path: /spec/template/spec/volumes/0/configMap/name + value: atenet-egress-agentgateway-config