Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions cmd/atenet/internal/router/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,8 @@ Ingress and egress are deployed separately today — `atenet-router` fronts the
ingress dataplane, `atenet-egress` the egress gateway — because the two scale
independently, not because they need separate binaries.

The `--atenet-router` choice only applies to the ingress dataplane. The egress
gateway is its own Deployment with a statically configured Envoy, so
`--atenet-router=agentgateway` leaves it untouched.
`--atenet-router` selects the dataplane for both Deployments. Each gateway has
its own static configuration because ingress and egress scale independently.

## status page

Expand Down
18 changes: 16 additions & 2 deletions cmd/atenet/internal/router/egress/egress.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ import (
)

const (
// agentgatewayClientCertificateAttribute is the PEM peer certificate agentgateway
// computes from the downstream TLS connection for ext_proc.
agentgatewayClientCertificateAttribute = "source.certificate"
// forwardedClientCertHeader is the header Envoy fills in with details of
// the mTLS peer, including the PEM chain it validated. The egress filter
// chain sets forward_client_cert_details: SANITIZE_SET, so whatever a
Expand Down Expand Up @@ -193,6 +196,13 @@ func (h *Handler) validateActor(ctx context.Context, identity *substratex509.Act
// on the request into a verified ActorIdentity, or an error describing why it
// cannot be trusted.
func (h *Handler) authenticateActorCertificate(md *extproc.RequestMetadata) (*substratex509.ActorIdentity, error) {
if certificate := md.Attribute(agentgatewayClientCertificateAttribute); certificate != "" {
chain, err := parseCertificateChainPEM([]byte(certificate))
if err != nil {
return nil, err
}
return h.verifyActorCertificate(chain)
}
header := md.Header(forwardedClientCertHeader)
if header == "" {
return nil, fmt.Errorf("request carries no %s header", forwardedClientCertHeader)
Expand Down Expand Up @@ -296,8 +306,12 @@ func parseXFCCChain(header string) ([]*x509.Certificate, error) {
return nil, fmt.Errorf("decoding the client certificate chain: %w", err)
}

return parseCertificateChainPEM([]byte(chainPEM))
}

func parseCertificateChainPEM(chainPEM []byte) ([]*x509.Certificate, error) {
var chain []*x509.Certificate
rest := []byte(chainPEM)
rest := chainPEM
for {
var block *pem.Block
block, rest = pem.Decode(rest)
Expand All @@ -314,7 +328,7 @@ func parseXFCCChain(header string) ([]*x509.Certificate, error) {
chain = append(chain, cert)
}
if len(chain) == 0 {
return nil, fmt.Errorf("%s carries no certificate", forwardedClientCertHeader)
return nil, fmt.Errorf("client certificate value carries no certificate")
}
return chain, nil
}
Expand Down
25 changes: 25 additions & 0 deletions cmd/atenet/internal/router/egress/egress_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/types/known/structpb"

"github.com/agent-substrate/substrate/cmd/atenet/internal/router/extproc"
"github.com/agent-substrate/substrate/internal/substratex509"
Expand Down Expand Up @@ -230,6 +231,19 @@ func egressMetadata(xfcc string) *extproc.RequestMetadata {
return extproc.NewRequestMetadata(headers, nil)
}

func agentgatewayEgressMetadata(certificate string) *extproc.RequestMetadata {
return extproc.NewRequestMetadata([]*corev3.HeaderValue{
{Key: ":method", RawValue: []byte("CONNECT")},
{Key: ":authority", RawValue: []byte("93.184.216.34:80")},
}, map[string]*structpb.Struct{
"envoy.filters.http.ext_proc": {
Fields: map[string]*structpb.Value{
agentgatewayClientCertificateAttribute: structpb.NewStringValue(certificate),
},
},
})
}

func wantStatus(t *testing.T, err error, want envoy_type.StatusCode) {
t.Helper()
if err == nil {
Expand Down Expand Up @@ -265,6 +279,17 @@ func TestHandleRequestHeadersAllowsVerifiedActor(t *testing.T) {
}
}

func TestHandleRequestHeadersAllowsAgentgatewayCertificateAttribute(t *testing.T) {
ca := newTestCA(t, "actor-identity-ca")
leaf := ca.issueActorCert(t, actorCertOptions{})
h := egressHandler(ca.roots(), runningActor(), nil)

certificate := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leaf.Raw})
if _, err := h.HandleRequestHeaders(context.Background(), agentgatewayEgressMetadata(string(certificate))); err != nil {
t.Fatalf("HandleRequestHeaders() error = %v, want nil", err)
}
}

// Every way an actor certificate can fail to prove an identity has to end in a
// denial, never in a tunnel.
func TestHandleRequestHeadersRejectsBadCertificates(t *testing.T) {
Expand Down
12 changes: 11 additions & 1 deletion cmd/atenet/internal/router/extproc/dispatch.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ const (
)

const (
// directionAttribute is set from a dataplane expression, not a client
// header, by dataplanes without Envoy filter chains.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

by dataplanes without Envoy filter chains.

who sets this request attribute, agw? Also I think we probably want some more specific namespace like ate.calllout(maybe a better key name?).direction

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, agentgateway sets this through the extProc requestAttributes config. I renamed the key to the more specific ate.extproc.direction.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I cant see the rename, did you commit it?

directionAttribute = "ate.extproc.direction"
// EgressFilterChainName is the Envoy filter chain that terminates actor
// egress CONNECTs, and so the one that selects the egress handler. It must
// stay in sync with the filter chain name in
Expand Down Expand Up @@ -63,6 +66,9 @@ const (
// an egress request misrouted to the ingress handler fails to parse as an actor
// DNS name and 404s, whereas the reverse leaks control-plane state.
func directionOf(req *extprocv3.ProcessingRequest) Direction {
if requestAttribute(req, directionAttribute) == string(DirectionEgress) {
return DirectionEgress
}
if filterChainName(req) == EgressFilterChainName {
return DirectionEgress
}
Expand All @@ -74,8 +80,12 @@ func directionOf(req *extprocv3.ProcessingRequest) Direction {
// attributes map is keyed by the ext_proc filter's name within the HCM chain,
// which we do not want to hardcode here, so scan every entry.
func filterChainName(req *extprocv3.ProcessingRequest) string {
return requestAttribute(req, FilterChainNameAttribute)
}

func requestAttribute(req *extprocv3.ProcessingRequest, name string) string {
for _, attrs := range req.GetAttributes() {
if v, ok := attrs.GetFields()[FilterChainNameAttribute]; ok {
if v, ok := attrs.GetFields()[name]; ok {
return v.GetStringValue()
}
}
Expand Down
14 changes: 14 additions & 0 deletions cmd/atenet/internal/router/extproc/dispatch_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,20 @@ func TestDirectionOf(t *testing.T) {
}
}

func TestDirectionOfAgentgatewayAttribute(t *testing.T) {
req := connectRequest("", "")
req.Attributes = map[string]*structpb.Struct{
"envoy.filters.http.ext_proc": {
Fields: map[string]*structpb.Value{
directionAttribute: structpb.NewStringValue(string(DirectionEgress)),
},
},
}
if got := directionOf(req); got != DirectionEgress {
t.Errorf("directionOf() = %v, want %v", got, DirectionEgress)
}
}

// A request the client dresses up to look like egress must not be enough: only
// the Envoy-asserted filter chain name selects the egress handler.
func TestDirectionOfIgnoresClientSuppliedAttributeHeader(t *testing.T) {
Expand Down
1 change: 0 additions & 1 deletion cmd/atenet/internal/router/extproc/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ func NewRequestMetadata(headers []*corev3.HeaderValue, attributes map[string]*st
method = val
}
}

return &RequestMetadata{
Headers: headersMap,
Path: path,
Expand Down
20 changes: 17 additions & 3 deletions hack/install-ate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ function usage() {
echo " --delete-ate-system Delete core system"
echo " --delete-all Delete core system and all registered demos"
echo " --ateapi-client-auth=cert|token Select how in-cluster clients authenticate to ateapi for --deploy-ate-system (default: cert; the server always accepts both)"
echo " --atenet-router=envoy|agentgateway Select the atenet router dataplane (default: envoy)"
echo " --atenet-router=envoy|agentgateway Select the ingress and egress dataplane (default: envoy)"

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

hmmm dont we want separate flags for that..? atenet-egress|atenet-ingress?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would prefer to keep one flag initially. We do not currently have a use case for mixing Envoy ingress with agentgateway egress or vice versa, and separate flags add configuration combinations we would need to support and test. We can split it later if that need appears.

echo " --store-backend=redis|postgres Configure the ateapi store backend (default: redis)"
echo " --otlp-endpoint URL Send all control plane telemetry to URL, not to the cluster default (see benchmarking/telemetry/README.md)"
echo ""
Expand Down Expand Up @@ -267,6 +267,15 @@ atenet_egress_manifest() {
fi
}

render_atenet_egress_manifest() {
if [[ "$(atenet_router)" == "agentgateway" ]]; then
kubectl kustomize manifests/ate-install/agentgateway-egress \
--load-restrictor LoadRestrictionsNone | run_ko resolve -f -
else
run_ko resolve -f "$(atenet_egress_manifest)"
fi
}

# Apply the ate-otel-config ConfigMap that every control plane component reads
# via envFrom. The full install gets it through render_ate_system_manifests, but
# the targeted single-component redeploys below apply raw manifests with no
Expand Down Expand Up @@ -452,6 +461,7 @@ create_egress_mitm_ca_pool_secret() {

# Only the sdsmint egress variant mounts this pool.
ensure_egress_mitm_ca_pool_secret() {
[[ "$(atenet_router)" != "agentgateway" ]] || return 0
[[ "${ATE_EXPERIMENTAL_USE_SDSMINT:-false}" == "true" ]] || return 0
run_kubectl get secret -n ate-system egress-mitm-ca-pool >/dev/null 2>&1 \
|| create_egress_mitm_ca_pool_secret
Expand Down Expand Up @@ -628,7 +638,9 @@ deploy_ate_system() {
# --experimental-use-sdsmint composes with every overlay instead of needing a
# variant of each.
ensure_egress_mitm_ca_pool_secret
run_ko apply -f "$(atenet_egress_manifest)"
local egress_manifests=""
egress_manifests="$(render_atenet_egress_manifest)"
echo "${egress_manifests}" | run_kubectl apply -f -

log_step "Waiting for ATE system components to be ready..."
case "$(store_backend)" in
Expand Down Expand Up @@ -726,7 +738,9 @@ deploy_atenet() {
echo "${router_manifest}" | run_kubectl apply -f -

ensure_egress_mitm_ca_pool_secret
run_ko apply -f "$(atenet_egress_manifest)"
local egress_manifests=""
egress_manifests="$(render_atenet_egress_manifest)"
echo "${egress_manifests}" | run_kubectl apply -f -
run_ko apply -f manifests/ate-install/atenet-dns.yaml
run_kubectl rollout status deployment/atenet-router -n ate-system --timeout=120s
run_kubectl rollout status deployment/atenet-egress -n ate-system --timeout=120s
Expand Down
40 changes: 19 additions & 21 deletions hack/verify-egress-demo.sh
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@
#
# The egress demo Actor accepts {"url":"..."} and performs an HTTP GET. With
# egress turned on, the Actor's outbound TCP is nftables-REDIRECTed
# into atunnel, wrapped in mTLS + HTTP CONNECT, and sent to the Envoy egress
# gateway, which terminates CONNECT and tunnels to the real destination. This
# script drives that path and shows the gateway's access log proving the actor's
# into atunnel, wrapped in mTLS + HTTP CONNECT, and sent to the egress gateway,
# which terminates CONNECT and tunnels to the real destination. This script
# drives that path and shows the ext_proc authentication log proving the actor's
# client certificate + CONNECT authority were seen.
set -o errexit -o nounset -o pipefail
ROOT="$(git rev-parse --show-toplevel)"; cd "${ROOT}"
Expand All @@ -42,10 +42,8 @@ kubectl-ate --context "${CTX}" create actor "${ACTOR}" \
--atespace "${ATESPACE}" --template ate-demo-egress/egress 2>/dev/null || true
${K} -n ate-system wait --for=condition=Ready "actor/${ACTOR}" 2>/dev/null || sleep 10

echo "== snapshot gateway log offset =="
# -c envoy explicitly: the gateway pod also runs the ext-proc sidecar, and the
# [egress] access log belongs to Envoy.
BEFORE=$(${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null | wc -l | tr -d ' ')
echo "== snapshot gateway authentication log offset =="
BEFORE=$(${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=-1 2>/dev/null | wc -l | tr -d ' ')

echo "== drive actor egress: GET ${TARGET_URL} via the actor =="
${K} -n ate-system port-forward service/atenet-router 18000:80 >/tmp/pf.log 2>&1 &
Expand All @@ -57,30 +55,30 @@ RESP=$(curl -s -o /dev/null -w "%{http_code}" -X POST http://localhost:18000/ \
-d "{\"url\":\"${TARGET_URL}\"}") || true
echo "actor round-trip HTTP ${RESP} (200 = the actor fetched ${TARGET_URL} through egress)"

echo "== NEW egress gateway access log lines (proof of CONNECT+mTLS+identity) =="
# Envoy emits the CONNECT entry asynchronously (an external dst can land seconds
# after the actor's response), so we poll. And the Actor's HTTP client keeps the
echo "== NEW egress authentication log lines (proof of CONNECT+mTLS+identity) =="
# The CONNECT can land seconds after the actor's response, so we poll. And the
# Actor's HTTP client keeps the
# tunnel alive: a repeat fetch to a host it already reached rides the open tunnel
# and produces no new access-log entry at all, so a run against a warm actor
# and produces no new authentication entry at all, so a run against a warm actor
# would fail even though egress is working. Fall back to any tunnel already open
# for this actor's SAN before declaring failure.
SAN="atespace/${ATESPACE}/actor/${ACTOR}"
# for this actor before declaring failure.
NEW=""
for _ in $(seq 1 15); do
NEW=$(${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null \
| tail -n +"$((BEFORE + 1))" | grep '\[egress\]' || true)
NEW=$(${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=-1 2>/dev/null \
| tail -n +"$((BEFORE + 1))" | grep 'egress identity authenticated' || true)
[ -n "${NEW}" ] && break
sleep 2
done
if [ -n "${NEW}" ]; then
echo "${NEW}"
elif ${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 2>/dev/null \
| grep -q "\[egress\].*${SAN}"; then
elif ${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=-1 2>/dev/null \
| grep 'egress identity authenticated' | grep -q "${ATESPACE}.*${ACTOR}\|${ACTOR}.*${ATESPACE}"; then
echo " no new CONNECT — the actor reused an already-open tunnel; its existing entries:"
${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=-1 | grep "\[egress\].*${SAN}" | tail -3
${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=-1 \
| grep 'egress identity authenticated' | grep "${ATESPACE}.*${ACTOR}\|${ACTOR}.*${ATESPACE}" | tail -3
else
echo "!! no [egress] lines for ${SAN} — dumping recent gateway logs:"
${K} -n ate-system logs deployment/atenet-egress -c envoy --tail=20
echo "!! no egress authentication lines for ${ATESPACE}/${ACTOR} — dumping recent ext_proc logs:"
${K} -n ate-system logs deployment/atenet-egress -c ext-proc --tail=20
exit 1
fi
echo "== PASS: actor egress traversed the Envoy egress gateway =="
echo "== PASS: actor egress traversed the egress gateway =="
22 changes: 22 additions & 0 deletions manifests/ate-install/agentgateway-egress/kustomization.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization

resources:
- ../atenet-egress.yaml

components:
- ../components/agentgateway
42 changes: 42 additions & 0 deletions manifests/ate-install/components/agentgateway/configmap.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -168,3 +168,45 @@ data:
key: /run/podidentity.podcert.ate.dev/credential-bundle.pem
root: /run/podidentity.podcert.ate.dev/trust-bundle.pem
insecureHost: true
---
apiVersion: v1
kind: ConfigMap
metadata:
name: atenet-egress-agentgateway-config
namespace: ate-system
data:
config.yaml: |
# yaml-language-server: $schema=https://agentgateway.dev/schema/config
frontendPolicies:
connect:
mode: route

gateways:
egress:
port: 8443
protocol: HTTPS
tls:
cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem
key: /run/servicedns.podcert.ate.dev/credential-bundle.pem
root: /run/actor-id-ca-certs/ca.crt

routes:
- name: substrate-egress
gateways:
- egress
policies:
extProc:
host: 127.0.0.1:50051
failureMode: failClosed
requestAttributes:
ate.extproc.direction: "'egress'"
source.certificate: source.certificate
processingOptions:
requestHeaderMode: send
responseHeaderMode: skip
requestBodyMode: none
responseBodyMode: none
requestTrailerMode: skip
responseTrailerMode: skip
backends:
- dynamic: {}
Loading
Loading