Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions .github/workflows/pr-workflow.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,30 @@ jobs:
env:
E2E_SANDBOX_CLASS: microvm
run: hack/run-e2e-kind.sh -v -args --no-color
- name: Deploy MITM egress (sdsmint)
# Swap the passthrough egress gateway for the sdsmint variant, which
# mints per-SNI leaves from the egress-mitm-ca-pool (created here if
# missing). Cluster-wide, so it must come AFTER the standard lanes:
# once egress TLS is intercepted, their passthrough assumptions
# (TestActorEgressHTTPS's end-to-end TLS with the origin) no longer hold.
run: hack/install-ate-kind.sh --deploy-ate-system --experimental-use-sdsmint
- name: Run E2E tests (egress MITM trust)
# The consumption half of the trust-bundle chain: an actor does TLS with
# the MITM gateway's minted leaf using ONLY the projected bundle, plus a
# system-roots negative control proving interception is real (see
# internal/e2e/suites/egressmitm).
env:
E2E_EGRESS_MITM: "1"
run: hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -v -args --no-color
- name: Run E2E tests (egress MITM trust, micro-VM)
# The same proof with the probe on the micro-VM runtime. Trust DELIVERY
# differs per sandbox class (gVisor RO bind vs the micro-VM unified
# virtio-fs share), so the handshake is proven on both. Uses the
# micro-VM deps staged earlier in this job.
env:
E2E_EGRESS_MITM: "1"
E2E_SANDBOX_CLASS: microvm
run: hack/run-e2e-kind.sh ./internal/e2e/suites/egressmitm -v -args --no-color
- name: Dump diagnostics on failure
if: failure()
run: |
Expand Down
3 changes: 2 additions & 1 deletion cmd/ateapi/internal/controlapi/functionaltest/common_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu
ateletFactory, ateletInformer := controlapi.AteletInformer(k8sClient)
scFactory := informers.NewSharedInformerFactory(k8sClient, 0)
scLister := scFactory.Storage().V1().StorageClasses().Lister()
ctbLister := scFactory.Certificates().V1beta1().ClusterTrustBundles().Lister()

substrateInformerFactory := externalversions.NewSharedInformerFactory(substrateClient, 0)
actorTemplateLister := substrateInformerFactory.Api().V1alpha1().ActorTemplates().Lister()
Expand Down Expand Up @@ -191,7 +192,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu
mockDriverName: mockPlugin,
}
}
service := controlapi.NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, dialer, instruments, "", volPlugins)
service := controlapi.NewService(persistence, wc, actorTemplateLister, workerPoolLister, sandboxConfigLister, csiDriverConfigLister, scLister, ctbLister, dialer, instruments, "", volPlugins)

// 5. Start REAL gRPC Server for ATE API
grpcServer := grpc.NewServer(grpc.UnaryInterceptor(ateinterceptors.ServerUnaryInterceptor))
Expand Down
4 changes: 3 additions & 1 deletion cmd/ateapi/internal/controlapi/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"github.com/agent-substrate/substrate/internal/volume/csi"
listersv1alpha1 "github.com/agent-substrate/substrate/pkg/client/listers/api/v1alpha1"
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
certlisters "k8s.io/client-go/listers/certificates/v1beta1"
storagev1listers "k8s.io/client-go/listers/storage/v1"
)

Expand Down Expand Up @@ -60,6 +61,7 @@ func NewService(
sandboxConfigLister listersv1alpha1.SandboxConfigLister,
csiDriverConfigLister listersv1alpha1.CSIDriverConfigLister,
storageClassLister storagev1listers.StorageClassLister,
clusterTrustBundleLister certlisters.ClusterTrustBundleLister,
dialer *AteletDialer,
instruments *Instruments,
egressGatewayAddress string,
Expand All @@ -76,7 +78,7 @@ func NewService(
instruments: instruments,
volumePlugins: volumePlugins,
}
s.actorWorkflow = NewActorWorkflow(persistence, workerCache, dialer, actorTemplateLister, workerPoolLister, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s)
s.actorWorkflow = NewActorWorkflow(persistence, workerCache, dialer, actorTemplateLister, workerPoolLister, sandboxConfigLister, storageClassLister, clusterTrustBundleLister, instruments, egressGatewayAddress, s)
return s
}

Expand Down
153 changes: 153 additions & 0 deletions cmd/ateapi/internal/controlapi/trust_bundle.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controlapi

import (
"fmt"
"slices"
"strings"

"github.com/agent-substrate/substrate/internal/pemutil"
"github.com/agent-substrate/substrate/internal/proto/ateletpb"
atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
certlisters "k8s.io/client-go/listers/certificates/v1beta1"
)

// EgressTrustBundleName is the well-known name of the egress gateway CA
// bundle (#823): the trust anchors for the per-SNI leaves the egress gateway
// mints, maintained by atecontroller from the egress-mitm-ca-pool.
const EgressTrustBundleName = "egress-mitm.ate.dev"

// supportedTrustBundles is the allowlist of bundle names the trustBundle
// SystemInfo data source may reference, mapping each to the Kubernetes
// ClusterTrustBundle object backing it. The template API deliberately names
// a bundle without saying where it comes from; this map is where substrate
// decides that. It is enforced here rather than in the CRD schema so a
// future configurable backend registry (#932) widens it without a template
// API change.
//
// The egress bundle's backing object is named by atecontroller's
// EgressMITMTrustReconciler, which derives it from the egress-mitm-ca-pool
// Secret; the k8s signer-linked naming convention
// (<signer-domain>:<signer-path>:<name>) is a backend detail the template
// name deliberately does not leak.
var supportedTrustBundles = map[string]string{
EgressTrustBundleName: "egress-mitm.ate.dev:mitm:primary-bundle",
}

// supportedTrustBundleNames returns the allowlist, sorted, for error text.
func supportedTrustBundleNames() string {
names := make([]string, 0, len(supportedTrustBundles))
for name := range supportedTrustBundles {
names = append(names, name)
}
slices.Sort(names)
return strings.Join(names, ", ")
}

// resolveTrustBundles fills the PemBundle bytes of every trustBundle data
// source in workloadSpec, resolving each named bundle through its backend and
// sanitizing its PEM the way kubelet does for projections (CERTIFICATE
// blocks only, deduplicated).
//
// The wire spec carries only {path, pem_bundle}; the bundle NAME lives in the
// ActorTemplate, so resolution walks the template and locates the matching
// wire entry by volume name + path. It must run on the resume path, before
// the spec is sent to atelet: an unsupported name, an unavailable backend,
// or a missing, empty, or unparseable bundle fails the actor start with an
// error naming the bundle, per the SystemInfo volume contract. atelet never
// talks to any bundle backend.
func resolveTrustBundles(lister certlisters.ClusterTrustBundleLister, template *atev1alpha1.ActorTemplate, workloadSpec *ateletpb.WorkloadSpec) error {
if template == nil {
return nil
}

// Wire entries indexed by (volume name, path) for filling in place.
type key struct{ volume, path string }
wire := map[key]*ateletpb.TrustBundleDataSource{}
for _, vol := range workloadSpec.GetVolumes() {
for _, ds := range vol.GetSystemInfo().GetDataSources() {
if tb := ds.GetTrustBundle(); tb != nil {
wire[key{vol.GetName(), tb.GetPath()}] = tb
}
}
}
if len(wire) == 0 {
return nil
}

// One resolution per distinct bundle name, shared across references.
resolved := map[string][]byte{}
for _, vol := range template.Spec.Volumes {
if vol.VolumeSource.SystemInfo == nil {
continue
}
for _, ds := range vol.VolumeSource.SystemInfo.DataSources {
if ds.TrustBundle == nil {
continue
}
name := ds.TrustBundle.Name
objectName, supported := supportedTrustBundles[name]
if !supported {
return fmt.Errorf("trust bundle %q is not supported by this deployment (supported: %s; referenced by volume %q)", name, supportedTrustBundleNames(), vol.Name)
}

pemBundle, ok := resolved[name]
if !ok {
var err error
pemBundle, err = fetchClusterTrustBundle(lister, objectName)
if err != nil {
return fmt.Errorf("trust bundle %q (referenced by volume %q): %w", name, vol.Name, err)
}
resolved[name] = pemBundle
}

entry, ok := wire[key{vol.Name, ds.TrustBundle.Path}]
if !ok {
// The wire spec is built from this same template, so a missing
// entry means the two views diverged — a bug, not user error.
return fmt.Errorf("internal error: no wire entry for trust bundle %q at volume %q path %q", name, vol.Name, ds.TrustBundle.Path)
}
entry.PemBundle = pemBundle
}
}
return nil
}

// fetchClusterTrustBundle is the Kubernetes backend for supported trust
// bundles: the bundle is read from the ClusterTrustBundle
// (certificates.k8s.io/v1beta1) named objectName, per the allowlist mapping.
// This is currently the only backend; the eventual backend registry (#932)
// slots in here.
func fetchClusterTrustBundle(lister certlisters.ClusterTrustBundleLister, objectName string) ([]byte, error) {
// A nil lister means ateapi found the ClusterTrustBundle API unavailable
// at startup (certificates.k8s.io/v1beta1 is feature-gated; see
// cmd/ateapi/main.go). Fail the start rather than panic on the lister.
if lister == nil {
return nil, fmt.Errorf("bundle backend unavailable in this deployment (the cluster does not serve certificates.k8s.io/v1beta1)")
}
bundle, err := lister.Get(objectName)
if apierrors.IsNotFound(err) {
return nil, fmt.Errorf("ClusterTrustBundle %q not found", objectName)
} else if err != nil {
return nil, fmt.Errorf("while reading ClusterTrustBundle %q: %w", objectName, err)
}
pemBundle, err := pemutil.SanitizeCertificateBundle([]byte(bundle.Spec.TrustBundle))
if err != nil {
return nil, fmt.Errorf("unusable trust bundle: %w", err)
}
return pemBundle, nil
}
169 changes: 169 additions & 0 deletions cmd/ateapi/internal/controlapi/trust_bundle_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package controlapi

import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"math/big"
"strings"
"testing"
"time"

atev1alpha1 "github.com/agent-substrate/substrate/pkg/api/v1alpha1"
certsv1beta1 "k8s.io/api/certificates/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
certlisters "k8s.io/client-go/listers/certificates/v1beta1"
"k8s.io/client-go/tools/cache"
)

// testCertPEM mints a throwaway self-signed certificate, PEM-encoded.
func testCertPEM(t *testing.T) []byte {
t.Helper()
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
t.Fatal(err)
}
der, err := x509.CreateCertificate(rand.Reader, &x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{CommonName: "test"},
NotBefore: time.Now(),
NotAfter: time.Now().Add(time.Hour),
}, &x509.Certificate{SerialNumber: big.NewInt(1)}, &key.PublicKey, key)
if err != nil {
t.Fatal(err)
}
return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
}

func ctbLister(t *testing.T, bundles ...*certsv1beta1.ClusterTrustBundle) certlisters.ClusterTrustBundleLister {
t.Helper()
indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{})
for _, b := range bundles {
if err := indexer.Add(b); err != nil {
t.Fatal(err)
}
}
return certlisters.NewClusterTrustBundleLister(indexer)
}

func trustBundleTemplate(volumeName, bundleName, path string) *atev1alpha1.ActorTemplate {
return &atev1alpha1.ActorTemplate{
Spec: atev1alpha1.ActorTemplateSpec{
Volumes: []atev1alpha1.Volume{{
Name: volumeName,
VolumeSource: atev1alpha1.VolumeSource{
SystemInfo: &atev1alpha1.SystemInfoVolumeSource{
DataSources: []atev1alpha1.SystemInfoDataSource{
{TrustBundle: &atev1alpha1.TrustBundleDataSource{Name: bundleName, Path: path}},
},
},
},
}},
},
}
}

// egressTrustBundleObjectName is the backing ClusterTrustBundle the allowlist
// maps EgressTrustBundleName to (named by atecontroller's reconciler).
const egressTrustBundleObjectName = "egress-mitm.ate.dev:mitm:primary-bundle"

func TestResolveTrustBundles(t *testing.T) {
certPEM := testCertPEM(t)
junk := "garbage\n" + string(pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: []byte("x")}))

template := trustBundleTemplate("system-info", EgressTrustBundleName, "trust/ca.pem")
spec, err := workloadSpecFromActorTemplate(template, nil)
if err != nil {
t.Fatalf("workloadSpecFromActorTemplate: %v", err)
}

t.Run("resolves and sanitizes into the wire spec", func(t *testing.T) {
lister := ctbLister(t, &certsv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{Name: egressTrustBundleObjectName},
// Junk around the certificate proves kubelet-style sanitization:
// only the CERTIFICATE block survives, and the duplicate is dropped.
Spec: certsv1beta1.ClusterTrustBundleSpec{TrustBundle: junk + string(certPEM) + string(certPEM)},
})
if err := resolveTrustBundles(lister, template, spec); err != nil {
t.Fatalf("resolveTrustBundles: %v", err)
}
got := spec.GetVolumes()[0].GetSystemInfo().GetDataSources()[0].GetTrustBundle()
if got.GetPath() != "trust/ca.pem" {
t.Errorf("path = %q, want %q", got.GetPath(), "trust/ca.pem")
}
if string(got.GetPemBundle()) != string(certPEM) {
t.Errorf("pem bundle = %q, want the sanitized certificate", got.GetPemBundle())
}
})

t.Run("unsupported bundle name fails naming it and the allowlist", func(t *testing.T) {
other := trustBundleTemplate("system-info", "my-own-bundle", "trust/ca.pem")
spec, _ := workloadSpecFromActorTemplate(other, nil)
// The lister has the bundle; the allowlist must still reject it —
// supported names are a substrate decision, not a cluster lookup.
lister := ctbLister(t, &certsv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{Name: "my-own-bundle"},
Spec: certsv1beta1.ClusterTrustBundleSpec{TrustBundle: string(certPEM)},
})
err := resolveTrustBundles(lister, other, spec)
if err == nil || !strings.Contains(err.Error(), `"my-own-bundle"`) || !strings.Contains(err.Error(), "not supported") || !strings.Contains(err.Error(), EgressTrustBundleName) {
t.Errorf("error = %v, want unsupported-name error listing the allowlist", err)
}
})

t.Run("missing bundle fails naming it", func(t *testing.T) {
spec, _ := workloadSpecFromActorTemplate(template, nil)
err := resolveTrustBundles(ctbLister(t), template, spec)
if err == nil || !strings.Contains(err.Error(), egressTrustBundleObjectName) || !strings.Contains(err.Error(), "not found") {
t.Errorf("error = %v, want not-found naming the bundle", err)
}
})

t.Run("unusable bundle fails naming it", func(t *testing.T) {
spec, _ := workloadSpecFromActorTemplate(template, nil)
lister := ctbLister(t, &certsv1beta1.ClusterTrustBundle{
ObjectMeta: metav1.ObjectMeta{Name: egressTrustBundleObjectName},
Spec: certsv1beta1.ClusterTrustBundleSpec{TrustBundle: junk},
})
err := resolveTrustBundles(lister, template, spec)
if err == nil || !strings.Contains(err.Error(), EgressTrustBundleName) || !strings.Contains(err.Error(), "unusable") {
t.Errorf("error = %v, want unusable-bundle naming the bundle", err)
}
})

t.Run("no trustBundle sources is a no-op even with a nil lister", func(t *testing.T) {
plain := &atev1alpha1.ActorTemplate{}
spec, _ := workloadSpecFromActorTemplate(plain, nil)
if err := resolveTrustBundles(nil, plain, spec); err != nil {
t.Fatalf("resolveTrustBundles: %v", err)
}
})

t.Run("trustBundle sources with a nil lister fail with a clear error, not a panic", func(t *testing.T) {
// nil lister = ateapi booted on a cluster without the feature-gated
// ClusterTrustBundle API (see cmd/ateapi/main.go) — the initial
// Kubernetes backend is unavailable in this deployment.
spec, _ := workloadSpecFromActorTemplate(template, nil)
err := resolveTrustBundles(nil, template, spec)
if err == nil || !strings.Contains(err.Error(), "backend unavailable in this deployment") {
t.Errorf("error = %v, want backend-unavailable error", err)
}
})
}
Loading
Loading