diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ef6038d..1b78fed 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -242,6 +242,101 @@ sequenceDiagram end ``` +## Decofile Delivery: S3 Target (etcd Offload) + +Content-heavy sites (many/large blocks — e.g. a blog with full-HTML posts inline) +can push the merged decofile's brotli+base64 size past the ~1MB etcd ConfigMap +ceiling, at which point the Kubernetes API server rejects the ConfigMap write and +the site can no longer publish. `spec.target: "s3"` is an **opt-in, per-site** +alternative: the operator uploads the decofile to S3 as plain JSON and points +pods at it over HTTP instead of mounting a ConfigMap. Fully reversible — flip +`target` back to `configmap` (the default) and the next reconcile reverts. + +### Flow 7a: Cold Start (target=s3) + +```mermaid +sequenceDiagram + participant User + participant K8s as Kubernetes API + participant Ctrl as Decofile Controller + participant S3 as S3 Bucket + participant Webhook as Mutating Webhook + participant Pod as Site Pod + + User->>K8s: Create/Update Decofile (spec.target: s3) + K8s->>Ctrl: Reconcile Event + Ctrl->>Ctrl: Retrieve source (github/inline) → JSON + Ctrl->>Ctrl: SHA-256 hash content + alt Hash unchanged (& commit unchanged for github) + Ctrl->>Ctrl: Skip upload — nothing to do + else Hash changed + Ctrl->>S3: PutObject (raw JSON, no compression) + Ctrl->>K8s: Update Status (ContentHash, S3URL) + end + + Note over User,Pod: Separately — Service creation/update + User->>K8s: Create Knative Service (deco.sites/decofile-inject) + K8s->>Webhook: Intercept CREATE/UPDATE + Webhook->>K8s: Get Decofile (target=s3) + Webhook->>Webhook: Derive S3 key + URL (same formula as controller —
Decofile.S3ObjectKey(), no dependency on reconcile order) + Webhook->>Webhook: Set env DECO_RELEASE=https://host/key
(NO volume/volumeMount injected) + Webhook->>Webhook: Merge host into DECO_ALLOWED_AUTHORITIES
(preserves runtime defaults if unset) + K8s->>Pod: Create Pod with injected env + Pod->>S3: GET https://host/key (plain HTTPS, no AWS auth) + S3->>Pod: 200 OK — decofile JSON + Pod->>Pod: fetch().then(JSON.parse) — no brotli/base64 decode +``` + +### Flow 7b: Hot Reload (target=s3) + +Identical to **Flow 4** (Change Notification) — the s3 target reuses +`NotifyPodsForDecofile` unchanged. The only difference from the ConfigMap path +is *what* changed (an S3 object, not a ConfigMap) and that the reconciler +pushes the **uncompressed** JSON in the notify payload (same as it always has — +`NotifyPodsForDecofile` was never brotli-aware; compression is a ConfigMap-side +concern only). Pods that implement `/.decofile/reload` don't need to know which +target delivered their *next* update; only the *initial* fetch path (mounted +file vs. HTTP GET) differs. + +### s3 vs configmap: what changes + +| | `configmap` (default) | `s3` | +|---|---|---| +| Storage | ConfigMap key `decofile.bin` (brotli+base64) | S3 object (plain JSON) | +| Size ceiling | ~1MB (etcd) | none (S3 object limit is 5TB) | +| Cold start | Volume mount, `DECO_RELEASE=file://…` | HTTP env only, `DECO_RELEASE=https://…` | +| Change detection | ConfigMap data diff | SHA-256 content hash (`Status.ContentHash`) | +| Hot reload | `NotifyPodsForDecofile` (unchanged) | `NotifyPodsForDecofile` (unchanged) | +| GC | Owned by Decofile (owner ref) — cascades | **Not GC'd** — a deleted Decofile leaves its S3 object; rely on a bucket lifecycle rule | +| `DECO_ALLOWED_AUTHORITIES` | untouched | webhook merges the S3/CDN host in, preserving runtime defaults | + +### Design decisions + +- **No compression.** The runtime's HTTP decofile provider + (`deco/engine/decofile/fetcher.ts`) does `fetch() → JSON.parse()` with no + brotli/base64 decode — that path only exists for `file://….bin`. Serving + compressed bytes would require a runtime change; S3 has no size pressure + forcing that trade, so the s3 target serves raw JSON. (`gzip` + + `Content-Encoding` — which `fetch` auto-decodes — is a possible future + optimization, not required.) +- **Content-hash gate, not ConfigMap-diff.** There's no "get the existing + object and compare" step for S3 (no cheap read-before-write like the + ConfigMap `Get`); a SHA-256 of the retrieved JSON, stored in + `Status.ContentHash`, decides whether to re-upload and re-notify. For a + `github` source this is checked *after* the (cheaper) commit-unchanged + short-circuit already used by the configmap path. +- **Deterministic key derivation shared by both sides.** `Decofile.S3ObjectKey()` + is called by both the controller (upload) and the webhook (URL for + `DECO_RELEASE`) so the webhook never needs to wait for a reconcile to know + where the object will be — same reasoning as `ConfigMapName()` for the + existing target. +- **`DECO_ALLOWED_AUTHORITIES` is additive, not overwritten.** Setting that env + var **replaces** the runtime's built-in default list (`configs.decocdn.com`, + `configs.deco.cx`, `admin.deco.cx`, `localhost`) rather than appending to it + — so the webhook always merges the s3 host into whatever's already on the + container (existing value or the defaults), never blindly setting it to just + the new host. + ## Key Design Decisions ### 1. Compression Strategy diff --git a/api/v1alpha1/decofile_types.go b/api/v1alpha1/decofile_types.go index 5105523..391974f 100644 --- a/api/v1alpha1/decofile_types.go +++ b/api/v1alpha1/decofile_types.go @@ -17,6 +17,8 @@ limitations under the License. package v1alpha1 import ( + "strings" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -37,6 +39,11 @@ const ( TargetConfigMap = "configmap" // TargetTanstackKV runs a Job that pushes the decofile to Cloudflare KV. TargetTanstackKV = "tanstack-kv" + // TargetS3 uploads the merged decofile JSON to S3 and points the runtime at + // it over HTTP (DECO_RELEASE=https://…) instead of mounting a ConfigMap. + // Escapes the ~1MB etcd ConfigMap ceiling for content-heavy sites. Bucket / + // region / public host come from operator env (DECOFILE_S3_*). + TargetS3 = "s3" ) // DecofileSpec defines the desired state of Decofile. @@ -65,7 +72,9 @@ type DecofileSpec struct { // "configmap" (default) writes a ConfigMap and notifies Knative pods. // "tanstack-kv" runs a self-cleaning Job that pushes the decofile to Cloudflare // KV — the fast-deploy content path for TanStack/Workers sites. - // +kubebuilder:validation:Enum=configmap;tanstack-kv + // "s3" uploads the decofile to S3 and serves it over HTTP (no ConfigMap) — + // the path for content-heavy sites that would exceed the etcd ConfigMap limit. + // +kubebuilder:validation:Enum=configmap;tanstack-kv;s3 // +kubebuilder:default=configmap // +optional Target string `json:"target,omitempty"` @@ -146,6 +155,15 @@ type DecofileStatus struct { // JobName is the K8s Job name for the current tanstack-kv sync (target=tanstack-kv). // +optional JobName string `json:"jobName,omitempty"` + + // ContentHash is the SHA-256 of the last delivered decofile JSON. Used by the + // s3 target to skip re-upload/notify when content is unchanged. + // +optional + ContentHash string `json:"contentHash,omitempty"` + + // S3URL is the HTTP URL the runtime reads from when target=s3. + // +optional + S3URL string `json:"s3URL,omitempty"` } // +kubebuilder:object:root=true @@ -165,6 +183,26 @@ func (d *Decofile) ConfigMapName() string { return "decofile-" + d.Name } +// DeploymentIdOrName returns spec.deploymentId, defaulting to the object name. +func (d *Decofile) DeploymentIdOrName() string { + if d.Spec.DeploymentId != "" { + return d.Spec.DeploymentId + } + return d.Name +} + +// S3ObjectKey returns the deterministic S3 key for this Decofile's config, +// derived the same way by the reconciler (upload) and the Service webhook +// (DECO_RELEASE URL) so neither depends on status. prefix is the operator-wide +// DECOFILE_S3_PREFIX (may be empty). +func (d *Decofile) S3ObjectKey(prefix string) string { + key := d.Namespace + "/" + d.DeploymentIdOrName() + "/decofile.json" + if prefix != "" { + return strings.Trim(prefix, "/") + "/" + key + } + return key +} + // +kubebuilder:object:root=true // DecofileList contains a list of Decofile. diff --git a/api/v1alpha1/decofile_types_test.go b/api/v1alpha1/decofile_types_test.go new file mode 100644 index 0000000..b554ad1 --- /dev/null +++ b/api/v1alpha1/decofile_types_test.go @@ -0,0 +1,41 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +*/ + +package v1alpha1 + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// The S3 object key must be derived identically by the reconciler (upload) and +// the Service webhook (DECO_RELEASE URL), so this pins the shared shape. +func TestDecofileS3ObjectKey(t *testing.T) { + df := &Decofile{ + ObjectMeta: metav1.ObjectMeta{Name: "dep-123", Namespace: "sites-econverse"}, + } + cases := []struct { + name string + deploymentId string + prefix string + want string + }{ + {"name as deploymentId, no prefix", "", "", "sites-econverse/dep-123/decofile.json"}, + {"explicit deploymentId", "abc", "", "sites-econverse/abc/decofile.json"}, + {"prefix trimmed", "", "decofiles/", "decofiles/sites-econverse/dep-123/decofile.json"}, + {"prefix without slash", "", "decofiles", "decofiles/sites-econverse/dep-123/decofile.json"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + df.Spec.DeploymentId = tc.deploymentId + if got := df.S3ObjectKey(tc.prefix); got != tc.want { + t.Fatalf("S3ObjectKey(%q) = %q, want %q", tc.prefix, got, tc.want) + } + }) + } +} diff --git a/chart/templates/customresourcedefinition-decofiles.deco.sites.yaml b/chart/templates/customresourcedefinition-decofiles.deco.sites.yaml index 002db38..1373d4c 100644 --- a/chart/templates/customresourcedefinition-decofiles.deco.sites.yaml +++ b/chart/templates/customresourcedefinition-decofiles.deco.sites.yaml @@ -114,9 +114,12 @@ spec: "configmap" (default) writes a ConfigMap and notifies Knative pods. "tanstack-kv" runs a self-cleaning Job that pushes the decofile to Cloudflare KV — the fast-deploy content path for TanStack/Workers sites. + "s3" uploads the decofile to S3 and serves it over HTTP (no ConfigMap) — + the path for content-heavy sites that would exceed the etcd ConfigMap limit. enum: - configmap - tanstack-kv + - s3 type: string required: - source @@ -191,6 +194,11 @@ spec: description: ConfigMapName is the name of the ConfigMap created for this Decofile type: string + contentHash: + description: |- + ContentHash is the SHA-256 of the last delivered decofile JSON. Used by the + s3 target to skip re-upload/notify when content is unchanged. + type: string githubCommit: description: GitHubCommit stores the commit SHA if using GitHub source type: string @@ -202,6 +210,9 @@ spec: description: LastUpdated is the timestamp of the last update format: date-time type: string + s3URL: + description: S3URL is the HTTP URL the runtime reads from when target=s3. + type: string sourceType: description: SourceType indicates which source was used (inline or github) diff --git a/chart/templates/deployment-operator-controller-manager.yaml b/chart/templates/deployment-operator-controller-manager.yaml index c483a74..1b70623 100644 --- a/chart/templates/deployment-operator-controller-manager.yaml +++ b/chart/templates/deployment-operator-controller-manager.yaml @@ -47,7 +47,7 @@ spec: command: - /manager image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) .Values.operatorApi.existingSecret (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn .Values.build.nodeSelector .Values.build.tolerations (and .Values.fastDeploy .Values.fastDeploy.syncerImage) }} + {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) .Values.operatorApi.existingSecret (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn .Values.build.nodeSelector .Values.build.tolerations (and .Values.fastDeploy .Values.fastDeploy.syncerImage) (and .Values.decofileS3 .Values.decofileS3.bucket) }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN @@ -135,6 +135,16 @@ spec: - name: DECOFILE_SYNCER_IMAGE value: {{ .Values.fastDeploy.syncerImage | quote }} {{- end }} + {{- if and .Values.decofileS3 .Values.decofileS3.bucket }} + - name: DECOFILE_S3_BUCKET + value: {{ .Values.decofileS3.bucket | quote }} + - name: DECOFILE_S3_REGION + value: {{ .Values.decofileS3.region | quote }} + - name: DECOFILE_S3_PUBLIC_HOST + value: {{ .Values.decofileS3.publicHost | quote }} + - name: DECOFILE_S3_PREFIX + value: {{ .Values.decofileS3.prefix | quote }} + {{- end }} {{- if .Values.operatorApi.existingSecret }} {{- if .Values.operatorApi.addr }} - name: OPERATOR_API_ADDR diff --git a/chart/values.yaml b/chart/values.yaml index 22663be..ff31ee8 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -116,6 +116,19 @@ s3: logsBucket: "" # S3 bucket for build logs stateBucket: "" # S3 bucket for site state +# Decofile S3 delivery (target=s3): serves the decofile to pods over HTTP from +# S3 instead of a ConfigMap, escaping the ~1MB etcd ConfigMap limit. Inert +# unless `bucket` is set. The controller writes to S3 with credentials from its +# EKS Pod Identity association (same mechanism as the build buckets) — that +# role needs s3:PutObject on the bucket. `publicHost` is the runtime-facing host +# (a CloudFront/VPC-restricted domain fronting the private bucket — the decofile +# can contain secrets, so the bucket must NOT be public-read). +decofileS3: + bucket: "" # S3 bucket name (required to enable) + region: "" # bucket region (co-locate with the EKS cluster) + publicHost: "" # host for DECO_RELEASE URL, e.g. configs.decocdn.com + prefix: "" # optional key prefix, e.g. "decofiles" + # Build job config — shared across all build platforms build: serviceAccount: "" # K8s ServiceAccount for builder pods (IRSA) diff --git a/cmd/main.go b/cmd/main.go index 5fa10db..143e959 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -357,11 +357,20 @@ func main() { decositesv1alpha1.TargetTanstackKV, deploy.NewTanstackKV(tkvCfg), ) + // s3 target: HTTP-delivered decofile for content-heavy sites. Inert + // (nil) unless DECOFILE_S3_BUCKET is set. + s3Uploader, s3err := controller.NewS3UploaderFromEnv(context.Background()) + if s3err != nil { + setupLog.Error(s3err, "decofile s3 target disabled: invalid DECOFILE_S3_* config") + } else if s3Uploader != nil { + setupLog.Info("decofile s3 target enabled") + } if err = (&controller.DecofileReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), HTTPClient: httpClient, FastDeploy: fastDeployRegistry, + S3: s3Uploader, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "Decofile") os.Exit(1) diff --git a/config/crd/bases/deco.sites_decofiles.yaml b/config/crd/bases/deco.sites_decofiles.yaml index 997a9fd..9d841e7 100644 --- a/config/crd/bases/deco.sites_decofiles.yaml +++ b/config/crd/bases/deco.sites_decofiles.yaml @@ -115,9 +115,12 @@ spec: "configmap" (default) writes a ConfigMap and notifies Knative pods. "tanstack-kv" runs a self-cleaning Job that pushes the decofile to Cloudflare KV — the fast-deploy content path for TanStack/Workers sites. + "s3" uploads the decofile to S3 and serves it over HTTP (no ConfigMap) — + the path for content-heavy sites that would exceed the etcd ConfigMap limit. enum: - configmap - tanstack-kv + - s3 type: string required: - source @@ -192,6 +195,11 @@ spec: description: ConfigMapName is the name of the ConfigMap created for this Decofile type: string + contentHash: + description: |- + ContentHash is the SHA-256 of the last delivered decofile JSON. Used by the + s3 target to skip re-upload/notify when content is unchanged. + type: string githubCommit: description: GitHubCommit stores the commit SHA if using GitHub source type: string @@ -203,6 +211,9 @@ spec: description: LastUpdated is the timestamp of the last update format: date-time type: string + s3URL: + description: S3URL is the HTTP URL the runtime reads from when target=s3. + type: string sourceType: description: SourceType indicates which source was used (inline or github) diff --git a/docs/decofile-s3-onboarding.md b/docs/decofile-s3-onboarding.md new file mode 100644 index 0000000..b8a0047 --- /dev/null +++ b/docs/decofile-s3-onboarding.md @@ -0,0 +1,143 @@ +# Decofile S3 Target — Onboarding a Site + +How to move a site's decofile off the etcd ConfigMap and onto S3, for sites +whose decofile is (or is close to) exceeding the **~1MB etcd ConfigMap limit** +(e.g. content-heavy sites with many/large blocks — blog posts with inline +HTML, large collections). Serves the decofile over **HTTP** instead of +mounting a ConfigMap — no size ceiling, same-VPC (no CDN, no internet hop). + +> Architecture + sequence diagrams: [`ARCHITECTURE.md`](../ARCHITECTURE.md#decofile-delivery-s3-target-etcd-offload). +> Companion PRs this feature shipped across: `decocms/operator#33` (this repo), +> `deco-sites/admin#3302`, `decocms/infra_applications#216`, +> `decocms/terraform-foundation-infra#27`, `decocms/terraform-eks-cluster#83`. + +## Mental model (read this first) + +**Nothing changes about how content is edited or published.** A site still +publishes the same way (git commit to `.deco/blocks/**` → admin upserts the +`Decofile` CR). What changes is **delivery**: + +| | `configmap` (default) | `s3` | +|---|---|---| +| Where the decofile lives | A ConfigMap key, mounted as a file | An S3 object, fetched over HTTP | +| How a pod gets it at boot | Reads the mounted file (`DECO_RELEASE=file://…`) | `fetch()`s the URL (`DECO_RELEASE=https://…`) | +| How a pod gets a **live update** | `POST /.decofile/reload` (operator pushes) | **identical** — same push, unchanged | +| Size ceiling | ~1MB (etcd) | none | + +**The only thing that's different for a running pod is the very first read at +boot.** Every subsequent update — someone edits a block and publishes — reaches +already-running pods exactly the same way regardless of target: the operator +detects the change and pushes a reload notification to `/.decofile/reload` on +every matching pod (see `ARCHITECTURE.md` Flow 4). The pod doesn't care whether +the *next* update source is S3 or a ConfigMap; it only mattered for how the pod +found its *first* copy when it started. + +**Nothing is manual per publish.** Once a site is onboarded (below), every +future publish flows through unchanged — the operator uploads the new content +to S3 automatically on each reconcile, exactly like it writes the ConfigMap +today. + +## Prerequisites (one-time, platform-wide) + +These need to exist before **any** site can use `target: s3`. Track via the PRs +above; do not repeat per site. + +1. **Operator ≥ the version shipping `decocms/operator#33`**, deployed with + `decofileS3.bucket`/`region`/`publicHost` set in the chart values + (`infra_applications` → `provisioning/deco-operator/main/values.yaml`). +2. **S3 bucket** `new-deco-decofiles` exists (`terraform-foundation-infra#27`), + with: + - a policy statement granting the operator's controller **Pod Identity** + role read/write (`terraform-eks-cluster#83`, module + `deco_operator_decofiles_pod_identity`) + - a policy statement granting **anonymous `GetObject`** scoped to the + eks-serverless VPC's S3 Gateway Endpoint (`aws:sourceVpce` condition) — + this is how pods read the object without any AWS credentials +3. Both terraform PRs applied with the **real VPC endpoint id** filled in + (see the `TODO` in `s3-setup/local.tf` — a placeholder until applied). + +Quick health check once deployed — create a Decofile with `target: s3` for a +throwaway site and confirm no ConfigMap is created: +```bash +kubectl -n sites- get decofile -o jsonpath='{.status.s3URL}{"\n"}' +kubectl -n sites- get configmap decofile- # should 404 +``` + +## Per-site steps + +### 1. Confirm the site actually needs this +Check the merged decofile's compressed size before flipping the switch — this +target is for sites that need it, not a default: +```bash +# rough check: sum of .deco/blocks/*.json, brotli+base64'd, vs ~1MB +du -sh .deco/blocks/ +``` +If a site is nowhere near the limit, leave it on `configmap` — no reason to +opt in early. + +### 2. Add the site to admin's opt-in list +Admin decides the target per site via the `DECOFILE_S3_SITES` env var +(comma-separated site slugs) on the `admin-env` Secret — see +`deco-sites/admin#3302` (`hosting/kubernetes/actions/decofile/upsert.ts`). +Add the site's slug there. No operator-side per-site config exists; the +`Decofile` CR's `spec.target` is set by admin on every publish. + +### 3. Publish +Trigger a normal publish for the site (git push / admin publish action). +Admin's next `Decofile` upsert will carry `spec.target: "s3"`. + +### 4. Verify +```bash +kubectl -n sites- get decofile -o yaml +# spec.target: s3 +# status.s3URL: https://new-deco-decofiles.s3.us-west-2.amazonaws.com/decofiles/... +# status.contentHash: + +kubectl -n sites- get configmap decofile- # 404 — no longer created + +# a new/rolled pod should show: +kubectl -n sites- get pod -o jsonpath='{.spec.containers[0].env}' | grep -A1 DECO_RELEASE +# DECO_RELEASE=https://new-deco-decofiles.s3.us-west-2.amazonaws.com/decofiles/... +``` +The pod's startup log line (`decofile has been loaded from https://…`, from the +deco runtime's `getProvider()`) confirms it read from S3, not a mounted file. + +### 5. Test the live-update path +Edit a block and publish again. The pod should reload **without a restart** +(same `/.decofile/reload` push as the ConfigMap path) — no extra step needed, +this isn't new behavior to configure. + +## Constraints + +- **Opt-in only, per site.** There is no automatic size-based promotion today — + a site stays on `configmap` until added to `DECOFILE_S3_SITES`. +- **Reads are unauthenticated, scoped by network path.** Any request reaching + the bucket via the eks-serverless VPC's S3 Gateway Endpoint can read any + object in the bucket (the `aws:sourceVpce` condition doesn't scope by key + prefix). Don't put anything in this bucket that shouldn't be readable by any + workload inside that VPC. +- **S3 objects are not Kubernetes-garbage-collected.** Deleting a `Decofile` + CR does not delete its S3 object (unlike the ConfigMap, which has an owner + reference). Rely on the bucket's lifecycle rule, or clean up manually if a + site is fully decommissioned. +- **No compression.** Objects are stored as plain JSON. A very large decofile + (tens of MB) will take proportionally longer to fetch at cold start than a + brotli'd ConfigMap would — there's no ceiling, but it isn't free either. + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| Pod stuck without a decofile / `decofile not defined` in logs | `DECOFILE_S3_PUBLIC_HOST` not set on the operator, or the host isn't reachable from the pod (VPC endpoint misconfigured) | check operator env `DECOFILE_S3_PUBLIC_HOST`; confirm the bucket policy's `aws:sourceVpce` matches the *actual* endpoint id (not the placeholder) | +| `AccessDenied` on the operator's `PutObject` | Pod Identity association missing/misconfigured, or bucket policy condition doesn't match the role | `kubectl -n deco-system describe pod ` — check for an AWS credential env/mount; verify `terraform-eks-cluster#83` applied for `workspace=serverless` | +| Still see a ConfigMap after publish | `spec.target` didn't get set to `s3` | check admin's `DECOFILE_S3_SITES` includes the site slug; check the `Decofile` CR's `spec.target` directly | +| Live edits not reaching the pod | Same failure modes as the ConfigMap path — this isn't s3-specific | see `ARCHITECTURE.md` notification troubleshooting; check `PodsNotified` condition on the `Decofile` | +| `authority ... is not allowed to be fetched from` in pod logs | Webhook didn't merge the S3 host into `DECO_ALLOWED_AUTHORITIES` (e.g. Service predates the webhook version) | re-roll the Knative Revision so the webhook re-runs; check the pod's `DECO_ALLOWED_AUTHORITIES` env includes the S3 host | + +## Disable / rollback + +Remove the site's slug from `DECOFILE_S3_SITES` and publish again. The next +`Decofile` upsert sets `spec.target: "configmap"`; the operator creates the +ConfigMap and the webhook mounts it on the next pod roll. The stale S3 object +is harmless (unauthenticated read-only, no owner reference — see Constraints) +but isn't automatically cleaned up. diff --git a/hack/helm-generator/main.go b/hack/helm-generator/main.go index 1f1d973..18dd04a 100644 --- a/hack/helm-generator/main.go +++ b/hack/helm-generator/main.go @@ -224,7 +224,7 @@ func addEnvVarsToDeployment(templatesDir string) error { contentStr := string(content) // Find the image line and add env vars after it - envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn .Values.build.nodeSelector .Values.build.tolerations (and .Values.fastDeploy .Values.fastDeploy.syncerImage) }} + envBlock := ` {{- if or (and .Values.github (or .Values.github.token .Values.github.existingSecret)) (and .Values.valkey (get .Values.valkey "sentinelUrls")) .Values.cfworkers.existingSecret .Values.cfworkers.builderImage .Values.cfworkers.artifactsBucket .Values.s3.region .Values.s3.logsBucket .Values.s3.stateBucket .Values.build.serviceAccount .Values.build.roleArn .Values.build.nodeSelector .Values.build.tolerations (and .Values.fastDeploy .Values.fastDeploy.syncerImage) (and .Values.decofileS3 .Values.decofileS3.bucket) }} env: {{- if and .Values.github .Values.github.existingSecret }} - name: GITHUB_TOKEN @@ -312,6 +312,16 @@ func addEnvVarsToDeployment(templatesDir string) error { - name: DECOFILE_SYNCER_IMAGE value: {{ .Values.fastDeploy.syncerImage | quote }} {{- end }} + {{- if and .Values.decofileS3 .Values.decofileS3.bucket }} + - name: DECOFILE_S3_BUCKET + value: {{ .Values.decofileS3.bucket | quote }} + - name: DECOFILE_S3_REGION + value: {{ .Values.decofileS3.region | quote }} + - name: DECOFILE_S3_PUBLIC_HOST + value: {{ .Values.decofileS3.publicHost | quote }} + - name: DECOFILE_S3_PREFIX + value: {{ .Values.decofileS3.prefix | quote }} + {{- end }} {{- end }}` imageLine := ` image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"` diff --git a/internal/controller/decofile_controller.go b/internal/controller/decofile_controller.go index 80456ae..73d437f 100644 --- a/internal/controller/decofile_controller.go +++ b/internal/controller/decofile_controller.go @@ -63,6 +63,9 @@ type DecofileReconciler struct { // FastDeploy dispatches non-configmap Decofile targets (e.g. tanstack-kv) to // a pluggable FastDeployment strategy. Nil = only the default ConfigMap path. FastDeploy *deploy.DeploymentRegistry + // S3 delivers the decofile via S3+HTTP for target=s3 (content-heavy sites + // that would exceed the etcd ConfigMap limit). Nil = s3 target unavailable. + S3 *S3Uploader } // +kubebuilder:rbac:groups=deco.sites,resources=decofiles,verbs=get;list;watch;create;update;patch;delete @@ -100,6 +103,13 @@ func (r *DecofileReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c log.V(1).Info("Fetched Decofile", "duration", time.Since(fetchStart)) + // s3 target: deliver over HTTP from S3 instead of a ConfigMap (escapes the + // etcd ConfigMap limit). Handled inline (not a FastDeployment) because it + // reuses this package's source retrieval + pod notifier. + if decofile.Spec.Target == decositesv1alpha1.TargetS3 { + return r.reconcileS3(ctx, req, decofile) + } + // Pluggable delivery: non-configmap targets (e.g. tanstack-kv KV sync) are // driven by a FastDeployment strategy that owns its own child resources + // status. The default/empty target falls through to the ConfigMap + Knative diff --git a/internal/controller/s3sink.go b/internal/controller/s3sink.go new file mode 100644 index 0000000..f89a848 --- /dev/null +++ b/internal/controller/s3sink.go @@ -0,0 +1,216 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + decositesv1alpha1 "github.com/deco-sites/decofile-operator/api/v1alpha1" +) + +// S3Uploader delivers the merged decofile as a plain-JSON object to S3, from +// which the deco runtime reads it over HTTP (DECO_RELEASE=https://…). It is the +// escape hatch for content-heavy sites whose decofile exceeds the ~1MB etcd +// ConfigMap limit. Nil when DECOFILE_S3_BUCKET is unset (s3 target disabled). +type S3Uploader struct { + client *s3.Client + bucket string + region string + prefix string + publicHost string // host used to build the runtime-facing HTTP URL +} + +// NewS3UploaderFromEnv builds an uploader from DECOFILE_S3_* env. Returns +// (nil, nil) when DECOFILE_S3_BUCKET is unset so the operator runs unchanged for +// clusters not using the s3 target. AWS credentials resolve via the default +// chain (IRSA web-identity in EKS). +func NewS3UploaderFromEnv(ctx context.Context) (*S3Uploader, error) { + bucket := os.Getenv("DECOFILE_S3_BUCKET") + if bucket == "" { + return nil, nil + } + region := os.Getenv("DECOFILE_S3_REGION") + cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(region)) + if err != nil { + return nil, fmt.Errorf("load aws config for decofile s3 target: %w", err) + } + return &S3Uploader{ + client: s3.NewFromConfig(cfg), + bucket: bucket, + region: region, + prefix: os.Getenv("DECOFILE_S3_PREFIX"), + publicHost: os.Getenv("DECOFILE_S3_PUBLIC_HOST"), + }, nil +} + +// Upload PUTs the raw decofile JSON. Served as plain JSON so the runtime's HTTP +// provider (fetch + JSON.parse) reads it directly — no compression, so no +// base64/brotli decode is needed on the runtime side. +// ponytail: uncompressed; add gzip + Content-Encoding if in-region egress/latency +// ever measures as a problem (Deno fetch auto-gunzips). +func (u *S3Uploader) Upload(ctx context.Context, key, jsonContent string) error { + _, err := u.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(u.bucket), + Key: aws.String(key), + Body: strings.NewReader(jsonContent), + ContentType: aws.String("application/json"), + CacheControl: aws.String("no-cache"), + }) + if err != nil { + return fmt.Errorf("put s3://%s/%s: %w", u.bucket, key, err) + } + return nil +} + +// URLFor builds the runtime-facing HTTP URL for an object key. Prefers the +// configured public host (a CloudFront/VPC-restricted domain); falls back to the +// regional virtual-hosted S3 endpoint. +func (u *S3Uploader) URLFor(key string) string { + host := u.publicHost + if host == "" { + host = fmt.Sprintf("%s.s3.%s.amazonaws.com", u.bucket, u.region) + } + return fmt.Sprintf("https://%s/%s", host, key) +} + +func sha256hex(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:]) +} + +// reconcileS3 delivers the decofile via S3+HTTP instead of a ConfigMap. It +// mirrors the ConfigMap path (retrieve → deliver → notify pods) but writes to +// S3 and gates re-work on a content hash so it never blows the etcd limit. +func (r *DecofileReconciler) reconcileS3(ctx context.Context, req ctrl.Request, decofile *decositesv1alpha1.Decofile) (ctrl.Result, error) { + log := logf.FromContext(ctx) + + if r.S3 == nil { + return ctrl.Result{}, fmt.Errorf("decofile target=s3 but operator has no S3 config (DECOFILE_S3_BUCKET unset)") + } + + deploymentId := decofile.DeploymentIdOrName() + + // GitHub gate: if the commit is unchanged and we've delivered before, there's + // nothing to do — skip the (expensive) repo download entirely. + if decofile.Spec.Source == SourceTypeGitHub && decofile.Spec.GitHub != nil && + decofile.Status.GitHubCommit == decofile.Spec.GitHub.Commit && + decofile.Status.ContentHash != "" { + log.V(1).Info("s3: github commit unchanged and already delivered, skipping") + return ctrl.Result{}, nil + } + + source, err := NewSource(r.Client, decofile) + if err != nil { + log.Error(err, "s3: failed to create source") + return ctrl.Result{}, err + } + jsonContent, err := source.Retrieve(ctx) + if err != nil { + log.Error(err, "s3: failed to retrieve source") + return ctrl.Result{}, err + } + + hash := sha256hex(jsonContent) + changed := hash != decofile.Status.ContentHash + key := decofile.S3ObjectKey(r.S3.prefix) + url := r.S3.URLFor(key) + + if changed { + if err := r.S3.Upload(ctx, key, jsonContent); err != nil { + log.Error(err, "s3: upload failed", "key", key) + return ctrl.Result{}, err + } + log.Info("s3: uploaded decofile", "url", url, "bytes", len(jsonContent)) + } else { + log.V(1).Info("s3: content unchanged, skipping upload", "url", url) + } + + // Notify running pods on change (same push path as the ConfigMap target; + // the mounted/URL source is only the cold-start read). + podsNotified := true + var notifyErr string + if changed { + ts := fmt.Sprintf("%d", time.Now().Unix()) + notifier := NewNotifier(r.Client, r.HTTPClient) + if err := notifier.NotifyPodsForDecofile(ctx, decofile.Namespace, deploymentId, ts, jsonContent); err != nil { + log.Error(err, "s3: failed to notify pods", "deploymentId", deploymentId) + podsNotified = false + notifyErr = err.Error() + } else { + log.Info("s3: notified pods", "deploymentId", deploymentId) + } + } + + // Update status on the freshest object to avoid conflicts. + fresh := &decositesv1alpha1.Decofile{} + if err := r.Get(ctx, req.NamespacedName, fresh); err != nil { + log.Error(err, "s3: failed to re-fetch Decofile for status update") + return ctrl.Result{}, err + } + fresh.Status.LastUpdated = metav1.Time{Time: time.Now()} + fresh.Status.SourceType = source.SourceType() + fresh.Status.ContentHash = hash + fresh.Status.S3URL = url + if fresh.Spec.Source == SourceTypeGitHub && fresh.Spec.GitHub != nil { + fresh.Status.GitHubCommit = fresh.Spec.GitHub.Commit + } + updateCondition(fresh, metav1.Condition{ + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "S3Uploaded", + Message: fmt.Sprintf("Decofile delivered to %s", url), + LastTransitionTime: metav1.Now(), + }) + if changed { + cond := metav1.Condition{ + Type: condTypePodsNotified, + LastTransitionTime: metav1.Now(), + } + if podsNotified { + cond.Status = metav1.ConditionTrue + cond.Reason = "NotificationSucceeded" + cond.Message = fmt.Sprintf("Notified pods for hash:%s", hash[:12]) + } else { + cond.Status = metav1.ConditionFalse + cond.Reason = "NotificationFailed" + cond.Message = fmt.Sprintf("Failed to notify pods for hash:%s: %s", hash[:12], notifyErr) + } + updateCondition(fresh, cond) + } + if err := r.Status().Update(ctx, fresh); err != nil { + log.Error(err, "s3: failed to update status") + return ctrl.Result{}, err + } + + if changed && !podsNotified { + return ctrl.Result{}, fmt.Errorf("s3: failed to notify pods: %s", notifyErr) + } + return ctrl.Result{}, nil +} diff --git a/internal/webhook/v1/allowed_authorities_test.go b/internal/webhook/v1/allowed_authorities_test.go new file mode 100644 index 0000000..b64eb59 --- /dev/null +++ b/internal/webhook/v1/allowed_authorities_test.go @@ -0,0 +1,63 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +*/ + +package v1 + +import "testing" + +// Run without envtest: go test -run TestMergeAllowedAuthorities ./internal/webhook/v1/ +func TestMergeAllowedAuthorities(t *testing.T) { + cases := []struct { + name string + existing string + exists bool + host string + wantValue string + wantSetEnv bool + }{ + { + name: "unset + custom host => defaults plus host", + exists: false, + host: "decofiles.example.com", + wantValue: "configs.decocdn.com,configs.deco.cx,admin.deco.cx,localhost,decofiles.example.com", + wantSetEnv: true, + }, + { + name: "unset + host already a default => no env needed", + exists: false, + host: "configs.decocdn.com", + wantSetEnv: false, + }, + { + name: "existing without host => appended, existing preserved", + existing: "foo.com,bar.com", + exists: true, + host: "decofiles.example.com", + wantValue: "foo.com,bar.com,decofiles.example.com", + wantSetEnv: true, + }, + { + name: "existing already contains host => unchanged (still written to preserve)", + existing: "foo.com,decofiles.example.com", + exists: true, + host: "decofiles.example.com", + wantValue: "foo.com,decofiles.example.com", + wantSetEnv: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + value, setEnv := mergeAllowedAuthorities(tc.existing, tc.exists, tc.host) + if setEnv != tc.wantSetEnv { + t.Fatalf("setEnv = %v, want %v", setEnv, tc.wantSetEnv) + } + if setEnv && value != tc.wantValue { + t.Fatalf("value = %q, want %q", value, tc.wantValue) + } + }) + } +} diff --git a/internal/webhook/v1/service_webhook.go b/internal/webhook/v1/service_webhook.go index 3351c0c..7df0f11 100644 --- a/internal/webhook/v1/service_webhook.go +++ b/internal/webhook/v1/service_webhook.go @@ -19,6 +19,8 @@ package v1 import ( "context" "fmt" + "os" + "strings" "github.com/google/uuid" corev1 "k8s.io/api/core/v1" @@ -138,6 +140,93 @@ func (d *ServiceCustomDefaulter) injectDecofileVolume(_ context.Context, service return nil } +// defaultAllowedAuthorities mirrors the deco runtime's built-in allowlist +// (engine/trustedAuthority.ts). Setting DECO_ALLOWED_AUTHORITIES replaces (not +// appends to) that default, so when we inject an S3/CloudFront host we must +// re-include the defaults or existing HTTP decofile fetches would break. +var defaultAllowedAuthorities = []string{ + "configs.decocdn.com", "configs.deco.cx", "admin.deco.cx", "localhost", +} + +// injectDecofileHTTP points DECO_RELEASE at the S3-backed HTTP URL and ensures +// the host is in DECO_ALLOWED_AUTHORITIES. No volume/mount is added — the +// decofile is fetched over HTTP, not read from a mounted ConfigMap. Key + host +// are derived deterministically (same as the reconciler) so this does not +// depend on the Decofile having been reconciled yet. +func (d *ServiceCustomDefaulter) injectDecofileHTTP(service *servingknativedevv1.Service, decofile *decositesv1alpha1.Decofile) error { + if len(service.Spec.Template.Spec.Containers) == 0 { + return fmt.Errorf("no containers found in Service spec") + } + host := os.Getenv("DECOFILE_S3_PUBLIC_HOST") + if host == "" { + return fmt.Errorf("decofile target=s3 but DECOFILE_S3_PUBLIC_HOST is not set on the operator") + } + key := decofile.S3ObjectKey(os.Getenv("DECOFILE_S3_PREFIX")) + url := fmt.Sprintf("https://%s/%s", host, key) + + idx := d.findTargetContainer(service) + // Reuses the ConfigMap path's env setter: sets DECO_RELEASE + reload token. + d.addOrUpdateEnvVars(service, idx, url) + d.ensureAllowedAuthority(service, idx, host) + return nil +} + +const allowedAuthoritiesEnv = "DECO_ALLOWED_AUTHORITIES" + +// mergeAllowedAuthorities computes the DECO_ALLOWED_AUTHORITIES value needed to +// let the runtime fetch the decofile from host. exists reports whether the +// container already sets the env (existing is its value). It returns the value +// to write and setEnv=false when no write is needed — i.e. the var is unset and +// host is already one of the runtime's built-in defaults, so the default +// allowlist already covers it. Pure (no k8s types) so it is unit-testable. +func mergeAllowedAuthorities(existing string, exists bool, host string) (value string, setEnv bool) { + var vals []string + if exists { + for _, v := range strings.Split(existing, ",") { + if v = strings.TrimSpace(v); v != "" { + vals = append(vals, v) + } + } + } else { + vals = append(vals, defaultAllowedAuthorities...) + } + for _, v := range vals { + if v == host { + // Already allowed. Only rewrite if the var already exists (to keep + // it); otherwise the runtime default covers it and no env is needed. + return existing, exists + } + } + return strings.Join(append(vals, host), ","), true +} + +// ensureAllowedAuthority makes sure host is present in the container's +// DECO_ALLOWED_AUTHORITIES env var, preserving any existing entries (or the +// runtime defaults when the var is unset). +func (d *ServiceCustomDefaulter) ensureAllowedAuthority(service *servingknativedevv1.Service, containerIdx int, host string) { + container := &service.Spec.Template.Spec.Containers[containerIdx] + + existingIdx := -1 + var existing string + for i, env := range container.Env { + if env.Name == allowedAuthoritiesEnv { + existingIdx = i + existing = env.Value + break + } + } + + value, setEnv := mergeAllowedAuthorities(existing, existingIdx >= 0, host) + if !setEnv { + return + } + if existingIdx >= 0 { + container.Env[existingIdx].Value = value + } else { + container.Env = append(container.Env, corev1.EnvVar{Name: allowedAuthoritiesEnv, Value: value}) + } +} + // addOrUpdateVolume adds or updates the decofile volume func (d *ServiceCustomDefaulter) addOrUpdateVolume(service *servingknativedevv1.Service, configMapName string) { volumeName := "decofile-config" @@ -273,15 +362,23 @@ func (d *ServiceCustomDefaulter) Default(ctx context.Context, obj runtime.Object return nil // Allow Service creation } - // Get mount path from annotation or use default directory - mountDir := "/app/decofile" - if customPath, exists := service.Annotations[decofileMountPathAnnot]; exists { - mountDir = customPath - } + // s3 target: point the runtime at the HTTP URL instead of mounting a + // ConfigMap volume (the decofile lives in S3, not etcd). + if decofile.Spec.Target == decositesv1alpha1.TargetS3 { + if err := d.injectDecofileHTTP(service, decofile); err != nil { + return err + } + } else { + // Get mount path from annotation or use default directory + mountDir := "/app/decofile" + if customPath, exists := service.Annotations[decofileMountPathAnnot]; exists { + mountDir = customPath + } - // Inject Decofile volume and env vars - if err := d.injectDecofileVolume(ctx, service, decofile, mountDir); err != nil { - return err + // Inject Decofile volume and env vars + if err := d.injectDecofileVolume(ctx, service, decofile, mountDir); err != nil { + return err + } } // Explicitly add deploymentId label to pod template for notification @@ -296,7 +393,7 @@ func (d *ServiceCustomDefaulter) Default(ctx context.Context, obj runtime.Object // falling back to deco's FILE_SYSTEM cache in the meantime. d.addOrUpdateValkeyEnvFrom(service) - servicelog.Info("Successfully injected Decofile into Service", "service", service.Name, "deploymentId", deploymentId, "configmap", decofile.ConfigMapName()) + servicelog.Info("Successfully injected Decofile into Service", "service", service.Name, "deploymentId", deploymentId, "target", decofile.Spec.Target) return nil }