From a3bc3fecb286be78a35eee5f1021fe647565f751 Mon Sep 17 00:00:00 2001 From: Brian McMahon Date: Mon, 29 Jun 2026 14:26:04 -0700 Subject: [PATCH 1/6] Un-fork operator-cloud: bring cloud differences behind a runtime Cloud gate Migrate the tigera/operator-cloud fork's differences into tigera/operator, gated by the CALICO_CLOUD env var so enterprise/OSS behavior is unchanged unless cloud mode is active. - Foundation: pkg/cloud (CALICO_CLOUD gate, tolerant Load), render/common/cloudconfig, controller/utils/cloudconfig, options.{Cloud,ESMigration}, cmd/main wiring. - Shared helpers: key_validator cloud tenancy claim (self-gating), auth_cloud, GetKeyValidatorConfig addTenancyClaim param, meta.DefaultCertificateDuration, elasticsearch.AddTenantId. - Per-component cloud paths (all gated): manager, logstorage (linseed/esgateway/ kibana/elastic/external-elastic/dashboards/kubecontrollers + ES ILM), compliance, fluentd, tiers, intrusiondetection, policyrecommendation, monitor, packetcapture. - apiserver + kube-controllers RBAC divergences gated behind Cloud. - Intrusion detection: clean up orphaned Image Assurance resources. - Cloud version-gen wiring: -cloud-versions flag, cloud.go.tpl, gen-versions-cloud target, generated pkg/components/cloud.go. - Cloud-path tests for manager/fluentd/kibana render and elastic/compliance/manager controllers. Plan + status in docs/cloud-unfork-migration-plan.md. Co-Authored-By: Claude Opus 4.8 --- Makefile | 11 +- cmd/main.go | 34 ++- config/cloud_versions.yml | 4 + docs/cloud-unfork-migration-plan.md | 289 ++++++++++++++++++ hack/gen-versions/cloud.go.tpl | 35 +++ hack/gen-versions/main.go | 45 +-- pkg/cloud/cloud.go | 192 ++++++++++++ pkg/cloud/cloud_test.go | 213 +++++++++++++ pkg/cloud/watch.go | 82 +++++ pkg/components/cloud.go | 33 ++ pkg/components/cloud_images.go | 18 ++ .../apiserver/apiserver_controller.go | 3 +- .../compliance/compliance_controller.go | 63 +++- .../compliance_controller_cloud_test.go | 214 +++++++++++++ .../intrusiondetection_controller.go | 11 + .../logcollector/logcollector_controller.go | 2 + .../dashboards/dashboards_controller.go | 23 +- .../logstorage/elastic/elastic_controller.go | 23 +- .../elastic/elastic_controller_cloud_test.go | 223 ++++++++++++++ .../elastic/external_elastic_controller.go | 25 ++ pkg/controller/logstorage/elastic/mock.go | 2 +- .../logstorage/kubecontrollers/cloud.go | 90 ++++++ .../kubecontrollers/es_kube_controllers.go | 15 + .../logstorage/kubecontrollers/esgateway.go | 7 + .../logstorage/linseed/linseed_controller.go | 32 +- pkg/controller/manager/manager_controller.go | 35 ++- .../manager/manager_controller_cloud.go | 184 +++++++++++ .../manager/manager_controller_cloud_test.go | 98 ++++++ pkg/controller/monitor/monitor_controller.go | 4 +- pkg/controller/options/options.go | 9 + .../packetcapture/packetcapture_controller.go | 2 +- .../policyrecommendation_controller.go | 19 ++ pkg/controller/tiers/tiers_controller.go | 7 + .../tiers/tiers_controller_cloud.go | 47 +++ pkg/controller/utils/auth.go | 14 +- pkg/controller/utils/auth_cloud.go | 45 +++ pkg/controller/utils/cloudconfig.go | 69 +++++ pkg/controller/utils/elasticsearch.go | 10 +- pkg/controller/utils/elasticsearch_cloud.go | 29 ++ pkg/render/apiserver.go | 146 ++++++--- .../key_validator_cloud.go | 49 +++ .../key_validator_config.go | 5 +- pkg/render/common/cloudconfig/cloudconfig.go | 135 ++++++++ .../cloudconfig/cloudconfig_suite_test.go | 33 ++ .../common/cloudconfig/cloudconfig_test.go | 125 ++++++++ .../common/elasticsearch/multitenancy.go | 21 ++ pkg/render/common/meta/meta.go | 2 + pkg/render/common/test/testing.go | 2 +- pkg/render/compliance.go | 18 ++ pkg/render/compliance_cloud.go | 60 ++++ pkg/render/fluentd.go | 24 ++ pkg/render/fluentd_cloud_test.go | 136 +++++++++ pkg/render/intrusion_detection.go | 14 + pkg/render/intrusion_detection_test.go | 4 +- .../kubecontrollers/kube-controllers.go | 22 +- pkg/render/logstorage/esgateway/cloud.go | 198 ++++++++++++ pkg/render/logstorage/esgateway/esgateway.go | 8 +- .../logstorage/esgateway/esgateway_test.go | 48 +++ pkg/render/logstorage/kibana/kibana.go | 12 + .../logstorage/kibana/kibana_cloud_test.go | 132 ++++++++ pkg/render/logstorage/linseed/cloud.go | 121 ++++++++ pkg/render/logstorage/linseed/linseed.go | 19 ++ pkg/render/logstorage/linseed/linseed_test.go | 21 ++ pkg/render/manager.go | 20 +- pkg/render/manager_cloud.go | 158 ++++++++++ pkg/render/manager_cloud_test.go | 117 +++++++ pkg/render/manager_test.go | 14 +- 67 files changed, 3826 insertions(+), 99 deletions(-) create mode 100644 config/cloud_versions.yml create mode 100644 docs/cloud-unfork-migration-plan.md create mode 100644 hack/gen-versions/cloud.go.tpl create mode 100644 pkg/cloud/cloud.go create mode 100644 pkg/cloud/cloud_test.go create mode 100644 pkg/cloud/watch.go create mode 100644 pkg/components/cloud.go create mode 100644 pkg/components/cloud_images.go create mode 100644 pkg/controller/compliance/compliance_controller_cloud_test.go create mode 100644 pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go create mode 100644 pkg/controller/logstorage/kubecontrollers/cloud.go create mode 100644 pkg/controller/manager/manager_controller_cloud.go create mode 100644 pkg/controller/manager/manager_controller_cloud_test.go create mode 100644 pkg/controller/tiers/tiers_controller_cloud.go create mode 100644 pkg/controller/utils/auth_cloud.go create mode 100644 pkg/controller/utils/cloudconfig.go create mode 100644 pkg/controller/utils/elasticsearch_cloud.go create mode 100644 pkg/render/common/authentication/tigera/key_validator_config/key_validator_cloud.go create mode 100644 pkg/render/common/cloudconfig/cloudconfig.go create mode 100644 pkg/render/common/cloudconfig/cloudconfig_suite_test.go create mode 100644 pkg/render/common/cloudconfig/cloudconfig_test.go create mode 100644 pkg/render/common/elasticsearch/multitenancy.go create mode 100644 pkg/render/compliance_cloud.go create mode 100644 pkg/render/fluentd_cloud_test.go create mode 100644 pkg/render/logstorage/esgateway/cloud.go create mode 100644 pkg/render/logstorage/kibana/kibana_cloud_test.go create mode 100644 pkg/render/logstorage/linseed/cloud.go create mode 100644 pkg/render/manager_cloud.go create mode 100644 pkg/render/manager_cloud_test.go diff --git a/Makefile b/Makefile index 2bbf1f1aeb..f091a3be6a 100644 --- a/Makefile +++ b/Makefile @@ -609,8 +609,9 @@ gen-files: manifests generate OS_VERSIONS?=config/calico_versions.yml EE_VERSIONS?=config/enterprise_versions.yml +CLOUD_VERSIONS?=config/cloud_versions.yml -.PHONY: gen-versions gen-versions-calico gen-versions-enterprise +.PHONY: gen-versions gen-versions-calico gen-versions-enterprise gen-versions-cloud gen-versions: gen-versions-calico gen-versions-enterprise @@ -620,6 +621,14 @@ gen-versions-calico: $(BINDIR)/gen-versions update-calico-crds gen-versions-enterprise: $(BINDIR)/gen-versions update-enterprise-crds $(BINDIR)/gen-versions -ee-versions=$(EE_VERSIONS) > pkg/components/enterprise.go +# gen-versions-cloud regenerates pkg/components/cloud.go from config/cloud_versions.yml. It is a +# Calico Cloud concern and is intentionally NOT part of the default `gen-versions` aggregate, so the +# enterprise build's generated output is unchanged. The cloud release pipeline invokes this target. +# Unlike the calico/enterprise targets, it does not fetch CRDs (cloud relies on the upstream operator +# repo for CRD updates). +gen-versions-cloud: $(BINDIR)/gen-versions + $(BINDIR)/gen-versions -cloud-versions=$(CLOUD_VERSIONS) > pkg/components/cloud.go + $(BINDIR)/gen-versions: $(shell find ./hack/gen-versions -type f) mkdir -p $(BINDIR) $(CONTAINERIZED) $(CALICO_BUILD) \ diff --git a/cmd/main.go b/cmd/main.go index 20ebc767c6..d8c0fa5009 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -34,6 +34,7 @@ import ( "github.com/tigera/operator/pkg/apigroup" "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/awssgsetup" + "github.com/tigera/operator/pkg/cloud" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/common/discovery" "github.com/tigera/operator/pkg/components" @@ -220,6 +221,15 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe os.Exit(1) } + // Load cloud options. For non-cloud (regular Calico Enterprise) installs this returns + // cloud.Options{Cloud: false} and is a no-op; cloud behavior is only activated when the + // cloud-operator-config ConfigMap (or the relevant env vars) are present. + cloudOpts, err := cloud.Load(ctx, cs) + if err != nil { + setupLog.Error(err, "failed to parse cloud options") + os.Exit(1) + } + v3CRDs, err := apis.UseV3CRDS(cfg) if err != nil { log.Error(err, "Failed to determine CRD version to use") @@ -304,7 +314,7 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe } } - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + mgrOpts := ctrl.Options{ Scheme: scheme, Metrics: metricsOpts, WebhookServer: webhook.NewServer(webhook.Options{ @@ -331,7 +341,17 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe // not being this mapper (which has since been rectified). It was a tough issue to figure out when the default // had changed out from under us, so better to continue to explicitly set it as we know this is the mapper we want. MapperProvider: apiutil.NewDynamicRESTMapper, - }) + } + + if cloudOpts.Cloud { + // Cloud tweaks: multiply the default leader-election timings by 4 to tolerate the higher + // API-server latency seen in Calico Cloud management clusters. + mgrOpts.LeaseDuration = cloud.ToPtr(60 * time.Second) + mgrOpts.RenewDeadline = cloud.ToPtr(40 * time.Second) + mgrOpts.RetryPeriod = cloud.ToPtr(8 * time.Second) + } + + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), mgrOpts) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) @@ -518,7 +538,9 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ShutdownContext: ctx, K8sClientset: clientset, MultiTenant: multiTenant, - ElasticExternal: discovery.UseExternalElastic(bootConfig), + ElasticExternal: discovery.UseExternalElastic(bootConfig) || cloudOpts.ElasticExternal, + Cloud: cloudOpts.Cloud, + ESMigration: cloudOpts.ESMigration, UseV3CRDs: v3CRDs, APIDiscovery: apiDiscovery, } @@ -665,6 +687,12 @@ func executePreDeleteHook(ctx context.Context, c client.Client) error { // verifyConfiguration verifies that the final configuration of the operator is correct before starting any controllers. func verifyConfiguration(ctx context.Context, cs kubernetes.Interface, opts options.ControllerOptions) error { + if opts.ESMigration { + // During the final phase of an ES migration both internal and external ES exist + // simultaneously, so the internal/external cert exclusivity checks below do not apply. + return nil + } + if opts.ElasticExternal { // There should not be an internal-es cert if _, err := cs.CoreV1().Secrets(render.ElasticsearchNamespace).Get(ctx, render.TigeraElasticsearchInternalCertSecret, metav1.GetOptions{}); err != nil { diff --git a/config/cloud_versions.yml b/config/cloud_versions.yml new file mode 100644 index 0000000000..a30aeb06c4 --- /dev/null +++ b/config/cloud_versions.yml @@ -0,0 +1,4 @@ +components: + cloud-rbac-api: + image: tigera/cc-rbac-api + version: v0.1.3-0-g79a7349 diff --git a/docs/cloud-unfork-migration-plan.md b/docs/cloud-unfork-migration-plan.md new file mode 100644 index 0000000000..7aa7c7613d --- /dev/null +++ b/docs/cloud-unfork-migration-plan.md @@ -0,0 +1,289 @@ +# Un-forking `operator-cloud` into `tigera/operator` + +Status: **in progress** — foundation landed, per-component work scoped below. + +## Goal & context + +`tigera/operator-cloud` is a long-lived fork of `tigera/operator` that adds Calico Cloud +("tesla") behavior. We want to delete the fork and bring its differences into `tigera/operator`, +gated so that cloud code only activates on a Calico Cloud install and **enterprise/OSS behavior is +unchanged**. + +Prerequisite: [operator-cloud#1059](https://github.com/tigera/operator-cloud/pull/1059) (remove +Image Assurance + runtime security) must merge first. This plan is written against the +post-#1059 fork state (branch `bm-remove-container-security`), which is fully caught up to +`tigera/operator` master — the merge-base equals master tip, so **every** diff below is genuine +cloud-specific divergence, not version drift. + +Total divergence: **125 files, +7299 / -232** (57 added, 67 modified, 1 deleted). + +## Gating design (decided) + +**Runtime `Cloud` flag**, not a build tag and not a CRD variant. + +- Cloud mode is gated **solely by the `CALICO_CLOUD` environment variable** (`cloud.EnableCloudEnvVar`), + set on the operator Deployment by the cloud installer. The operator never infers cloud mode by + sniffing for a ConfigMap. `cloud.Load(ctx, cs)` returns `Options{Cloud: false}` (no error, no + ConfigMap read, no watch) unless `CALICO_CLOUD` is truthy; a set-but-unparseable value errors. +- Only once cloud mode is enabled does `cloud.Load` read the `cloud-operator-config` ConfigMap (and + `ELASTIC_EXTERNAL`/`ELASTIC_MIGRATION` env) for cloud config *values* — those drive behavior, they + do not gate it. +- `cloud.Options{Cloud, ElasticExternal, ESMigration}` is folded into + `options.ControllerOptions{Cloud, ElasticExternal, ESMigration}` in `cmd/main.go`. +- Controllers and render code activate cloud paths only when `opts.Cloud` is true. The fork ran + these paths unconditionally because the whole binary was cloud — **the central migration task is + to add `if opts.Cloud { ... }` guards** around each ported cloud behavior so it is inert in + enterprise. +- Cloud render customizations follow the fork's existing pattern: a typed per-component extension + struct (e.g. `render.ManagerCloudResources`) populated by the controller only in cloud mode and + consumed by `*_cloud.go` "decorator" methods in the render package. (This matches the typed + per-component extension-struct direction already preferred for the OSS/enterprise split.) + +### Gating gotchas (must NOT be ported verbatim) + +Several fork edits change shared code paths and would alter enterprise behavior if copied directly. +Each must be guarded by `opts.Cloud` (or an equivalent cloud-only field): + +- `key_validator_config.go`: `RequiredEnv` unconditionally appends + `CALICO_CLOUD_REQUIRE_TENANT_CLAIM=true` via `addCloudEnvs`. Must only emit when a cloud tenant + claim is configured. +- `utils/auth.go`: `GetKeyValidatorConfig` gains an `addTenancyClaim bool` param — ripples to every + caller. In the unified repo derive the bool from cloud mode (e.g. `opts.Cloud && !opts.MultiTenant`). +- `manager.go`: the OIDC `CNX_WEB_OIDC_AUTHORITY`/`CNX_WEB_OIDC_AUDIENCE` changes are cloud-only + workarounds and must be gated. (The image-assurance/runtime-security namespace + deprecated-resource + cleanups in `manager.go` are enterprise-safe and already partly upstream via #1059.) +- `cmd/main.go` leader-election timing (`LeaseDuration`/`RenewDeadline`/`RetryPeriod` ×4) — gated + behind `cloudOpts.Cloud` (done). + +## Progress log + +- **Phase 0 (foundation)** — DONE, builds + unit-tested. Gate is the `CALICO_CLOUD` env var. +- **Phase 1 (shared helpers)** — DONE: `meta.DefaultCertificateDuration`, `elasticsearch.AddTenantId`, + `test.ExpectVolumeMount` offset fix, `key_validator_cloud.go` (self-gating `addCloudEnvs` — emits + cloud OIDC envs only when a tenant claim is configured), `auth_cloud.go` + `GetKeyValidatorConfig` + `addTenancyClaim` param (all 5 callers updated: apiserver/packetcapture pass `false`; + manager/compliance pass `opts.Cloud && !opts.MultiTenant`; monitor gained a `cloud` field). Tests pass. +- **Phase 2 manager** — DONE, builds + render/controller tests pass (incl. a new gated cloud render + spec and a non-cloud negative test). `ManagerConfiguration.Cloud`/`CloudResources`; all decorators + in `manager_cloud.go` early-return when `!cfg.Cloud`; controller runs `handleCloudReconcile` and + `addCloudWatch` only when `opts.Cloud`. **Skipped** the fork's Image-Assurance-removal hunks in + `manager.go`/`manager_test.go` — enterprise still ships IA (#1059 removed it from the fork only). +- **Phase 2 logstorage** — DONE, builds + all logstorage render/controller tests pass. Gated: + linseed (`Config.Cloud`, cloud objects/deployment/`ELASTIC_INDICES_CREATION_DISABLED`), esgateway + (`CloudConfig.Enabled` gate, cloud objects + deployment mods), kibana (`CloudKibanaConfigOverrides` + global only populated by the cloud-gated elastic controller), kube-controllers (`TenantId` field + + gated `TENANT_ID` env — **skipped** the fork's RBAC `create`/`update` widening on + clusterroles/bindings as an unexplained privilege escalation), elastic controller (cloud-kibana + override read behind `r.cloud`), external-elastic controller (`AddTenantId` behind + `opts.Cloud && !MultiTenant`), dashboards controller (cloud single-tenant tenant-from-CloudConfig), + and ES ILM (`SetILMPolicies(ctx, ls, cloud)` param gates `tweakILMPoliciesForCloud`). Each cloud + controller gained a `cloud` field set from `opts.Cloud`; cloud ConfigMap watches gated on + `opts.Cloud`. +- **Phase 2 remaining** — DONE, builds + all affected controller/render tests pass: + - compliance: gated cloud OIDC-issuer egress NetworkPolicy (`Cloud` field + `compliance_cloud.go`). + - fluentd: gated `setFluentdCloudEnvs` (`DISABLE_ES_*_LOG`); `Cloud` wired from logcollector controller (both Linux & Windows configs). + - tiers: gated `cloudPatchTier` (strips `app.kubernetes.io/instance` label from allow-tigera tier). + - intrusiondetection / policyrecommendation: gated single-tenant tenant-from-CloudConfig + cloud ConfigMap watch. + - packetcapture / monitor: kvc `addTenancyClaim` gating (done in Phase 1). + - **RBAC divergences — now GATED behind `Cloud` (resolved):** + - `apiserver.go` `tigeraUserClusterRole` + `tigeraNetworkAdminClusterRole`: added `Cloud` field; + when cloud, UISettings RBAC is per-user only (no `cluster-settings` group) and the `lma.tigera.io` + resources include `runtime`; when not cloud, the RBAC is byte-identical to enterprise (verified by + the existing apiserver render specs still passing). Controller sets `Cloud: r.opts.Cloud`. + - `kube-controllers.go` `NewElasticsearchKubeControllers`: added `Cloud` field; cloud grants + `create`/`update` on clusterroles/clusterrolebindings (cloud es-kube-controllers provisions + managed-cluster RBAC), enterprise keeps read-only. es-kube-controllers controller sets `Cloud: r.cloud`. + - **SKIPPED (not RBAC, not a cloud feature):** `intrusion_detection.go` render diff is only + Image-Assurance cleanup (#1059); enterprise keeps IA, so it stays out. + +### Cloud-path tests — DONE (core set) +Ported + passing (adapted to the runtime `Cloud` gate): +- `pkg/render/manager_cloud_test.go` (+ non-cloud negative case). +- `pkg/render/fluentd_cloud_test.go` (`Cloud: true`; + non-cloud negative case). +- `pkg/render/logstorage/kibana/kibana_cloud_test.go` (drives `CloudKibanaConfigOverrides`; AfterEach + resets the global to avoid leaking into enterprise specs). +- `pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go` (`r.opts.Cloud = true` on the shim). +- `pkg/controller/compliance/compliance_controller_cloud_test.go` (`opts.Cloud: true`). +- `pkg/controller/manager/manager_controller_cloud_test.go` (`TestEnv` — `cloudConfigOverride`). + +While porting the compliance controller test I found **three gated `compliance_controller.go` changes +I'd missed** and added them: the cloud ConfigMap watch, the system-root trusted-bundle creation for +external-OIDC management clusters, and the tenant-from-CloudConfig block. Added an exported +`compliance.NewReconcilerWithShims` test constructor. + +### Still outstanding +- Lower-value `_test.go` deltas (esgateway/linseed/logstorage/kube-controllers/users/external-elastic/ + es-kube-controllers/policyrecommendation cloud test cases) and the golden-YAML RBAC test infra + (`render_suite_test.go` TestMain, `testsupport/golden_yaml.go`, `clusterrole` pkg, `.golangci.yml`, + `manager-cloud-rbac-all-golden.yaml`). Functional code + enterprise safety already verified. +- Phase 3 (cloud version-gen wiring) and Phase 4 (cloud build/release pipeline). +- Decisions on the three SKIPPED items above. +- Run full `make ut` / CI before PR. + +### Per-file judgment calls discovered (apply to remaining work) + +- **Skip incidental refactors that aren't cloud features.** e.g. `pkg/render/logstorage.go` only changes + `curatorDecommissionedResources` to use a helper — not cloud, would shift enterprise output. Don't port. +- **`elasticsearch.go` ILM gating is a real trap.** The fork calls `tweakILMPoliciesForCloud` from + `SetILMPolicies` unconditionally, adding a `tigera_secure_ee_runtime` policy. Must be gated (thread a + cloud flag onto `esClient`) or it changes enterprise ILM. +- **`kibana.go` uses a global var `CloudKibanaConfigOverrides`** the fork itself flags as wrong. Prefer + threading it through the kibana `Configuration` rather than porting the global (empty default is + enterprise-safe, but the pattern is bad and racy). +- **External-ES already skips the ES storage-class requirement** (`elastic_controller.go` returns early + when `opts.ElasticExternal`), so cloud external-ES installs never need `tigera-elasticsearch`. + +## Phased plan + +### Phase 0 — Foundation ✅ DONE (this branch) + +Behavior-neutral scaffolding; compiles and unit-tested, zero enterprise impact. + +| File | Notes | +|---|---| +| `pkg/cloud/cloud.go`, `watch.go`, `cloud_test.go` | Ported; `Load` made tolerant — returns `Cloud:false` (no error, no watch) when no cloud config present. Added `Cloud` field + non-cloud test case. | +| `pkg/render/common/cloudconfig/{cloudconfig,*_test}.go` | Verbatim port (pure data type). | +| `pkg/controller/utils/cloudconfig.go` | `GetCloudConfig`, `GetTenantFromCloudAuthConfig`, `CloudAuthConfig` const. Unused until controllers are wired. | +| `pkg/components/cloud_images.go` | `CloudRegistry` const. | +| `config/cloud_versions.yml` | `cloud-rbac-api` source-of-truth. | +| `pkg/controller/options/options.go` | `+Cloud`, `+ESMigration` fields. | +| `cmd/main.go` | `cloud.Load` wired; `ElasticExternal ||= cloudOpts.ElasticExternal`; `Cloud`/`ESMigration` set; leader-election tweak gated; `verifyConfiguration` ESMigration early-return. | + +Verified: `go build` + `go test ./pkg/cloud/... ./pkg/render/common/cloudconfig/...` pass; `go vet ./cmd/...` clean. + +### Phase 1 — Shared cloud helpers (additive, low risk) + +New packages/files used by later phases; harmless to enterprise on their own. + +- `pkg/render/common/clusterrole/clusterroles.go` (new) +- `pkg/render/common/elasticsearch/multitenancy.go` (new) +- `pkg/render/common/meta/meta.go` (small additions) +- `pkg/render/testsupport/golden_yaml.go` + `pkg/render/common/test/testing.go` + `.golangci.yml` + (test helper for golden-YAML cloud RBAC tests; the lint config only exempts this helper's dot-import) +- `pkg/controller/utils/elasticsearch_cloud.go` (new) + `elasticsearch.go` (gated edits) +- `pkg/controller/utils/auth_cloud.go` + `auth.go` `GetKeyValidatorConfig` signature (see gotchas) +- `pkg/render/common/authentication/.../key_validator_cloud.go` + `key_validator_config.go` (gated) + +### Phase 2 — Per-component render + controller cloud paths (the bulk) + +For each component: add the `*_cloud.go` decorator/helpers, add the typed `…CloudResources` field +to the render config, populate it in the controller **only when `opts.Cloud`**, and add cloud +ConfigMap watches. Gate every shared-file edit. + +| Component | Render files | Controller files | +|---|---|---| +| **manager** | `manager.go` (M), `manager_cloud.go` (A) | `manager_controller.go` (M), `manager_controller_cloud.go` (A) | +| **logstorage / linseed** | `linseed/{linseed.go (M), cloud.go (A)}` | `linseed/linseed_controller.go` (M) | +| **logstorage / esgateway** | `esgateway/{esgateway.go (M), cloud.go (A)}` | `kubecontrollers/{esgateway.go, es_kube_controllers.go, cloud.go (A)}` | +| **logstorage / kibana** | `kibana/kibana.go` (M) | — | +| **logstorage (core)** | `logstorage.go` (M) | `elastic/{elastic_controller.go, external_elastic_controller.go}`, `dashboards/dashboards_controller.go`, `users` | +| **apiserver** | `apiserver.go` (M) | `apiserver/apiserver_controller.go` (M) | +| **compliance** | `compliance.go` (M), `compliance_cloud.go` (A) | `compliance/compliance_controller.go` (M) | +| **fluentd** | `fluentd.go` (M) | (tests: `fluentd_cloud_test.go`) | +| **intrusion detection** | `intrusion_detection.go` (M) | `intrusiondetection/intrusiondetection_controller.go` (M) | +| **kube-controllers** | `kubecontrollers/kube-controllers.go` (M) | — | +| **monitor / packetcapture / policyrecommendation / tiers** | — | respective `*_controller.go` (M) — mostly cloud config-map watches / tenant plumbing | + +Plus the test files for each (`*_cloud_test.go`, `*_test.go` deltas, `render_suite_test.go`, +`test/mainline_test.go`, `pkg/render/testdata/manager-cloud-rbac-all-golden.yaml`). + +Recommended ordering: manager → logstorage (linseed/esgateway/kibana/elastic) → apiserver → +compliance → fluentd/intrusiondetection/kubecontrollers → remaining controllers. Land each +component as its own PR with its tests, verifying enterprise golden files are unchanged. + +### Phase 3 — Version generation wiring ✅ DONE + +- `hack/gen-versions/main.go`: ported the `-cloud-versions` flag (and fixed the swapped os/ee help + text); generation now requires exactly one of `-os-versions`/`-ee-versions`/`-cloud-versions`. +- Added `hack/gen-versions/cloud.go.tpl` (the fork's `-cloud-versions` flag was dead — no template + existed). It generates `pkg/components/cloud.go` with `ComponentCloudRBACAPI` + `CloudImages` from + `config/cloud_versions.yml`, referencing the hand-written `CloudRegistry` const in `cloud_images.go`. +- `Makefile`: added a standalone `gen-versions-cloud` target (`CLOUD_VERSIONS?=config/cloud_versions.yml`). + **Intentionally NOT in the default `gen-versions` aggregate** so the enterprise build's generated + output is byte-identical (verified: os/ee generation still match the committed calico.go/enterprise.go). + No CRD-fetch dep (cloud relies on upstream operator for CRDs). The cloud release pipeline invokes it. +- Generated `pkg/components/cloud.go` is committed, gofmt-clean, compiles, and idempotent (dirty-check safe). +- `cloud-rbac-api` (cc-rbac-api) is not consumed by operator render code — it's a release-tooling image + pin (the `cloud.go` component pin + the `CloudRegistry` const in `cloud_images.go`, which the cloud + release tool patches). **Did NOT** edit the shared `enterprise.go.tpl` (the fork's tesla-prefixed + Kibana/Manager + CloudRegistry kube-controllers edits would globally tesla-fy enterprise images). + Cloud image overrides happen at runtime where wired (e.g. manager via `CloudResources.ManagerImage`); + any further cloud image pinning (kibana/kube-controllers) is a Phase 4 build/release concern. + +### Phase 4 — Cloud build/release pipeline (separate effort) + +This is infrastructure, not operator code. See the classification below; re-implement cloud build +targets and Semaphore/Argo pipelines so they **coexist** with enterprise's quay/multi-arch `v*` +flow rather than clobbering it. Track as a follow-up; not required for the code merge. + +--- + +## CI / build / release / tooling classification + +Each infra file sorted by disposition. **Do not bulk-copy** — most either don't apply or must be +made cloud-conditional. + +### OBSOLETE — delete with the fork, do not migrate +- `.argoci/cron/operator-cloud-fork-maintenance-nightly.yaml` — nightly upstream→fork merge cron. +- `.argoci/templates/sync-operator-cloud-branches.yaml` — runs `hack/update_fork.sh` merge-back. +- `.argoci/templates/hashrelease/sync-operator-hashrelease-fork.yaml` — fork-sync half (build-kickoff survives elsewhere). +- `hack/update_fork.sh`, `hack/hashrelease/update_hashrelease_fork.sh` — fork merge-back automation. +- `.claude/skills/merge-operator-into-operator-cloud/SKILL.md` — fork-merge runbook. +- `.github/README.md` + `CLOUD_README.md` (symlink) — fork branch-maintenance docs. +- `.semaphore/approve_check.yml` — auto-approves bot fork-sync PRs. +- `.bulldozer.yml`, `.ccbot/config.yaml` — automerge config existing to service fork-sync (drop unless automerge is independently wanted). + +### KEEP-ENTERPRISE — drop the fork's divergence +- `.github/workflows/codeql-analysis.yml` (fork deleted it — keep enterprise's CodeQL). +- `.github/CODEOWNERS` (fork made it `* @tigera/calico-cloud` — keep enterprise; add path-scoped cloud owners if desired). +- `.github/workflows/sync-versions.yml` (fork disabled the cron — keep enterprise's). +- `config/calico_versions.yml` (only a duplicated comment), `config/enterprise_versions.yml` (tesla wiring). +- `hack/gen-versions/enterprise.go.tpl` (tesla image edits — handle as cloud-scoped overrides instead). +- `git-hooks/pre-commit-in-container` (fork disabled the copyright check globally — don't). +- `api/go.mod` (module rename to `operator-cloud/api` — don't carry). + +### TWEAK — migrate but make cloud-conditional (don't globally override enterprise) +- `Makefile` — cloud `REPO`/`BUILD_IMAGE`/`IMAGE_REGISTRY=gcr.io`/`PUSH_IMAGE_PREFIXES`/ + `EXCLUDE_MANIFEST_REGISTRIES`, `VALIDARCHES=amd64`, `gen-bundle`/`bundle-crd-options` (RH v1beta1). + Gate behind a cloud variant so enterprise quay + multi-arch defaults survive. +- `.semaphore/{push_images,release,semaphore}.yml` — GCR auth/push, `cloud-v*` release trigger, + `staging` promotions, disabled multi-arch block. Make additive/coexisting, not replacements. +- `hack/release/{checks,flags}.go` — swapped OSS version validation to cloud format globally; + make variant-conditional. +- `hack/gen-versions/main.go` — `-cloud-versions` flag (additive but currently unwired; finish in Phase 3). +- `git-hooks/files-to-skip` — adds cloud `_cloud` files to deepcopy skip; migrate with the Go code. + +### REPLACE — cloud capability needed, fork impl is fork-specific +- `.argoci/templates/hashrelease/build-hashrelease.yaml` — re-target build+push at unified repo. +- `.argoci/templates/hashrelease/update-cluster-with-hashrelease.yaml` + `hack/hashrelease/*.py` — + live cloud-cluster rollout automation (single/multi-tenant/managed); re-home as a cloud pipeline. +- `hack/release/cloud.go`, `hack/release/internal/versions/cloud.go`, `hack/release/cloud_test.go`, + `hack/release/utils_test.go`, `hack/release/CLOUD.md` — cloud extension of the shared release + tool (GCR/tesla images, `cloud-v*` format, hashrelease flags, disabled GH release). Re-introduce + as a cloud-gated extension rather than always-on `init()` registration. + +### MIGRATE-ASIS — harmless, bring over +- `.github/workflows/secret_scanners.yml` (TruffleHog/gitleaks PR scan). +- `.golangci.yml` (only exempts the cloud golden-YAML test helper's dot-import). +- `.argoci/config.yaml` (`version: v2` marker). + +### Cloud build/release capabilities to (re)implement (Phase 4 scope) +1. Cloud image identity & GCR registry overrides (Makefile), amd64-only, without touching enterprise quay/multi-arch. +2. Cloud GCR push/release Semaphore blocks triggered by `cloud-v*`, coexisting with enterprise `v*`. +3. Cloud release-tool extension (flags, `cloud-v*` validation, GCR/tesla images, hashrelease outputs). +4. Cloud version generation (`cloud.go.tpl` + `gen-versions-cloud`), cloud-scoped image overrides. +5. Cloud hashrelease build pipeline (Argo), re-targeted at the unified repo. +6. Cloud cluster rollout automation (Argo + `hack/hashrelease` python). + +--- + +## Risks & verification + +- **Enterprise regression** is the primary risk. Mitigation: every shared-file edit guarded by + `opts.Cloud`; enterprise golden YAML / `_test.go` fixtures must be byte-identical before/after each + PR. Run `make ut` and the render golden tests per component. +- **Cross-cutting signature changes** (`GetKeyValidatorConfig`) touch many callers — land with all + call sites updated in one PR. +- **Dead/unwired fork code** (the `-cloud-versions` gen path) — finish wiring in Phase 3, don't + copy it half-done. +- Keep `make gen-files` / `make dirty-check` green after any API/CRD-adjacent changes. diff --git a/hack/gen-versions/cloud.go.tpl b/hack/gen-versions/cloud.go.tpl new file mode 100644 index 0000000000..258df94b2a --- /dev/null +++ b/hack/gen-versions/cloud.go.tpl @@ -0,0 +1,35 @@ +// Copyright (c) 2023-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Components defined here are required to be kept in sync with +// config/cloud_versions.yml. Generated by `make gen-versions-cloud`; do not edit directly. +// The CloudRegistry constant these components reference lives in cloud_images.go. + +package components + +var ( +{{- with index .Components "cloud-rbac-api" }} + ComponentCloudRBACAPI = Component{ + Version: "{{ .Version }}", + Image: "{{ .Image }}", + Registry: CloudRegistry, + imagePath: "{{ .ImagePath }}", + variant: enterpriseVariant, + } +{{- end }} + + CloudImages = []Component{ + ComponentCloudRBACAPI, + } +) diff --git a/hack/gen-versions/main.go b/hack/gen-versions/main.go index 93d95e746c..52c1b8b6ef 100644 --- a/hack/gen-versions/main.go +++ b/hack/gen-versions/main.go @@ -22,22 +22,25 @@ import ( ) const ( - eeVersionsTpl = "enterprise.go.tpl" - osVersionsTpl = "calico.go.tpl" + eeVersionsTpl = "enterprise.go.tpl" + osVersionsTpl = "calico.go.tpl" + cloudVersionsTpl = "cloud.go.tpl" ) var ( - templateDir string - debug bool - eeVersionsPath string - osVersionsPath string + templateDir string + debug bool + eeVersionsPath string + osVersionsPath string + cloudVersionsPath string ) func main() { - flag.StringVar(&templateDir, "template-dir", "hack/gen-versions/", "path to directory containing templates files named calico.go.tpl and enterprise.go.tpl") + flag.StringVar(&templateDir, "template-dir", "hack/gen-versions/", "path to directory containing templates files named calico.go.tpl, enterprise.go.tpl and cloud.go.tpl") flag.BoolVar(&debug, "debug", false, "enable debug logging") - flag.StringVar(&eeVersionsPath, "ee-versions", "", "path to calico versions file") - flag.StringVar(&osVersionsPath, "os-versions", "", "path to enterprise versions file") + flag.StringVar(&eeVersionsPath, "ee-versions", "", "path to enterprise versions file") + flag.StringVar(&osVersionsPath, "os-versions", "", "path to calico versions file") + flag.StringVar(&cloudVersionsPath, "cloud-versions", "", "path to cloud versions file") flag.Parse() if debug { @@ -45,24 +48,32 @@ func main() { log.Println("debug logging enabled") } - if osVersionsPath != "" && eeVersionsPath != "" { - log.Println("must only set one of either -os-versions or -ee-versions") + // Exactly one of the versions flags must be set. + set := 0 + for _, p := range []string{osVersionsPath, eeVersionsPath, cloudVersionsPath} { + if p != "" { + set++ + } + } + if set != 1 { + log.Println("must specify exactly one of -os-versions, -ee-versions or -cloud-versions") flag.PrintDefaults() os.Exit(1) } - if osVersionsPath != "" { + switch { + case osVersionsPath != "": if err := run(osVersionsPath, filepath.Join(templateDir, osVersionsTpl)); err != nil { log.Fatalln(err) } - } else if eeVersionsPath != "" { + case eeVersionsPath != "": if err := run(eeVersionsPath, filepath.Join(templateDir, eeVersionsTpl)); err != nil { log.Fatalln(err) } - } else { - log.Println("must specify either -os-versions or -ee-versions") - flag.PrintDefaults() - os.Exit(1) + case cloudVersionsPath != "": + if err := run(cloudVersionsPath, filepath.Join(templateDir, cloudVersionsTpl)); err != nil { + log.Fatalln(err) + } } } diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go new file mode 100644 index 0000000000..6a3e5da91a --- /dev/null +++ b/pkg/cloud/cloud.go @@ -0,0 +1,192 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloud + +import ( + "context" + "fmt" + "os" + "strconv" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" + "github.com/tigera/operator/pkg/render/logstorage" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" +) + +var setupLog = ctrl.Log.WithName("cloud_setup") + +const ( + configMapName = "cloud-operator-config" + + // EnableCloudEnvVar is the environment variable that gates cloud mode. When set to a truthy + // value (parseable by strconv.ParseBool, e.g. "true"/"1") the operator runs as a Calico Cloud + // install and cloud-specific behavior is activated. When unset or false the operator behaves as + // a regular Calico/Calico Enterprise install. This is set on the operator Deployment by the + // cloud installer; the operator never infers cloud mode by sniffing for ConfigMaps. + EnableCloudEnvVar = "CALICO_CLOUD" +) + +// Options holds the cloud-specific configuration parsed at operator startup. When Cloud is false the +// operator is running as a regular (non-cloud) Calico Enterprise install and the remaining fields are +// not meaningful. +type Options struct { + // Cloud indicates that this operator is running as a Calico Cloud install. It is true when the + // cloud-operator-config ConfigMap is present or the relevant cloud env vars are set. + Cloud bool + ElasticExternal bool + ESMigration bool +} + +// Load determines whether the operator is running in cloud mode and, if so, parses the cloud options. +// +// Cloud mode is gated solely by the CALICO_CLOUD environment variable (see EnableCloudEnvVar). When +// it is unset or false the operator is a regular Calico/Calico Enterprise install and Load returns +// Options{Cloud: false} with no error, leaving enterprise behavior unchanged. Only once cloud mode +// is enabled does Load read the cloud-operator-config ConfigMap (and env) for cloud config values. +func Load(ctx context.Context, cs kubernetes.Interface) (Options, error) { + cloudEnabled, err := parseEnableCloud() + if err != nil { + return Options{}, err + } + if !cloudEnabled { + setupLog.Info("cloud mode not enabled; running in non-cloud mode", "envVar", EnableCloudEnvVar) + return Options{Cloud: false}, nil + } + + var cmData map[string]string + cloudConfig, err := cs.CoreV1().ConfigMaps(common.OperatorNamespace()).Get(ctx, configMapName, metav1.GetOptions{}) + if err != nil { + if !kerrors.IsNotFound(err) { + return Options{}, fmt.Errorf("failed to read configmap '%s': %v", configMapName, err) + } + setupLog.Info("missing configmap. reading config from env.", "name", configMapName) + } else { + cmData = cloudConfig.Data + } + + elasticExternal, err := parseBoolFromEnvOrConfigMap("ELASTIC_EXTERNAL", cmData) + if err != nil { + return Options{}, fmt.Errorf("error in parsing ELASTIC_EXTERNAL: %v", err) + } + + esMigration, err := parseBoolFromEnvOrConfigMap("ELASTIC_MIGRATION", cmData) + if err != nil { + setupLog.Info("error in parsing ELASTIC_MIGRATION: %v. Defaulting to false", err) + esMigration = false + } + + opts := Options{ + Cloud: true, + ElasticExternal: elasticExternal, + ESMigration: esMigration, + } + + if err := verify(ctx, cs, opts); err != nil { + return Options{}, err + } + + if err = watch(cs, cmData); err != nil { + return Options{}, err + } + + return opts, nil +} + +// parseEnableCloud reads the CALICO_CLOUD gate env var. Unset or empty means non-cloud (false). A +// set-but-unparseable value is an error so misconfiguration fails loudly rather than silently +// disabling cloud. +func parseEnableCloud() (bool, error) { + strVal := os.Getenv(EnableCloudEnvVar) + if strVal == "" { + return false, nil + } + val, err := strconv.ParseBool(strVal) + if err != nil { + return false, fmt.Errorf("unable to convert env %s=%s to bool: %v", EnableCloudEnvVar, strVal, err) + } + return val, nil +} + +func parseBoolFromEnvOrConfigMap(key string, configMap map[string]string) (bool, error) { + var cmVal *bool + if strVal := configMap[key]; strVal != "" { + val, err := strconv.ParseBool(configMap[key]) + if err != nil { + return false, fmt.Errorf("unable to convert configmap %s=%s to bool: %v", key, strVal, err) + } + setupLog.Info("parsed config from cloud configmap", key, val) + cmVal = &val + } + + var envVal *bool + if strVal := os.Getenv(key); strVal != "" { + val, err := strconv.ParseBool(strVal) + if err != nil { + return false, fmt.Errorf("unable to convert env %s=%s to bool: %v", key, strVal, err) + } + setupLog.Info("parsed config from env", key, val) + envVal = &val + } + + if cmVal == nil && envVal == nil { + return false, fmt.Errorf("value %s not found in configmap or env", key) + } + + if cmVal != nil && envVal != nil && *cmVal != *envVal { + return false, fmt.Errorf("value for %s differs: set to %t in configmap and %t in env", key, *cmVal, *envVal) + } + + if cmVal != nil { + return *cmVal, nil + } else { + return *envVal, nil + } +} + +func verify(ctx context.Context, cs kubernetes.Interface, opts Options) error { + if opts.ESMigration { + return nil + } + if opts.ElasticExternal { + // there should not be an internal-es cert + _, err := cs.CoreV1().Secrets(render.ElasticsearchNamespace).Get(ctx, render.TigeraElasticsearchInternalCertSecret, metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("unexpected error encountered when confirming elastic is not currently internal: %v", err) + } + return fmt.Errorf("refusing to run: operator configured as external-es but secret/%s found which suggests its internal-es", render.TigeraElasticsearchInternalCertSecret) + } else { + // there should not be an external-es cert + _, err := cs.CoreV1().Secrets(render.ElasticsearchNamespace).Get(ctx, logstorage.ExternalCertsSecret, metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("unexpected error encountered when confirming elastic is not currently external: %v", err) + } + return fmt.Errorf("refusing to run: operator configured as internal-es but configmap/%s found which suggests its external-es", cloudconfig.CloudConfigConfigMapName) + } +} + +// Redefine ptr.To as ToPtr since aliases to a generic function are unsupported. See https://github.com/golang/go/issues/52654 +func ToPtr[T any](v T) *T { + return ptr.To(v) +} diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go new file mode 100644 index 0000000000..e70f3c98ae --- /dev/null +++ b/pkg/cloud/cloud_test.go @@ -0,0 +1,213 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloud + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/logstorage" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func init() { + // stub the watcher for testing + watch = func(cs kubernetes.Interface, cmData map[string]string) error { return nil } +} + +func TestLoad(t *testing.T) { + var ( + ctx = context.Background() + + // helper function to produce a fake kubernetes client which will return a configmap + // containing the specified map data. + clientWithConfigMapData = func(data map[string]string) kubernetes.Interface { + return fake.NewSimpleClientset(&corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: configMapName, + Namespace: common.OperatorNamespace(), + }, + Data: data, + }) + } + ) + + for _, tc := range []struct { + name string + // noConfigMap, when true, builds a client with no cloud-operator-config configmap at all. + noConfigMap bool + // env is a function to set desired env vars + env func() + // cmData is the map of data which the test harness should load into a configmap + cmData map[string]string + + assert func(*testing.T, Options, error) + }{ + { + name: "non-cloud when CALICO_CLOUD unset (even with configmap present)", + noConfigMap: false, + cmData: map[string]string{"ELASTIC_EXTERNAL": "true"}, + assert: func(t *testing.T, opts Options, err error) { + require.NoError(t, err) + require.False(t, opts.Cloud) + require.False(t, opts.ElasticExternal) + }, + }, + { + name: "error if CALICO_CLOUD is not a valid bool", + env: func() { + _ = os.Setenv(EnableCloudEnvVar, "yes-please") + }, + noConfigMap: true, + assert: func(t *testing.T, opts Options, err error) { + require.ErrorContains(t, err, "unable to convert env CALICO_CLOUD") + }, + }, + { + name: "error if ELASTIC_EXTERNAL omitted", + env: func() { + _ = os.Setenv(EnableCloudEnvVar, "true") + }, + cmData: map[string]string{}, + assert: func(t *testing.T, opts Options, err error) { + require.ErrorContains(t, err, "value ELASTIC_EXTERNAL not found") + }, + }, + { + name: "ELASTIC_EXTERNAL parsed from configmap", + env: func() { + _ = os.Setenv(EnableCloudEnvVar, "true") + }, + cmData: map[string]string{ + "ELASTIC_EXTERNAL": "true", + }, + assert: func(t *testing.T, opts Options, err error) { + require.NoError(t, err) + require.True(t, opts.Cloud) + require.True(t, opts.ElasticExternal) + }, + }, + { + name: "ELASTIC_EXTERNAL parsed from env", + noConfigMap: true, + env: func() { + _ = os.Setenv(EnableCloudEnvVar, "true") + _ = os.Setenv("ELASTIC_EXTERNAL", "true") + }, + assert: func(t *testing.T, opts Options, err error) { + require.NoError(t, err) + require.True(t, opts.Cloud) + require.True(t, opts.ElasticExternal) + }, + }, + { + name: "ELASTIC_EXTERNAL errors if set to different value", + env: func() { + _ = os.Setenv(EnableCloudEnvVar, "true") + _ = os.Setenv("ELASTIC_EXTERNAL", "false") + }, + cmData: map[string]string{ + "ELASTIC_EXTERNAL": "true", + }, + assert: func(t *testing.T, opts Options, err error) { + require.ErrorContains(t, err, "value for ELASTIC_EXTERNAL differs: set to true in configmap and false in env") + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + os.Clearenv() + + var c kubernetes.Interface + if tc.noConfigMap { + c = fake.NewSimpleClientset() + } else { + c = clientWithConfigMapData(tc.cmData) + } + if tc.env != nil { + tc.env() + } + + opts, err := Load(ctx, c) + tc.assert(t, opts, err) + }) + } +} + +func TestVerify(t *testing.T) { + ctx := context.Background() + + t.Run("external-es", func(t *testing.T) { + t.Run("pass verification to run if no secret present", func(t *testing.T) { + err := verify(ctx, fake.NewSimpleClientset(), Options{ElasticExternal: true}) + require.NoError(t, err) + }) + t.Run("pass verification to run if external-es secret is present", func(t *testing.T) { + cs := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: logstorage.ExternalCertsSecret, + Namespace: render.ElasticsearchNamespace, + }, + }) + err := verify(ctx, cs, Options{ElasticExternal: true}) + require.NoError(t, err) + }) + + t.Run("fail verification to run if internal-es secret is present", func(t *testing.T) { + cs := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: render.TigeraElasticsearchInternalCertSecret, + Namespace: render.ElasticsearchNamespace, + }, + }) + err := verify(ctx, cs, Options{ElasticExternal: true}) + require.ErrorContains(t, err, "refusing to run: operator configured as external-es") + }) + }) + + t.Run("internal-es", func(t *testing.T) { + t.Run("pass verification to run if no external-es secret present", func(t *testing.T) { + err := verify(ctx, fake.NewSimpleClientset(), Options{ElasticExternal: false}) + require.NoError(t, err) + }) + + t.Run("pass verification to run if internal-es secret is present", func(t *testing.T) { + cs := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: render.TigeraElasticsearchInternalCertSecret, + Namespace: render.ElasticsearchNamespace, + }, + }) + err := verify(ctx, cs, Options{ElasticExternal: false}) + require.NoError(t, err) + }) + + t.Run("fail verification to run if external-es secret is present", func(t *testing.T) { + cs := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: logstorage.ExternalCertsSecret, + Namespace: render.ElasticsearchNamespace, + }, + }) + err := verify(ctx, cs, Options{ElasticExternal: false}) + require.ErrorContains(t, err, "refusing to run: operator configured as internal-es but configmap") + }) + }) +} diff --git a/pkg/cloud/watch.go b/pkg/cloud/watch.go new file mode 100644 index 0000000000..806695275e --- /dev/null +++ b/pkg/cloud/watch.go @@ -0,0 +1,82 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloud + +import ( + "os" + "time" + + "github.com/tigera/operator/pkg/common" + ctrl "sigs.k8s.io/controller-runtime" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +var configWatchLog = ctrl.Log.WithName("cloud_config_watch") + +// watch spawns a goroutine which should exit if a configmap's data is changed. +// it is stubbed for testing. +var watch = func(cs kubernetes.Interface, cmData map[string]string) error { + informer := cache.NewSharedInformer( + cache.NewListWatchFromClient( + cs.CoreV1().RESTClient(), + "configmaps", + common.OperatorNamespace(), + fields.OneTermEqualSelector("metadata.name", configMapName), + ), + &v1.ConfigMap{}, + 0, // no resync period + ) + _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + UpdateFunc: func(_, newObj interface{}) { + if !compareMap(cmData, newObj.(*v1.ConfigMap).Data) { + configWatchLog.Info("detected config change. rebooting") + os.Exit(0) + } else { + configWatchLog.Info("ignoring configmap update as data was not modified") + } + }, + AddFunc: func(obj interface{}) { + if !compareMap(cmData, obj.(*v1.ConfigMap).Data) { + configWatchLog.Info("detected config creation change. rebooting") + os.Exit(0) + } else { + configWatchLog.Info("ignoring configmap creation as data was not modified") + } + }, + }) + if err != nil { + return err + } + + go informer.Run(make(chan struct{})) + for !informer.HasSynced() { + time.Sleep(1 * time.Second) + } + return nil +} + +func compareMap(m1, m2 map[string]string) bool { + if len(m1) != len(m2) { + return false + } + for k, v := range m1 { + if m2[k] != v { + return false + } + } + return true +} diff --git a/pkg/components/cloud.go b/pkg/components/cloud.go new file mode 100644 index 0000000000..03accd5b18 --- /dev/null +++ b/pkg/components/cloud.go @@ -0,0 +1,33 @@ +// Copyright (c) 2023-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Components defined here are required to be kept in sync with +// config/cloud_versions.yml. Generated by `make gen-versions-cloud`; do not edit directly. +// The CloudRegistry constant these components reference lives in cloud_images.go. + +package components + +var ( + ComponentCloudRBACAPI = Component{ + Version: "v0.1.3-0-g79a7349", + Image: "cc-rbac-api", + Registry: CloudRegistry, + imagePath: "tigera/", + variant: enterpriseVariant, + } + + CloudImages = []Component{ + ComponentCloudRBACAPI, + } +) diff --git a/pkg/components/cloud_images.go b/pkg/components/cloud_images.go new file mode 100644 index 0000000000..91d95bdb0f --- /dev/null +++ b/pkg/components/cloud_images.go @@ -0,0 +1,18 @@ +// Copyright (c) 2023 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package components + +const ( + CloudRegistry = "gcr.io/tigera-tesla/" +) diff --git a/pkg/controller/apiserver/apiserver_controller.go b/pkg/controller/apiserver/apiserver_controller.go index c2b34492c0..6340b77ef4 100644 --- a/pkg/controller/apiserver/apiserver_controller.go +++ b/pkg/controller/apiserver/apiserver_controller.go @@ -450,7 +450,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re trustedBundle.AddCertificates(certificate) } - keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain) + keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain, false) if err != nil { r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to get KeyValidator Config", err, reqLogger) return reconcile.Result{}, err @@ -510,6 +510,7 @@ func (r *ReconcileAPIServer) Reconcile(ctx context.Context, request reconcile.Re KeyValidatorConfig: keyValidatorConfig, KubernetesVersion: r.opts.KubernetesVersion, ClusterDomain: r.opts.ClusterDomain, + Cloud: r.opts.Cloud, RequiresAggregationServer: !r.opts.UseV3CRDs, RBACManagementEnabled: managerCR.RBACManagementEnabled(), QueryServerTLSKeyPairCertificateManagementOnly: queryServerTLSSecretCertificateManagementOnly, diff --git a/pkg/controller/compliance/compliance_controller.go b/pkg/controller/compliance/compliance_controller.go index 3cc740956a..200fc06577 100644 --- a/pkg/controller/compliance/compliance_controller.go +++ b/pkg/controller/compliance/compliance_controller.go @@ -43,6 +43,7 @@ import ( "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" + "github.com/tigera/operator/pkg/render/common/cloudconfig" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -91,6 +92,13 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { {Name: networkpolicy.CalicoComponentDefaultDenyPolicyName, Namespace: installNS}, }) + if opts.Cloud && opts.ElasticExternal { + // This ConfigMap is needed for utils.GetCloudConfig + if err = utils.AddConfigMapWatch(complianceController, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("compliance-controller failed to watch the ConfigMap resource: %w", err) + } + } + // Watch for changes to primary resource Compliance err = complianceController.WatchObject(&operatorv1.Compliance{}, &handler.EnqueueRequestForObject{}) if err != nil { @@ -158,6 +166,28 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions, licenseA return r } +// NewReconcilerWithShims constructs a ReconcileCompliance with the provided dependencies. It is +// intended for tests that need to inject a fake client, scheme, and status manager. +func NewReconcilerWithShims( + cli client.Client, + scheme *runtime.Scheme, + status status.StatusManager, + opts options.ControllerOptions, + licenseAPIReady *utils.ReadyFlag, + tierWatchReady *utils.ReadyFlag, +) *ReconcileCompliance { + r := &ReconcileCompliance{ + client: cli, + scheme: scheme, + status: status, + licenseAPIReady: licenseAPIReady, + tierWatchReady: tierWatchReady, + opts: opts, + } + r.status.Run(opts.ShutdownContext) + return r +} + // blank assignment to verify that ReconcileCompliance implements reconcile.Reconciler var _ reconcile.Reconciler = &ReconcileCompliance{} @@ -411,6 +441,20 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R return reconcile.Result{}, nil } + // Calico Cloud: update the trusted bundle with system root certificates on management clusters, + // as they may be needed by compliance-server when the OIDC provider is external. + // TODO: we should do this closer to instantiation of the trustedBundle but can't because that code + // is shared with upstream; in Cloud we need access to the authenticationCR. + if r.opts.Cloud && managementCluster != nil && authenticationCR != nil && authenticationCR.Spec.OIDC != nil && + authenticationCR.Spec.OIDC.Type == operatorv1.OIDCTypeTigera && !r.opts.MultiTenant { + bundleMaker, err = certificateManager.CreateTrustedBundleWithSystemRootCertificates(managerInternalTLSSecret, linseedCertificate) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "failed to create trusted bundle with system root certs", err, reqLogger) + return reconcile.Result{}, err + } + trustedBundle = bundleMaker.(certificatemanagement.TrustedBundleRO) + } + // Determine the namespaces to which we must bind the cluster role. // For multi-tenant, the cluster role will be bind to the service account in the tenant namespace // For single-tenant or zero-tenant, the cluster role will be bind to the service account in the tigera-compliance @@ -423,12 +467,28 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R // Create a component handler to manage the rendered component. handler := utils.NewComponentHandler(log, r.client, r.scheme, instance) - keyValidatorConfig, err := utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain) + keyValidatorConfig, err := utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain, r.opts.Cloud && !r.opts.MultiTenant) if err != nil { r.status.SetDegraded(operatorv1.ResourceValidationError, "Failed to process the authentication CR.", err, reqLogger) return reconcile.Result{}, err } + if r.opts.Cloud && r.opts.ElasticExternal && !r.opts.MultiTenant { + // For Calico Cloud single-tenant clusters sharing an external ES, extract the tenant + // information from the cloud config map. + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + if errors.IsNotFound(err) { + reqLogger.Info("Failed to retrieve External Elasticsearch config map") + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to retrieve External Elasticsearch config map", err, reqLogger) + return reconcile.Result{}, nil + } + r.status.SetDegraded(operatorv1.ResourceReadError, "Unable to read External Elasticsearch config map", err, reqLogger) + return reconcile.Result{}, err + } + tenant = cloudConfig.ToTenant() + } + reqLogger.V(3).Info("rendering components") setUp := render.NewSetup(&render.SetUpConfiguration{ @@ -462,6 +522,7 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R Tenant: tenant, Compliance: instance, ExternalElastic: r.opts.ElasticExternal, + Cloud: r.opts.Cloud, } // Render the desired objects from the CRD and create or update them. diff --git a/pkg/controller/compliance/compliance_controller_cloud_test.go b/pkg/controller/compliance/compliance_controller_cloud_test.go new file mode 100644 index 0000000000..8cf01458d0 --- /dev/null +++ b/pkg/controller/compliance/compliance_controller_cloud_test.go @@ -0,0 +1,214 @@ +// Copyright (c) 2024-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compliance_test + +import ( + "context" + "crypto/x509" + "encoding/pem" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/compliance" + "github.com/tigera/operator/pkg/controller/options" + "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/controller/utils" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/test" +) + +var _ = Describe("Cloud Compliance controller tests", func() { + var c client.Client + var ctx context.Context + var cr *operatorv1.Compliance + var r *compliance.ReconcileCompliance + var mockStatus *status.MockStatus + var scheme *runtime.Scheme + var installation *operatorv1.Installation + var licenseReadyFlag *utils.ReadyFlag + var tierReadyFlag *utils.ReadyFlag + + BeforeEach(func() { + // The schema contains all objects that should be known to the fake client when the test runs. + scheme = runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + Expect(appsv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(rbacv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(operatorv1.SchemeBuilder.AddToScheme(scheme)).NotTo(HaveOccurred()) + + // Create a client that will have a crud interface of k8s objects. + c = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + ctx = context.Background() + + mockStatus = &status.MockStatus{} + mockStatus.On("AddDaemonsets", mock.Anything).Return() + mockStatus.On("AddDeployments", mock.Anything).Return() + mockStatus.On("RemoveDeployments", mock.Anything).Return() + mockStatus.On("RemoveDaemonsets", mock.Anything).Return() + mockStatus.On("AddStatefulSets", mock.Anything).Return() + mockStatus.On("RemoveCertificateSigningRequests", mock.Anything).Return() + mockStatus.On("AddCronJobs", mock.Anything) + mockStatus.On("IsAvailable").Return(true) + mockStatus.On("OnCRFound").Return() + mockStatus.On("AddCertificateSigningRequests", mock.Anything).Return() + mockStatus.On("ClearDegraded") + mockStatus.On("SetDegraded", "Waiting for LicenseKeyAPI to be ready", "").Return().Maybe() + mockStatus.On("ReadyToMonitor") + mockStatus.On("SetMetaData", mock.Anything).Return() + mockStatus.On("ClearWarning", mock.Anything) + mockStatus.On("Run").Return() + + // The compliance reconcile loop depends on a ton of objects that should be available in your client as + // prerequisites. Without them, compliance will not even start creating objects. Let's create them now. + installation = &operatorv1.Installation{ + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "some.registry.org/", + }, + Status: operatorv1.InstallationStatus{ + Variant: operatorv1.CalicoEnterprise, + Computed: &operatorv1.InstallationSpec{ + Registry: "my-reg", + // The test is provider agnostic. + KubernetesProvider: operatorv1.ProviderNone, + }, + }, + } + Expect(c.Create( + ctx, + installation)).NotTo(HaveOccurred()) + + Expect(c.Create(ctx, &operatorv1.APIServer{ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, Status: operatorv1.APIServerStatus{State: operatorv1.TigeraStatusReady}})).NotTo(HaveOccurred()) + Expect(c.Create(ctx, &v3.Tier{ObjectMeta: metav1.ObjectMeta{Name: "calico-system"}})).NotTo(HaveOccurred()) + Expect(c.Create(ctx, &v3.LicenseKey{ObjectMeta: metav1.ObjectMeta{Name: "default"}, Status: v3.LicenseKeyStatus{Features: []string{common.ComplianceFeature}}})).NotTo(HaveOccurred()) + Expect(c.Create(ctx, &operatorv1.Authentication{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + Spec: operatorv1.AuthenticationSpec{ + OIDC: &operatorv1.AuthenticationOIDC{ + IssuerURL: "https://auth.dev.calicocloud.io/", + Type: operatorv1.OIDCTypeTigera, + }, + }, + Status: operatorv1.AuthenticationStatus{ + State: operatorv1.TigeraStatusReady, + }, + })).ToNot(HaveOccurred()) + + certificateManager, err := certificatemanager.Create(c, nil, dns.DefaultClusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Create(context.Background(), certificateManager.KeyPair().Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + + esDNSNames := dns.GetServiceDNSNames(render.TigeraElasticsearchGatewaySecret, render.ElasticsearchNamespace, dns.DefaultClusterDomain) + linseedKeyPair, err := certificateManager.GetOrCreateKeyPair(c, render.TigeraLinseedSecret, render.ElasticsearchNamespace, esDNSNames) + Expect(err).NotTo(HaveOccurred()) + + // For managed clusters, we also need the public cert for Linseed. + linseedPublicCert, err := certificateManager.GetOrCreateKeyPair(c, render.VoltronLinseedPublicCert, common.OperatorNamespace(), esDNSNames) + Expect(err).NotTo(HaveOccurred()) + + Expect(c.Create(ctx, linseedKeyPair.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + Expect(c.Create(ctx, linseedPublicCert.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + + // Apply the compliance CR to the fake cluster. + cr = &operatorv1.Compliance{ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}} + Expect(c.Create(ctx, cr)).NotTo(HaveOccurred()) + + // Mark that watches were successful. + licenseReadyFlag = &utils.ReadyFlag{} + tierReadyFlag = &utils.ReadyFlag{} + + licenseReadyFlag.MarkAsReady() + tierReadyFlag.MarkAsReady() + }) + + Context("Single tenant management cluster", func() { + BeforeEach(func() { + Expect(c.Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: utils.CloudAuthConfig, Namespace: "tigera-operator"}, + Data: map[string]string{ + "tenantID": "anyTenant", + }, + })).NotTo(HaveOccurred()) + Expect(c.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: render.OIDCSecretName, Namespace: "tigera-operator"}, + Data: map[string][]byte{ + "clientID": []byte("anyID"), + "clientSecret": []byte("anySecret"), + }, + })).NotTo(HaveOccurred()) + + Expect(c.Create( + ctx, + &operatorv1.ManagementCluster{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + })).NotTo(HaveOccurred()) + + // Create an object we can use throughout the test to do the compliance reconcile loops. + // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. + opts := options.ControllerOptions{ + DetectedProvider: operatorv1.ProviderNone, + ClusterDomain: dns.DefaultClusterDomain, + ShutdownContext: context.TODO(), + Cloud: true, + } + r = compliance.NewReconcilerWithShims(c, scheme, mockStatus, opts, licenseReadyFlag, tierReadyFlag) + }) + + It("should create a trusted bundle with external certificates", func() { + result, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(Equal(0 * time.Second)) + + trustedBundleConfigMap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-ca-bundle", + Namespace: render.ComplianceNamespace, + }, + } + err = test.GetResource(c, &trustedBundleConfigMap) + Expect(err).ShouldNot(HaveOccurred()) + + publicCerts, _ := pem.Decode([]byte(trustedBundleConfigMap.Data["ca-bundle.crt"])) + Expect(publicCerts).NotTo(BeNil()) + _, err = x509.ParseCertificate(publicCerts.Bytes) + Expect(err).To(BeNil(), "Error parsing bytes from ca-bundle.crt into certificate") + + tigeraCerts, _ := pem.Decode([]byte(trustedBundleConfigMap.Data["tigera-ca-bundle.crt"])) + Expect(tigeraCerts).NotTo(BeNil()) + tigeraX509Certs, err := x509.ParseCertificate(tigeraCerts.Bytes) + Expect(err).To(BeNil(), "Error parsing bytes from tigera-ca-bundle.crt into certificate") + Expect(tigeraX509Certs.Subject.CommonName).To(Equal("tigera-operator-signer")) + }) + }) +}) diff --git a/pkg/controller/intrusiondetection/intrusiondetection_controller.go b/pkg/controller/intrusiondetection/intrusiondetection_controller.go index d42744433b..d1334da2b1 100644 --- a/pkg/controller/intrusiondetection/intrusiondetection_controller.go +++ b/pkg/controller/intrusiondetection/intrusiondetection_controller.go @@ -429,6 +429,17 @@ func (r *ReconcileIntrusionDetection) Reconcile(ctx context.Context, request rec bundleMaker = nil } + if r.opts.Cloud && r.opts.ElasticExternal && !r.opts.MultiTenant { + // For Calico Cloud single-tenant clusters sharing an external ES, extract the tenant + // information from the cloud config map. + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) + return reconcile.Result{}, err + } + tenant = cloudConfig.ToTenant() + } + // Create a component handler to manage the rendered component. handler := utils.NewComponentHandler(log, r.client, r.scheme, instance) diff --git a/pkg/controller/logcollector/logcollector_controller.go b/pkg/controller/logcollector/logcollector_controller.go index 60e88431fe..d780be1a2c 100644 --- a/pkg/controller/logcollector/logcollector_controller.go +++ b/pkg/controller/logcollector/logcollector_controller.go @@ -609,6 +609,7 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile PacketCapture: packetcaptureapi, NonClusterHost: nonclusterhost, LicenseExpired: licenseExpired, + Cloud: r.opts.Cloud, } // Render the fluentd component for Linux comp := render.Fluentd(fluentdCfg) @@ -681,6 +682,7 @@ func (r *ReconcileLogCollector) Reconcile(ctx context.Context, request reconcile FluentdKeyPair: fluentdKeyPair, EKSLogForwarderKeyPair: eksLogForwarderKeyPair, LicenseExpired: licenseExpired, + Cloud: r.opts.Cloud, } comp = render.Fluentd(fluentdCfg) diff --git a/pkg/controller/logstorage/dashboards/dashboards_controller.go b/pkg/controller/logstorage/dashboards/dashboards_controller.go index d18ecf2a6d..98b3051b93 100644 --- a/pkg/controller/logstorage/dashboards/dashboards_controller.go +++ b/pkg/controller/logstorage/dashboards/dashboards_controller.go @@ -65,6 +65,7 @@ type DashboardsSubController struct { clusterDomain string multiTenant bool elasticExternal bool + cloud bool tierWatchReady *utils.ReadyFlag } @@ -82,6 +83,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { tierWatchReady: &utils.ReadyFlag{}, multiTenant: opts.MultiTenant, elasticExternal: opts.ElasticExternal, + cloud: opts.Cloud, } r.status.Run(opts.ShutdownContext) @@ -272,11 +274,22 @@ func (d DashboardsSubController) Reconcile(ctx context.Context, request reconcil return reconcile.Result{RequeueAfter: utils.StandardRetry}, nil } } else { - // If we're using an external ES and Kibana, the Tenant resource must specify the Kibana endpoint. - if tenant == nil || tenant.Spec.Elastic == nil || tenant.Spec.Elastic.KibanaURL == "" { - reqLogger.Error(nil, "Kibana URL must be specified for this tenant") - d.status.SetDegraded(operatorv1.ResourceValidationError, "Kibana URL must be specified for this tenant", nil, reqLogger) - return reconcile.Result{}, nil + if d.multiTenant || !d.cloud { + // If we're using an external ES and Kibana, the Tenant resource must specify the Kibana endpoint. + if tenant == nil || tenant.Spec.Elastic == nil || tenant.Spec.Elastic.KibanaURL == "" { + reqLogger.Error(nil, "Kibana URL must be specified for this tenant") + d.status.SetDegraded(operatorv1.ResourceValidationError, "Kibana URL must be specified for this tenant", nil, reqLogger) + return reconcile.Result{}, nil + } + } else { + // This is a Calico Cloud single-tenant cluster connected to an external ES & Kibana. Read + // the tenant configuration from the CloudConfig ConfigMap. + cloudConfig, err := utils.GetCloudConfig(ctx, d.client) + if err != nil { + d.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) + return reconcile.Result{}, err + } + tenant = cloudConfig.ToTenant() } // Determine the host and port from the URL. diff --git a/pkg/controller/logstorage/elastic/elastic_controller.go b/pkg/controller/logstorage/elastic/elastic_controller.go index cccb5f1ba2..1e026da598 100644 --- a/pkg/controller/logstorage/elastic/elastic_controller.go +++ b/pkg/controller/logstorage/elastic/elastic_controller.go @@ -16,6 +16,7 @@ package elastic import ( "context" + "encoding/json" "fmt" "net/url" @@ -80,6 +81,7 @@ type ElasticSubController struct { clusterDomain string tierWatchReady *utils.ReadyFlag multiTenant bool + cloud bool } func Add(mgr manager.Manager, opts options.ControllerOptions) error { @@ -102,6 +104,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { clusterDomain: opts.ClusterDomain, provider: opts.DetectedProvider, multiTenant: opts.MultiTenant, + cloud: opts.Cloud, } r.status.Run(opts.ShutdownContext) @@ -480,6 +483,24 @@ func (r *ElasticSubController) Reconcile(ctx context.Context, request reconcile. r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to retrieve the Kibana service", err, reqLogger) return reconcile.Result{}, err } + + if r.cloud { + // Read the cloud-kibana-config ConfigMap and parse it into a global map in the render package. The render code will read this map + // and use it to override the default Kibana configuration. + // TODO: We should instead be passing this via arguments to the render code. + kbCm := &corev1.ConfigMap{} + if err = r.client.Get(ctx, types.NamespacedName{Name: "cloud-kibana-config", Namespace: common.OperatorNamespace()}, kbCm); err != nil { + if !errors.IsNotFound(err) { + return reconcile.Result{}, fmt.Errorf("failed to read cloud-kibana-config ConfigMap: %s", err.Error()) + } + } else { + kibana.CloudKibanaConfigOverrides = map[string]interface{}{} + if err = json.Unmarshal([]byte(kbCm.Data["config"]), &kibana.CloudKibanaConfigOverrides); err != nil { + r.status.SetDegraded(operatorv1.InvalidConfigurationError, "Failed to unmarshal config in cloud-kibana-config ConfigMap", err, reqLogger) + return reconcile.Result{}, err + } + } + } } // Query the trusted bundle from the namespace. @@ -656,7 +677,7 @@ func (r *ElasticSubController) applyILMPolicies(ls *operatorv1.LogStorage, reqLo return err } - if err = esClient.SetILMPolicies(ctx, ls); err != nil { + if err = esClient.SetILMPolicies(ctx, ls, r.cloud); err != nil { return err } return nil diff --git a/pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go b/pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go new file mode 100644 index 0000000000..27ce7460c7 --- /dev/null +++ b/pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go @@ -0,0 +1,223 @@ +// Copyright (c) 2022-2026 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package elastic + +import ( + "context" + + "github.com/stretchr/testify/mock" + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/operator/pkg/render/logstorage" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" + relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/secret" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + admissionv1beta1 "k8s.io/api/admissionregistration/v1beta1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/status" + "github.com/tigera/operator/pkg/render/logstorage/kibana" + "github.com/tigera/operator/pkg/render/monitor" +) + +var _ = Describe("External ES controller (Cloud))", func() { + var ( + cli client.Client + mockStatus *status.MockStatus + scheme *runtime.Scheme + ctx context.Context + certificateManager certificatemanager.CertificateManager + install *operatorv1.Installation + ) + + BeforeEach(func() { + scheme = runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).ShouldNot(HaveOccurred()) + Expect(storagev1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(appsv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(rbacv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(batchv1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + Expect(admissionv1beta1.SchemeBuilder.AddToScheme(scheme)).ShouldNot(HaveOccurred()) + + ctx = context.Background() + cli = fake.NewClientBuilder().WithScheme(scheme).Build() + var err error + certificateManager, err = certificatemanager.Create(cli, nil, "", common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + Expect(cli.Create(ctx, certificateManager.KeyPair().Secret(common.OperatorNamespace()))) // Persist the root-ca in the operator namespace. + + // Create secrets necessary for reconcile to complete. These are typically created by the secrets controller. + esKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, render.TigeraElasticsearchInternalCertSecret, common.OperatorNamespace(), []string{render.TigeraElasticsearchInternalCertSecret}) + Expect(err).NotTo(HaveOccurred()) + Expect(cli.Create(ctx, esKeyPair.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + dnsNames := dns.GetServiceDNSNames(kibana.ServiceName, kibana.Namespace, dns.DefaultClusterDomain) + kbKeyPair, err := certificateManager.GetOrCreateKeyPair(cli, kibana.TigeraKibanaCertSecret, common.OperatorNamespace(), dnsNames) + Expect(err).NotTo(HaveOccurred()) + Expect(cli.Create(ctx, kbKeyPair.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + + // Create the trusted bundle configmap. This is normally created out of band by the secret controller. + bundle := certificateManager.CreateTrustedBundle(esKeyPair) + Expect(cli.Create(ctx, bundle.ConfigMap(render.ElasticsearchNamespace))).NotTo(HaveOccurred()) + + prometheusTLS, err := certificateManager.GetOrCreateKeyPair(cli, monitor.PrometheusClientTLSSecretName, common.OperatorNamespace(), []string{monitor.PrometheusServerTLSSecretName}) + Expect(err).NotTo(HaveOccurred()) + + Expect(cli.Create(ctx, prometheusTLS.Secret(common.OperatorNamespace()))).NotTo(HaveOccurred()) + + Expect(cli.Create(ctx, &operatorv1.APIServer{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + Status: operatorv1.APIServerStatus{State: operatorv1.TigeraStatusReady}, + })).NotTo(HaveOccurred()) + + Expect(cli.Create(ctx, &v3.Tier{ + ObjectMeta: metav1.ObjectMeta{Name: "allow-tigera"}, + })).NotTo(HaveOccurred()) + + install = &operatorv1.Installation{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Status: operatorv1.InstallationStatus{ + Variant: operatorv1.CalicoEnterprise, + Computed: &operatorv1.InstallationSpec{}, + }, + Spec: operatorv1.InstallationSpec{ + Variant: operatorv1.CalicoEnterprise, + Registry: "some.registry.org/", + }, + } + Expect(cli.Create(ctx, install)).ShouldNot(HaveOccurred()) + + Expect(cli.Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{Name: render.ElasticsearchNamespace}, + })).NotTo(HaveOccurred()) + // Create the public certs used to verify the Elasticsearch and Kibana. + esPublicCert, err := secret.CreateTLSSecret( + nil, + "tigera-secure-es-http-certs-public", + common.OperatorNamespace(), + "tls.key", + "tls.crt", + rmeta.DefaultCertificateDuration, + nil, + ) + Expect(err).ShouldNot(HaveOccurred()) + Expect(cli.Create(ctx, esPublicCert)).ShouldNot(HaveOccurred()) + + kbPublicCert, err := secret.CreateTLSSecret( + nil, + "tigera-secure-kb-http-certs-public", + common.OperatorNamespace(), + "tls.key", + "tls.crt", + rmeta.DefaultCertificateDuration, + nil, + ) + Expect(err).ShouldNot(HaveOccurred()) + Expect(cli.Create(ctx, kbPublicCert)).ShouldNot(HaveOccurred()) + + // Create the ES admin username and password. + esAdminUserSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: render.ElasticsearchAdminUserSecret, + Namespace: common.OperatorNamespace(), + }, + Data: map[string][]byte{"tigera-mgmt": []byte("password")}, + } + Expect(cli.Create(ctx, esAdminUserSecret)).ShouldNot(HaveOccurred()) + + // Create the ExternalCertsSecret which contains the client certificate for connecting to the external ES cluster. + externalCertsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: logstorage.ExternalCertsSecret, + Namespace: common.OperatorNamespace(), + }, + Data: map[string][]byte{ + "tls.crt": {}, + }, + } + Expect(cli.Create(ctx, externalCertsSecret)).ShouldNot(HaveOccurred()) + + Expect(cli.Create( + ctx, + &operatorv1.ManagementCluster{ + ObjectMeta: metav1.ObjectMeta{Name: utils.DefaultEnterpriseInstanceKey.Name}, + })).NotTo(HaveOccurred()) + + mockStatus = &status.MockStatus{} + mockStatus.On("Run").Return() + mockStatus.On("OnCRFound").Return() + }) + + It("reconciles successfully", func() { + CreateLogStorage(cli, &operatorv1.LogStorage{ + ObjectMeta: metav1.ObjectMeta{Name: "tigera-secure"}, + Spec: operatorv1.LogStorageSpec{}, + Status: operatorv1.LogStorageStatus{State: operatorv1.TigeraStatusReady}, + }) + + // Run the reconciler. + r, err := NewExternalESReconcilerWithShims(cli, scheme, mockStatus, operatorv1.ProviderNone, dns.DefaultClusterDomain) + Expect(err).ShouldNot(HaveOccurred()) + // Enable cloud mode so the cloud (tenant-from-CloudConfig) path is exercised. + r.opts.Cloud = true + + // Cloud config doesn't exist, so we should be degraded. + mockStatus.On("SetDegraded", operatorv1.ResourceReadError, "Failed to retrieve tigera-secure-cloud-config config map", mock.Anything, mock.Anything) + result, err := r.Reconcile(ctx, reconcile.Request{}) + Expect(err).NotTo(HaveOccurred()) + Expect(result).Should(Equal(reconcile.Result{})) + mockStatus.AssertExpectations(GinkgoT()) + + // Create the cloud config ConfigMap, which contains external ES information and tenant ID for this cluster. + cloudConfig := cloudconfig.NewCloudConfig("tenantId", "tenantName", "externalES.com", "externalKb.com", true) + Expect(cli.Create(ctx, cloudConfig.ConfigMap())).ShouldNot(HaveOccurred()) + + mockStatus.On("ClearDegraded") + mockStatus.On("ReadyToMonitor") + result, err = r.Reconcile(ctx, reconcile.Request{}) + Expect(err).ShouldNot(HaveOccurred()) + Expect(result).Should(Equal(reconcile.Result{})) + + mockStatus.AssertExpectations(GinkgoT()) + + // Verify that the tenantId has been added to the clusterName. + clusterConfig := &corev1.ConfigMap{} + Expect(cli.Get(ctx, client.ObjectKey{Name: relasticsearch.ClusterConfigConfigMapName, Namespace: common.OperatorNamespace()}, clusterConfig)).ToNot(HaveOccurred()) + Expect(clusterConfig.Data["clusterName"]).To(Equal("tenantId.cluster")) + }) +}) diff --git a/pkg/controller/logstorage/elastic/external_elastic_controller.go b/pkg/controller/logstorage/elastic/external_elastic_controller.go index 32d03cdddf..de4d265720 100644 --- a/pkg/controller/logstorage/elastic/external_elastic_controller.go +++ b/pkg/controller/logstorage/elastic/external_elastic_controller.go @@ -37,6 +37,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/logstorage/externalelasticsearch" ) @@ -101,6 +102,13 @@ func AddExternalES(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("log-storage-external-es-controller failed to watch tigera-elasticsearch namespace: %w", err) } + if opts.Cloud { + // Calico Cloud addition. + if err := utils.AddConfigMapWatch(c, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("log-storage-controller failed to watch the ConfigMap resource: %w", err) + } + } + return nil } @@ -152,6 +160,23 @@ func (r *ExternalESController) Reconcile(ctx context.Context, request reconcile. flowShards := logstoragecommon.CalculateFlowShards(ls.Spec.Nodes, logstoragecommon.DefaultElasticsearchShards) clusterConfig := relasticsearch.NewClusterConfig(render.DefaultElasticsearchClusterName, ls.Replicas(), logstoragecommon.DefaultElasticsearchShards, flowShards) + // Calico Cloud addition. For Calico Cloud single-tenant management clusters connected to a + // multi-tenant external ES, augment the cluster config with this management cluster's tenant ID. + if r.opts.Cloud && !r.opts.MultiTenant { + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + if errors.IsNotFound(err) { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to retrieve tigera-secure-cloud-config config map", err, reqLogger) + return reconcile.Result{}, nil + } + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to retrieve tigera-secure-cloud-config config map", err, reqLogger) + return reconcile.Result{}, err + } + if cloudConfig.TenantId() != "" { + clusterConfig.AddTenantId(cloudConfig.TenantId()) + } + } + hdler := utils.NewComponentHandler(reqLogger, r.client, r.scheme, ls) externalElasticsearch := externalelasticsearch.ExternalElasticsearch(installationSpec, clusterConfig, pullSecrets, r.opts.MultiTenant) for _, component := range []render.Component{externalElasticsearch} { diff --git a/pkg/controller/logstorage/elastic/mock.go b/pkg/controller/logstorage/elastic/mock.go index 394595ee0f..cdc6a32991 100644 --- a/pkg/controller/logstorage/elastic/mock.go +++ b/pkg/controller/logstorage/elastic/mock.go @@ -41,7 +41,7 @@ func (m *MockESClient) CreateUser(_ context.Context, _ *utils.User) error { return fmt.Errorf("CreateUser not implemented in mock client") } -func (m *MockESClient) SetILMPolicies(_ context.Context, _ *operatorv1.LogStorage) error { +func (m *MockESClient) SetILMPolicies(_ context.Context, _ *operatorv1.LogStorage, _ bool) error { return nil } diff --git a/pkg/controller/logstorage/kubecontrollers/cloud.go b/pkg/controller/logstorage/kubecontrollers/cloud.go new file mode 100644 index 0000000000..1a73e30a31 --- /dev/null +++ b/pkg/controller/logstorage/kubecontrollers/cloud.go @@ -0,0 +1,90 @@ +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubecontrollers + +import ( + "context" + + "github.com/go-logr/logr" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/logstorage" + "github.com/tigera/operator/pkg/render/logstorage/esgateway" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// esGatewayAddCloudModificationsToConfig modifies the provided *esgateway.Config to include Calico Cloud specific configuration. +func (r *ESKubeControllersController) esGatewayAddCloudModificationsToConfig(c *esgateway.Config, esAdminUserSecret *corev1.Secret, reqLogger logr.Logger, ctx context.Context) (bool, error) { + c.Cloud.Enabled = true + c.Cloud.EsAdminUserSecret = esAdminUserSecret + c.Cloud.ExternalElastic = true + + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + if errors.IsNotFound(err) { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to retrieve tigera-secure-cloud-config config map", err, reqLogger) + return false, nil + } + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to retrieve tigera-secure-cloud-config config map", err, reqLogger) + return false, err + } + + c.Cloud.ExternalESDomain = cloudConfig.ExternalESDomain() + c.Cloud.ExternalKibanaDomain = cloudConfig.ExternalKibanaDomain() + + if cloudConfig.EnableMTLS() { + c.Cloud.ExternalCertsSecret, err = utils.GetSecret(ctx, r.client, logstorage.ExternalCertsSecret, common.OperatorNamespace()) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Waiting for external Elasticsearch certs secret to be available", err, reqLogger) + return false, err + } + if c.Cloud.ExternalCertsSecret == nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Waiting for external Elasticsearch certs secret to be available", err, reqLogger) + return false, nil + } + c.Cloud.EnableMTLS = cloudConfig.EnableMTLS() + } + + if cloudConfig.TenantId() != "" { + c.Cloud.TenantId = cloudConfig.TenantId() + } + + return true, nil +} + +// esKubeControllersAddCloudModificationsToConfig modifies the provided *kubecontrollers.KubeControllersConfiguration to include Calico Cloud specific configuration. +func (r *ESKubeControllersController) esKubeControllersAddCloudModificationsToConfig(c *kubecontrollers.KubeControllersConfiguration, reqLogger logr.Logger, ctx context.Context) (reconcile.Result, bool, error) { + if r.cloud && r.elasticExternal && !r.multiTenant { + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + if errors.IsNotFound(err) { + r.status.SetDegraded(operatorv1.ResourceReadError, "tigera-secure-cloud-config config map not found", err, reqLogger) + return reconcile.Result{}, false, nil + } + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to retrieve tigera-secure-cloud-config config map", err, reqLogger) + return reconcile.Result{}, false, err + } + + if cloudConfig.TenantId() != "" { + c.TenantId = cloudConfig.TenantId() + } + } + + return reconcile.Result{}, true, nil +} diff --git a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go index c51cbc076a..2c2ae2c0a1 100644 --- a/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go +++ b/pkg/controller/logstorage/kubecontrollers/es_kube_controllers.go @@ -45,6 +45,7 @@ import ( "github.com/tigera/operator/pkg/controller/utils/imageset" "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/kubecontrollers" "github.com/tigera/operator/pkg/render/logstorage/esgateway" @@ -61,6 +62,7 @@ type ESKubeControllersController struct { clusterDomain string elasticExternal bool multiTenant bool + cloud bool tierWatchReady *utils.ReadyFlag } @@ -85,6 +87,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { status: status.New(mgr.GetClient(), initializer.TigeraStatusLogStorageKubeController, opts.KubernetesVersion), elasticExternal: opts.ElasticExternal, multiTenant: opts.MultiTenant, + cloud: opts.Cloud, tierWatchReady: &utils.ReadyFlag{}, } r.status.Run(opts.ShutdownContext) @@ -143,6 +146,12 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { if err := utils.AddDeploymentWatch(c, kubecontrollers.EsKubeController, esKubeControllersNamespace.InstallNamespace()); err != nil { return fmt.Errorf("log-storage-access-controller failed to watch the Service resource: %w", err) } + if opts.Cloud && opts.ElasticExternal { + // This ConfigMap is needed for utils.GetCloudConfig + if err = utils.AddConfigMapWatch(c, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("log-storage-kubecontrollers failed to watch the ConfigMap resource: %w", err) + } + } // Perform periodic reconciliation. This acts as a backstop to catch reconcile issues, // and also makes sure we spot when things change that might not trigger a reconciliation. @@ -343,6 +352,12 @@ func (r *ESKubeControllersController) Reconcile(ctx context.Context, request rec Namespace: helper.InstallNamespace(), BindingNamespaces: namespaces, Tenant: nil, + Cloud: r.cloud, + } + if r.cloud { + if result, proceed, err := r.esKubeControllersAddCloudModificationsToConfig(&kubeControllersCfg, reqLogger, ctx); err != nil || !proceed { + return result, err + } } esKubeControllerComponents := kubecontrollers.NewElasticsearchKubeControllers(&kubeControllersCfg) diff --git a/pkg/controller/logstorage/kubecontrollers/esgateway.go b/pkg/controller/logstorage/kubecontrollers/esgateway.go index 1cb1d525a9..feca28bbba 100644 --- a/pkg/controller/logstorage/kubecontrollers/esgateway.go +++ b/pkg/controller/logstorage/kubecontrollers/esgateway.go @@ -112,6 +112,13 @@ func (r *ESKubeControllersController) createESGateway( LogStorage: logStorage, } + // Calico Cloud modifications. Only applied for cloud external-ES installs. + if r.cloud && r.elasticExternal { + if proceed, err := r.esGatewayAddCloudModificationsToConfig(cfg, esAdminUserSecret, reqLogger, ctx); err != nil || !proceed { + return err + } + } + esGatewayComponent := esgateway.EsGateway(cfg) if err = imageset.ApplyImageSet(ctx, r.client, variant, esGatewayComponent); err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Error with images from ImageSet", err, reqLogger) diff --git a/pkg/controller/logstorage/linseed/linseed_controller.go b/pkg/controller/logstorage/linseed/linseed_controller.go index 3175bc4415..3f6451a12e 100644 --- a/pkg/controller/logstorage/linseed/linseed_controller.go +++ b/pkg/controller/logstorage/linseed/linseed_controller.go @@ -47,6 +47,7 @@ import ( "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/dns" "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" relasticsearch "github.com/tigera/operator/pkg/render/common/elasticsearch" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/render/logstorage" @@ -67,6 +68,7 @@ type LinseedSubController struct { dpiAPIReady *utils.ReadyFlag multiTenant bool elasticExternal bool + cloud bool } func Add(mgr manager.Manager, opts options.ControllerOptions) error { @@ -84,6 +86,7 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { multiTenant: opts.MultiTenant, status: status.New(mgr.GetClient(), "log-storage-access", opts.KubernetesVersion), elasticExternal: opts.ElasticExternal, + cloud: opts.Cloud, } r.status.Run(opts.ShutdownContext) @@ -177,6 +180,13 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { }) go utils.WaitToAddResourceWatch(c, opts.K8sClientset, log, r.dpiAPIReady, []client.Object{&v3.DeepPacketInspection{TypeMeta: metav1.TypeMeta{Kind: v3.KindDeepPacketInspection}}}) + if opts.Cloud && opts.ElasticExternal { + // This ConfigMap is needed for utils.GetCloudConfig + if err = utils.AddConfigMapWatch(c, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("log-storage-linseed-controller failed to watch the ConfigMap resource: %w", err) + } + } + // Perform periodic reconciliation. This acts as a backstop to catch reconcile issues, // and also makes sure we spot when things change that might not trigger a reconciliation. err = utils.AddPeriodicReconcile(c, utils.PeriodicReconcileTime, eventHandler) @@ -310,11 +320,22 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. return reconcile.Result{RequeueAfter: utils.StandardRetry}, nil } } else { - // If we're using an external ES, the Tenant resource must specify the ES endpoint. - if tenant == nil || tenant.Spec.Elastic == nil || tenant.Spec.Elastic.URL == "" { - reqLogger.Error(nil, "Elasticsearch URL must be specified for this tenant") - r.status.SetDegraded(operatorv1.ResourceValidationError, "Elasticsearch URL must be specified for this tenant", nil, reqLogger) - return reconcile.Result{}, nil + if r.multiTenant || !r.cloud { + // If we're using an external ES, the Tenant resource must specify the ES endpoint. + if tenant == nil || tenant.Spec.Elastic == nil || tenant.Spec.Elastic.URL == "" { + reqLogger.Error(nil, "Elasticsearch URL must be specified for this tenant") + r.status.SetDegraded(operatorv1.ResourceValidationError, "Elasticsearch URL must be specified for this tenant", nil, reqLogger) + return reconcile.Result{}, nil + } + } else { + // This is a Calico Cloud single-tenant cluster connected to a multi-tenant ES. Read the + // tenant configuration from the CloudConfig ConfigMap. + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) + return reconcile.Result{}, err + } + tenant = cloudConfig.ToTenant() } // Determine the host and port from the URL. @@ -447,6 +468,7 @@ func (r *LinseedSubController) Reconcile(ctx context.Context, request reconcile. ElasticClientSecret: esClientSecret, ElasticClientCredentialsSecret: &credentials, LogStorage: logStorage, + Cloud: r.cloud, } linseedComponent := linseed.Linseed(cfg) diff --git a/pkg/controller/manager/manager_controller.go b/pkg/controller/manager/manager_controller.go index 5f637b92d7..993e229821 100644 --- a/pkg/controller/manager/manager_controller.go +++ b/pkg/controller/manager/manager_controller.go @@ -202,6 +202,12 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { } } + if opts.Cloud { + if err = addCloudWatch(c, eventHandler, opts.ElasticExternal); err != nil { + return fmt.Errorf("manager-controller failed to add CC watches: %v", err) + } + } + return nil } @@ -467,6 +473,31 @@ func (r *ReconcileManager) Reconcile(ctx context.Context, request reconcile.Requ if err != nil { r.status.SetDegraded(operatorv1.ResourceCreateError, "Error creating trusted bundle for manager", err, logc) } + + // Handle all the resources that are specific to Calico Cloud. For non-cloud installs this is + // skipped entirely, leaving mcr at its zero value and enterprise behavior unchanged. + var mcr render.ManagerCloudResources + if r.opts.Cloud { + var reconcileResult *reconcile.Result + bundleMaker, mcr, tenant, reconcileResult, err = r.handleCloudReconcile( + ctx, + logc, + helper, + tenant, + authenticationCR, + certificateManager, + bundleMaker, + trustedSecretNames, + request.Namespace, + ) + if err != nil { + // status degraded should already be set by r.handleCloudReconcile + return reconcile.Result{}, err + } else if reconcileResult != nil { + return *reconcileResult, nil + } + } + certificateManager.AddToStatusManager(r.status, helper.InstallNamespace()) // Check that Prometheus is running @@ -582,7 +613,7 @@ func (r *ReconcileManager) Reconcile(ctx context.Context, request reconcile.Requ tunnelSecretPassthrough = render.NewCreationPassthrough(tunnelCASecret) } - keyValidatorConfig, err := utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain) + keyValidatorConfig, err := utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain, r.opts.Cloud && !r.opts.MultiTenant) if err != nil { r.status.SetDegraded(operatorv1.ResourceValidationError, "Failed to process the authentication CR.", err, logc) return reconcile.Result{}, err @@ -705,6 +736,8 @@ func (r *ReconcileManager) Reconcile(ctx context.Context, request reconcile.Requ Authentication: authenticationCR, KibanaEnabled: kibanaEnabled, CACertCommonName: certificateManager.CACertCommonName(), + Cloud: r.opts.Cloud, + CloudResources: mcr, } // Render the desired objects from the CRD and create or update them. diff --git a/pkg/controller/manager/manager_controller_cloud.go b/pkg/controller/manager/manager_controller_cloud.go new file mode 100644 index 0000000000..5f0c56591b --- /dev/null +++ b/pkg/controller/manager/manager_controller_cloud.go @@ -0,0 +1,184 @@ +// Copyright (c) 2022-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "context" + "fmt" + + "github.com/tigera/operator/pkg/controller/certificatemanager" + "github.com/tigera/operator/pkg/ctrlruntime" + "github.com/tigera/operator/pkg/tls/certificatemanagement" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/utils" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/cloudconfig" +) + +var ( + CloudManagerConfigOverrideName = "cloud-manager-config" + CloudVoltronConfigOverrideName = "cloud-voltron-config" +) + +func addCloudWatch(c ctrlruntime.Controller, eventHandler handler.EventHandler, elasticExternal bool) error { + if elasticExternal { + if err := utils.AddConfigMapWatch(c, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), eventHandler); err != nil { + return fmt.Errorf("manager-controller failed to watch the ConfigMap resource: %v", err) + } + } else { + if err := utils.AddConfigMapWatch(c, utils.CloudAuthConfig, common.OperatorNamespace(), eventHandler); err != nil { + return fmt.Errorf("manager-controller failed to watch the ConfigMap resource: %v", err) + } + } + + if err := utils.AddConfigMapWatch(c, CloudVoltronConfigOverrideName, common.OperatorNamespace(), eventHandler); err != nil { + return err + } + + if err := utils.AddConfigMapWatch(c, CloudManagerConfigOverrideName, common.OperatorNamespace(), eventHandler); err != nil { + return fmt.Errorf("manager-controller failed to watch the ConfigMap resource: %v", err) + } + + return nil +} + +// handleCloudReconcile reconciles cloud resources and returns the cloud-ready certificate trusted bundle and resources. +// It returns a non-nil reconcile.Result when it's waiting for resources to be available. +func (r *ReconcileManager) handleCloudReconcile( + ctx context.Context, + reqLogger logr.Logger, + helper utils.NamespaceHelper, + tenant *operatorv1.Tenant, + authenticationCR *operatorv1.Authentication, + certificateManager certificatemanager.CertificateManager, + bundleMaker certificatemanagement.TrustedBundle, + trustedSecretNames []string, + requestNamespace string, +) (certificatemanagement.TrustedBundle, render.ManagerCloudResources, *operatorv1.Tenant, *reconcile.Result, error) { + + if authenticationCR != nil && authenticationCR.Spec.OIDC != nil && authenticationCR.Spec.OIDC.Type == operatorv1.OIDCTypeTigera { + var err error + bundleMaker, err = certificateManager.CreateTrustedBundleWithSystemRootCertificates() + if err != nil { + r.status.SetDegraded(operatorv1.ResourceCreateError, "failed to create trusted bundle with system root certs", err, reqLogger) + return nil, render.ManagerCloudResources{}, nil, nil, err + } + } + + for _, secret := range trustedSecretNames { + certificate, err := certificateManager.GetCertificate(r.client, secret, helper.TruthNamespace()) + if err != nil { + r.status.SetDegraded(operatorv1.CertificateError, fmt.Sprintf("Failed to retrieve %s", secret), err, reqLogger) + return nil, render.ManagerCloudResources{}, nil, nil, err + } else if certificate == nil { + reqLogger.Info(fmt.Sprintf("Waiting for secret '%s' to become available", secret)) + r.status.SetDegraded(operatorv1.ResourceNotReady, fmt.Sprintf("Waiting for secret '%s' to become available", secret), nil, reqLogger) + // stop reconciler iteration with no error as it is waiting for a resource to become available + return nil, render.ManagerCloudResources{}, nil, &reconcile.Result{}, nil + } + + if bundleMaker != nil { + bundleMaker.AddCertificates(certificate) + } + } + + mcr := render.ManagerCloudResources{ + VoltronMetricsEnabled: true, + VoltronInternalHttpsPort: 9444, + ManagerExtraEnv: map[string]string{}, + } + + if err := r.cloudConfigOverride(ctx, helper.TruthNamespace(), &mcr); err != nil { + return nil, render.ManagerCloudResources{}, nil, nil, err + } + + if !r.opts.MultiTenant { + if r.opts.ElasticExternal { + // For single-tenant clusters sharing an external ES, extract the tenant information from + // the cloud config map. + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) + return nil, render.ManagerCloudResources{}, nil, nil, err + } + tenant = cloudConfig.ToTenant() + } else { + var err error + tenant, err = utils.GetTenantFromCloudAuthConfig(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to fetch tenant information from config map", err, reqLogger) + } + } + } + + return bundleMaker, mcr, tenant, nil, nil +} + +// cloudConfigOverride set manager and voltron renderer env override +func (r *ReconcileManager) cloudConfigOverride(ctx context.Context, namespace string, mcr *render.ManagerCloudResources) error { + var err error + managerCfg, err := r.getConfigMapData(ctx, types.NamespacedName{Namespace: namespace, Name: CloudManagerConfigOverrideName}) + if err != nil { + return err + } + for key, val := range managerCfg { + switch key { + case "portalAPIURL": + // support legacy functionality where 'portalAPIURL' was a special field used to set + // the portal url and enable support. + mcr.ManagerExtraEnv["CNX_PORTAL_URL"] = val + mcr.ManagerExtraEnv["ENABLE_PORTAL_SUPPORT"] = "true" + + // support legacy functionality where 'auth0OrgID' was a special field used to set the org ID + case "auth0OrgID": + mcr.ManagerExtraEnv["CNX_AUTH0_ORG_ID"] = val + + // special key used to control which image of manager is used + case "managerImage": + mcr.ManagerImage = val + + // add any other fields as-is + default: + mcr.ManagerExtraEnv[key] = val + } + } + + mcr.VoltronExtraEnv, err = r.getConfigMapData(ctx, types.NamespacedName{Namespace: namespace, Name: CloudVoltronConfigOverrideName}) + if err != nil { + return err + } + + return nil +} + +func (r *ReconcileManager) getConfigMapData(ctx context.Context, name types.NamespacedName) (map[string]string, error) { + configMap := &corev1.ConfigMap{} + if err := r.client.Get(ctx, name, configMap); err != nil { + if !errors.IsNotFound(err) { + return nil, fmt.Errorf("failed to read %s ConfigMap: %s", name, err.Error()) + } + return nil, nil + } + return configMap.Data, nil +} diff --git a/pkg/controller/manager/manager_controller_cloud_test.go b/pkg/controller/manager/manager_controller_cloud_test.go new file mode 100644 index 0000000000..e28cfb45ff --- /dev/null +++ b/pkg/controller/manager/manager_controller_cloud_test.go @@ -0,0 +1,98 @@ +// Copyright (c) 2022-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package manager + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/render" +) + +func TestEnv(t *testing.T) { + var ( + ctx = context.Background() + ) + + for _, tc := range []struct { + name string + configMapData map[string]string + expectedEnvVars map[string]string + expectedManagerImage string + }{ + { + name: "no data", + configMapData: map[string]string{}, + expectedEnvVars: map[string]string{}, + }, + { + name: "manager image", + configMapData: map[string]string{"managerImage": "tigera/manager:foo"}, + expectedEnvVars: map[string]string{}, + expectedManagerImage: "tigera/manager:foo", + }, + { + name: "arbitrary var set", + configMapData: map[string]string{"FOO": "BAR"}, + expectedEnvVars: map[string]string{"FOO": "BAR"}, + }, + { + name: "arbitrary known usage value set", + configMapData: map[string]string{"ENABLE_CC_USAGE": "true"}, + expectedEnvVars: map[string]string{"ENABLE_CC_USAGE": "true"}, + }, + { + name: "legacy portal api settings set", + configMapData: map[string]string{"portalAPIURL": "test.calicocloud.io"}, + expectedEnvVars: map[string]string{ + "CNX_PORTAL_URL": "test.calicocloud.io", + "ENABLE_PORTAL_SUPPORT": "true", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var ( + c = fake.NewClientBuilder().Build() + + rm = &ReconcileManager{ + client: c, + } + + mcr = &render.ManagerCloudResources{ + ManagerExtraEnv: make(map[string]string), + } + ) + + require.NoError(t, c.Create(ctx, &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: CloudManagerConfigOverrideName, + Namespace: common.OperatorNamespace(), + }, + Data: tc.configMapData, + })) + + err := rm.cloudConfigOverride(ctx, common.OperatorNamespace(), mcr) + require.NoError(t, err) + require.Equal(t, tc.expectedEnvVars, mcr.ManagerExtraEnv) + require.Equal(t, tc.expectedManagerImage, mcr.ManagerImage) + }) + } +} diff --git a/pkg/controller/monitor/monitor_controller.go b/pkg/controller/monitor/monitor_controller.go index 028c85059e..362e5df5f9 100644 --- a/pkg/controller/monitor/monitor_controller.go +++ b/pkg/controller/monitor/monitor_controller.go @@ -109,6 +109,7 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions, promethe licenseAPIReady: licenseAPIReady, clusterDomain: opts.ClusterDomain, multiTenant: opts.MultiTenant, + cloud: opts.Cloud, } r.status.AddStatefulSets([]types.NamespacedName{ @@ -191,6 +192,7 @@ type ReconcileMonitor struct { licenseAPIReady *utils.ReadyFlag clusterDomain string multiTenant bool + cloud bool } func (r *ReconcileMonitor) getMonitor(ctx context.Context) (*operatorv1.Monitor, error) { @@ -368,7 +370,7 @@ func (r *ReconcileMonitor) Reconcile(ctx context.Context, request reconcile.Requ } var keyValidatorConfig rauth.KeyValidatorConfig if authenticationCR != nil && authenticationCR.Status.State == operatorv1.TigeraStatusReady { - keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.clusterDomain) + keyValidatorConfig, err = utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.clusterDomain, r.cloud && !r.multiTenant) if err != nil { r.status.SetDegraded(operatorv1.ResourceUpdateError, "Failed to process the authentication CR.", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 14acd47230..51959effb3 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -48,6 +48,15 @@ type ControllerOptions struct { // and instead will configure the cluster to use an external Elasticsearch. ElasticExternal bool + // Cloud indicates the operator is running as a Calico Cloud install. When set, controllers + // activate cloud-specific behavior (cloud render decorations, cloud config maps, etc.). When + // false the operator behaves as a regular Calico/Calico Enterprise install. + Cloud bool + + // ESMigration is enabled in the last phase of an ES migration, when we need to keep both an + // LSS configuration and internal elasticsearch running. Only meaningful when Cloud is set. + ESMigration bool + // Whether or not to use crd.projectcalico.org/v1 or projectcalico.org/v3 for Calico CRDs. UseV3CRDs bool diff --git a/pkg/controller/packetcapture/packetcapture_controller.go b/pkg/controller/packetcapture/packetcapture_controller.go index 03eb828f9b..c075178d50 100644 --- a/pkg/controller/packetcapture/packetcapture_controller.go +++ b/pkg/controller/packetcapture/packetcapture_controller.go @@ -232,7 +232,7 @@ func (r *ReconcilePacketCapture) Reconcile(ctx context.Context, request reconcil return reconcile.Result{}, err } - keyValidatorConfig, err := utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain) + keyValidatorConfig, err := utils.GetKeyValidatorConfig(ctx, r.client, authenticationCR, r.opts.ClusterDomain, false) if err != nil { r.status.SetDegraded(operatorv1.ResourceValidationError, "Failed to process the authentication CR.", err, reqLogger) return reconcile.Result{}, err diff --git a/pkg/controller/policyrecommendation/policyrecommendation_controller.go b/pkg/controller/policyrecommendation/policyrecommendation_controller.go index 57df1bf030..1fca4018cc 100644 --- a/pkg/controller/policyrecommendation/policyrecommendation_controller.go +++ b/pkg/controller/policyrecommendation/policyrecommendation_controller.go @@ -44,6 +44,7 @@ import ( "github.com/tigera/operator/pkg/ctrlruntime" "github.com/tigera/operator/pkg/render" rcertificatemanagement "github.com/tigera/operator/pkg/render/certificatemanagement" + "github.com/tigera/operator/pkg/render/common/cloudconfig" "github.com/tigera/operator/pkg/render/common/networkpolicy" "github.com/tigera/operator/pkg/tls/certificatemanagement" ) @@ -138,6 +139,13 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error { return fmt.Errorf("policy-recommendation-controller failed to watch policy-recommendation Tigerastatus: %w", err) } + if opts.Cloud && opts.ElasticExternal { + // This ConfigMap is needed for utils.GetCloudConfig + if err = utils.AddConfigMapWatch(c, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("policy-recommendation-controller failed to watch the ConfigMap resource: %w", err) + } + } + return nil } @@ -333,6 +341,17 @@ func (r *ReconcilePolicyRecommendation) Reconcile(ctx context.Context, request r return reconcile.Result{}, err } + if r.opts.Cloud && r.opts.ElasticExternal && !r.opts.MultiTenant { + // For Calico Cloud single-tenant clusters sharing an external ES, extract the tenant + // information from the cloud config map. + cloudConfig, err := utils.GetCloudConfig(ctx, r.client) + if err != nil { + r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, logc) + return reconcile.Result{}, err + } + tenant = cloudConfig.ToTenant() + } + // Create a component handler to manage the rendered component. defaultHandler := utils.NewComponentHandler(log, r.client, r.scheme, policyRecommendation) diff --git a/pkg/controller/tiers/tiers_controller.go b/pkg/controller/tiers/tiers_controller.go index e7e7bcbb43..0b154a9dd7 100644 --- a/pkg/controller/tiers/tiers_controller.go +++ b/pkg/controller/tiers/tiers_controller.go @@ -122,6 +122,13 @@ func (r *ReconcileTiers) Reconcile(ctx context.Context, request reconcile.Reques return reconcile.Result{RequeueAfter: utils.StandardRetry}, nil } + if r.opts.Cloud { + if err := r.cloudPatchTier(ctx); err != nil { + r.status.SetDegraded(operatorv1.ResourcePatchError, "Error patching tier", err, reqLogger) + return reconcile.Result{}, nil + } + } + tiersConfig, reconcileResult := r.prepareTiersConfig(ctx, reqLogger) if reconcileResult != nil { return *reconcileResult, nil diff --git a/pkg/controller/tiers/tiers_controller_cloud.go b/pkg/controller/tiers/tiers_controller_cloud.go new file mode 100644 index 0000000000..60766d4182 --- /dev/null +++ b/pkg/controller/tiers/tiers_controller_cloud.go @@ -0,0 +1,47 @@ +// Copyright (c) 2023-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tiers + +import ( + "context" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// cloudPatchTier removes the allow-tigera tier's app.kubernetes.io/instance label to fix Calico +// Cloud CD sync. Only invoked for cloud installs. +func (r *ReconcileTiers) cloudPatchTier(ctx context.Context) error { + tier := &v3.Tier{} + err := r.client.Get(ctx, client.ObjectKey{Name: networkpolicy.CalicoTierName}, tier) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + return err + } + + if tier.Labels["app.kubernetes.io/instance"] != "" { + tierPatchFrom := client.MergeFrom(tier.DeepCopy()) + delete(tier.Labels, "app.kubernetes.io/instance") + + if err = r.client.Patch(ctx, tier, tierPatchFrom); err != nil { + return err + } + } + return nil +} diff --git a/pkg/controller/utils/auth.go b/pkg/controller/utils/auth.go index a8ba89d4fa..fdb3773c10 100644 --- a/pkg/controller/utils/auth.go +++ b/pkg/controller/utils/auth.go @@ -38,7 +38,11 @@ import ( // GetKeyValidatorConfig uses the operatorv1.Authentication CR given to create the KeyValidatorConfig. This may be // either a DexKeyValidatorConfig or a tigerakvc.KeyValidatorConfig. -func GetKeyValidatorConfig(ctx context.Context, cli client.Client, authenticationCR *operatorv1.Authentication, clusterDomain string) (rauth.KeyValidatorConfig, error) { +// +// addTenancyClaim must only be set for Calico Cloud single-tenant management clusters; when true the +// KeyValidatorConfig is configured to require the cloud tenant claim (read from the cloud-auth-config +// ConfigMap). Regular Calico/Calico Enterprise callers pass false, leaving behavior unchanged. +func GetKeyValidatorConfig(ctx context.Context, cli client.Client, authenticationCR *operatorv1.Authentication, clusterDomain string, addTenancyClaim bool) (rauth.KeyValidatorConfig, error) { var keyValidatorConfig rauth.KeyValidatorConfig if authenticationCR != nil { _, idpSecret, err := GetSecretOrProviderClass(ctx, cli, authenticationCR) @@ -50,6 +54,14 @@ func GetKeyValidatorConfig(ctx context.Context, cli client.Client, authenticatio if oidc != nil && oidc.Type == operatorv1.OIDCTypeTigera { var kvcOptions []tigerakvc.Option + if addTenancyClaim { + cloudOption, err := getCloudKeyValidatorOption(ctx, cli) + if err != nil { + return nil, err + } + kvcOptions = append(kvcOptions, cloudOption) + } + if oidc.UsernameClaim != "" { kvcOptions = append(kvcOptions, tigerakvc.WithUsernameClaim(oidc.UsernameClaim)) } diff --git a/pkg/controller/utils/auth_cloud.go b/pkg/controller/utils/auth_cloud.go new file mode 100644 index 0000000000..d10b1bfd4b --- /dev/null +++ b/pkg/controller/utils/auth_cloud.go @@ -0,0 +1,45 @@ +// Copyright (c) 2022-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "fmt" + + "github.com/tigera/operator/pkg/common" + tigerakvc "github.com/tigera/operator/pkg/render/common/authentication/tigera/key_validator_config" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// getCloudKeyValidatorOption reads the cloud-auth-config ConfigMap (present only on Calico Cloud +// single-tenant management clusters) and returns an Option that configures the KeyValidatorConfig +// to require the cloud tenant claim. It is only called when cloud tenancy is enabled. +func getCloudKeyValidatorOption(ctx context.Context, cli client.Client) (tigerakvc.Option, error) { + cm := &corev1.ConfigMap{} + if err := cli.Get(ctx, types.NamespacedName{Name: CloudAuthConfig, Namespace: common.OperatorNamespace()}, cm); err != nil { + return nil, fmt.Errorf("missing config map %s/%s: %w", common.OperatorNamespace(), CloudAuthConfig, err) + } + + tenantID, ok := cm.Data["tenantID"] + if !ok { + return nil, fmt.Errorf("cloud config map %s/%s is missing the tenantID field", common.OperatorNamespace(), CloudAuthConfig) + } else if tenantID == "" { + return nil, fmt.Errorf("cloud config map %s/%s has empty tenantID field", common.OperatorNamespace(), CloudAuthConfig) + } + + return tigerakvc.WithTenantClaim(tenantID), nil +} diff --git a/pkg/controller/utils/cloudconfig.go b/pkg/controller/utils/cloudconfig.go new file mode 100644 index 0000000000..6f9570ea75 --- /dev/null +++ b/pkg/controller/utils/cloudconfig.go @@ -0,0 +1,69 @@ +// Copyright (c) 2022-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "context" + "fmt" + + v1 "github.com/tigera/operator/api/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/render/common/cloudconfig" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// CloudAuthConfig is the name of the ConfigMap holding cloud authentication configuration +// (e.g. the tenantID) for a single-tenant cloud management cluster backed by internal Elasticsearch. +const CloudAuthConfig = "cloud-auth-config" + +// GetCloudConfig retrieves the config map containing the configuration values needed to set up communications with +// external Elasticsearch and Kibana, such as the externalESDomain and externalKibanaDomain. +func GetCloudConfig(ctx context.Context, cli client.Client) (*cloudconfig.CloudConfig, error) { + configMap := &corev1.ConfigMap{} + if err := cli.Get(ctx, client.ObjectKey{Name: cloudconfig.CloudConfigConfigMapName, Namespace: common.OperatorNamespace()}, configMap); err != nil { + return nil, err + } + + return cloudconfig.NewCloudConfigFromConfigMap(configMap) +} + +func GetTenantFromCloudAuthConfig(ctx context.Context, cli client.Client) (*v1.Tenant, error) { + cm := &corev1.ConfigMap{} + if err := cli.Get(ctx, types.NamespacedName{Name: CloudAuthConfig, Namespace: common.OperatorNamespace()}, cm); err != nil { + return nil, fmt.Errorf("missing config map %s/%s: %w", common.OperatorNamespace(), CloudAuthConfig, err) + } + + tenantID, ok := cm.Data["tenantID"] + if !ok { + return nil, fmt.Errorf("cloud config map %s/%s is missing the tenantID field", common.OperatorNamespace(), CloudAuthConfig) + } else if tenantID == "" { + return nil, fmt.Errorf("cloud config map %s/%s has empty tenantID field", common.OperatorNamespace(), CloudAuthConfig) + } + + return &v1.Tenant{ + // We don't specify a Namespace for this tenant because it represents a singular tenant installed + // in this management cluster. The signals to the render code that this is a single-tenant cluster and not + // a cluster capable of multi-tenancy. We are also omitting elastic configuration since this setup maps + // to internal elastic + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: v1.TenantSpec{ + ID: tenantID, + }, + }, nil +} diff --git a/pkg/controller/utils/elasticsearch.go b/pkg/controller/utils/elasticsearch.go index ee604dde2c..0a977f291b 100644 --- a/pkg/controller/utils/elasticsearch.go +++ b/pkg/controller/utils/elasticsearch.go @@ -112,7 +112,7 @@ func GetElasticsearchClusterConfig(ctx context.Context, cli client.Client) (*rel type ElasticsearchClientCreator func(client client.Client, ctx context.Context, elasticHTTPSEndpoint string, external bool) (ElasticClient, error) type ElasticClient interface { - SetILMPolicies(context.Context, *operatorv1.LogStorage) error + SetILMPolicies(context.Context, *operatorv1.LogStorage, bool) error CreateUser(context.Context, *User) error DeleteUser(context.Context, *User) error GetUsers(ctx context.Context) ([]User, error) @@ -422,9 +422,13 @@ func (es *esClient) GetUsers(ctx context.Context) ([]User, error) { return users, nil } -// SetILMPolicies creates ILM policies for each timeseries based index using the retention period and storage size in LogStorage -func (es *esClient) SetILMPolicies(ctx context.Context, ls *operatorv1.LogStorage) error { +// SetILMPolicies creates ILM policies for each timeseries based index using the retention period and storage size in LogStorage. +// When cloud is true, additional Calico Cloud specific ILM policies are added. +func (es *esClient) SetILMPolicies(ctx context.Context, ls *operatorv1.LogStorage, cloud bool) error { policyList := es.listILMPolicies(ls) + if cloud { + es.tweakILMPoliciesForCloud(ls, policyList) + } return es.createOrUpdatePolicies(ctx, policyList) } diff --git a/pkg/controller/utils/elasticsearch_cloud.go b/pkg/controller/utils/elasticsearch_cloud.go new file mode 100644 index 0000000000..dda5ae395b --- /dev/null +++ b/pkg/controller/utils/elasticsearch_cloud.go @@ -0,0 +1,29 @@ +// Copyright (c) 2023-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import operatorv1 "github.com/tigera/operator/api/v1" + +// tweakILMPoliciesForCloud tweaks the ILM policies generated by listILMPolicies() for Calico Cloud. +// It adds a policy for the runtime logs. There is no need to reduce the size of the flow policy. +// The percentages are only used to come up with a max_size of an index and it is not critical +// that we account for the size taken up by the runtime index when coming up with that max_size. +// This allocates 16.5% of the 70% ES disk space to runtime logs. +func (es *esClient) tweakILMPoliciesForCloud(ls *operatorv1.LogStorage, policies map[string]policyDetail) { + totalEsStorage := getTotalEsDisk(ls) + majorPctOfTotalDisk := 0.7 + + policies["tigera_secure_ee_runtime"] = buildILMPolicy(totalEsStorage, majorPctOfTotalDisk, 0.165, 8, true) +} diff --git a/pkg/render/apiserver.go b/pkg/render/apiserver.go index 1d7e6ff4b6..46e978cd1d 100644 --- a/pkg/render/apiserver.go +++ b/pkg/render/apiserver.go @@ -147,6 +147,12 @@ type APIServerConfiguration struct { // tigera-network-admin. RBACManagementEnabled bool + // Cloud indicates the API server is being rendered for a Calico Cloud install. It gates + // cloud-specific RBAC in the tigera-ui-user / tigera-network-admin cluster roles (Calico Cloud + // exposes only per-user UISettings and grants access to runtime logs). When false the RBAC is + // exactly the regular Calico/Calico Enterprise RBAC. + Cloud bool + // Whether or not we should run the aggregation API server for projectcalico.org/v3 APIs // as part of this component. RequiresAggregationServer bool @@ -1759,22 +1765,40 @@ func (c *apiServerComponent) tigeraUserClusterRole() *rbacv1.ClusterRole { }, Verbs: []string{"get", "watch", "list"}, }, - // User can: - // - read UISettings in the cluster-settings group - // - read and write UISettings in the user-settings group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"get", "list", "watch"}, - ResourceNames: []string{"cluster-settings"}, - }, + } + + // User can: + // - read UISettings in the cluster-settings group (non-cloud only) + // - read and write UISettings in the user-settings group + // Default settings group and settings are created in manager.go. + // Calico Cloud exposes only per-user UISettings, so the cluster-settings group is omitted there. + if c.cfg.Cloud { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"user-settings"}, + }, + ) + } else { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"get", "list", "watch"}, + ResourceNames: []string{"cluster-settings"}, + }, + ) + } + + rules = append(rules, []rbacv1.PolicyRule{ { APIGroups: []string{"projectcalico.org"}, Resources: []string{"uisettingsgroups/data"}, @@ -1823,19 +1847,22 @@ func (c *apiServerComponent) tigeraUserClusterRole() *rbacv1.ClusterRole { Resources: []string{"securityeventwebhooks"}, Verbs: []string{"get", "list"}, }, - } + }...) // Privileges for lma.tigera.io have no effect on managed clusters. if c.cfg.ManagementClusterConnection == nil { // Access to flow logs, audit logs, and statistics. // Access to log into Kibana for oidc users. + // Calico Cloud additionally grants access to runtime logs. + resourceNames := []string{"flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations"} + if c.cfg.Cloud { + resourceNames = append([]string{"runtime"}, resourceNames...) + } rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "recommendations", - }, - Verbs: []string{"get"}, + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: resourceNames, + Verbs: []string{"get"}, }) } @@ -1981,22 +2008,46 @@ func (c *apiServerComponent) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole }, Verbs: []string{"create", "update", "delete", "patch", "get", "watch", "list"}, }, - // User can: - // - read and write UISettings in the cluster-settings group, and rename the group - // - read and write UISettings in the user-settings group, and rename the group - // Default settings group and settings are created in manager.go. - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups"}, - Verbs: []string{"get", "patch", "update"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, - { - APIGroups: []string{"projectcalico.org"}, - Resources: []string{"uisettingsgroups/data"}, - Verbs: []string{"*"}, - ResourceNames: []string{"cluster-settings", "user-settings"}, - }, + } + + // User can: + // - read and write UISettings in the cluster-settings group, and rename the group (non-cloud only) + // - read and write UISettings in the user-settings group, and rename the group + // Default settings group and settings are created in manager.go. + // Calico Cloud exposes only per-user UISettings, so the cluster-settings group is omitted there. + if c.cfg.Cloud { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get"}, + ResourceNames: []string{"user-settings"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"user-settings"}, + }, + ) + } else { + rules = append(rules, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups"}, + Verbs: []string{"get", "patch", "update"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + rbacv1.PolicyRule{ + APIGroups: []string{"projectcalico.org"}, + Resources: []string{"uisettingsgroups/data"}, + Verbs: []string{"*"}, + ResourceNames: []string{"cluster-settings", "user-settings"}, + }, + ) + } + + rules = append(rules, []rbacv1.PolicyRule{ // Allow the user to read and write applicationlayers to enable/disable WAF. { APIGroups: []string{"operator.tigera.io"}, @@ -2058,7 +2109,7 @@ func (c *apiServerComponent) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole }, Verbs: []string{"patch"}, }, - } + }...) // Role/binding access for the RBAC management UI. ui-apis writes these // impersonating the caller, so the apiserver enforces escalation against the @@ -2083,13 +2134,16 @@ func (c *apiServerComponent) tigeraNetworkAdminClusterRole() *rbacv1.ClusterRole if c.cfg.ManagementClusterConnection == nil { // Access to flow logs, audit logs, and statistics. // Elasticsearch superuser access once logged into Kibana. + // Calico Cloud additionally grants access to runtime logs. + resourceNames := []string{"flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations"} + if c.cfg.Cloud { + resourceNames = append([]string{"runtime"}, resourceNames...) + } rules = append(rules, rbacv1.PolicyRule{ - APIGroups: []string{"lma.tigera.io"}, - Resources: []string{"*"}, - ResourceNames: []string{ - "flows", "audit*", "l7", "events", "dns", "waf", "kibana_login", "elasticsearch_superuser", "recommendations", - }, - Verbs: []string{"get"}, + APIGroups: []string{"lma.tigera.io"}, + Resources: []string{"*"}, + ResourceNames: resourceNames, + Verbs: []string{"get"}, }) } diff --git a/pkg/render/common/authentication/tigera/key_validator_config/key_validator_cloud.go b/pkg/render/common/authentication/tigera/key_validator_config/key_validator_cloud.go new file mode 100644 index 0000000000..fb5c721f4f --- /dev/null +++ b/pkg/render/common/authentication/tigera/key_validator_config/key_validator_cloud.go @@ -0,0 +1,49 @@ +// Copyright (c) 2022-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tigerakvc + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" +) + +// CloudAuthenticationConfig holds Calico Cloud specific authentication settings. It is only +// populated for cloud installs (via WithTenantClaim); for regular Calico/Calico Enterprise +// installs requireTenantClaim is false and addCloudEnvs is a no-op, so no cloud env vars are +// emitted and enterprise behavior is unchanged. +type CloudAuthenticationConfig struct { + requireTenantClaim bool + tenantID string +} + +// WithTenantClaim configures the KeyValidatorConfig to require and validate a Calico Cloud tenant +// claim. It must only be supplied for cloud installs. +func WithTenantClaim(tenantID string) Option { + return func(config *KeyValidatorConfig) { + config.cloud.requireTenantClaim = true + config.cloud.tenantID = tenantID + } +} + +func (kvc *KeyValidatorConfig) addCloudEnvs(prefix string, envs []corev1.EnvVar) []corev1.EnvVar { + if !kvc.cloud.requireTenantClaim { + return envs + } + return append(envs, + corev1.EnvVar{Name: fmt.Sprintf("%sCALICO_CLOUD_REQUIRE_TENANT_CLAIM", prefix), Value: "true"}, + corev1.EnvVar{Name: fmt.Sprintf("%sCALICO_CLOUD_TENANT_CLAIM", prefix), Value: kvc.cloud.tenantID}, + ) +} diff --git a/pkg/render/common/authentication/tigera/key_validator_config/key_validator_config.go b/pkg/render/common/authentication/tigera/key_validator_config/key_validator_config.go index 57b9c02682..0a597256b3 100644 --- a/pkg/render/common/authentication/tigera/key_validator_config/key_validator_config.go +++ b/pkg/render/common/authentication/tigera/key_validator_config/key_validator_config.go @@ -44,6 +44,7 @@ type KeyValidatorConfig struct { usernamePrefix string groupsPrefix string rootCA []byte + cloud CloudAuthenticationConfig } func New(issuerURL string, clientID string, options ...Option) (authentication.KeyValidatorConfig, error) { @@ -83,7 +84,7 @@ func (kvc *KeyValidatorConfig) ClientID() string { } func (kvc *KeyValidatorConfig) RequiredEnv(prefix string) []corev1.EnvVar { - return []corev1.EnvVar{ + return kvc.addCloudEnvs(prefix, []corev1.EnvVar{ {Name: fmt.Sprintf("%sOIDC_AUTH_ENABLED", prefix), Value: strconv.FormatBool(true)}, {Name: fmt.Sprintf("%sOIDC_AUTH_ISSUER", prefix), Value: kvc.Issuer()}, {Name: fmt.Sprintf("%sOIDC_AUTH_JWKSURL", prefix), Value: kvc.wellKnownConfig.JWKSURL}, @@ -92,7 +93,7 @@ func (kvc *KeyValidatorConfig) RequiredEnv(prefix string) []corev1.EnvVar { {Name: fmt.Sprintf("%sOIDC_AUTH_GROUPS_CLAIM", prefix), Value: kvc.groupsClaim}, {Name: fmt.Sprintf("%sOIDC_AUTH_USERNAME_PREFIX", prefix), Value: kvc.usernamePrefix}, {Name: fmt.Sprintf("%sOIDC_AUTH_GROUPS_PREFIX", prefix), Value: kvc.groupsPrefix}, - } + }) } func (kvc *KeyValidatorConfig) RequiredAnnotations() map[string]string { diff --git a/pkg/render/common/cloudconfig/cloudconfig.go b/pkg/render/common/cloudconfig/cloudconfig.go new file mode 100644 index 0000000000..33641551fc --- /dev/null +++ b/pkg/render/common/cloudconfig/cloudconfig.go @@ -0,0 +1,135 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudconfig + +import ( + "fmt" + "strconv" + + v1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/common" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + CloudConfigConfigMapName = "tigera-secure-cloud-config" +) + +func NewCloudConfig(tenantId string, tenantName string, externalESDomain string, externalKibanaDomain string, enableMTLS bool) *CloudConfig { + return &CloudConfig{ + tenantId: tenantId, + tenantName: tenantName, + externalESDomain: externalESDomain, + externalKibanaDomain: externalKibanaDomain, + enableMTLS: enableMTLS, + } +} + +func NewCloudConfigFromConfigMap(configMap *corev1.ConfigMap) (*CloudConfig, error) { + var enableMTLS bool + var err error + + if configMap.Data["tenantId"] == "" { + return nil, fmt.Errorf("'tenantId' is not set") + } + + if configMap.Data["tenantName"] == "" { + return nil, fmt.Errorf("'tenantName' is not set") + } + + if configMap.Data["externalESDomain"] == "" { + return nil, fmt.Errorf("'externalESDomain' is not set") + } + + if configMap.Data["externalKibanaDomain"] == "" { + return nil, fmt.Errorf("'externalKibanaDomain' is not set") + } + + if configMap.Data["enableMTLS"] == "" { + enableMTLS = false + } else { + if enableMTLS, err = strconv.ParseBool(configMap.Data["enableMTLS"]); err != nil { + return nil, errors.Wrap(err, "'enableMTLS' must be a bool") + } + } + + return NewCloudConfig(configMap.Data["tenantId"], configMap.Data["tenantName"], configMap.Data["externalESDomain"], configMap.Data["externalKibanaDomain"], enableMTLS), nil +} + +type CloudConfig struct { + tenantId string + tenantName string + externalESDomain string + externalKibanaDomain string + enableMTLS bool +} + +// ToTenant converts the given CloudConfig structure to a Tenant object. +// This allows controllers that have been converted to support multi-tenancy to still leverage +// the single-tenant CloudConfig structure using the same code path as in multi-tenancy. +func (c CloudConfig) ToTenant() *v1.Tenant { + return &v1.Tenant{ + // We don't specify a Namespace for this tenant because it represents a singular tenant installed + // in this management cluster. The signals to the render code that this is a single-tenant cluster and not + // a cluster capable of multi-tenancy. + ObjectMeta: metav1.ObjectMeta{Name: "default"}, + Spec: v1.TenantSpec{ + ID: c.tenantId, + Name: c.tenantName, + Elastic: &v1.TenantElasticSpec{ + URL: fmt.Sprintf("https://%s:443", c.externalESDomain), + KibanaURL: fmt.Sprintf("https://%s:443", c.externalKibanaDomain), + MutualTLS: c.enableMTLS, + }, + }, + } +} + +func (c CloudConfig) TenantId() string { + return c.tenantId +} + +func (c CloudConfig) TenantName() string { + return c.tenantName +} + +func (c CloudConfig) ExternalESDomain() string { + return c.externalESDomain +} + +func (c CloudConfig) ExternalKibanaDomain() string { + return c.externalKibanaDomain +} + +func (c CloudConfig) EnableMTLS() bool { + return c.enableMTLS +} + +func (c CloudConfig) ConfigMap() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: CloudConfigConfigMapName, + Namespace: common.OperatorNamespace(), + }, + Data: map[string]string{ + "tenantId": c.tenantId, + "tenantName": c.tenantName, + "externalESDomain": c.externalESDomain, + "externalKibanaDomain": c.externalKibanaDomain, + "enableMTLS": strconv.FormatBool(c.enableMTLS), + }, + } +} diff --git a/pkg/render/common/cloudconfig/cloudconfig_suite_test.go b/pkg/render/common/cloudconfig/cloudconfig_suite_test.go new file mode 100644 index 0000000000..5d7605767d --- /dev/null +++ b/pkg/render/common/cloudconfig/cloudconfig_suite_test.go @@ -0,0 +1,33 @@ +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudconfig + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +func TestStatus(t *testing.T) { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter))) + RegisterFailHandler(Fail) + suiteConfig, reporterConfig := GinkgoConfiguration() + reporterConfig.JUnitReport = "../../../report/cloudconfig_suite.xml" + RunSpecs(t, "pkg/render/common/cloudconfig/CloudConfig Suite", suiteConfig, reporterConfig) +} diff --git a/pkg/render/common/cloudconfig/cloudconfig_test.go b/pkg/render/common/cloudconfig/cloudconfig_test.go new file mode 100644 index 0000000000..da125c5d73 --- /dev/null +++ b/pkg/render/common/cloudconfig/cloudconfig_test.go @@ -0,0 +1,125 @@ +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudconfig + +import ( + "strconv" + + "github.com/tigera/operator/pkg/common" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("CloudConfig ConfigMap tests", func() { + Context("NewCloudConfigFromConfigMap", func() { + var configMap *corev1.ConfigMap + + BeforeEach(func() { + configMap = &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: CloudConfigConfigMapName, + Namespace: common.OperatorNamespace(), + }, + Data: map[string]string{ + "tenantId": "abc123", + "tenantName": "tenant1", + "externalESDomain": "externalES.com", + "externalKibanaDomain": "externalKibana.com", + "enableMTLS": strconv.FormatBool(false), + }, + } + }) + + It("should return a valid CloudConfig", func() { + expectedCloudConfig := &CloudConfig{ + tenantId: "abc123", + tenantName: "tenant1", + externalESDomain: "externalES.com", + externalKibanaDomain: "externalKibana.com", + enableMTLS: false, + } + + cc, err := NewCloudConfigFromConfigMap(configMap) + Expect(err).ShouldNot(HaveOccurred()) + Expect(cc).Should(Equal(expectedCloudConfig)) + }) + + It("should return an error when tenantId is not set", func() { + configMap.Data["tenantId"] = "" + _, err := NewCloudConfigFromConfigMap(configMap) + Expect(err).Should(HaveOccurred()) + }) + + It("should return an error when tenantName is not set", func() { + configMap.Data["tenantName"] = "" + _, err := NewCloudConfigFromConfigMap(configMap) + Expect(err).Should(HaveOccurred()) + }) + + It("should return an error when externalESDomain is not set", func() { + configMap.Data["externalESDomain"] = "" + _, err := NewCloudConfigFromConfigMap(configMap) + Expect(err).Should(HaveOccurred()) + }) + + It("should return an error when externalKibanaDomain is not set", func() { + configMap.Data["externalKibanaDomain"] = "" + _, err := NewCloudConfigFromConfigMap(configMap) + Expect(err).Should(HaveOccurred()) + }) + + It("should return an error when enableMTLS is not a valid boolean", func() { + configMap.Data["enableMTLS"] = "truee" + _, err := NewCloudConfigFromConfigMap(configMap) + Expect(err).Should(HaveOccurred()) + }) + }) + + Context("ConfigMap from CloudConfig", func() { + var cloudConfig *CloudConfig + + BeforeEach(func() { + cloudConfig = &CloudConfig{ + tenantId: "abc123", + tenantName: "tenant1", + externalESDomain: "externalES.com", + externalKibanaDomain: "externalKibana.com", + enableMTLS: false, + } + }) + + It("should return a valid ConfigMap from CloudConfig", func() { + expectedConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: CloudConfigConfigMapName, + Namespace: common.OperatorNamespace(), + }, + Data: map[string]string{ + "tenantId": "abc123", + "tenantName": "tenant1", + "externalESDomain": "externalES.com", + "externalKibanaDomain": "externalKibana.com", + "enableMTLS": strconv.FormatBool(false), + }, + } + cm := cloudConfig.ConfigMap() + Expect(cm).Should(Equal(expectedConfigMap)) + }) + }) +}) diff --git a/pkg/render/common/elasticsearch/multitenancy.go b/pkg/render/common/elasticsearch/multitenancy.go new file mode 100644 index 0000000000..a4a3b65333 --- /dev/null +++ b/pkg/render/common/elasticsearch/multitenancy.go @@ -0,0 +1,21 @@ +// Copyright (c) 2021 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package elasticsearch + +import "fmt" + +func (c *ClusterConfig) AddTenantId(tenantId string) { + c.clusterName = fmt.Sprintf("%s.%s", tenantId, c.clusterName) +} diff --git a/pkg/render/common/meta/meta.go b/pkg/render/common/meta/meta.go index 99af4f5690..54087a5848 100644 --- a/pkg/render/common/meta/meta.go +++ b/pkg/render/common/meta/meta.go @@ -27,6 +27,8 @@ import ( type OSType string const ( + DefaultCertificateDuration = 825 * 24 * time.Hour + OSTypeAny OSType = "any" OSTypeLinux OSType = "linux" OSTypeWindows OSType = "windows" diff --git a/pkg/render/common/test/testing.go b/pkg/render/common/test/testing.go index 2530fb9ed9..979c6d73e8 100644 --- a/pkg/render/common/test/testing.go +++ b/pkg/render/common/test/testing.go @@ -260,7 +260,7 @@ func ExpectVolumeMount(vms []corev1.VolumeMount, name, path string) { return } } - Expect(false).To(BeTrue(), fmt.Sprintf("Missing expected volume mount %s", name)) + ExpectWithOffset(1, false).To(BeTrue(), fmt.Sprintf("Missing expected volume mount %s", name)) } // CreateCertSecret creates a secret that is not signed by the certificate manager, making it useful for testing legacy diff --git a/pkg/render/compliance.go b/pkg/render/compliance.go index e9743400a8..f4f9bd02fa 100644 --- a/pkg/render/compliance.go +++ b/pkg/render/compliance.go @@ -117,6 +117,10 @@ type ComplianceConfiguration struct { Tenant *operatorv1.Tenant ExternalElastic bool Compliance *operatorv1.Compliance + + // Cloud indicates compliance is being rendered for a Calico Cloud install. When false the cloud + // NetworkPolicy below is not created and enterprise behavior is unchanged. + Cloud bool } type complianceComponent struct { @@ -225,6 +229,20 @@ func (c *complianceComponent) Objects() ([]client.Object, []client.Object) { c.complianceServerServiceAccount(), c.complianceServerClusterRoleBinding(), ) + + if c.cfg.Cloud { + // Cloud modifications: add a network policy to talk to the OIDC issuer if one is being used. + // Note this is only strictly required when not using dex and the provider is outside the + // cluster, but that knowledge is not currently available in the compliance renderer, so + // enable it for all OIDC providers. + if c.cfg.KeyValidatorConfig != nil { + complianceObjs = append(complianceObjs, c.cloudComplianceServerAllowTigeraNetworkPolicy(c.cfg.KeyValidatorConfig.Issuer())) + } + // allow-tigera Tier was renamed to calico-system + objsToDelete = append(objsToDelete, + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("cloud-compliance-server", c.cfg.Namespace), + ) + } } else { // Compliance server is only for Standalone or Management clusters objsToDelete = append(objsToDelete, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: ComplianceServerName, Namespace: c.cfg.Namespace}}) diff --git a/pkg/render/compliance_cloud.go b/pkg/render/compliance_cloud.go new file mode 100644 index 0000000000..6cad14353e --- /dev/null +++ b/pkg/render/compliance_cloud.go @@ -0,0 +1,60 @@ +// Copyright (c) 2023-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render + +import ( + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + "github.com/tigera/operator/pkg/render/common/networkpolicy" +) + +const ( + CloudComplianceServerPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "cloud-" + ComplianceServerName +) + +func (c *complianceComponent) cloudComplianceServerAllowTigeraNetworkPolicy(issuerURL string) *v3.NetworkPolicy { + issuerDomain := strings.TrimPrefix(issuerURL, "https://") + issuerDomain = strings.TrimSuffix(issuerDomain, "/") + + egressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: []numorstring.Port{{MinPort: 443, MaxPort: 443}}, + Domains: []string{issuerDomain}, + }, + }, + } + + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: CloudComplianceServerPolicyName, + Namespace: c.cfg.Namespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(ComplianceServerName), + Types: []v3.PolicyType{v3.PolicyTypeEgress}, + Egress: egressRules, + }, + } +} diff --git a/pkg/render/fluentd.go b/pkg/render/fluentd.go index ac651f2db0..66dbd4df1f 100644 --- a/pkg/render/fluentd.go +++ b/pkg/render/fluentd.go @@ -188,6 +188,10 @@ type FluentdConfiguration struct { // LicenseExpired indicates the license has expired and fluentd DaemonSet should be removed. LicenseExpired bool + + // Cloud indicates fluentd is being rendered for a Calico Cloud install. When false the cloud + // env vars below are not added and enterprise behavior is unchanged. + Cloud bool } type fluentdComponent struct { @@ -856,11 +860,31 @@ func (c *fluentdComponent) envvars() []corev1.EnvVar { } } + if c.cfg.Cloud { + envs = setFluentdCloudEnvs(envs, c.cfg.Tenant.MultiTenant()) + } + envs = append(envs, corev1.EnvVar{Name: "CA_CRT_PATH", Value: c.trustedBundlePath()}) return envs } +// setFluentdCloudEnvs adds the Calico Cloud specific fluentd env vars. Done as a separate function +// to make future updates easier. Only called for cloud installs. +func setFluentdCloudEnvs(envs []corev1.EnvVar, multiTenant bool) []corev1.EnvVar { + envs = append(envs, + corev1.EnvVar{Name: "DISABLE_ES_DNS_LOG", Value: "true"}, + corev1.EnvVar{Name: "DISABLE_ES_AUDIT_EE_LOG", Value: "true"}, + corev1.EnvVar{Name: "DISABLE_ES_AUDIT_KUBE_LOG", Value: "true"}, + corev1.EnvVar{Name: "DISABLE_ES_BGP_LOG", Value: "true"}, + ) + + if !multiTenant { + envs = append(envs, corev1.EnvVar{Name: "DISABLE_ES_FLOW_LOG", Value: "true"}) + } + return envs +} + func (c *fluentdComponent) trustedBundlePath() string { if c.cfg.OSType == rmeta.OSTypeWindows { return certificatemanagement.TrustedCertBundleMountPathWindows diff --git a/pkg/render/fluentd_cloud_test.go b/pkg/render/fluentd_cloud_test.go new file mode 100644 index 0000000000..0bc316200b --- /dev/null +++ b/pkg/render/fluentd_cloud_test.go @@ -0,0 +1,136 @@ +// Copyright (c) 2024-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/tigera/operator/test" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/apis" + "github.com/tigera/operator/pkg/common" + "github.com/tigera/operator/pkg/controller/certificatemanager" + ctrlrfake "github.com/tigera/operator/pkg/ctrlruntime/client/fake" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + rtest "github.com/tigera/operator/pkg/render/common/test" +) + +var _ = Describe("Tigera Secure Fluentd Cloud rendering tests", func() { + var cfg *render.FluentdConfiguration + var cli client.Client + + BeforeEach(func() { + // Initialize a default instance to use. Each test can override this to its + // desired configuration. + scheme := runtime.NewScheme() + Expect(apis.AddToScheme(scheme, false)).NotTo(HaveOccurred()) + cli = ctrlrfake.DefaultFakeClientBuilder(scheme).Build() + + certificateManager, err := certificatemanager.Create(cli, nil, clusterDomain, common.OperatorNamespace(), certificatemanager.AllowCACreation()) + Expect(err).NotTo(HaveOccurred()) + + metricsSecret, err := certificateManager.GetOrCreateKeyPair(cli, render.FluentdPrometheusTLSSecretName, common.OperatorNamespace(), []string{""}) + Expect(err).NotTo(HaveOccurred()) + + cfg = &render.FluentdConfiguration{ + LogCollector: &operatorv1.LogCollector{}, + ClusterDomain: dns.DefaultClusterDomain, + OSType: rmeta.OSTypeLinux, + Installation: &operatorv1.InstallationSpec{ + KubernetesProvider: operatorv1.ProviderNone, + }, + FluentdKeyPair: metricsSecret, + TrustedBundle: certificateManager.CreateTrustedBundle(), + Cloud: true, + } + }) + + Context("single tenant", func() { + It("should render fluentd Daemonset with all log collection disabled", func() { + component := render.Fluentd(cfg) + resources, _ := component.Objects() + + ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) + + container := test.GetContainer(ds.Spec.Template.Spec.Containers, "fluentd") + Expect(container).NotTo(BeNil()) + + Expect(container.Env).To(ContainElements([]v1.EnvVar{ + {Name: "DISABLE_ES_DNS_LOG", Value: "true"}, + {Name: "DISABLE_ES_AUDIT_EE_LOG", Value: "true"}, + {Name: "DISABLE_ES_AUDIT_KUBE_LOG", Value: "true"}, + {Name: "DISABLE_ES_BGP_LOG", Value: "true"}, + {Name: "DISABLE_ES_FLOW_LOG", Value: "true"}, + })) + }) + }) + + Context("multi tenant", func() { + It("should render fluentd Daemonset with flow log collection enabled and the rest disabled", func() { + cfg.Tenant = &operatorv1.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenantA", + Namespace: "tenant-a-namespace", + }, + Spec: operatorv1.TenantSpec{ + ID: "tenant-a-id", + }, + } + component := render.Fluentd(cfg) + resources, _ := component.Objects() + + ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1)) + + container := test.GetContainer(ds.Spec.Template.Spec.Containers, "fluentd") + Expect(container).NotTo(BeNil()) + Expect(container.Env).NotTo(ContainElements([]v1.EnvVar{ + {Name: "DISABLE_ES_FLOW_LOG", Value: "true"}, + })) + + Expect(container.Env).To(ContainElements([]v1.EnvVar{ + {Name: "DISABLE_ES_DNS_LOG", Value: "true"}, + {Name: "DISABLE_ES_AUDIT_EE_LOG", Value: "true"}, + {Name: "DISABLE_ES_AUDIT_KUBE_LOG", Value: "true"}, + {Name: "DISABLE_ES_BGP_LOG", Value: "true"}, + })) + }) + }) + + Context("non-cloud", func() { + It("should not add the cloud DISABLE_ES_* env vars when Cloud is false", func() { + cfg.Cloud = false + component := render.Fluentd(cfg) + resources, _ := component.Objects() + + ds := rtest.GetResource(resources, "fluentd-node", "tigera-fluentd", "apps", "v1", "DaemonSet").(*appsv1.DaemonSet) + container := test.GetContainer(ds.Spec.Template.Spec.Containers, "fluentd") + Expect(container).NotTo(BeNil()) + Expect(container.Env).NotTo(ContainElements([]v1.EnvVar{ + {Name: "DISABLE_ES_DNS_LOG", Value: "true"}, + {Name: "DISABLE_ES_FLOW_LOG", Value: "true"}, + })) + }) + }) +}) diff --git a/pkg/render/intrusion_detection.go b/pkg/render/intrusion_detection.go index f0aedbad0a..ffbdebdce3 100644 --- a/pkg/render/intrusion_detection.go +++ b/pkg/render/intrusion_detection.go @@ -187,6 +187,20 @@ func (c *intrusionDetectionComponent) Objects() ([]client.Object, []client.Objec // allow-tigera Tier was renamed to calico-system networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("intrusion-detection-controller", c.cfg.Namespace), networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("default-deny", c.cfg.Namespace), + // Image Assurance was removed; clean up the resources that prior versions copied + // into the intrusion-detection namespace. + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-image-assurance-api-cert", Namespace: c.cfg.Namespace}, + }, + &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "tigera-image-assurance-config", Namespace: c.cfg.Namespace}, + }, + &corev1.Secret{ + TypeMeta: metav1.TypeMeta{Kind: "Secret", APIVersion: "v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "image-assurance-api-token", Namespace: c.cfg.Namespace}, + }, } if !c.cfg.ManagementCluster { diff --git a/pkg/render/intrusion_detection_test.go b/pkg/render/intrusion_detection_test.go index 8b1084b928..117d5a9eae 100644 --- a/pkg/render/intrusion_detection_test.go +++ b/pkg/render/intrusion_detection_test.go @@ -862,9 +862,9 @@ var _ = Describe("Intrusion Detection rendering tests", func() { func assertEnvVarlistMatch(envVars []corev1.EnvVar, expectedEnvVars []expectedEnvVar) { for _, expected := range expectedEnvVars { if expected.val != "" { - Expect(envVars).To(ContainElement(corev1.EnvVar{Name: expected.name, Value: expected.val})) + ExpectWithOffset(1, envVars).To(ContainElement(corev1.EnvVar{Name: expected.name, Value: expected.val})) } else { - Expect(envVars).To(ContainElement(corev1.EnvVar{ + ExpectWithOffset(1, envVars).To(ContainElement(corev1.EnvVar{ Name: expected.name, ValueFrom: &corev1.EnvVarSource{ SecretKeyRef: &corev1.SecretKeySelector{ diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index f8f2dae650..b71f3ca19f 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -116,6 +116,14 @@ type KubeControllersConfiguration struct { WASMCACert *corev1.ConfigMap TrustedBundle certificatemanagement.TrustedBundleRO + // Calico Cloud additions. TenantId is only set by the cloud-gated controller path; when empty + // (regular Calico/Calico Enterprise) no cloud env is emitted. + TenantId string + + // Cloud indicates kube-controllers is being rendered for a Calico Cloud install. When false the + // cloud-specific RBAC below is not granted and enterprise RBAC is unchanged. + Cloud bool + MetricsServerTLS certificatemanagement.KeyPairInterface // Namespace to be installed into. @@ -232,6 +240,14 @@ func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeCon if cfg.Installation.Variant.IsEnterprise() { kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, kubeControllersRoleEnterpriseCommonRules(cfg)...) + + // Calico Cloud's es-kube-controllers provisions RBAC for managed-cluster access, so it needs + // to create/update cluster roles and bindings. Enterprise only needs read access. + clusterRoleVerbs := []string{"watch", "list", "get"} + if cfg.Cloud { + clusterRoleVerbs = append(clusterRoleVerbs, "create", "update") + } + kubeControllerRolePolicyRules = append(kubeControllerRolePolicyRules, rbacv1.PolicyRule{ APIGroups: []string{"elasticsearch.k8s.elastic.co"}, @@ -241,7 +257,7 @@ func NewElasticsearchKubeControllers(cfg *KubeControllersConfiguration) *kubeCon rbacv1.PolicyRule{ APIGroups: []string{"rbac.authorization.k8s.io"}, Resources: []string{"clusterroles", "clusterrolebindings"}, - Verbs: []string{"watch", "list", "get"}, + Verbs: clusterRoleVerbs, }, ) @@ -951,6 +967,10 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.Tenant.MultiTenant() && c.kubeControllerConfigName == "elasticsearch")}, } + if c.cfg.TenantId != "" { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.TenantId}) + } + env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) if c.cfg.Installation.Variant.IsEnterprise() { diff --git a/pkg/render/logstorage/esgateway/cloud.go b/pkg/render/logstorage/esgateway/cloud.go new file mode 100644 index 0000000000..a91fe0c12b --- /dev/null +++ b/pkg/render/logstorage/esgateway/cloud.go @@ -0,0 +1,198 @@ +// Copyright (c) 2022-2026 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package esgateway + +import ( + "strconv" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + + "github.com/tigera/operator/pkg/render" + rmeta "github.com/tigera/operator/pkg/render/common/meta" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + "github.com/tigera/operator/pkg/render/common/secret" + "github.com/tigera/operator/pkg/render/logstorage" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + CloudPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "cloud-es-gateway-access" +) + +// CloudConfig holds Calico Cloud specific es-gateway configuration. Enabled gates all cloud +// behavior: when false (regular Calico/Calico Enterprise) the cloud decorators are no-ops. +type CloudConfig struct { + Enabled bool + EsAdminUserSecret *corev1.Secret + ExternalCertsSecret *corev1.Secret + TenantId string + EnableMTLS bool + ExternalElastic bool + ExternalESDomain string + ExternalKibanaDomain string +} + +func removeEnv(evs []corev1.EnvVar, name string) []corev1.EnvVar { + for i, ev := range evs { + if ev.Name == name { + evs = append(evs[:i], evs[i+1:]...) + } + } + return evs +} + +func (e *esGateway) modifyDeploymentForCloud(d *appsv1.Deployment) { + if !e.cfg.Cloud.Enabled { + return + } + + envs := d.Spec.Template.Spec.Containers[0].Env + // Enable prometheus metrics endpoint at :METRICS_PORT/metrics (Default is 9091). + envs = append(envs, corev1.EnvVar{Name: "ES_GATEWAY_METRICS_ENABLED", Value: "true"}) + + if e.cfg.Cloud.ExternalElastic { + // Find the following Envs and remove them + envs = removeEnv(envs, "ES_GATEWAY_ELASTIC_ENDPOINT") + envs = removeEnv(envs, "ES_GATEWAY_KIBANA_ENDPOINT") + envs = append(envs, corev1.EnvVar{ + Name: "ES_GATEWAY_ELASTIC_ENDPOINT", + Value: "https://" + e.cfg.Cloud.ExternalESDomain + ":443", + }) + envs = append(envs, corev1.EnvVar{ + Name: "ES_GATEWAY_KIBANA_ENDPOINT", + Value: "https://" + e.cfg.Cloud.ExternalKibanaDomain + ":443", + }) + // Enable this so that fluentd cannot modify ILM but the POSTs fluentd performs + // still succeed. + envs = append(envs, corev1.EnvVar{ + Name: "ES_GATEWAY_ILM_DUMMY_ROUTE_ENABLED", + Value: "true", + }) + } + + if e.cfg.Cloud.EnableMTLS { + // todo: delete these from the envVars + envs = removeEnv(envs, "ES_GATEWAY_KIBANA_CLIENT_CERT_PATH") + envs = removeEnv(envs, "ES_GATEWAY_ELASTIC_CLIENT_CERT_PATH") + + envs = append(envs, []corev1.EnvVar{ + {Name: "ES_GATEWAY_ELASTIC_CLIENT_CERT_PATH", Value: "/certs/elasticsearch/mtls/client.crt"}, + {Name: "ES_GATEWAY_ELASTIC_CLIENT_KEY_PATH", Value: "/certs/elasticsearch/mtls/client.key"}, + {Name: "ES_GATEWAY_ENABLE_ELASTIC_MUTUAL_TLS", Value: strconv.FormatBool(e.cfg.Cloud.EnableMTLS)}, + {Name: "ES_GATEWAY_KIBANA_CLIENT_CERT_PATH", Value: "/certs/kibana/mtls/client.crt"}, + {Name: "ES_GATEWAY_KIBANA_CLIENT_KEY_PATH", Value: "/certs/kibana/mtls/client.key"}, + {Name: "ES_GATEWAY_ENABLE_KIBANA_MUTUAL_TLS", Value: strconv.FormatBool(e.cfg.Cloud.EnableMTLS)}, + }...) + + d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, corev1.Volume{ + Name: logstorage.ExternalCertsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: logstorage.ExternalCertsSecret, + }, + }, + }) + + d.Spec.Template.Spec.Containers[0].VolumeMounts = append(d.Spec.Template.Spec.Containers[0].VolumeMounts, + []corev1.VolumeMount{ + {Name: logstorage.ExternalCertsVolumeName, MountPath: "/certs/elasticsearch/mtls", ReadOnly: true}, + {Name: logstorage.ExternalCertsVolumeName, MountPath: "/certs/kibana/mtls", ReadOnly: true}, + }...) + } + + if e.cfg.Cloud.TenantId != "" { + envs = append(envs, corev1.EnvVar{Name: "ES_GATEWAY_TENANT_ID", Value: e.cfg.Cloud.TenantId}) + } + + d.Spec.Template.Spec.Containers[0].Env = envs + if e.cfg.Cloud.ExternalCertsSecret != nil { + d.Spec.Template.Annotations["hash.operator.tigera.io/cloud-external-es-secrets"] = rmeta.SecretsAnnotationHash(e.cfg.Cloud.ExternalCertsSecret) + } +} + +func (e *esGateway) getCloudObjects() (toCreate, toDelete []client.Object) { + if !e.cfg.Cloud.Enabled { + return nil, nil + } + + s := []client.Object{} + + if e.cfg.Cloud.ExternalCertsSecret != nil { + s = append(s, secret.ToRuntimeObjects(secret.CopyToNamespace(render.ElasticsearchNamespace, e.cfg.Cloud.ExternalCertsSecret)...)...) + } + + if e.cfg.Cloud.EsAdminUserSecret != nil { + s = append(s, secret.ToRuntimeObjects(secret.CopyToNamespace(render.ElasticsearchNamespace, e.cfg.Cloud.EsAdminUserSecret)...)...) + } + s = append(s, e.allowTigeraPolicyForCloud()) + + // allow-tigera Tier was renamed to calico-system + toDelete = append(toDelete, + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("cloud-es-gateway-access", render.ElasticsearchNamespace), + ) + return s, toDelete +} + +func (e *esGateway) allowTigeraPolicyForCloud() *v3.NetworkPolicy { + egressRules := []v3.Rule{} + if e.cfg.Cloud.ExternalElastic { + egressRules = append(egressRules, + v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: []numorstring.Port{{MinPort: 443, MaxPort: 443}}, + Domains: []string{e.cfg.Cloud.ExternalESDomain, e.cfg.Cloud.ExternalKibanaDomain}, + }, + }, + ) + } + + ingressRules := []v3.Rule{ + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Source: v3.EntityRule{ + NamespaceSelector: "projectcalico.org/name == 'monitoring'", + Selector: "app == 'prometheus'", + }, + // This matches the default. The metrics are enabled only on cloud (see + // ES_GATEWAY_METRICS_ENABLED which is added in this file). + Destination: v3.EntityRule{ + Ports: []numorstring.Port{{MinPort: 9091, MaxPort: 9091}}, + }, + }, + } + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: CloudPolicyName, + Namespace: render.ElasticsearchNamespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(DeploymentName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, + } +} diff --git a/pkg/render/logstorage/esgateway/esgateway.go b/pkg/render/logstorage/esgateway/esgateway.go index d8673fae21..98e3423493 100644 --- a/pkg/render/logstorage/esgateway/esgateway.go +++ b/pkg/render/logstorage/esgateway/esgateway.go @@ -83,6 +83,8 @@ type Config struct { Namespace string TruthNamespace string LogStorage *operatorv1.LogStorage + // Cloud holds Calico Cloud specific configuration. Inert unless Cloud.Enabled is set. + Cloud CloudConfig } func (e *esGateway) ResolveImages(is *operatorv1.ImageSet) error { @@ -116,6 +118,10 @@ func (e *esGateway) Objects() (toCreate, toDelete []client.Object) { toCreate = append(toCreate, e.esGatewayRoleBinding()) toCreate = append(toCreate, e.esGatewayServiceAccount()) + ccToCreate, ccToDelete := e.getCloudObjects() + toCreate = append(toCreate, ccToCreate...) + toDelete = append(toDelete, ccToDelete...) + // allow-tigera Tier was renamed to calico-system toDelete = append(toDelete, networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("es-gateway-access", e.cfg.Namespace)) @@ -304,8 +310,8 @@ func (e *esGateway) esGatewayDeployment() *appsv1.Deployment { } } + e.modifyDeploymentForCloud(&d) return &d - } func (e *esGateway) esGatewayServiceAccount() *corev1.ServiceAccount { diff --git a/pkg/render/logstorage/esgateway/esgateway_test.go b/pkg/render/logstorage/esgateway/esgateway_test.go index 7f29d298d7..e108557663 100644 --- a/pkg/render/logstorage/esgateway/esgateway_test.go +++ b/pkg/render/logstorage/esgateway/esgateway_test.go @@ -43,6 +43,7 @@ import ( "github.com/tigera/operator/pkg/render/common/podaffinity" rtest "github.com/tigera/operator/pkg/render/common/test" "github.com/tigera/operator/pkg/render/kubecontrollers" + "github.com/tigera/operator/pkg/render/logstorage" "github.com/tigera/operator/pkg/render/testutils" "github.com/tigera/operator/pkg/tls" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -128,6 +129,53 @@ var _ = Describe("ES Gateway rendering tests", func() { })) }) + It("should render Calico Cloud resources and deployment tweaks when Cloud is enabled", func() { + cfg.Cloud = CloudConfig{ + Enabled: true, + EsAdminUserSecret: &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: render.ElasticsearchAdminUserSecret, Namespace: common.OperatorNamespace()}}, + ExternalCertsSecret: &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: logstorage.ExternalCertsSecret, Namespace: common.OperatorNamespace()}}, + TenantId: "tenantId", + EnableMTLS: true, + ExternalElastic: true, + ExternalESDomain: "externalEs.com", + ExternalKibanaDomain: "externalKb.com", + } + + expectedResources := []client.Object{ + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: PolicyName, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersUserSecret, Namespace: common.OperatorNamespace()}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersVerificationUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: kubecontrollers.ElasticsearchKubeControllersSecureUserSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: ServiceName, Namespace: render.ElasticsearchNamespace}}, + &rbacv1.Role{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, + &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: RoleName, Namespace: render.ElasticsearchNamespace}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: ServiceAccountName, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: logstorage.ExternalCertsSecret, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: render.ElasticsearchAdminUserSecret, Namespace: render.ElasticsearchNamespace}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: CloudPolicyName, Namespace: render.ElasticsearchNamespace}}, + &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: DeploymentName, Namespace: render.ElasticsearchNamespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: relasticsearch.PublicCertSecret, Namespace: common.OperatorNamespace()}}, + } + createResources, _ := EsGateway(cfg).Objects() + rtest.ExpectResources(createResources, expectedResources) + + deploy, ok := rtest.GetResource(createResources, DeploymentName, render.ElasticsearchNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(ok).To(BeTrue()) + env := deploy.Spec.Template.Spec.Containers[0].Env + Expect(env).To(ContainElement(corev1.EnvVar{Name: "ES_GATEWAY_METRICS_ENABLED", Value: "true"})) + Expect(env).To(ContainElement(corev1.EnvVar{Name: "ES_GATEWAY_ELASTIC_ENDPOINT", Value: "https://externalEs.com:443"})) + Expect(env).To(ContainElement(corev1.EnvVar{Name: "ES_GATEWAY_KIBANA_ENDPOINT", Value: "https://externalKb.com:443"})) + Expect(env).To(ContainElement(corev1.EnvVar{Name: "ES_GATEWAY_TENANT_ID", Value: "tenantId"})) + Expect(env).To(ContainElement(corev1.EnvVar{Name: "ES_GATEWAY_ENABLE_ELASTIC_MUTUAL_TLS", Value: "true"})) + }) + + It("should not render Calico Cloud resources when Cloud is disabled", func() { + createResources, _ := EsGateway(cfg).Objects() + Expect(rtest.GetResource(createResources, CloudPolicyName, render.ElasticsearchNamespace, "projectcalico.org", "v3", "NetworkPolicy")).To(BeNil()) + deploy := rtest.GetResource(createResources, DeploymentName, render.ElasticsearchNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(deploy.Spec.Template.Spec.Containers[0].Env).NotTo(ContainElement(corev1.EnvVar{Name: "ES_GATEWAY_METRICS_ENABLED", Value: "true"})) + }) + It("should render an ES Gateway deployment and all supporting resources when CertificateManagement is enabled", func() { secret, err := certificatemanagement.CreateSelfSignedSecret("", "", "", nil) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/render/logstorage/kibana/kibana.go b/pkg/render/logstorage/kibana/kibana.go index 540b055c59..adb1c4d18e 100644 --- a/pkg/render/logstorage/kibana/kibana.go +++ b/pkg/render/logstorage/kibana/kibana.go @@ -66,6 +66,11 @@ const ( var EntityRule = networkpolicy.CreateEntityRule(Namespace, CRName, Port) +// CloudKibanaConfigOverrides holds Calico Cloud Kibana config overrides. It is populated only by the +// cloud-gated controller path; for regular Calico/Calico Enterprise it stays empty and is a no-op. +// TODO: This shouldn't be done with a global variable set by the controller; thread it through Configuration instead. +var CloudKibanaConfigOverrides = map[string]interface{}{} + // Kibana renders the components necessary for kibana and elasticsearch func Kibana(cfg *Configuration) render.Component { return &kibana{ @@ -256,6 +261,13 @@ func (k *kibana) kibanaCR() *kbv1.Kibana { "xpack.productDocBase.artifactRepositoryUrl": "http://localhost:5601", } + // TODO: This shouldn't be done with a global variable set by the controller. + if len(CloudKibanaConfigOverrides) != 0 { + for k, v := range CloudKibanaConfigOverrides { + config[k] = v + } + } + var initContainers []corev1.Container var volumes []corev1.Volume var automountToken bool diff --git a/pkg/render/logstorage/kibana/kibana_cloud_test.go b/pkg/render/logstorage/kibana/kibana_cloud_test.go new file mode 100644 index 0000000000..ce2cbaba69 --- /dev/null +++ b/pkg/render/logstorage/kibana/kibana_cloud_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2024-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kibana_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + kbv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/kibana/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/dns" + "github.com/tigera/operator/pkg/render" + "github.com/tigera/operator/pkg/render/common/networkpolicy" + rtest "github.com/tigera/operator/pkg/render/common/test" + "github.com/tigera/operator/pkg/render/logstorage/kibana" +) + +var _ = Describe("Kibana cloud rendering tests", func() { + // CloudKibanaConfigOverrides is a package-global populated only by the cloud-gated controller. + // Reset it after each test so a leaked value cannot affect the non-cloud kibana specs. + AfterEach(func() { + kibana.CloudKibanaConfigOverrides = map[string]interface{}{} + }) + + Context("single-tenant rendering with internal elastic", func() { + var installation *operatorv1.InstallationSpec + var replicas int32 + var cfg *kibana.Configuration + + expectedResources := []client.Object{ + &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: kibana.Namespace}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: kibana.PolicyName, Namespace: kibana.Namespace}}, + &v3.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: networkpolicy.CalicoComponentDefaultDenyPolicyName, Namespace: kibana.Namespace}}, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "tigera-kibana", Namespace: kibana.Namespace}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "tigera-pull-secret", Namespace: kibana.Namespace}}, + &kbv1.Kibana{ObjectMeta: metav1.ObjectMeta{Name: kibana.CRName, Namespace: kibana.Namespace}}, + &rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: render.TigeraOperatorSecrets, Namespace: kibana.Namespace}}, + } + + BeforeEach(func() { + logStorage := &operatorv1.LogStorage{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-secure", + }, + Spec: operatorv1.LogStorageSpec{ + Nodes: &operatorv1.Nodes{ + Count: 1, + ResourceRequirements: nil, + }, + }, + Status: operatorv1.LogStorageStatus{ + State: "", + }, + } + + installation = &operatorv1.InstallationSpec{ + ControlPlaneReplicas: &replicas, + KubernetesProvider: operatorv1.ProviderNone, + Registry: "testregistry.com/", + } + + replicas = 2 + kibanaKeyPair, bundle := getX509Certs(installation) + + cfg = &kibana.Configuration{ + LogStorage: logStorage, + Installation: installation, + KibanaKeyPair: kibanaKeyPair, + PullSecrets: []*corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "tigera-pull-secret"}}, + }, + Provider: operatorv1.ProviderNone, + ClusterDomain: dns.DefaultClusterDomain, + TrustedBundle: bundle, + Enabled: true, + } + }) + + It("should apply the cloud Kibana config overrides", func() { + googleTagManagerConfig := map[string]interface{}{ + "enabled": true, + "container": "XYZ", + } + tigeraConfig := map[string]interface{}{ + "enabled": true, + "licenseEdition": "cloudEdition", + } + kibana.CloudKibanaConfigOverrides = map[string]interface{}{ + "googletagmanager": googleTagManagerConfig, + "tigera": tigeraConfig, + } + component := kibana.Kibana(cfg) + createResources, _ := component.Objects() + rtest.ExpectResources(createResources, expectedResources) + + kb := rtest.GetResource(createResources, kibana.CRName, kibana.Namespace, "kibana.k8s.elastic.co", "v1", "Kibana") + Expect(kb).NotTo(BeNil()) + kibanaCR := kb.(*kbv1.Kibana) + + Expect(kibanaCR.Spec.Config.Data["googletagmanager"]).To(Equal(googleTagManagerConfig)) + Expect(kibanaCR.Spec.Config.Data["tigera"]).To(Equal(tigeraConfig)) + }) + + It("should not include the googletagmanager override key when no overrides are set", func() { + // googletagmanager is a cloud-only key not present in the default Kibana config, so it's a + // safe signal that the cloud overrides were not applied. + component := kibana.Kibana(cfg) + createResources, _ := component.Objects() + kb := rtest.GetResource(createResources, kibana.CRName, kibana.Namespace, "kibana.k8s.elastic.co", "v1", "Kibana").(*kbv1.Kibana) + Expect(kb.Spec.Config.Data).NotTo(HaveKey("googletagmanager")) + }) + }) +}) diff --git a/pkg/render/logstorage/linseed/cloud.go b/pkg/render/logstorage/linseed/cloud.go new file mode 100644 index 0000000000..e081f3523e --- /dev/null +++ b/pkg/render/logstorage/linseed/cloud.go @@ -0,0 +1,121 @@ +// Copyright (c) 2023-2026 Tigera, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package linseed + +import ( + "strconv" + + v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" + + "github.com/tigera/operator/pkg/render/common/networkpolicy" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // Name of the network policy that adds CC specific rules to Linseed. + CloudPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "cloud-linseed-access" +) + +// modifyDeploymentForCloud applies Calico Cloud specific tweaks to the Linseed deployment. It is +// only called when rendering for cloud (gated by Config.Cloud), so it never affects enterprise. +func (c *linseed) modifyDeploymentForCloud(d *appsv1.Deployment) { + envs := d.Spec.Template.Spec.Containers[0].Env + + // Replica count for the policy activity index, kept consistent with the other index replicas. + envs = append(envs, corev1.EnvVar{Name: "ELASTIC_POLICY_ACTIVITY_INDEX_REPLICAS", Value: strconv.Itoa(c.cfg.ESClusterConfig.Replicas())}) + + // Enable prometheus metrics endpoint at :METRICS_PORT/metrics (Default is 9095). + // We use the same certificate for TLS on the metrics endpoint as we do for the main API. + envs = append(envs, corev1.EnvVar{Name: "LINSEED_ENABLE_METRICS", Value: "true"}) + envs = append(envs, corev1.EnvVar{Name: "LINSEED_METRICS_CERT", Value: c.cfg.KeyPair.VolumeMountCertificateFilePath()}) + envs = append(envs, corev1.EnvVar{Name: "LINSEED_METRICS_KEY", Value: c.cfg.KeyPair.VolumeMountKeyFilePath()}) + + if c.cfg.Tenant != nil { + if c.cfg.ExternalElastic { + // Overwrite policy activity index name until we create the tenant CR on all environments + envs = append(envs, corev1.EnvVar{Name: "ELASTIC_POLICY_ACTIVITY_BASE_INDEX_NAME", Value: "calico_policy_activity_standard"}) + } + } + d.Spec.Template.Spec.Containers[0].Env = envs +} + +func (c *linseed) getCloudObjects() (toCreate, toDelete []client.Object) { + s := []client.Object{} + s = append(s, c.allowTigeraPolicyForCloud()) + + // allow-tigera Tier was renamed to calico-system + toDelete = append(toDelete, + networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("cloud-linseed-access", c.namespace), + ) + + return s, toDelete +} + +func (c *linseed) allowTigeraPolicyForCloud() *v3.NetworkPolicy { + egressRules := []v3.Rule{} + if c.cfg.ElasticClientSecret != nil { + // TODO: At the moment, we only support mTLS for Elasticsearch when using an external ES cluster. + // That allows us to use the presence of the secret as a proxy for whether we should append this egress rule. + // In the future, we should support mTLS for internal ES clusters as well and switch this to a better check. + + // Allow egress traffic to the external Elasticsearch. + egressRules = append(egressRules, + v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: []numorstring.Port{{MinPort: 443, MaxPort: 443}}, + Domains: []string{c.cfg.ElasticHost}, + }, + }, + ) + } + + ingressRules := []v3.Rule{ + { + // Allow ingress traffic from Calico Cloud monitoring stack to the Linseed + // metrics port. + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Source: v3.EntityRule{ + NamespaceSelector: "projectcalico.org/name == 'monitoring'", + Selector: "app == 'prometheus'", + }, + Destination: v3.EntityRule{ + Ports: []numorstring.Port{{MinPort: 9095, MaxPort: 9095}}, + }, + }, + } + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{ + Name: CloudPolicyName, + Namespace: c.namespace, + }, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(DeploymentName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: ingressRules, + Egress: egressRules, + }, + } +} diff --git a/pkg/render/logstorage/linseed/linseed.go b/pkg/render/logstorage/linseed/linseed.go index 7f976280f4..3b07b8f61f 100644 --- a/pkg/render/logstorage/linseed/linseed.go +++ b/pkg/render/logstorage/linseed/linseed.go @@ -127,6 +127,10 @@ type Config struct { ElasticPort string LogStorage *operatorv1.LogStorage + + // Cloud indicates Linseed is being rendered for a Calico Cloud install. When false (regular + // Calico/Calico Enterprise) all cloud decorations are inert. + Cloud bool } func (l *linseed) ResolveImages(is *operatorv1.ImageSet) error { @@ -172,6 +176,13 @@ func (l *linseed) Objects() (toCreate, toDelete []client.Object) { toCreate = append(toCreate, secret.ToRuntimeObjects(secret.CopyToNamespace(l.cfg.Namespace, l.cfg.ElasticClientSecret)...)...) } + if l.cfg.Cloud { + // Add in Calico Cloud resources. + ccToCreate, ccToDelete := l.getCloudObjects() + toCreate = append(toCreate, ccToCreate...) + toDelete = append(toDelete, ccToDelete...) + } + return toCreate, toDelete } @@ -387,6 +398,10 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { // tenant ID in the name. Disable tenant suffix in index names to preserve // backward compatibility while still enforcing tenant isolation at the query level. envVars = append(envVars, corev1.EnvVar{Name: "ELASTIC_MULTI_INDEX_TENANT_SUFFIX_ENABLED", Value: "false"}) + } else if l.cfg.Cloud { + // For Calico Cloud's external Elasticsearch, single-index-format indices are created + // out of band, so Linseed should not create them itself. + envVars = append(envVars, corev1.EnvVar{Name: "ELASTIC_INDICES_CREATION_DISABLED", Value: "true"}) } if l.cfg.Tenant.MultiTenant() { @@ -511,6 +526,10 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { }, } + if l.cfg.Cloud { + l.modifyDeploymentForCloud(&d) + } + if l.cfg.Tenant.MultiTenant() { if overrides := l.cfg.Tenant.Spec.LinseedDeployment; overrides != nil { rcomponents.ApplyDeploymentOverrides(&d, overrides) diff --git a/pkg/render/logstorage/linseed/linseed_test.go b/pkg/render/logstorage/linseed/linseed_test.go index 936db2f3bc..7271c51ed4 100644 --- a/pkg/render/logstorage/linseed/linseed_test.go +++ b/pkg/render/logstorage/linseed/linseed_test.go @@ -128,6 +128,27 @@ var _ = Describe("Linseed rendering tests", func() { compareResources(createResources, expectedResources, false) }) + It("should render Calico Cloud network policy and deployment tweaks when Cloud is enabled", func() { + cfg.Cloud = true + component := Linseed(cfg) + createResources, _ := component.Objects() + + Expect(rtest.GetResource(createResources, CloudPolicyName, render.ElasticsearchNamespace, "projectcalico.org", "v3", "NetworkPolicy")).NotTo(BeNil()) + + d := rtest.GetResource(createResources, DeploymentName, render.ElasticsearchNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + env := d.Spec.Template.Spec.Containers[0].Env + Expect(env).To(ContainElement(corev1.EnvVar{Name: "LINSEED_ENABLE_METRICS", Value: "true"})) + Expect(env).To(ContainElement(corev1.EnvVar{Name: "ELASTIC_POLICY_ACTIVITY_INDEX_REPLICAS", Value: "1"})) + }) + + It("should not render Calico Cloud resources when Cloud is disabled", func() { + component := Linseed(cfg) + createResources, _ := component.Objects() + Expect(rtest.GetResource(createResources, CloudPolicyName, render.ElasticsearchNamespace, "projectcalico.org", "v3", "NetworkPolicy")).To(BeNil()) + d := rtest.GetResource(createResources, DeploymentName, render.ElasticsearchNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(d.Spec.Template.Spec.Containers[0].Env).NotTo(ContainElement(corev1.EnvVar{Name: "LINSEED_ENABLE_METRICS", Value: "true"})) + }) + It("should render Secrets RBAC permissions as part of ClusterRole", func() { component := Linseed(cfg) createResources, _ := component.Objects() diff --git a/pkg/render/manager.go b/pkg/render/manager.go index d634e22478..a952084cd0 100644 --- a/pkg/render/manager.go +++ b/pkg/render/manager.go @@ -224,6 +224,14 @@ type ManagerConfiguration struct { // CACertCommonName is the CommonName from the CA certificate used for operator-managed certificates. // Passed to Voltron so it can identify the correct CA issuer public key. CACertCommonName string + + // Cloud indicates the manager is being rendered for a Calico Cloud install. When false (regular + // Calico/Calico Enterprise) all cloud decorations below are inert and CloudResources is ignored. + Cloud bool + + // CloudResources holds Calico Cloud specific manager/voltron customizations. Only consumed when + // Cloud is true. + CloudResources ManagerCloudResources } type managerComponent struct { @@ -254,6 +262,10 @@ func (c *managerComponent) ResolveImages(is *operatorv1.ImageSet) error { if len(errMsgs) != 0 { return fmt.Errorf("%s", strings.Join(errMsgs, ",")) } + + // run cloud image customizations (no-op when not in cloud mode) + c.resolveCloudImages() + return nil } @@ -354,7 +366,7 @@ func (c *managerComponent) managerDeployment() *appsv1.Deployment { initContainers = append(initContainers, c.cfg.VoltronLinseedKeyPair.InitContainer(ManagerNamespace, securitycontext.NewNonRootContext())) } - managerPodContainers := []corev1.Container{c.managerUIAPIsContainer(), c.voltronContainer()} + managerPodContainers := []corev1.Container{c.managerUIAPIsContainer(), c.decorateCloudVoltronContainer(c.voltronContainer())} if c.cfg.Tenant == nil { managerPodContainers = append(managerPodContainers, c.dashboardContainer(), c.managerContainer()) } @@ -397,7 +409,7 @@ func (c *managerComponent) managerDeployment() *appsv1.Deployment { Strategy: appsv1.DeploymentStrategy{ Type: appsv1.RecreateDeploymentStrategyType, }, - Template: *podTemplate, + Template: c.decorateCloudDeploymentSpec(*podTemplate), }, } @@ -505,6 +517,7 @@ func (c *managerComponent) managerEnvVars() []corev1.EnvVar { } envs = append(envs, c.managerOAuth2EnvVars()...) + envs = c.setManagerCloudEnvs(envs) return envs } @@ -538,6 +551,9 @@ func (c *managerComponent) managerOAuth2EnvVars() []corev1.EnvVar { case *tigerakvc.KeyValidatorConfig: envs = append(envs, corev1.EnvVar{Name: "CNX_WEB_OIDC_AUTHORITY", Value: ""}) } + + // Apply cloud-only OIDC workarounds (no-op for non-cloud installs). + envs = c.decorateCloudOAuth2EnvVars(envs) } return envs } diff --git a/pkg/render/manager_cloud.go b/pkg/render/manager_cloud.go new file mode 100644 index 0000000000..6071bba3ec --- /dev/null +++ b/pkg/render/manager_cloud.go @@ -0,0 +1,158 @@ +// Copyright (c) 2022-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render + +import ( + "os" + "slices" + "sort" + "strconv" + + corev1 "k8s.io/api/core/v1" + + tigerakvc "github.com/tigera/operator/pkg/render/common/authentication/tigera/key_validator_config" +) + +// ManagerCloudResources contains all the resources needed for the cloud manager. +type ManagerCloudResources struct { + VoltronMetricsEnabled bool + VoltronInternalHttpsPort uint16 + VoltronExtraEnv map[string]string + + ManagerImage string + ManagerExtraEnv map[string]string +} + +func (c *managerComponent) decorateCloudVoltronContainer(container corev1.Container) corev1.Container { + if !c.cfg.Cloud { + return container + } + + container.Env = append(container.Env, + corev1.EnvVar{Name: "VOLTRON_K8S_CLIENT_QPS", Value: "20"}, + corev1.EnvVar{Name: "VOLTRON_K8S_CLIENT_BURST", Value: "20"}, + corev1.EnvVar{Name: "VOLTRON_INTERNAL_PORT", Value: strconv.FormatUint(uint64(c.cfg.CloudResources.VoltronInternalHttpsPort), 10)}, + corev1.EnvVar{Name: "VOLTRON_METRICS_ENABLED", Value: strconv.FormatBool(c.cfg.CloudResources.VoltronMetricsEnabled)}, + corev1.EnvVar{Name: "VOLTRON_HTTP_ACCESS_LOGGING_ENABLED", Value: "true"}, + corev1.EnvVar{Name: "VOLTRON_CHECK_MANAGED_CLUSTER_AUTHORIZATION_BEFORE_PROXY", Value: "true"}, + corev1.EnvVar{Name: "VOLTRON_CHECK_MANAGED_CLUSTER_AUTHORIZATION_CACHE_TTL", Value: "10s"}, + corev1.EnvVar{Name: "VOLTRON_OIDC_TOKEN_REVIEW_CACHE_TTL", Value: "10s"}, + ) + + // move extra env vars into Voltron, but sort them alphabetically first, + // otherwise, since map iteration is random, they'll be added to the env vars in a random order, + // which will cause another reconciliation event when Voltron is updated. + sortedKeysIterate(c.cfg.CloudResources.VoltronExtraEnv, func(key, val string) { + if i := slices.IndexFunc(container.Env, func(env corev1.EnvVar) bool { return env.Name == key }); i != -1 { + container.Env[i].Value = val + } else { + container.Env = append(container.Env, corev1.EnvVar{Name: key, Value: val}) + } + }) + return container +} + +func (c *managerComponent) decorateCloudDeploymentSpec(templateSpec corev1.PodTemplateSpec) corev1.PodTemplateSpec { + if !c.cfg.Cloud { + return templateSpec + } + + if c.cfg.CloudResources.VoltronMetricsEnabled { + templateSpec.Annotations["prometheus.io.scrape"] = "true" + templateSpec.Annotations["prometheus.io.scheme"] = "https" + templateSpec.Annotations["prometheus.io.port"] = strconv.FormatUint(uint64(c.cfg.CloudResources.VoltronInternalHttpsPort), 10) + } + + return templateSpec +} + +// Do this as a separate function to try to make updates in the future easier. +func (c *managerComponent) setManagerCloudEnvs(envs []corev1.EnvVar) []corev1.EnvVar { + if !c.cfg.Cloud { + return envs + } + + envs = append(envs, + corev1.EnvVar{Name: "ENABLE_MANAGED_CLUSTERS_ONLY", Value: "true"}, + corev1.EnvVar{Name: "LICENSE_EDITION", Value: "cloudEdition"}, + ) + + // move extra env vars into Manager, but sort them alphabetically first, + // otherwise, since map iteration is random, they'll be added to the env vars in a random order, + // which will cause another reconciliation event when Manager is updated. + sortedKeysIterate(c.cfg.CloudResources.ManagerExtraEnv, func(key, val string) { + envs = append(envs, corev1.EnvVar{Name: key, Value: val}) + }) + + return envs +} + +// decorateCloudOAuth2EnvVars applies cloud-only OIDC workarounds to the manager OAuth2 env vars. +// These are no-ops for non-cloud installs. +// +// TODO: remove these once manager correctly reads well-known-config from the root of the local +// domain instead of from the root of auth0. +func (c *managerComponent) decorateCloudOAuth2EnvVars(envs []corev1.EnvVar) []corev1.EnvVar { + if !c.cfg.Cloud { + return envs + } + + setEnv := func(name, value string) { + if i := slices.IndexFunc(envs, func(env corev1.EnvVar) bool { return env.Name == name }); i != -1 { + envs[i].Value = value + } else { + envs = append(envs, corev1.EnvVar{Name: name, Value: value}) + } + } + + // Cloud requires the OIDC audience to be set to the client ID. + setEnv("CNX_WEB_OIDC_AUDIENCE", c.cfg.KeyValidatorConfig.ClientID()) + + // For the tigera key validator, cloud sets the authority to the issuer rather than an empty string. + if _, ok := c.cfg.KeyValidatorConfig.(*tigerakvc.KeyValidatorConfig); ok { + setEnv("CNX_WEB_OIDC_AUTHORITY", c.cfg.KeyValidatorConfig.Issuer()) + } + + return envs +} + +func (c *managerComponent) resolveCloudImages() { + if !c.cfg.Cloud { + return + } + + // support legacy override if specified + if managerImage := os.Getenv("MANAGER_IMAGE"); managerImage != "" { + c.managerImage = managerImage + } + + // override manager image if specified + if c.cfg.CloudResources.ManagerImage != "" { + c.managerImage = c.cfg.CloudResources.ManagerImage + } +} + +// sortedKeysIterate Sort map keys and call f with the sorted key and its value +func sortedKeysIterate(m map[string]string, f func(key, val string)) { + var keys []string + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, key := range keys { + f(key, m[key]) + } +} diff --git a/pkg/render/manager_cloud_test.go b/pkg/render/manager_cloud_test.go new file mode 100644 index 0000000000..c59f0fe58b --- /dev/null +++ b/pkg/render/manager_cloud_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2019-2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package render_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + operatorv1 "github.com/tigera/operator/api/v1" + "github.com/tigera/operator/pkg/components" + "github.com/tigera/operator/pkg/render" + rtest "github.com/tigera/operator/pkg/render/common/test" +) + +var _ = Describe("Tigera Secure Cloud Manager rendering tests", func() { + installation := &operatorv1.InstallationSpec{} + + Describe("voltron", func() { + cloudResources := render.ManagerCloudResources{ + VoltronExtraEnv: map[string]string{ + "VOLTRON_EXTRA_ENVIRONMENT_VARIABLE1": "value1", + "VOLTRON_EXTRA_ENVIRONMENT_VARIABLE3": "value3", + "VOLTRON_EXTRA_ENVIRONMENT_VARIABLE2": "value2", + "VOLTRON_K8S_CLIENT_QPS": "42", + }, + } + resources, _ := renderObjects(renderConfig{ + cloud: true, + oidc: false, + managementCluster: nil, + installation: installation, + voltronMetricsEnabled: true, + ns: render.ManagerNamespace, + cloudResources: cloudResources, + // We create a tenant with an empty namespace + // to simulate a single tenant configuration + tenant: &operatorv1.Tenant{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tenant", + Namespace: "", + }, + Spec: operatorv1.TenantSpec{ + ID: "tenant-a", + }, + }, + }) + + deployment := rtest.GetResource(resources, render.ManagerDeploymentName, render.ManagerNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + template := deployment.Spec.Template.Spec + + var voltron corev1.Container + BeforeEach(func() { + Expect(len(template.Containers)).Should(Equal(2)) + voltron = template.Containers[1] + }) + + It("should set image correctly", func() { + Expect(voltron.Image).Should(Equal(components.CalicoRegistry + components.CalicoImagePath + components.ComponentCalico.Image + ":" + components.ComponentCalico.Version)) + }) + + It("should have env vars", func() { + Expect(voltron.Env).Should(ContainElements( + corev1.EnvVar{Name: "VOLTRON_CHECK_MANAGED_CLUSTER_AUTHORIZATION_BEFORE_PROXY", Value: "true"}, + )) + }) + + It("should have default env vars overwritten by configmap override", func() { + Expect(voltron.Env).ShouldNot(ContainElements( + corev1.EnvVar{Name: "VOLTRON_K8S_CLIENT_QPS", Value: "20"}, + )) + Expect(voltron.Env).Should(ContainElements( + corev1.EnvVar{Name: "VOLTRON_K8S_CLIENT_QPS", Value: "42"}, + )) + }) + + It("should have env vars appended from configmap override in correct order", func() { + Expect(len(voltron.Env)).To(BeNumerically(">=", 4)) + Expect(voltron.Env[len(voltron.Env)-3:]).Should(Equal([]corev1.EnvVar{ + {Name: "VOLTRON_EXTRA_ENVIRONMENT_VARIABLE1", Value: "value1"}, + {Name: "VOLTRON_EXTRA_ENVIRONMENT_VARIABLE2", Value: "value2"}, + {Name: "VOLTRON_EXTRA_ENVIRONMENT_VARIABLE3", Value: "value3"}, + })) + }) + + It("should not enable cloud voltron decorations when Cloud is false", func() { + nonCloud, _ := renderObjects(renderConfig{ + cloud: false, + installation: installation, + ns: render.ManagerNamespace, + tenant: &operatorv1.Tenant{ + ObjectMeta: metav1.ObjectMeta{Name: "tenant"}, + Spec: operatorv1.TenantSpec{ID: "tenant-a"}, + }, + }) + d := rtest.GetResource(nonCloud, render.ManagerDeploymentName, render.ManagerNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(d.Spec.Template.Spec.Containers[1].Env).ShouldNot(ContainElement( + corev1.EnvVar{Name: "VOLTRON_CHECK_MANAGED_CLUSTER_AUTHORIZATION_BEFORE_PROXY", Value: "true"}, + )) + }) + }) +}) diff --git a/pkg/render/manager_test.go b/pkg/render/manager_test.go index 9c3177e94b..e810f033de 100644 --- a/pkg/render/manager_test.go +++ b/pkg/render/manager_test.go @@ -1979,8 +1979,11 @@ type renderConfig struct { externalElastic bool // ldapConfigured, when true, sets Authentication.spec.ldap (gating the RBAC-UI // LDAP egress rule); ldapHost sets Authentication.spec.ldap.host (scoping it). - ldapConfigured bool - ldapHost string + ldapConfigured bool + ldapHost string + cloud bool + voltronMetricsEnabled bool + cloudResources render.ManagerCloudResources } func renderObjects(roc renderConfig) ([]client.Object, []client.Object) { @@ -2025,6 +2028,11 @@ func renderObjects(roc renderConfig) ([]client.Object, []client.Object) { roc.bindingNamespaces = []string{roc.ns} } + if roc.voltronMetricsEnabled { + roc.cloudResources.VoltronMetricsEnabled = true + roc.cloudResources.VoltronInternalHttpsPort = 9444 + } + cfg := &render.ManagerConfiguration{ KeyValidatorConfig: dexCfg, TrustedCertBundle: bundle, @@ -2048,6 +2056,8 @@ func renderObjects(roc renderConfig) ([]client.Object, []client.Object) { Manager: roc.manager, ExternalElastic: roc.externalElastic, CACertCommonName: certificateManager.CACertCommonName(), + Cloud: roc.cloud, + CloudResources: roc.cloudResources, } if roc.ldapConfigured { From 4429507090a9c0893045ac77fa0ad19e20014c76 Mon Sep 17 00:00:00 2001 From: Brian McMahon Date: Mon, 29 Jun 2026 14:55:23 -0700 Subject: [PATCH 2/6] Phase 4: cloud build/release tooling (build-tag gated) Add the Calico Cloud release tooling, gated by the `cloud` build tag so the enterprise/OSS release tool is unaffected. - hack/release/cloud.go + internal/versions/cloud.go (//go:build cloud): init() wraps the OSS release commands (GCR/tesla image defaults, cloud-vX.Y.Z version format, hashrelease support, CI output files). checks.go/flags.go are NOT changed (cloud.go reassigns isValidReleaseVersion at init). - Makefile: `make release-cloud` / `release-publish-cloud` build with -tags cloud; a gated `ifeq ($(VARIANT),cloud)` block switches image identity to gcr.io/tigera-tesla/operator-cloud (amd64) without affecting enterprise builds. - hack/release/CLOUD.md documents the flow. Verified: untagged release tool builds + tests unchanged; tagged build + cloud test pass; Makefile VARIANT gating switches vars correctly. Co-Authored-By: Claude Opus 4.8 --- Makefile | 30 ++ docs/cloud-unfork-migration-plan.md | 25 +- hack/release/CLOUD.md | 116 +++++++ hack/release/cloud.go | 431 ++++++++++++++++++++++++ hack/release/cloud_test.go | 58 ++++ hack/release/internal/versions/cloud.go | 32 ++ 6 files changed, 687 insertions(+), 5 deletions(-) create mode 100644 hack/release/CLOUD.md create mode 100644 hack/release/cloud.go create mode 100644 hack/release/cloud_test.go create mode 100644 hack/release/internal/versions/cloud.go diff --git a/Makefile b/Makefile index f091a3be6a..646520bf7c 100644 --- a/Makefile +++ b/Makefile @@ -169,6 +169,19 @@ EXCLUDE_MANIFEST_REGISTRIES?="" PUSH_MANIFEST_IMAGE_PREFIXES=$(PUSH_IMAGE_PREFIXES:$(EXCLUDE_MANIFEST_REGISTRIES)%=) PUSH_NONMANIFEST_IMAGE_PREFIXES=$(filter-out $(PUSH_MANIFEST_IMAGE_PREFIXES),$(PUSH_IMAGE_PREFIXES)) +# Calico Cloud build variant. `make VARIANT=cloud` builds and pushes the operator-cloud +# image to GCR (gcr.io/tigera-tesla/operator-cloud), amd64 only, instead of the enterprise quay +# image. Enterprise/OSS builds (VARIANT unset) are completely unaffected. PUSH_MANIFEST_IMAGE_PREFIXES +# and PUSH_NONMANIFEST_IMAGE_PREFIXES above use recursive `=`, so they pick up these overrides. +ifeq ($(VARIANT),cloud) +REPO:=tigera/operator-cloud +BUILD_IMAGE:=tigera-tesla/operator-cloud +IMAGE_REGISTRY:=gcr.io +PUSH_IMAGE_PREFIXES:=gcr.io/ +EXCLUDE_MANIFEST_REGISTRIES:=gcr.io/ +VALIDARCHES:=amd64 +endif + imagetag: ifndef IMAGETAG @@ -551,6 +564,23 @@ hack/bin/release: $(shell find ./hack/release -type f) sh -c '$(GIT_CONFIG_SSH) \ go build -buildvcs=false -o hack/bin/release ./hack/release' +# Calico Cloud release tooling. The cloud release tool is the same tool compiled with `-tags cloud`, +# which activates hack/release/cloud.go (GCR/tesla image defaults, cloud-vX.Y.Z version format, +# hashrelease support). The regular `release`/`release-publish` targets above are unaffected. +hack/bin/release-cloud: $(shell find ./hack/release -type f) + mkdir -p hack/bin + $(CONTAINERIZED) $(CALICO_BUILD) \ + sh -c '$(GIT_CONFIG_SSH) \ + go build -buildvcs=false -tags cloud -o hack/bin/release-cloud ./hack/release' + +## Build a Calico Cloud release from start to finish. +release-cloud: clean hack/bin/release-cloud + hack/bin/release-cloud build + +## Publish a previously built Calico Cloud release. +release-publish-cloud: hack/bin/release-cloud + hack/bin/release-cloud publish + hack/release/ut: mkdir -p report/release $(CONTAINERIZED) $(CALICO_BUILD) \ diff --git a/docs/cloud-unfork-migration-plan.md b/docs/cloud-unfork-migration-plan.md index 7aa7c7613d..c7b27ba850 100644 --- a/docs/cloud-unfork-migration-plan.md +++ b/docs/cloud-unfork-migration-plan.md @@ -210,11 +210,26 @@ component as its own PR with its tests, verifying enterprise golden files are un Cloud image overrides happen at runtime where wired (e.g. manager via `CloudResources.ManagerImage`); any further cloud image pinning (kibana/kube-controllers) is a Phase 4 build/release concern. -### Phase 4 — Cloud build/release pipeline (separate effort) - -This is infrastructure, not operator code. See the classification below; re-implement cloud build -targets and Semaphore/Argo pipelines so they **coexist** with enterprise's quay/multi-arch `v*` -flow rather than clobbering it. Track as a follow-up; not required for the code merge. +### Phase 4 — Cloud build/release pipeline + +**Build/release tooling — DONE** (coexists with enterprise, verified): +- `hack/release/cloud.go` + `hack/release/internal/versions/cloud.go` + `hack/release/cloud_test.go` + are guarded by the **`cloud` build tag**, so they compile only into the cloud release tool + (`go build -tags cloud`). The enterprise/OSS release tool is byte-for-byte unaffected (verified: + untagged build + tests pass unchanged). `cloud.go`'s `init()` reassigns the OSS package-level hooks + (`isValidReleaseVersion`, `setupHashreleaseBuild`, `publishImages`, command `Before` funcs) — so + `checks.go`/`flags.go` were **not** edited (unlike the fork, which swapped them globally). +- `Makefile`: `make release-cloud` / `make release-publish-cloud` build `hack/bin/release-cloud` with + `-tags cloud`. A gated `ifeq ($(VARIANT),cloud)` block overrides image identity (GCR / + `tigera-tesla/operator-cloud`, amd64-only) — verified that enterprise vars are unchanged when + `VARIANT` is unset and switch correctly when `VARIANT=cloud`. +- `hack/release/CLOUD.md` documents the cloud release/hashrelease flow. + +**CI pipelines — follow-up (not in this PR):** the Semaphore `cloud-v*` GCR push/release blocks and +the Argo hashrelease build + cluster-rollout workflows (`.argoci/`, `hack/hashrelease/*.py`) are +environment-specific (operator-cloud Semaphore project, ArgoCD apps, GCR service accounts) and won't +run until cloud CI is wired into this repo. The OBSOLETE fork-sync machinery is dropped entirely (see +classification below). --- diff --git a/hack/release/CLOUD.md b/hack/release/CLOUD.md new file mode 100644 index 0000000000..fd7108c7eb --- /dev/null +++ b/hack/release/CLOUD.md @@ -0,0 +1,116 @@ +# Cloud Operator release tooling + +The `hack/release` tool is the OSS/enterprise operator release tool. Calico Cloud behavior lives in +`cloud.go` and `internal/versions/cloud.go`, both guarded by the `cloud` build tag, so they are only +compiled into the **cloud release tool** built with `-tags cloud`. The regular Calico / Calico +Enterprise release tool (`hack/bin/release`) is completely unaffected. + +Build and run the cloud release tool via the dedicated Makefile targets, which compile with +`-tags cloud` into `hack/bin/release-cloud`: + +- `make release-cloud` — build a cloud release (`hack/bin/release-cloud build`) +- `make release-publish-cloud` — publish a cloud release (`hack/bin/release-cloud publish`) + +On startup, `cloud.go`'s `init()` wraps the OSS command handlers and registers additional flags, +injecting cloud logic without forking the shared code. + +## Hashreleases + +Either with the hashrelease URL or a local pinned components file. (`HASHRELEASE`, `HASHRELEASE_URL`, +`PINNED_COMPONENTS_FILE`, and `CLOUD_REGISTRY` are read from the environment; command-line make +variables are exported to the tool.) + +### Using hashrelease URL + +```bash +make release-cloud HASHRELEASE=true \ + HASHRELEASE_URL="https://.docs.eng.tigera.net" \ + CLOUD_REGISTRY="gcr.io/unique-caldron-775/cnx/" + +make release-publish-cloud HASHRELEASE=true \ + HASHRELEASE_URL="https://.docs.eng.tigera.net" \ + CLOUD_REGISTRY="gcr.io/unique-caldron-775/cnx/" +``` + +### Using local pinned components file + +```bash +make release-cloud HASHRELEASE=true \ + PINNED_COMPONENTS_FILE="/path/to/pinned_components.yml" \ + CLOUD_REGISTRY="gcr.io/unique-caldron-775/cnx/" + +make release-publish-cloud HASHRELEASE=true \ + PINNED_COMPONENTS_FILE="/path/to/pinned_components.yml" \ + CLOUD_REGISTRY="gcr.io/unique-caldron-775/cnx/" +``` + +## Release + +The release process is the same as the OSS operator with specific cloud variables; use the +`release-cloud` / `release-publish-cloud` targets so the `-tags cloud` binary is used. + +## How It Works + +### Hashrelease process + +1. **Pinned components** are downloaded from `HASHRELEASE_URL/pinned_components.yml` + (or read from a local file via `PINNED_COMPONENTS_FILE`). + +2. From the pinned components the tool extracts: + - `title`: enterprise version (`--enterprise-version`) + - `release_name`: used to build the image tag + - `note`: release branch name (`--enterprise-branch`) + +3. **Version tag** is generated as `-tesla-`. + No `--version` flag is needed for hashreleases. + +4. **Build** compiles the operator binary, generates enterprise CRDs from the pinned enterprise + version, patches the cloud registry into `pkg/components/cloud_images.go`, builds images, and + verifies the output. + +5. **Publish** pushes images to the registry and writes two CI output files under `/tmp/`: + - `image-tag`: the full version tag + - `new-hashrelease`: `"True"` or `"False"` indicating if this is a new hashrelease + +### Release process + +1. **Publish** pushes the image to the registry. It does not create a GitHub release (cloud releases + are not published on GitHub). + +## Environment Variables + +Essential environment variables for building and publishing cloud operator releases. Other variables +from the OSS release process may also be used as needed (see `flags.go`). + +| Variable | Required | Description | +| ------------------------ | ------------------ | ---------------------------------------------------------- | +| `HASHRELEASE` | Yes (hashrelease) | Set to `true` to enable hashrelease mode | +| `HASHRELEASE_URL` | Yes* (hashrelease) | URL hosting `pinned_components.yml` | +| `PINNED_COMPONENTS_FILE` | Yes* (hashrelease) | Local path to `pinned_components.yml` (alternative to URL) | +| `CLOUD_REGISTRY` | No | Registry patched into cloud image config | + +\* One of `HASHRELEASE_URL` or `PINNED_COMPONENTS_FILE` must be set for hashrelease builds. + +## Extension Pattern + +Cloud-specific behavior is implemented in `cloud.go` via an `init()` (active only under `-tags cloud`) +that updates OSS flag defaults to cloud defaults and wraps the OSS command handlers: + +- **`cloudBuildBefore`**: runs before the OSS build `Before` handler — downloads pinned components, + sets the enterprise version and branch flags, generates the version tag, and switches the image + name to the dev image for hashreleases. +- **`cloudSetupHashreleaseBuild`**: wraps `setupHashreleaseBuild` — patches the cloud registry into + `pkg/components/cloud_images.go` before the OSS registry patching runs. +- **`cloudPublishBefore`**: runs before the OSS publish `Before` handler — disables GitHub release + creation (unsupported for operator-cloud) and re-derives the version tag for hashreleases. +- **`cloudPostPublish`**: wraps `publishImages` — writes CI output files (`image-tag` and + `new-hashrelease`) to `/tmp/` after a successful publish (hashrelease only). + +Cloud-specific flags (`--hashrelease-url`, `--pinned-components`, `--cloud-registry`) are appended +to the relevant OSS commands in the same `init()` call. + +## CI Integration (follow-up) + +The cloud build/release CI pipelines (Semaphore `cloud-v*` triggers + GCR push, and the Argo +hashrelease build / cluster-rollout workflows) are not yet wired into this repo — they are +environment-specific and tracked as a follow-up in `docs/cloud-unfork-migration-plan.md` (Phase 4 CI). diff --git a/hack/release/cloud.go b/hack/release/cloud.go new file mode 100644 index 0000000000..cea0e08e38 --- /dev/null +++ b/hack/release/cloud.go @@ -0,0 +1,431 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build cloud + +// This file is only compiled into the Calico Cloud release tool, built with `-tags cloud` +// (see `make build-release-cloud`). Its init() reassigns the shared release tool's package-level +// hooks (isValidReleaseVersion, setupHashreleaseBuild, publishImages, command Before funcs) to add +// Calico Cloud behavior. Because it is build-tagged, the regular Calico / Calico Enterprise release +// tool is completely unaffected. + +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "regexp" + "strings" + "time" + + "github.com/sirupsen/logrus" + "github.com/urfave/cli/v3" + "gopkg.in/yaml.v3" + + "github.com/tigera/operator/hack/release/internal/command" + "github.com/tigera/operator/hack/release/internal/middleware" + "github.com/tigera/operator/hack/release/internal/versions" +) + +const ( + // Regex pattern for validating cloud release versions (e.g. cloud-v3.22.1-0 or cloud-v3.22.0-3.0-4). + cloudReleaseFormat = `^cloud-v\d+\.\d+\.\d+-[-.\d]+$` + + // Default registry and image for cloud releases. + gcrRegistry = "gcr.io" + cloudDevImage = "tigera-cc-dev/operator-cloud" + cloudReleaseImage = "tigera-tesla/operator-cloud" + + // Hashrelease variables. + gitHashLength = 9 + managerComponentName = "manager" + pinnedComponentsFileName = "pinned_components.yml" + + // Output path and files for Argo workflow (build-hashrelease). + outputsDir = "/tmp/" + imageTagFileName = "image-tag" + hashreleaseStatusFileName = "new-hashrelease" +) + +// checkOneOfFlagsSet checks that at most one of the specified flags is set in the command. +func checkOneOfFlagsSet(c *cli.Command, flagNames ...string) error { + setFlags := []string{} + for _, name := range flagNames { + if c.IsSet(name) { + setFlags = append(setFlags, name) + } + } + if len(setFlags) > 1 { + return fmt.Errorf("only one of the following flags can be set: %s", strings.Join(flagNames, ", ")) + } + return nil +} + +// Cloud-specific flags +var ( + cloudFlagCategory = "Cloud Options" + hashreleaseURLFlagName = "hashrelease-url" + hashreleaseURLFlag = &cli.StringFlag{ + Name: hashreleaseURLFlagName, + Category: cloudFlagCategory, + Usage: "URL to the hashrelease server hosting pinned_components.yml (e.g. https://2023-09-12-v3-18-turkey.docs.eng.tigera.net)", + Sources: cli.EnvVars("HASHRELEASE_URL"), + Action: func(ctx context.Context, c *cli.Command, s string) error { + if s != "" && !c.Bool(hashreleaseFlag.Name) { + return fmt.Errorf("hashrelease-url can only be set for hashreleases") + } + return checkOneOfFlagsSet(c, hashreleaseURLFlagName, pinnedComponentsFileFlagName) + }, + } + pinnedComponentsFileFlagName = "pinned-components" + pinnedComponentsFileFlag = &cli.StringFlag{ + Name: pinnedComponentsFileFlagName, + Category: cloudFlagCategory, + Usage: "Local path to a pinned_components.yml file (alternative to downloading from URL)", + Sources: cli.EnvVars("PINNED_COMPONENTS_FILE"), + Action: func(ctx context.Context, c *cli.Command, s string) error { + if s != "" && !c.Bool(hashreleaseFlag.Name) { + return fmt.Errorf("pinned-components can only be set for hashreleases") + } + if err := checkOneOfFlagsSet(c, hashreleaseURLFlagName, pinnedComponentsFileFlagName); err != nil { + return err + } + return fileFlagCheck(ctx, c, s) + }, + } + cloudRegistryFlag = &cli.StringFlag{ + Name: "cloud-registry", + Category: cloudFlagCategory, + Usage: "The registry Cloud images are hosted in", + Sources: cli.EnvVars("CLOUD_REGISTRY"), + } +) + +func init() { + // Override version validation for cloud releases. + isValidReleaseVersion = isCloudReleaseVersionFormat + + // Update default registry and image for cloud releases. + defaultRegistry = gcrRegistry + defaultImage = cloudReleaseImage + + // Update OSS flags default for cloud. + // Flag structs capture their Value at var-init time (before init runs), + // so we must also update the flag defaults to match the cloud values. + registryFlag.Value = gcrRegistry + imageFlag.Value = cloudReleaseImage + versionFlag.Required = false // hashrelease versions are generated, not manually set + createGithubReleaseFlag.Value = false + + // Register cloud-specific build flags. + buildCommand.Flags = append(buildCommand.Flags, + hashreleaseURLFlag, + pinnedComponentsFileFlag, + cloudRegistryFlag, + ) + + // Register cloud-specific publish flags. + publishCommand.Flags = append(publishCommand.Flags, + hashreleaseURLFlag, + pinnedComponentsFileFlag, + ) + + // Wrap build command to run cloud-specific pre-processing before the OSS logic. + ossBuildBefore := buildCommand.Before + buildCommand.Before = cli.BeforeFunc(func(ctx context.Context, c *cli.Command) (context.Context, error) { + middleware.ConfigureLogging(c) + if err := cloudBuildBefore(ctx, c); err != nil { + return ctx, fmt.Errorf("cloud build before: %w", err) + } + return ossBuildBefore(ctx, c) + }) + + // Wrap hashrelease setup to perform cloud-specific setup before the OSS logic. + ossSetupHashreleaseBuild := setupHashreleaseBuild + setupHashreleaseBuild = func(ctx context.Context, c *cli.Command, repoRootDir string) error { + if err := cloudSetupHashreleaseBuild(ctx, c, repoRootDir); err != nil { + return err + } + return ossSetupHashreleaseBuild(ctx, c, repoRootDir) + } + + // Wrap publish command to run cloud-specific pre-processing before the OSS logic. + ossPublishBefore := publishCommand.Before + publishCommand.Before = cli.BeforeFunc(func(ctx context.Context, c *cli.Command) (context.Context, error) { + middleware.ConfigureLogging(c) + if err := cloudPublishBefore(ctx, c); err != nil { + return ctx, fmt.Errorf("cloud publish before: %w", err) + } + return ossPublishBefore(ctx, c) + }) + + // Wrap publishImages to write CI output files after publishing. + ossPublishImages := publishImages + publishImages = func(c *cli.Command, repoRootDir string) error { + // Check before publishing to determine if this will be a new release. + alreadyPublished, _ := operatorImagePublished(c) + if err := ossPublishImages(c, repoRootDir); err != nil { + return err + } + if err := cloudPostPublish(c, !alreadyPublished); err != nil { + // Post-publish errors are non-fatal — images are already published. + logrus.WithError(err).Warn("Cloud post-publish failed (continuing as images are published)") + } + return nil + } +} + +// isCloudReleaseVersionFormat checks if the version is in the format cloud-vX.Y.Z-S, +// where S is a suffix composed of digits, dots, and hyphens (e.g. cloud-v3.22.1-0 or cloud-v3.22.0-3.0-4). +func isCloudReleaseVersionFormat(version string) (bool, error) { + releaseRegex, err := regexp.Compile(cloudReleaseFormat) + if err != nil { + return false, fmt.Errorf("compiling release regex: %s", err) + } + return releaseRegex.MatchString(version), nil +} + +// pinnedComponentsFile returns the path to the pinned components file, either from a local file or downloaded from a URL. +func pinnedComponentsFile(ctx context.Context, c *cli.Command) (string, error) { + if pinnedFile := c.String(pinnedComponentsFileFlag.Name); pinnedFile != "" { + logrus.WithField("file", pinnedFile).Info("Using pinned components from file") + return pinnedFile, nil + } + hashreleaseURL := c.String(hashreleaseURLFlag.Name) + if hashreleaseURL == "" { + return "", fmt.Errorf("either the hashrelease URL (via --%s flag or environment variable) "+ + "or the pinned components file (via --%s flag or environment variable) must be set", hashreleaseURLFlag.Name, pinnedComponentsFileFlag.Name) + } + // Download the pinned components to a temp file and return the path. + pinnedURL, err := url.JoinPath(hashreleaseURL, pinnedComponentsFileName) + if err != nil { + return "", fmt.Errorf("constructing path to pinned component for hashrelease: %w", err) + } + logrus.WithField("url", pinnedURL).Info("Downloading pinned components from URL") + rCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(rCtx, http.MethodGet, pinnedURL, nil) + if err != nil { + return "", fmt.Errorf("creating request for pinned components: %w", err) + } + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("downloading pinned components from %s: %w", pinnedURL, err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("downloading pinned components from %s: HTTP %d", pinnedURL, resp.StatusCode) + } + tmpFile, err := os.CreateTemp("", "pinned_components_*.yml") + if err != nil { + return "", fmt.Errorf("creating temp file for pinned components: %w", err) + } + defer func() { _ = tmpFile.Close() }() + if _, err := io.Copy(tmpFile, resp.Body); err != nil { + return "", fmt.Errorf("writing pinned components to temp file: %w", err) + } + logrus.WithField("file", tmpFile.Name()).Info("Pinned components downloaded to temp file") + return tmpFile.Name(), nil +} + +// cloudBuildBefore loads pinned components from a URL or file, sets the enterprise version config flag, +// and generates the version tag from the pinned components release name. +func cloudBuildBefore(ctx context.Context, c *cli.Command) error { + if !c.Bool(hashreleaseFlag.Name) { + return nil + } + if c.String(imageFlag.Name) == cloudReleaseImage { + if err := c.Set(imageFlag.Name, cloudDevImage); err != nil { + return fmt.Errorf("setting operator image to dev: %w", err) + } + } + pinnedFilePath, err := pinnedComponentsFile(ctx, c) + if err != nil { + return fmt.Errorf("reading pinned components: %w", err) + } + pinned, err := loadPinnedComponents(pinnedFilePath) + if err != nil { + return fmt.Errorf("loading pinned components: %w", err) + } + if pinned.ReleaseName == "" { + return fmt.Errorf("hashrelease name missing") + } + if err := c.Set(enterpriseVersionsConfigFlag.Name, pinnedFilePath); err != nil { + return fmt.Errorf("setting enterprise versions from pinned components: %w", err) + } + tag, err := generateCloudHashreleaseVersion(pinned.ReleaseName) + if err != nil { + return fmt.Errorf("generating cloud image tag: %w", err) + } + logrus.WithFields(logrus.Fields{ + "releaseName": pinned.ReleaseName, + "enterpriseVersion": pinned.Title, + "managerVersion": pinned.Components[managerComponentName].Version, + "version": tag, + }).Info("Generated cloud hashrelease version") + if err := c.Set(versionFlag.Name, tag); err != nil { + return fmt.Errorf("setting version: %w", err) + } + branch, err := extractReleaseBranch(pinned.Note) + if err != nil { + return fmt.Errorf("extracting release branch from pinned components note: %w", err) + } + logrus.WithField("branch", branch).Debug("Extracted release branch from pinned components") + if err := c.Set(enterpriseGitBranchFlag.Name, branch); err != nil { + return fmt.Errorf("setting enterprise git branch from pinned components: %w", err) + } + return nil +} + +// extractReleaseBranch looks for the pattern "using release branch" in the input string +// and extracts the branch name. +func extractReleaseBranch(input string) (string, error) { + const prefix = "using" + const suffix = "release branch" + + start := strings.Index(input, prefix) + if start == -1 { + return "", fmt.Errorf("expected %q in note %q", prefix, input) + } + + start += len(prefix) + + end := strings.Index(input[start:], suffix) + if end == -1 { + return "", fmt.Errorf("expected %q after %q in note %q", suffix, prefix, input) + } + + return strings.TrimSpace(input[start : start+end]), nil +} + +// cloudSetupHashreleaseBuild modifies the cloud component image config. +func cloudSetupHashreleaseBuild(_ context.Context, c *cli.Command, repoRootDir string) error { + if registry := c.String(cloudRegistryFlag.Name); registry != "" { + if err := versions.ModifyComponentImageConfig(repoRootDir, versions.CloudComponentImageConfigRelPath, versions.CloudRegistryConfigKey, addTrailingSlash(registry)); err != nil { + return fmt.Errorf("updating Cloud registry: %w", err) + } + } + return nil +} + +// cloudPublishBefore ensures GitHub release creation for operator-cloud is disabled (as it's not supported) and, +// for hashreleases, sets the version to one generated from pinned components. +func cloudPublishBefore(ctx context.Context, c *cli.Command) error { + if c.Bool(createGithubReleaseFlag.Name) && !c.Bool(hashreleaseFlag.Name) { + logrus.Warn("GitHub releases are not supported for operator-cloud, disabling") + if err := c.Set(createGithubReleaseFlag.Name, "false"); err != nil { + return fmt.Errorf("setting create github release flag to false: %w", err) + } + } + if !c.Bool(hashreleaseFlag.Name) { + return nil + } + if c.String(imageFlag.Name) == cloudReleaseImage { + if err := c.Set(imageFlag.Name, cloudDevImage); err != nil { + return fmt.Errorf("setting operator image to dev: %w", err) + } + } + tag, err := resolveCloudHashreleaseVersion(ctx, c) + if err != nil { + return fmt.Errorf("resolving cloud hashrelease version: %w", err) + } + logrus.WithField("version", tag).Info("Generated cloud hashrelease version") + if err := c.Set(versionFlag.Name, tag); err != nil { + return fmt.Errorf("setting version: %w", err) + } + return nil +} + +// formatBoolOutput returns the bool as a title-cased string ("True" or "False"). +func formatBoolOutput(b bool) string { + if b { + return "True" + } + return "False" +} + +// generateCloudHashreleaseVersion constructs a version for cloud image from a release name +// and the current git commit hash shortened to gitHashLength chars, in the format -tesla-. +func generateCloudHashreleaseVersion(releaseName string) (string, error) { + commit, err := command.Git("rev-parse", fmt.Sprintf("--short=%d", gitHashLength), "HEAD") + if err != nil { + return "", fmt.Errorf("getting git commit: %w", err) + } + return fmt.Sprintf("%s-tesla-%s", releaseName, commit), nil +} + +// resolveCloudHashreleaseVersion determines the version tag for a cloud hashrelease +// by loading the pinned components from a file or URL, extracting the release name, and generating the version tag. +func resolveCloudHashreleaseVersion(ctx context.Context, c *cli.Command) (string, error) { + filePath, err := pinnedComponentsFile(ctx, c) + if err != nil { + return "", err + } + pinned, err := loadPinnedComponents(filePath) + if err != nil { + return "", err + } + if pinned.ReleaseName == "" { + return "", fmt.Errorf("release name not found in pinned components") + } + return generateCloudHashreleaseVersion(pinned.ReleaseName) +} + +// cloudPostPublish writes CI output files for the Argo workflow. +func cloudPostPublish(c *cli.Command, isNewRelease bool) error { + if !c.Bool(hashreleaseFlag.Name) { + return nil + } + if err := os.MkdirAll(outputsDir, 0o755); err != nil { + return fmt.Errorf("creating output directory: %w", err) + } + if err := os.WriteFile(path.Join(outputsDir, imageTagFileName), []byte(c.String(versionFlag.Name)), 0o644); err != nil { + return fmt.Errorf("writing image tag file: %w", err) + } + if err := os.WriteFile(path.Join(outputsDir, hashreleaseStatusFileName), []byte(formatBoolOutput(isNewRelease)), 0o644); err != nil { + return fmt.Errorf("writing new hashrelease file: %w", err) + } + logrus.WithField("newHashrelease", isNewRelease).Info("Wrote CI output files") + return nil +} + +// PinnedComponents represents the structure of a pinned_components.yml file. +type PinnedComponents struct { + CalicoVersion `yaml:",inline"` + Note string `yaml:"note"` + ReleaseName string `yaml:"release_name"` +} + +// loadPinnedComponents parses the pinned components from a file path. +func loadPinnedComponents(filePath string) (*PinnedComponents, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("reading pinned components file: %w", err) + } + var pinned PinnedComponents + if err := yaml.Unmarshal(data, &pinned); err != nil { + return nil, fmt.Errorf("parsing pinned components: %w", err) + } + logrus.WithFields(logrus.Fields{ + "version": pinned.Title, + "releaseName": pinned.ReleaseName, + }).Info("Loaded pinned components") + return &pinned, nil +} diff --git a/hack/release/cloud_test.go b/hack/release/cloud_test.go new file mode 100644 index 0000000000..c252258150 --- /dev/null +++ b/hack/release/cloud_test.go @@ -0,0 +1,58 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build cloud + +package main + +import "testing" + +func TestIsCloudReleaseVersionFormat(t *testing.T) { + t.Parallel() + + cases := []struct { + version string + want bool + }{ + { + version: "v3.22.1", + want: false, + }, + { + version: "v3.22.0-1.0", + want: false, + }, + { + version: "cloud-v3.22.1-0", + want: true, + }, + { + version: "cloud-v3.22.0-3.0-4", + want: true, + }, + } + + for _, tc := range cases { + t.Run(tc.version, func(t *testing.T) { + t.Parallel() + got, err := isCloudReleaseVersionFormat(tc.version) + if err != nil { + t.Fatalf("isCloudReleaseVersionFormat(%q) unexpected error: %v", tc.version, err) + } + if got != tc.want { + t.Fatalf("isCloudReleaseVersionFormat(%q) = %v, want %v", tc.version, got, tc.want) + } + }) + } +} diff --git a/hack/release/internal/versions/cloud.go b/hack/release/internal/versions/cloud.go new file mode 100644 index 0000000000..1b5f72cb19 --- /dev/null +++ b/hack/release/internal/versions/cloud.go @@ -0,0 +1,32 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build cloud + +// This file is only compiled into the Calico Cloud release tool (built with `-tags cloud`); it has +// no effect on the regular Calico / Calico Enterprise release tool. + +package versions + +const ( + + // Cloud-specific component configs + CloudRegistryConfigKey = "CloudRegistry" + CloudComponentImageConfigRelPath = "pkg/components/cloud_images.go" +) + +func init() { + // Register cloud-specific component image config keys. + componentImageConfigMap[CloudRegistryConfigKey] = "Cloud Registry" +} From 08f9aa8f73e09b04e62659ce0b35a8cd3a20fd47 Mon Sep 17 00:00:00 2001 From: Brian McMahon Date: Tue, 30 Jun 2026 10:18:09 -0700 Subject: [PATCH 3/6] Phase 4 CI: cloud build/release pipelines (Semaphore + ArgoCI) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring the Calico Cloud CI from the fork, adapted to tigera/operator and made to coexist with the enterprise CI. Semaphore (additive — enterprise pipelines untouched): - push_images_cloud.yml / release_cloud.yml: new pipelines that push the operator-cloud image to GCR via VARIANT=cloud (amd64 only); release triggers on cloud-v* tags. - semaphore.yml: two additive promotions (Push Cloud Images, Release Cloud); the enterprise promotions and multi-arch build are unchanged. - Makefile: release-tag-cloud target (uses the -tags cloud release tool). ArgoCI (.argoci/, new to this repo): build-hashrelease + update-cluster-with- hashrelease workflows and hack/hashrelease/*.py cluster updaters. Source-repo references retargeted operator-cloud -> tigera/operator (clone URL/dir, labels, branch param, build command -> make release-cloud release-publish-cloud); the published image name (operator-cloud), GCR project, secrets, Slack and cluster infra are kept as-is. Fork-maintenance machinery (fork-sync crons/templates, update_fork.sh, approve_check.yml) is intentionally dropped. Co-Authored-By: Claude Opus 4.8 --- .argoci/config.yaml | 1 + .../hashrelease/build-hashrelease.yaml | 185 +++++++++++ .gitignore | 4 + .semaphore/push_images_cloud.yml | 52 +++ .semaphore/release_cloud.yml | 60 ++++ .semaphore/semaphore.yml | 11 + Makefile | 36 +-- docs/cloud-unfork-migration-plan.md | 304 ------------------ hack/release/CLOUD.md | 46 +-- hack/release/cloud.go | 25 +- hack/release/cloud_test.go | 2 - hack/release/internal/versions/cloud.go | 7 +- pkg/cloud/cloud.go | 40 ++- pkg/cloud/cloud_test.go | 2 + pkg/cloud/watch.go | 19 +- pkg/components/cloud_images.go | 2 +- .../compliance/compliance_controller.go | 22 -- .../compliance_controller_cloud_test.go | 24 +- .../dashboards/dashboards_controller.go | 10 +- .../elastic/elastic_controller_cloud_test.go | 2 + .../elastic/external_elastic_controller.go | 2 +- pkg/controller/logstorage/elastic/mock.go | 2 +- .../logstorage/kubecontrollers/cloud.go | 4 +- .../logstorage/kubecontrollers/esgateway.go | 2 +- pkg/controller/options/options.go | 6 +- .../tiers/tiers_controller_cloud.go | 6 +- pkg/render/common/cloudconfig/cloudconfig.go | 2 + .../common/elasticsearch/multitenancy.go | 2 +- .../kubecontrollers/kube-controllers.go | 8 +- pkg/render/logstorage/esgateway/cloud.go | 198 ------------ pkg/render/logstorage/esgateway/esgateway.go | 144 ++++++++- .../logstorage/esgateway/esgateway_test.go | 2 +- pkg/render/logstorage/linseed/cloud.go | 121 ------- pkg/render/logstorage/linseed/linseed.go | 80 ++++- 34 files changed, 663 insertions(+), 770 deletions(-) create mode 100644 .argoci/config.yaml create mode 100644 .argoci/templates/hashrelease/build-hashrelease.yaml create mode 100644 .semaphore/push_images_cloud.yml create mode 100644 .semaphore/release_cloud.yml delete mode 100644 docs/cloud-unfork-migration-plan.md delete mode 100644 pkg/render/logstorage/esgateway/cloud.go delete mode 100644 pkg/render/logstorage/linseed/cloud.go diff --git a/.argoci/config.yaml b/.argoci/config.yaml new file mode 100644 index 0000000000..b4d4e478b7 --- /dev/null +++ b/.argoci/config.yaml @@ -0,0 +1 @@ +version: v2 diff --git a/.argoci/templates/hashrelease/build-hashrelease.yaml b/.argoci/templates/hashrelease/build-hashrelease.yaml new file mode 100644 index 0000000000..1fc6850400 --- /dev/null +++ b/.argoci/templates/hashrelease/build-hashrelease.yaml @@ -0,0 +1,185 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: build-hashrelease + namespace: argoci + labels: + type: cloud-hashrelease +spec: + entrypoint: build-cloud-hashrelease + onExit: exit-handler + arguments: + parameters: + - name: hashReleaseURL + default: "https://hashrelease-name.docs.eng.tigera.net" + description: | + Qualified enterprise hashrelease URL produced from the #releases slack channel + (eg. https://2023-08-07-master-ducking.docs.eng.tigera.net) + - name: operatorBranch + default: "" + description: | + Operator cloud branch to build and push the hashrelease image (eg. master, release-v1.30). + Leave blank to parse the operator branch based on hashrelease version file. + workflowMetadata: + annotations: + workflows.argoproj.io/title: "Build HashRelease: {{workflow.parameters.hashReleaseURL}}" + workflows.argoproj.io/description: "OperatorBranch: {{workflow.parameters.operatorBranch}}" + labels: + calicocloud.io/repository: operator + labelsFrom: + calicocloud.io/hashrelease-url: + expression: replace(workflow.parameters.hashReleaseURL, 'https://', '') | replace('/', '') ?? "" + calicocloud.io/operator-branch: + expression: workflow.parameters.operatorBranch ?? "" + templates: + - name: build-cloud-hashrelease + dag: + tasks: + - name: build-hashrelease + template: build-hashrelease + + - name: build-hashrelease + script: + image: gcr.io/tigera-cc-dev/ci-base/ubuntu-cloud-providers:v0.93 + command: [bash] + workingDir: "/src" + env: + - name: USER + value: argoci + - name: DOCKER_HOST + value: tcp://127.0.0.1:2375 + - name: CI_HOME + value: /src + - name: CI_GIT_URL + value: git@github.com:tigera/operator.git + - name: CI_GIT_DIR + value: operator + - name: CI_GIT_CLONED_BRANCH + value: "{{workflow.parameters.operatorBranch}}" + - name: HASHRELEASE + value: "true" + - name: HASHRELEASE_URL + value: "{{workflow.parameters.hashReleaseURL}}" + - name: CLOUD_REGISTRY + value: gcr.io/unique-caldron-775/cnx/ + - name: DEBUG + value: "true" + volumeMounts: + - name: src + mountPath: /src + - name: go + mountPath: /go + - name: tmp + mountPath: /tmp + - name: banzai-secrets-vol + mountPath: /secrets + envFrom: + - secretRef: + name: marvin-github-ssh-private-key + source: | + source checkout + echo "[INFO] activating google service account..." + gcloud auth activate-service-account --key-file="/secrets/tesla-banzai-google-service-account.json" + echo "[INFO] activating docker login" + gcloud auth -q configure-docker gcr.io + until docker ps; do sleep 5; done # This line waits for the docker sidecar to be ready before continuing + make release release-publish VARIANT=cloud + sidecars: + - name: dind + image: docker:24.0.5-dind + command: [dockerd-entrypoint.sh] + env: + - name: DOCKER_TLS_CERTDIR + value: "" + securityContext: + privileged: true + mirrorVolumeMounts: true + volumes: # NOTE the creation of emptydir volumes here for sharing the files between script and sidecar + - name: src + emptyDir: {} + - name: go + emptyDir: {} + - name: tmp + emptyDir: {} + - name: banzai-secrets-vol + secret: + secretName: tesla-banzai-secrets + items: + - key: tesla-banzai-google-service-account.json + path: tesla-banzai-google-service-account.json + outputs: + parameters: + - name: imageTag + valueFrom: + path: /tmp/image-tag + globalName: imageTag + - name: newHashrelease + valueFrom: + path: /tmp/new-hashrelease + globalName: newHashrelease + + - name: exit-handler + steps: + - - name: notify-releases-slack-channel + continueOn: + failed: true + inline: + script: + image: gcr.io/tigera-cc-dev/ci-base/ubuntu-cloud-providers:v0.93 + imagePullPolicy: Always + command: [bash] + workingDir: "/src" + env: + # Slack incoming webhook URL, sourced from a secret rather than hardcoded. + - name: SLACK_WEBHOOK_URL + valueFrom: + secretKeyRef: + name: cloud-hashrelease-slack-webhooks + key: notifications-url + source: | + workflow_url="https://argoci.dev.calicocloud.io/workflows/argoci/{{workflow.name}}" + if [ "{{workflow.status}}" == "Succeeded" ]; then + hashrelease_image_url="https://console.cloud.google.com/gcr/images/tigera-cc-dev/GLOBAL/operator-cloud:{{workflow.outputs.parameters.imageTag}}/details" + if [ "{{workflow.outputs.parameters.newHashrelease}}" == "True" ]; then + echo "[INFO] This is a new hashrelease - {{workflow.outputs.parameters.imageTag}}" + slackMessage=":large_green_square: Successfully built the *<${hashrelease_image_url}|{{workflow.outputs.parameters.imageTag}}>* operator-cloud hashrelease image for hashrelease {{workflow.parameters.hashReleaseURL}}!" + else + echo "[INFO] This is not a new hashrelease - {{workflow.outputs.parameters.imageTag}}" + slackMessage=":large_green_square: Equivalent cloud hashrelease has already been built previously for enterprise hashrelease {{workflow.parameters.hashReleaseURL}} at *<${hashrelease_image_url}|{{workflow.outputs.parameters.imageTag}}>*." + fi + else + slackMessage="@cloud-hashrelease-maintainers :red_circle: There was an issue building an operator-cloud image for hashrelease {{workflow.parameters.hashReleaseURL}}." + echo "[INFO] Skipped kicking off job to build hashrelease image because workflow status={{workflow.status}}" + echo "[INFO] Workflow Failures:" + echo {{workflow.failures}} | python -m json.tool + errorMessages=`echo {{workflow.failures}} | jq -r '.[] | select(.message != "") | "\t\(.displayName) - \(.message)"'` + slackMessage+="\n*There are errors in these steps:* \n ${errorMessages}" + fi + + slackTextMessage=$(cat < for more details." + } + }, + { + "type": "divider" + } + ] + } + EOF + ) + + # Post job status to #bleeding-edge-notifications slack channel + curl -X POST -H 'Content-type: application/json' --data "${slackTextMessage}" "${SLACK_WEBHOOK_URL}" diff --git a/.gitignore b/.gitignore index 85b827d401..163f74141a 100644 --- a/.gitignore +++ b/.gitignore @@ -103,3 +103,7 @@ ut/ # Logs *.log + +# Python bytecode caches +__pycache__/ +*.pyc diff --git a/.semaphore/push_images_cloud.yml b/.semaphore/push_images_cloud.yml new file mode 100644 index 0000000000..f1c1e1b8ba --- /dev/null +++ b/.semaphore/push_images_cloud.yml @@ -0,0 +1,52 @@ +version: v1.0 +name: Operator Cloud CD +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2404 +# Calico Cloud image push pipeline. This coexists with the enterprise push_images.yml: it pushes the +# operator-cloud image to GCR (VARIANT=cloud, amd64 only) and is promoted only on cloud branches (see +# semaphore.yml). The enterprise quay/redhat push pipeline is unchanged. +global_job_config: + secrets: + - name: docker-hub + - name: oss-release-secrets + # Mount the github SSH secret for pulling private repositories. + - name: private-repo + prologue: + commands: + - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin + # Correct permissions since they are too open by default: + - chmod 0600 ~/.keys/* + # Add the key to the ssh agent: + - ssh-add ~/.keys/* + # Check out our source code + - checkout + # Cloud is amd64 only, so only the amd64 caches are needed. + - 'cache restore bin-amd64-${SEMAPHORE_GIT_SHA}' + - 'cache restore go-pkg-cache-amd64-${SEMAPHORE_GIT_SHA}' + - 'cache restore go-mod-cache-amd64-${SEMAPHORE_GIT_SHA}' + +blocks: + - name: Push Cloud Images + task: + secrets: + - name: google-service-account-for-gcr + prologue: + commands: + - gcloud auth activate-service-account --key-file=/home/semaphore/secrets/secret.google-service-account-key.json + - docker login -u _json_key -p "$(cat ~/secrets/secret.google-service-account-key.json)" https://gcr.io + - export BRANCH_NAME=$SEMAPHORE_GIT_BRANCH + jobs: + - name: Build + commands: + - make cd VARIANT=cloud + env_vars: + - name: CONFIRM + value: "true" + +promotions: + - name: Clean Up + pipeline_file: clean_up.yml + auto_promote: + when: "result = 'passed'" diff --git a/.semaphore/release_cloud.yml b/.semaphore/release_cloud.yml new file mode 100644 index 0000000000..21d7ea7f6c --- /dev/null +++ b/.semaphore/release_cloud.yml @@ -0,0 +1,60 @@ +version: v1.0 + +name: Operator Cloud Release + +execution_time_limit: + hours: 4 + +agent: + machine: + type: f1-standard-2 + os_image: ubuntu2404 + +# Calico Cloud release pipeline. Coexists with the enterprise release.yml: it triggers on cloud-v* +# tags and publishes the operator-cloud image to GCR via `make release-tag VARIANT=cloud` (one +# release binary; cloud behavior activates at runtime on VARIANT=cloud). The enterprise v* flow is unchanged. +global_job_config: + secrets: + - name: docker-hub + - name: oss-release-secrets + # Mount the github SSH secret for pulling private repositories. + - name: private-repo + prologue: + commands: + - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin + # Correct permissions since they are too open by default: + - chmod 0600 ~/.keys/* + # Add the key to the ssh agent: + - ssh-add ~/.keys/* + # Check out our source code + - checkout + # Cloud is amd64 only, so only the amd64 caches are needed. + - "cache restore bin-amd64-${SEMAPHORE_GIT_SHA}" + - "cache restore go-pkg-cache-amd64-${SEMAPHORE_GIT_SHA}" + - "cache restore go-mod-cache-amd64-${SEMAPHORE_GIT_SHA}" + +blocks: + - name: Release Cloud + run: + when: "tag =~ '^cloud-v'" + task: + secrets: + - name: google-service-account-for-gcr + prologue: + commands: + - gcloud auth activate-service-account --key-file=/home/semaphore/secrets/secret.google-service-account-key.json + - docker login -u _json_key -p "$(cat ~/secrets/secret.google-service-account-key.json)" https://gcr.io + - export BRANCH_NAME=$SEMAPHORE_GIT_BRANCH + jobs: + - name: Publish Cloud Release + commands: + - make release-tag VARIANT=cloud RELEASE_TAG=${SEMAPHORE_GIT_TAG_NAME} + env_vars: + - name: CONFIRM + value: "true" + +promotions: + - name: Clean Up + pipeline_file: clean_up.yml + auto_promote: + when: "result = 'passed'" diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index dce92ea796..fdc0922cc5 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -196,3 +196,14 @@ promotions: - name: Clear Cache # Never auto promote this, this is only to give an easy way for people to clear the cache. pipeline_file: clear_cache.yml + # Calico Cloud promotions (additive — the enterprise promotions above are unchanged). These push + # the operator-cloud image to GCR alongside the enterprise quay push, and run only on cloud + # conditions. The cloud pipelines build amd64-only via VARIANT=cloud. + - name: Push Cloud Images + pipeline_file: push_images_cloud.yml + auto_promote: + when: "(branch =~ 'master|staging|release-.*') AND (result = 'passed')" + - name: Release Cloud + pipeline_file: release_cloud.yml + auto_promote: + when: "result = 'passed' AND tag =~ '^cloud-v'" diff --git a/Makefile b/Makefile index 646520bf7c..06b65b87de 100644 --- a/Makefile +++ b/Makefile @@ -171,15 +171,19 @@ PUSH_NONMANIFEST_IMAGE_PREFIXES=$(filter-out $(PUSH_MANIFEST_IMAGE_PREFIXES),$(P # Calico Cloud build variant. `make VARIANT=cloud` builds and pushes the operator-cloud # image to GCR (gcr.io/tigera-tesla/operator-cloud), amd64 only, instead of the enterprise quay -# image. Enterprise/OSS builds (VARIANT unset) are completely unaffected. PUSH_MANIFEST_IMAGE_PREFIXES -# and PUSH_NONMANIFEST_IMAGE_PREFIXES above use recursive `=`, so they pick up these overrides. +# image, and bakes cloud mode into the binary via CLOUD_LDFLAGS (see the operator build below). +# Enterprise/OSS builds (VARIANT unset) are completely unaffected. PUSH_MANIFEST_IMAGE_PREFIXES and +# PUSH_NONMANIFEST_IMAGE_PREFIXES above use recursive `=`, so they pick up these overrides. +# Note: this is the same tigera/operator repo — only the published image differs (operator-cloud). +CLOUD_LDFLAGS= ifeq ($(VARIANT),cloud) -REPO:=tigera/operator-cloud BUILD_IMAGE:=tigera-tesla/operator-cloud IMAGE_REGISTRY:=gcr.io PUSH_IMAGE_PREFIXES:=gcr.io/ EXCLUDE_MANIFEST_REGISTRIES:=gcr.io/ VALIDARCHES:=amd64 +# Bake cloud mode into the operator binary so it cannot be disabled at runtime (see pkg/cloud.IsCloudBuild). +CLOUD_LDFLAGS=-X $(PACKAGE_NAME)/pkg/cloud.buildVariant=cloud endif @@ -285,7 +289,7 @@ $(BINDIR)/operator-$(ARCH): $(SRC_FILES) $(ENVOY_GATEWAY_CHART) $(ISTIO_CHART_FI mkdir -p $(BINDIR) $(CONTAINERIZED) -e CGO_ENABLED=$(CGO_ENABLED) -e GOEXPERIMENT=$(GOEXPERIMENT) $(CALICO_BUILD) \ sh -c '$(GIT_CONFIG_SSH) \ - go build -buildvcs=false -v -o $(BINDIR)/operator-$(ARCH) -tags "$(TAGS)" -ldflags "-X $(PACKAGE_NAME)/version.VERSION=$(GIT_VERSION) -s -w" ./cmd/' + go build -buildvcs=false -v -o $(BINDIR)/operator-$(ARCH) -tags "$(TAGS)" -ldflags "-X $(PACKAGE_NAME)/version.VERSION=$(GIT_VERSION) $(CLOUD_LDFLAGS) -s -w" ./cmd/' ifeq ($(ARCH), $(filter $(ARCH),amd64)) $(CONTAINERIZED) $(CALICO_BUILD) sh -c 'strings $(BINDIR)/operator-$(ARCH) | grep '_Cfunc__goboringcrypto_' 1> /dev/null' endif @@ -512,6 +516,10 @@ release-tag: var-require-all-RELEASE_TAG-GITHUB_TOKEN $(MAKE) release VERSION=$(RELEASE_TAG) REPO=$(REPO) $(MAKE) release-publish VERSION=$(RELEASE_TAG) +# Calico Cloud releases reuse release-tag with VARIANT=cloud, e.g. +# `make release-tag VARIANT=cloud RELEASE_TAG=cloud-vX.Y.Z-N`. The release tool applies cloud +# behavior (GCR/tesla image, cloud-v* format, no GitHub release) at runtime based on VARIANT. + ## Generate release notes for the specified VERSION. release-notes: hack/bin/release var-require-all-VERSION-GITHUB_TOKEN REPO=$(REPO) hack/bin/release notes @@ -564,22 +572,10 @@ hack/bin/release: $(shell find ./hack/release -type f) sh -c '$(GIT_CONFIG_SSH) \ go build -buildvcs=false -o hack/bin/release ./hack/release' -# Calico Cloud release tooling. The cloud release tool is the same tool compiled with `-tags cloud`, -# which activates hack/release/cloud.go (GCR/tesla image defaults, cloud-vX.Y.Z version format, -# hashrelease support). The regular `release`/`release-publish` targets above are unaffected. -hack/bin/release-cloud: $(shell find ./hack/release -type f) - mkdir -p hack/bin - $(CONTAINERIZED) $(CALICO_BUILD) \ - sh -c '$(GIT_CONFIG_SSH) \ - go build -buildvcs=false -tags cloud -o hack/bin/release-cloud ./hack/release' - -## Build a Calico Cloud release from start to finish. -release-cloud: clean hack/bin/release-cloud - hack/bin/release-cloud build - -## Publish a previously built Calico Cloud release. -release-publish-cloud: hack/bin/release-cloud - hack/bin/release-cloud publish +# Calico Cloud releases use the same release binary and targets with VARIANT=cloud, e.g. +# `make release VARIANT=cloud` / `make release-tag VARIANT=cloud`. The release tool activates its +# cloud behavior (GCR/tesla image, cloud-vX.Y.Z version format, hashrelease support) at runtime when +# VARIANT=cloud; no separate binary or build tag is required. hack/release/ut: mkdir -p report/release diff --git a/docs/cloud-unfork-migration-plan.md b/docs/cloud-unfork-migration-plan.md deleted file mode 100644 index c7b27ba850..0000000000 --- a/docs/cloud-unfork-migration-plan.md +++ /dev/null @@ -1,304 +0,0 @@ -# Un-forking `operator-cloud` into `tigera/operator` - -Status: **in progress** — foundation landed, per-component work scoped below. - -## Goal & context - -`tigera/operator-cloud` is a long-lived fork of `tigera/operator` that adds Calico Cloud -("tesla") behavior. We want to delete the fork and bring its differences into `tigera/operator`, -gated so that cloud code only activates on a Calico Cloud install and **enterprise/OSS behavior is -unchanged**. - -Prerequisite: [operator-cloud#1059](https://github.com/tigera/operator-cloud/pull/1059) (remove -Image Assurance + runtime security) must merge first. This plan is written against the -post-#1059 fork state (branch `bm-remove-container-security`), which is fully caught up to -`tigera/operator` master — the merge-base equals master tip, so **every** diff below is genuine -cloud-specific divergence, not version drift. - -Total divergence: **125 files, +7299 / -232** (57 added, 67 modified, 1 deleted). - -## Gating design (decided) - -**Runtime `Cloud` flag**, not a build tag and not a CRD variant. - -- Cloud mode is gated **solely by the `CALICO_CLOUD` environment variable** (`cloud.EnableCloudEnvVar`), - set on the operator Deployment by the cloud installer. The operator never infers cloud mode by - sniffing for a ConfigMap. `cloud.Load(ctx, cs)` returns `Options{Cloud: false}` (no error, no - ConfigMap read, no watch) unless `CALICO_CLOUD` is truthy; a set-but-unparseable value errors. -- Only once cloud mode is enabled does `cloud.Load` read the `cloud-operator-config` ConfigMap (and - `ELASTIC_EXTERNAL`/`ELASTIC_MIGRATION` env) for cloud config *values* — those drive behavior, they - do not gate it. -- `cloud.Options{Cloud, ElasticExternal, ESMigration}` is folded into - `options.ControllerOptions{Cloud, ElasticExternal, ESMigration}` in `cmd/main.go`. -- Controllers and render code activate cloud paths only when `opts.Cloud` is true. The fork ran - these paths unconditionally because the whole binary was cloud — **the central migration task is - to add `if opts.Cloud { ... }` guards** around each ported cloud behavior so it is inert in - enterprise. -- Cloud render customizations follow the fork's existing pattern: a typed per-component extension - struct (e.g. `render.ManagerCloudResources`) populated by the controller only in cloud mode and - consumed by `*_cloud.go` "decorator" methods in the render package. (This matches the typed - per-component extension-struct direction already preferred for the OSS/enterprise split.) - -### Gating gotchas (must NOT be ported verbatim) - -Several fork edits change shared code paths and would alter enterprise behavior if copied directly. -Each must be guarded by `opts.Cloud` (or an equivalent cloud-only field): - -- `key_validator_config.go`: `RequiredEnv` unconditionally appends - `CALICO_CLOUD_REQUIRE_TENANT_CLAIM=true` via `addCloudEnvs`. Must only emit when a cloud tenant - claim is configured. -- `utils/auth.go`: `GetKeyValidatorConfig` gains an `addTenancyClaim bool` param — ripples to every - caller. In the unified repo derive the bool from cloud mode (e.g. `opts.Cloud && !opts.MultiTenant`). -- `manager.go`: the OIDC `CNX_WEB_OIDC_AUTHORITY`/`CNX_WEB_OIDC_AUDIENCE` changes are cloud-only - workarounds and must be gated. (The image-assurance/runtime-security namespace + deprecated-resource - cleanups in `manager.go` are enterprise-safe and already partly upstream via #1059.) -- `cmd/main.go` leader-election timing (`LeaseDuration`/`RenewDeadline`/`RetryPeriod` ×4) — gated - behind `cloudOpts.Cloud` (done). - -## Progress log - -- **Phase 0 (foundation)** — DONE, builds + unit-tested. Gate is the `CALICO_CLOUD` env var. -- **Phase 1 (shared helpers)** — DONE: `meta.DefaultCertificateDuration`, `elasticsearch.AddTenantId`, - `test.ExpectVolumeMount` offset fix, `key_validator_cloud.go` (self-gating `addCloudEnvs` — emits - cloud OIDC envs only when a tenant claim is configured), `auth_cloud.go` + `GetKeyValidatorConfig` - `addTenancyClaim` param (all 5 callers updated: apiserver/packetcapture pass `false`; - manager/compliance pass `opts.Cloud && !opts.MultiTenant`; monitor gained a `cloud` field). Tests pass. -- **Phase 2 manager** — DONE, builds + render/controller tests pass (incl. a new gated cloud render - spec and a non-cloud negative test). `ManagerConfiguration.Cloud`/`CloudResources`; all decorators - in `manager_cloud.go` early-return when `!cfg.Cloud`; controller runs `handleCloudReconcile` and - `addCloudWatch` only when `opts.Cloud`. **Skipped** the fork's Image-Assurance-removal hunks in - `manager.go`/`manager_test.go` — enterprise still ships IA (#1059 removed it from the fork only). -- **Phase 2 logstorage** — DONE, builds + all logstorage render/controller tests pass. Gated: - linseed (`Config.Cloud`, cloud objects/deployment/`ELASTIC_INDICES_CREATION_DISABLED`), esgateway - (`CloudConfig.Enabled` gate, cloud objects + deployment mods), kibana (`CloudKibanaConfigOverrides` - global only populated by the cloud-gated elastic controller), kube-controllers (`TenantId` field + - gated `TENANT_ID` env — **skipped** the fork's RBAC `create`/`update` widening on - clusterroles/bindings as an unexplained privilege escalation), elastic controller (cloud-kibana - override read behind `r.cloud`), external-elastic controller (`AddTenantId` behind - `opts.Cloud && !MultiTenant`), dashboards controller (cloud single-tenant tenant-from-CloudConfig), - and ES ILM (`SetILMPolicies(ctx, ls, cloud)` param gates `tweakILMPoliciesForCloud`). Each cloud - controller gained a `cloud` field set from `opts.Cloud`; cloud ConfigMap watches gated on - `opts.Cloud`. -- **Phase 2 remaining** — DONE, builds + all affected controller/render tests pass: - - compliance: gated cloud OIDC-issuer egress NetworkPolicy (`Cloud` field + `compliance_cloud.go`). - - fluentd: gated `setFluentdCloudEnvs` (`DISABLE_ES_*_LOG`); `Cloud` wired from logcollector controller (both Linux & Windows configs). - - tiers: gated `cloudPatchTier` (strips `app.kubernetes.io/instance` label from allow-tigera tier). - - intrusiondetection / policyrecommendation: gated single-tenant tenant-from-CloudConfig + cloud ConfigMap watch. - - packetcapture / monitor: kvc `addTenancyClaim` gating (done in Phase 1). - - **RBAC divergences — now GATED behind `Cloud` (resolved):** - - `apiserver.go` `tigeraUserClusterRole` + `tigeraNetworkAdminClusterRole`: added `Cloud` field; - when cloud, UISettings RBAC is per-user only (no `cluster-settings` group) and the `lma.tigera.io` - resources include `runtime`; when not cloud, the RBAC is byte-identical to enterprise (verified by - the existing apiserver render specs still passing). Controller sets `Cloud: r.opts.Cloud`. - - `kube-controllers.go` `NewElasticsearchKubeControllers`: added `Cloud` field; cloud grants - `create`/`update` on clusterroles/clusterrolebindings (cloud es-kube-controllers provisions - managed-cluster RBAC), enterprise keeps read-only. es-kube-controllers controller sets `Cloud: r.cloud`. - - **SKIPPED (not RBAC, not a cloud feature):** `intrusion_detection.go` render diff is only - Image-Assurance cleanup (#1059); enterprise keeps IA, so it stays out. - -### Cloud-path tests — DONE (core set) -Ported + passing (adapted to the runtime `Cloud` gate): -- `pkg/render/manager_cloud_test.go` (+ non-cloud negative case). -- `pkg/render/fluentd_cloud_test.go` (`Cloud: true`; + non-cloud negative case). -- `pkg/render/logstorage/kibana/kibana_cloud_test.go` (drives `CloudKibanaConfigOverrides`; AfterEach - resets the global to avoid leaking into enterprise specs). -- `pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go` (`r.opts.Cloud = true` on the shim). -- `pkg/controller/compliance/compliance_controller_cloud_test.go` (`opts.Cloud: true`). -- `pkg/controller/manager/manager_controller_cloud_test.go` (`TestEnv` — `cloudConfigOverride`). - -While porting the compliance controller test I found **three gated `compliance_controller.go` changes -I'd missed** and added them: the cloud ConfigMap watch, the system-root trusted-bundle creation for -external-OIDC management clusters, and the tenant-from-CloudConfig block. Added an exported -`compliance.NewReconcilerWithShims` test constructor. - -### Still outstanding -- Lower-value `_test.go` deltas (esgateway/linseed/logstorage/kube-controllers/users/external-elastic/ - es-kube-controllers/policyrecommendation cloud test cases) and the golden-YAML RBAC test infra - (`render_suite_test.go` TestMain, `testsupport/golden_yaml.go`, `clusterrole` pkg, `.golangci.yml`, - `manager-cloud-rbac-all-golden.yaml`). Functional code + enterprise safety already verified. -- Phase 3 (cloud version-gen wiring) and Phase 4 (cloud build/release pipeline). -- Decisions on the three SKIPPED items above. -- Run full `make ut` / CI before PR. - -### Per-file judgment calls discovered (apply to remaining work) - -- **Skip incidental refactors that aren't cloud features.** e.g. `pkg/render/logstorage.go` only changes - `curatorDecommissionedResources` to use a helper — not cloud, would shift enterprise output. Don't port. -- **`elasticsearch.go` ILM gating is a real trap.** The fork calls `tweakILMPoliciesForCloud` from - `SetILMPolicies` unconditionally, adding a `tigera_secure_ee_runtime` policy. Must be gated (thread a - cloud flag onto `esClient`) or it changes enterprise ILM. -- **`kibana.go` uses a global var `CloudKibanaConfigOverrides`** the fork itself flags as wrong. Prefer - threading it through the kibana `Configuration` rather than porting the global (empty default is - enterprise-safe, but the pattern is bad and racy). -- **External-ES already skips the ES storage-class requirement** (`elastic_controller.go` returns early - when `opts.ElasticExternal`), so cloud external-ES installs never need `tigera-elasticsearch`. - -## Phased plan - -### Phase 0 — Foundation ✅ DONE (this branch) - -Behavior-neutral scaffolding; compiles and unit-tested, zero enterprise impact. - -| File | Notes | -|---|---| -| `pkg/cloud/cloud.go`, `watch.go`, `cloud_test.go` | Ported; `Load` made tolerant — returns `Cloud:false` (no error, no watch) when no cloud config present. Added `Cloud` field + non-cloud test case. | -| `pkg/render/common/cloudconfig/{cloudconfig,*_test}.go` | Verbatim port (pure data type). | -| `pkg/controller/utils/cloudconfig.go` | `GetCloudConfig`, `GetTenantFromCloudAuthConfig`, `CloudAuthConfig` const. Unused until controllers are wired. | -| `pkg/components/cloud_images.go` | `CloudRegistry` const. | -| `config/cloud_versions.yml` | `cloud-rbac-api` source-of-truth. | -| `pkg/controller/options/options.go` | `+Cloud`, `+ESMigration` fields. | -| `cmd/main.go` | `cloud.Load` wired; `ElasticExternal ||= cloudOpts.ElasticExternal`; `Cloud`/`ESMigration` set; leader-election tweak gated; `verifyConfiguration` ESMigration early-return. | - -Verified: `go build` + `go test ./pkg/cloud/... ./pkg/render/common/cloudconfig/...` pass; `go vet ./cmd/...` clean. - -### Phase 1 — Shared cloud helpers (additive, low risk) - -New packages/files used by later phases; harmless to enterprise on their own. - -- `pkg/render/common/clusterrole/clusterroles.go` (new) -- `pkg/render/common/elasticsearch/multitenancy.go` (new) -- `pkg/render/common/meta/meta.go` (small additions) -- `pkg/render/testsupport/golden_yaml.go` + `pkg/render/common/test/testing.go` + `.golangci.yml` - (test helper for golden-YAML cloud RBAC tests; the lint config only exempts this helper's dot-import) -- `pkg/controller/utils/elasticsearch_cloud.go` (new) + `elasticsearch.go` (gated edits) -- `pkg/controller/utils/auth_cloud.go` + `auth.go` `GetKeyValidatorConfig` signature (see gotchas) -- `pkg/render/common/authentication/.../key_validator_cloud.go` + `key_validator_config.go` (gated) - -### Phase 2 — Per-component render + controller cloud paths (the bulk) - -For each component: add the `*_cloud.go` decorator/helpers, add the typed `…CloudResources` field -to the render config, populate it in the controller **only when `opts.Cloud`**, and add cloud -ConfigMap watches. Gate every shared-file edit. - -| Component | Render files | Controller files | -|---|---|---| -| **manager** | `manager.go` (M), `manager_cloud.go` (A) | `manager_controller.go` (M), `manager_controller_cloud.go` (A) | -| **logstorage / linseed** | `linseed/{linseed.go (M), cloud.go (A)}` | `linseed/linseed_controller.go` (M) | -| **logstorage / esgateway** | `esgateway/{esgateway.go (M), cloud.go (A)}` | `kubecontrollers/{esgateway.go, es_kube_controllers.go, cloud.go (A)}` | -| **logstorage / kibana** | `kibana/kibana.go` (M) | — | -| **logstorage (core)** | `logstorage.go` (M) | `elastic/{elastic_controller.go, external_elastic_controller.go}`, `dashboards/dashboards_controller.go`, `users` | -| **apiserver** | `apiserver.go` (M) | `apiserver/apiserver_controller.go` (M) | -| **compliance** | `compliance.go` (M), `compliance_cloud.go` (A) | `compliance/compliance_controller.go` (M) | -| **fluentd** | `fluentd.go` (M) | (tests: `fluentd_cloud_test.go`) | -| **intrusion detection** | `intrusion_detection.go` (M) | `intrusiondetection/intrusiondetection_controller.go` (M) | -| **kube-controllers** | `kubecontrollers/kube-controllers.go` (M) | — | -| **monitor / packetcapture / policyrecommendation / tiers** | — | respective `*_controller.go` (M) — mostly cloud config-map watches / tenant plumbing | - -Plus the test files for each (`*_cloud_test.go`, `*_test.go` deltas, `render_suite_test.go`, -`test/mainline_test.go`, `pkg/render/testdata/manager-cloud-rbac-all-golden.yaml`). - -Recommended ordering: manager → logstorage (linseed/esgateway/kibana/elastic) → apiserver → -compliance → fluentd/intrusiondetection/kubecontrollers → remaining controllers. Land each -component as its own PR with its tests, verifying enterprise golden files are unchanged. - -### Phase 3 — Version generation wiring ✅ DONE - -- `hack/gen-versions/main.go`: ported the `-cloud-versions` flag (and fixed the swapped os/ee help - text); generation now requires exactly one of `-os-versions`/`-ee-versions`/`-cloud-versions`. -- Added `hack/gen-versions/cloud.go.tpl` (the fork's `-cloud-versions` flag was dead — no template - existed). It generates `pkg/components/cloud.go` with `ComponentCloudRBACAPI` + `CloudImages` from - `config/cloud_versions.yml`, referencing the hand-written `CloudRegistry` const in `cloud_images.go`. -- `Makefile`: added a standalone `gen-versions-cloud` target (`CLOUD_VERSIONS?=config/cloud_versions.yml`). - **Intentionally NOT in the default `gen-versions` aggregate** so the enterprise build's generated - output is byte-identical (verified: os/ee generation still match the committed calico.go/enterprise.go). - No CRD-fetch dep (cloud relies on upstream operator for CRDs). The cloud release pipeline invokes it. -- Generated `pkg/components/cloud.go` is committed, gofmt-clean, compiles, and idempotent (dirty-check safe). -- `cloud-rbac-api` (cc-rbac-api) is not consumed by operator render code — it's a release-tooling image - pin (the `cloud.go` component pin + the `CloudRegistry` const in `cloud_images.go`, which the cloud - release tool patches). **Did NOT** edit the shared `enterprise.go.tpl` (the fork's tesla-prefixed - Kibana/Manager + CloudRegistry kube-controllers edits would globally tesla-fy enterprise images). - Cloud image overrides happen at runtime where wired (e.g. manager via `CloudResources.ManagerImage`); - any further cloud image pinning (kibana/kube-controllers) is a Phase 4 build/release concern. - -### Phase 4 — Cloud build/release pipeline - -**Build/release tooling — DONE** (coexists with enterprise, verified): -- `hack/release/cloud.go` + `hack/release/internal/versions/cloud.go` + `hack/release/cloud_test.go` - are guarded by the **`cloud` build tag**, so they compile only into the cloud release tool - (`go build -tags cloud`). The enterprise/OSS release tool is byte-for-byte unaffected (verified: - untagged build + tests pass unchanged). `cloud.go`'s `init()` reassigns the OSS package-level hooks - (`isValidReleaseVersion`, `setupHashreleaseBuild`, `publishImages`, command `Before` funcs) — so - `checks.go`/`flags.go` were **not** edited (unlike the fork, which swapped them globally). -- `Makefile`: `make release-cloud` / `make release-publish-cloud` build `hack/bin/release-cloud` with - `-tags cloud`. A gated `ifeq ($(VARIANT),cloud)` block overrides image identity (GCR / - `tigera-tesla/operator-cloud`, amd64-only) — verified that enterprise vars are unchanged when - `VARIANT` is unset and switch correctly when `VARIANT=cloud`. -- `hack/release/CLOUD.md` documents the cloud release/hashrelease flow. - -**CI pipelines — follow-up (not in this PR):** the Semaphore `cloud-v*` GCR push/release blocks and -the Argo hashrelease build + cluster-rollout workflows (`.argoci/`, `hack/hashrelease/*.py`) are -environment-specific (operator-cloud Semaphore project, ArgoCD apps, GCR service accounts) and won't -run until cloud CI is wired into this repo. The OBSOLETE fork-sync machinery is dropped entirely (see -classification below). - ---- - -## CI / build / release / tooling classification - -Each infra file sorted by disposition. **Do not bulk-copy** — most either don't apply or must be -made cloud-conditional. - -### OBSOLETE — delete with the fork, do not migrate -- `.argoci/cron/operator-cloud-fork-maintenance-nightly.yaml` — nightly upstream→fork merge cron. -- `.argoci/templates/sync-operator-cloud-branches.yaml` — runs `hack/update_fork.sh` merge-back. -- `.argoci/templates/hashrelease/sync-operator-hashrelease-fork.yaml` — fork-sync half (build-kickoff survives elsewhere). -- `hack/update_fork.sh`, `hack/hashrelease/update_hashrelease_fork.sh` — fork merge-back automation. -- `.claude/skills/merge-operator-into-operator-cloud/SKILL.md` — fork-merge runbook. -- `.github/README.md` + `CLOUD_README.md` (symlink) — fork branch-maintenance docs. -- `.semaphore/approve_check.yml` — auto-approves bot fork-sync PRs. -- `.bulldozer.yml`, `.ccbot/config.yaml` — automerge config existing to service fork-sync (drop unless automerge is independently wanted). - -### KEEP-ENTERPRISE — drop the fork's divergence -- `.github/workflows/codeql-analysis.yml` (fork deleted it — keep enterprise's CodeQL). -- `.github/CODEOWNERS` (fork made it `* @tigera/calico-cloud` — keep enterprise; add path-scoped cloud owners if desired). -- `.github/workflows/sync-versions.yml` (fork disabled the cron — keep enterprise's). -- `config/calico_versions.yml` (only a duplicated comment), `config/enterprise_versions.yml` (tesla wiring). -- `hack/gen-versions/enterprise.go.tpl` (tesla image edits — handle as cloud-scoped overrides instead). -- `git-hooks/pre-commit-in-container` (fork disabled the copyright check globally — don't). -- `api/go.mod` (module rename to `operator-cloud/api` — don't carry). - -### TWEAK — migrate but make cloud-conditional (don't globally override enterprise) -- `Makefile` — cloud `REPO`/`BUILD_IMAGE`/`IMAGE_REGISTRY=gcr.io`/`PUSH_IMAGE_PREFIXES`/ - `EXCLUDE_MANIFEST_REGISTRIES`, `VALIDARCHES=amd64`, `gen-bundle`/`bundle-crd-options` (RH v1beta1). - Gate behind a cloud variant so enterprise quay + multi-arch defaults survive. -- `.semaphore/{push_images,release,semaphore}.yml` — GCR auth/push, `cloud-v*` release trigger, - `staging` promotions, disabled multi-arch block. Make additive/coexisting, not replacements. -- `hack/release/{checks,flags}.go` — swapped OSS version validation to cloud format globally; - make variant-conditional. -- `hack/gen-versions/main.go` — `-cloud-versions` flag (additive but currently unwired; finish in Phase 3). -- `git-hooks/files-to-skip` — adds cloud `_cloud` files to deepcopy skip; migrate with the Go code. - -### REPLACE — cloud capability needed, fork impl is fork-specific -- `.argoci/templates/hashrelease/build-hashrelease.yaml` — re-target build+push at unified repo. -- `.argoci/templates/hashrelease/update-cluster-with-hashrelease.yaml` + `hack/hashrelease/*.py` — - live cloud-cluster rollout automation (single/multi-tenant/managed); re-home as a cloud pipeline. -- `hack/release/cloud.go`, `hack/release/internal/versions/cloud.go`, `hack/release/cloud_test.go`, - `hack/release/utils_test.go`, `hack/release/CLOUD.md` — cloud extension of the shared release - tool (GCR/tesla images, `cloud-v*` format, hashrelease flags, disabled GH release). Re-introduce - as a cloud-gated extension rather than always-on `init()` registration. - -### MIGRATE-ASIS — harmless, bring over -- `.github/workflows/secret_scanners.yml` (TruffleHog/gitleaks PR scan). -- `.golangci.yml` (only exempts the cloud golden-YAML test helper's dot-import). -- `.argoci/config.yaml` (`version: v2` marker). - -### Cloud build/release capabilities to (re)implement (Phase 4 scope) -1. Cloud image identity & GCR registry overrides (Makefile), amd64-only, without touching enterprise quay/multi-arch. -2. Cloud GCR push/release Semaphore blocks triggered by `cloud-v*`, coexisting with enterprise `v*`. -3. Cloud release-tool extension (flags, `cloud-v*` validation, GCR/tesla images, hashrelease outputs). -4. Cloud version generation (`cloud.go.tpl` + `gen-versions-cloud`), cloud-scoped image overrides. -5. Cloud hashrelease build pipeline (Argo), re-targeted at the unified repo. -6. Cloud cluster rollout automation (Argo + `hack/hashrelease` python). - ---- - -## Risks & verification - -- **Enterprise regression** is the primary risk. Mitigation: every shared-file edit guarded by - `opts.Cloud`; enterprise golden YAML / `_test.go` fixtures must be byte-identical before/after each - PR. Run `make ut` and the render golden tests per component. -- **Cross-cutting signature changes** (`GetKeyValidatorConfig`) touch many callers — land with all - call sites updated in one PR. -- **Dead/unwired fork code** (the `-cloud-versions` gen path) — finish wiring in Phase 3, don't - copy it half-done. -- Keep `make gen-files` / `make dirty-check` green after any API/CRD-adjacent changes. diff --git a/hack/release/CLOUD.md b/hack/release/CLOUD.md index fd7108c7eb..92145ef5d5 100644 --- a/hack/release/CLOUD.md +++ b/hack/release/CLOUD.md @@ -1,18 +1,18 @@ # Cloud Operator release tooling -The `hack/release` tool is the OSS/enterprise operator release tool. Calico Cloud behavior lives in -`cloud.go` and `internal/versions/cloud.go`, both guarded by the `cloud` build tag, so they are only -compiled into the **cloud release tool** built with `-tags cloud`. The regular Calico / Calico -Enterprise release tool (`hack/bin/release`) is completely unaffected. +The `hack/release` tool is a single binary that serves both the OSS/enterprise and Calico Cloud +release flows. Calico Cloud behavior lives in `cloud.go` and `internal/versions/cloud.go` and is +activated **at runtime** when `VARIANT=cloud` (there is no separate `-tags cloud` build). Use the +regular release targets with `VARIANT=cloud`: -Build and run the cloud release tool via the dedicated Makefile targets, which compile with -`-tags cloud` into `hack/bin/release-cloud`: +- `make release VARIANT=cloud` — build a cloud release +- `make release-publish VARIANT=cloud` — publish a cloud release +- `make release-tag VARIANT=cloud RELEASE_TAG=cloud-vX.Y.Z-N` — build + publish a tagged cloud release -- `make release-cloud` — build a cloud release (`hack/bin/release-cloud build`) -- `make release-publish-cloud` — publish a cloud release (`hack/bin/release-cloud publish`) - -On startup, `cloud.go`'s `init()` wraps the OSS command handlers and registers additional flags, -injecting cloud logic without forking the shared code. +`VARIANT=cloud` also switches the built operator image to `gcr.io/tigera-tesla/operator-cloud` +(amd64) and bakes cloud mode into the operator binary (`-X …/pkg/cloud.buildVariant=cloud`), so the +shipped cloud image is immutably cloud (see `pkg/cloud.IsCloudBuild`). When `VARIANT` is unset, +`cloud.go`'s `init()` returns immediately and the enterprise release flow is unchanged. ## Hashreleases @@ -23,11 +23,11 @@ variables are exported to the tool.) ### Using hashrelease URL ```bash -make release-cloud HASHRELEASE=true \ +make release VARIANT=cloud HASHRELEASE=true \ HASHRELEASE_URL="https://.docs.eng.tigera.net" \ CLOUD_REGISTRY="gcr.io/unique-caldron-775/cnx/" -make release-publish-cloud HASHRELEASE=true \ +make release-publish VARIANT=cloud HASHRELEASE=true \ HASHRELEASE_URL="https://.docs.eng.tigera.net" \ CLOUD_REGISTRY="gcr.io/unique-caldron-775/cnx/" ``` @@ -35,19 +35,19 @@ make release-publish-cloud HASHRELEASE=true \ ### Using local pinned components file ```bash -make release-cloud HASHRELEASE=true \ +make release VARIANT=cloud HASHRELEASE=true \ PINNED_COMPONENTS_FILE="/path/to/pinned_components.yml" \ CLOUD_REGISTRY="gcr.io/unique-caldron-775/cnx/" -make release-publish-cloud HASHRELEASE=true \ +make release-publish VARIANT=cloud HASHRELEASE=true \ PINNED_COMPONENTS_FILE="/path/to/pinned_components.yml" \ CLOUD_REGISTRY="gcr.io/unique-caldron-775/cnx/" ``` ## Release -The release process is the same as the OSS operator with specific cloud variables; use the -`release-cloud` / `release-publish-cloud` targets so the `-tags cloud` binary is used. +The release process is the same as the OSS operator, run with `VARIANT=cloud` (e.g. +`make release-tag VARIANT=cloud RELEASE_TAG=cloud-vX.Y.Z-N`). ## How It Works @@ -93,7 +93,7 @@ from the OSS release process may also be used as needed (see `flags.go`). ## Extension Pattern -Cloud-specific behavior is implemented in `cloud.go` via an `init()` (active only under `-tags cloud`) +Cloud-specific behavior is implemented in `cloud.go` via an `init()` (active only when `VARIANT=cloud`) that updates OSS flag defaults to cloud defaults and wraps the OSS command handlers: - **`cloudBuildBefore`**: runs before the OSS build `Before` handler — downloads pinned components, @@ -109,8 +109,10 @@ that updates OSS flag defaults to cloud defaults and wraps the OSS command handl Cloud-specific flags (`--hashrelease-url`, `--pinned-components`, `--cloud-registry`) are appended to the relevant OSS commands in the same `init()` call. -## CI Integration (follow-up) +## CI Integration -The cloud build/release CI pipelines (Semaphore `cloud-v*` triggers + GCR push, and the Argo -hashrelease build / cluster-rollout workflows) are not yet wired into this repo — they are -environment-specific and tracked as a follow-up in `docs/cloud-unfork-migration-plan.md` (Phase 4 CI). +The cloud build/release runs via: +- Semaphore: `.semaphore/push_images_cloud.yml` (GCR push on `master`/`staging`/`release-*`) and + `.semaphore/release_cloud.yml` (`cloud-v*` tags), promoted from `.semaphore/semaphore.yml`. +- ArgoCI: `.argoci/templates/hashrelease/build-hashrelease.yaml` builds+publishes a cloud image from + an enterprise hashrelease and exposes `imageTag`/`newHashrelease`. diff --git a/hack/release/cloud.go b/hack/release/cloud.go index cea0e08e38..967491d709 100644 --- a/hack/release/cloud.go +++ b/hack/release/cloud.go @@ -12,13 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build cloud - -// This file is only compiled into the Calico Cloud release tool, built with `-tags cloud` -// (see `make build-release-cloud`). Its init() reassigns the shared release tool's package-level -// hooks (isValidReleaseVersion, setupHashreleaseBuild, publishImages, command Before funcs) to add -// Calico Cloud behavior. Because it is build-tagged, the regular Calico / Calico Enterprise release -// tool is completely unaffected. +// This file is part of the single release tool binary. Its Calico Cloud behavior is activated at +// runtime only when VARIANT=cloud (see cloudVariantEnabled): init() then reassigns the shared release +// tool's package-level hooks (isValidReleaseVersion, setupHashreleaseBuild, publishImages, command +// Before funcs) and registers the cloud flags. When VARIANT is unset, init() returns immediately and +// the regular Calico / Calico Enterprise release tool is completely unaffected. This replaces the +// former `-tags cloud` build so one binary handles both (per PR review from @radTuti / @caseydavenport). package main @@ -117,7 +116,19 @@ var ( } ) +// cloudVariantEnabled reports whether the release tool is running as the Calico Cloud variant. It is +// driven by the VARIANT env var (set by `make ... VARIANT=cloud`), so a single release binary serves +// both the enterprise and cloud release flows. +func cloudVariantEnabled() bool { + return os.Getenv("VARIANT") == "cloud" +} + func init() { + // Cloud behavior is opt-in at runtime; leave the OSS/enterprise release tool untouched otherwise. + if !cloudVariantEnabled() { + return + } + // Override version validation for cloud releases. isValidReleaseVersion = isCloudReleaseVersionFormat diff --git a/hack/release/cloud_test.go b/hack/release/cloud_test.go index c252258150..c36d82b958 100644 --- a/hack/release/cloud_test.go +++ b/hack/release/cloud_test.go @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build cloud - package main import "testing" diff --git a/hack/release/internal/versions/cloud.go b/hack/release/internal/versions/cloud.go index 1b5f72cb19..1be0eb6960 100644 --- a/hack/release/internal/versions/cloud.go +++ b/hack/release/internal/versions/cloud.go @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build cloud - -// This file is only compiled into the Calico Cloud release tool (built with `-tags cloud`); it has -// no effect on the regular Calico / Calico Enterprise release tool. +// These cloud-specific config keys are always compiled in; they are only consumed by the release +// tool's cloud path (VARIANT=cloud), so they have no effect on the regular Calico / Calico +// Enterprise release flow. package versions diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index 6a3e5da91a..b408f25c6d 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -1,3 +1,5 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -34,14 +36,25 @@ var setupLog = ctrl.Log.WithName("cloud_setup") const ( configMapName = "cloud-operator-config" - // EnableCloudEnvVar is the environment variable that gates cloud mode. When set to a truthy - // value (parseable by strconv.ParseBool, e.g. "true"/"1") the operator runs as a Calico Cloud - // install and cloud-specific behavior is activated. When unset or false the operator behaves as - // a regular Calico/Calico Enterprise install. This is set on the operator Deployment by the - // cloud installer; the operator never infers cloud mode by sniffing for ConfigMaps. + // EnableCloudEnvVar can enable cloud mode for a non-cloud build. The shipped Calico Cloud + // operator image has cloud mode baked in at build time (see IsCloudBuild) and cannot be turned + // off; this env var is only a fallback for enabling cloud mode in a regular build, e.g. for + // local development or testing. EnableCloudEnvVar = "CALICO_CLOUD" ) +// buildVariant is injected at build time via -ldflags "-X .../pkg/cloud.buildVariant=cloud" when +// building the Calico Cloud operator image. It is empty for the regular Calico/Calico Enterprise +// image. Baking it into the binary means cloud mode is immutable: it cannot be disabled by editing +// the operator Deployment's environment. +var buildVariant string + +// IsCloudBuild reports whether this binary was built as the Calico Cloud variant. When true, cloud +// mode is baked in and cannot be disabled at runtime. +func IsCloudBuild() bool { + return buildVariant == "cloud" +} + // Options holds the cloud-specific configuration parsed at operator startup. When Cloud is false the // operator is running as a regular (non-cloud) Calico Enterprise install and the remaining fields are // not meaningful. @@ -55,17 +68,22 @@ type Options struct { // Load determines whether the operator is running in cloud mode and, if so, parses the cloud options. // -// Cloud mode is gated solely by the CALICO_CLOUD environment variable (see EnableCloudEnvVar). When -// it is unset or false the operator is a regular Calico/Calico Enterprise install and Load returns +// Cloud mode is enabled when this is a cloud build (IsCloudBuild, baked into the cloud image) or, +// for a regular build, when the CALICO_CLOUD env var is truthy (dev/testing fallback). When neither +// applies the operator is a regular Calico/Calico Enterprise install and Load returns // Options{Cloud: false} with no error, leaving enterprise behavior unchanged. Only once cloud mode // is enabled does Load read the cloud-operator-config ConfigMap (and env) for cloud config values. func Load(ctx context.Context, cs kubernetes.Interface) (Options, error) { - cloudEnabled, err := parseEnableCloud() - if err != nil { - return Options{}, err + cloudEnabled := IsCloudBuild() + if !cloudEnabled { + envEnabled, err := parseEnableCloud() + if err != nil { + return Options{}, err + } + cloudEnabled = envEnabled } if !cloudEnabled { - setupLog.Info("cloud mode not enabled; running in non-cloud mode", "envVar", EnableCloudEnvVar) + setupLog.Info("cloud mode not enabled; running in non-cloud mode") return Options{Cloud: false}, nil } diff --git a/pkg/cloud/cloud_test.go b/pkg/cloud/cloud_test.go index e70f3c98ae..2b33959f81 100644 --- a/pkg/cloud/cloud_test.go +++ b/pkg/cloud/cloud_test.go @@ -1,3 +1,5 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/pkg/cloud/watch.go b/pkg/cloud/watch.go index 806695275e..cf10bab1d1 100644 --- a/pkg/cloud/watch.go +++ b/pkg/cloud/watch.go @@ -1,3 +1,5 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -13,6 +15,7 @@ package cloud import ( + "maps" "os" "time" @@ -42,7 +45,7 @@ var watch = func(cs kubernetes.Interface, cmData map[string]string) error { ) _, err := informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ UpdateFunc: func(_, newObj interface{}) { - if !compareMap(cmData, newObj.(*v1.ConfigMap).Data) { + if !maps.Equal(cmData, newObj.(*v1.ConfigMap).Data) { configWatchLog.Info("detected config change. rebooting") os.Exit(0) } else { @@ -50,7 +53,7 @@ var watch = func(cs kubernetes.Interface, cmData map[string]string) error { } }, AddFunc: func(obj interface{}) { - if !compareMap(cmData, obj.(*v1.ConfigMap).Data) { + if !maps.Equal(cmData, obj.(*v1.ConfigMap).Data) { configWatchLog.Info("detected config creation change. rebooting") os.Exit(0) } else { @@ -68,15 +71,3 @@ var watch = func(cs kubernetes.Interface, cmData map[string]string) error { } return nil } - -func compareMap(m1, m2 map[string]string) bool { - if len(m1) != len(m2) { - return false - } - for k, v := range m1 { - if m2[k] != v { - return false - } - } - return true -} diff --git a/pkg/components/cloud_images.go b/pkg/components/cloud_images.go index 91d95bdb0f..cd9f0ef9ad 100644 --- a/pkg/components/cloud_images.go +++ b/pkg/components/cloud_images.go @@ -1,4 +1,4 @@ -// Copyright (c) 2023 Tigera, Inc. All rights reserved. +// Copyright (c) 2023-2026 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/controller/compliance/compliance_controller.go b/pkg/controller/compliance/compliance_controller.go index 200fc06577..a3f866dbe0 100644 --- a/pkg/controller/compliance/compliance_controller.go +++ b/pkg/controller/compliance/compliance_controller.go @@ -166,28 +166,6 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions, licenseA return r } -// NewReconcilerWithShims constructs a ReconcileCompliance with the provided dependencies. It is -// intended for tests that need to inject a fake client, scheme, and status manager. -func NewReconcilerWithShims( - cli client.Client, - scheme *runtime.Scheme, - status status.StatusManager, - opts options.ControllerOptions, - licenseAPIReady *utils.ReadyFlag, - tierWatchReady *utils.ReadyFlag, -) *ReconcileCompliance { - r := &ReconcileCompliance{ - client: cli, - scheme: scheme, - status: status, - licenseAPIReady: licenseAPIReady, - tierWatchReady: tierWatchReady, - opts: opts, - } - r.status.Run(opts.ShutdownContext) - return r -} - // blank assignment to verify that ReconcileCompliance implements reconcile.Reconciler var _ reconcile.Reconciler = &ReconcileCompliance{} diff --git a/pkg/controller/compliance/compliance_controller_cloud_test.go b/pkg/controller/compliance/compliance_controller_cloud_test.go index 8cf01458d0..f0913860d2 100644 --- a/pkg/controller/compliance/compliance_controller_cloud_test.go +++ b/pkg/controller/compliance/compliance_controller_cloud_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package compliance_test +package compliance import ( "context" @@ -37,7 +37,6 @@ import ( "github.com/tigera/operator/pkg/apis" "github.com/tigera/operator/pkg/common" "github.com/tigera/operator/pkg/controller/certificatemanager" - "github.com/tigera/operator/pkg/controller/compliance" "github.com/tigera/operator/pkg/controller/options" "github.com/tigera/operator/pkg/controller/status" "github.com/tigera/operator/pkg/controller/utils" @@ -51,7 +50,7 @@ var _ = Describe("Cloud Compliance controller tests", func() { var c client.Client var ctx context.Context var cr *operatorv1.Compliance - var r *compliance.ReconcileCompliance + var r *ReconcileCompliance var mockStatus *status.MockStatus var scheme *runtime.Scheme var installation *operatorv1.Installation @@ -176,13 +175,20 @@ var _ = Describe("Cloud Compliance controller tests", func() { // Create an object we can use throughout the test to do the compliance reconcile loops. // As the parameters in the client changes, we expect the outcomes of the reconcile loops to change. - opts := options.ControllerOptions{ - DetectedProvider: operatorv1.ProviderNone, - ClusterDomain: dns.DefaultClusterDomain, - ShutdownContext: context.TODO(), - Cloud: true, + r = &ReconcileCompliance{ + client: c, + scheme: scheme, + status: mockStatus, + licenseAPIReady: licenseReadyFlag, + tierWatchReady: tierReadyFlag, + opts: options.ControllerOptions{ + DetectedProvider: operatorv1.ProviderNone, + ClusterDomain: dns.DefaultClusterDomain, + ShutdownContext: context.TODO(), + Cloud: true, + }, } - r = compliance.NewReconcilerWithShims(c, scheme, mockStatus, opts, licenseReadyFlag, tierReadyFlag) + r.status.Run(r.opts.ShutdownContext) }) It("should create a trusted bundle with external certificates", func() { diff --git a/pkg/controller/logstorage/dashboards/dashboards_controller.go b/pkg/controller/logstorage/dashboards/dashboards_controller.go index 98b3051b93..9d60f714eb 100644 --- a/pkg/controller/logstorage/dashboards/dashboards_controller.go +++ b/pkg/controller/logstorage/dashboards/dashboards_controller.go @@ -274,16 +274,20 @@ func (d DashboardsSubController) Reconcile(ctx context.Context, request reconcil return reconcile.Result{RequeueAfter: utils.StandardRetry}, nil } } else { + // External ES & Kibana. Two ways to obtain the tenant's Kibana endpoint: + // - multi-tenant (incl. enterprise) and non-cloud single-tenant: the Tenant CR must carry it. + // This is the pre-existing enterprise behavior, preserved by the `!d.cloud` clause. + // - Calico Cloud single-tenant: there is no Tenant CR, so it's derived from the cloud config map. if d.multiTenant || !d.cloud { - // If we're using an external ES and Kibana, the Tenant resource must specify the Kibana endpoint. + // The Tenant resource must specify the Kibana endpoint. if tenant == nil || tenant.Spec.Elastic == nil || tenant.Spec.Elastic.KibanaURL == "" { reqLogger.Error(nil, "Kibana URL must be specified for this tenant") d.status.SetDegraded(operatorv1.ResourceValidationError, "Kibana URL must be specified for this tenant", nil, reqLogger) return reconcile.Result{}, nil } } else { - // This is a Calico Cloud single-tenant cluster connected to an external ES & Kibana. Read - // the tenant configuration from the CloudConfig ConfigMap. + // Calico Cloud single-tenant cluster connected to an external ES & Kibana. Read the tenant + // configuration from the CloudConfig ConfigMap. cloudConfig, err := utils.GetCloudConfig(ctx, d.client) if err != nil { d.status.SetDegraded(operatorv1.ResourceReadError, "Failed to read cloud config", err, reqLogger) diff --git a/pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go b/pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go index 27ce7460c7..a4a4b0681f 100644 --- a/pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go +++ b/pkg/controller/logstorage/elastic/elastic_controller_cloud_test.go @@ -181,6 +181,8 @@ var _ = Describe("External ES controller (Cloud))", func() { mockStatus = &status.MockStatus{} mockStatus.On("Run").Return() mockStatus.On("OnCRFound").Return() + mockStatus.On("SetWarning", mock.Anything, mock.Anything).Return().Maybe() + mockStatus.On("ClearWarning", mock.Anything).Return().Maybe() }) It("reconciles successfully", func() { diff --git a/pkg/controller/logstorage/elastic/external_elastic_controller.go b/pkg/controller/logstorage/elastic/external_elastic_controller.go index de4d265720..2b18d64e15 100644 --- a/pkg/controller/logstorage/elastic/external_elastic_controller.go +++ b/pkg/controller/logstorage/elastic/external_elastic_controller.go @@ -103,7 +103,7 @@ func AddExternalES(mgr manager.Manager, opts options.ControllerOptions) error { } if opts.Cloud { - // Calico Cloud addition. + // Watched so single-tenant external-ES clusters re-reconcile when the cloud config map changes. if err := utils.AddConfigMapWatch(c, cloudconfig.CloudConfigConfigMapName, common.OperatorNamespace(), &handler.EnqueueRequestForObject{}); err != nil { return fmt.Errorf("log-storage-controller failed to watch the ConfigMap resource: %w", err) } diff --git a/pkg/controller/logstorage/elastic/mock.go b/pkg/controller/logstorage/elastic/mock.go index cdc6a32991..a27891495f 100644 --- a/pkg/controller/logstorage/elastic/mock.go +++ b/pkg/controller/logstorage/elastic/mock.go @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2024 Tigera, Inc. All rights reserved. +// Copyright (c) 2020-2026 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/controller/logstorage/kubecontrollers/cloud.go b/pkg/controller/logstorage/kubecontrollers/cloud.go index 1a73e30a31..93a4c4110d 100644 --- a/pkg/controller/logstorage/kubecontrollers/cloud.go +++ b/pkg/controller/logstorage/kubecontrollers/cloud.go @@ -62,7 +62,7 @@ func (r *ESKubeControllersController) esGatewayAddCloudModificationsToConfig(c * } if cloudConfig.TenantId() != "" { - c.Cloud.TenantId = cloudConfig.TenantId() + c.Cloud.TenantID = cloudConfig.TenantId() } return true, nil @@ -82,7 +82,7 @@ func (r *ESKubeControllersController) esKubeControllersAddCloudModificationsToCo } if cloudConfig.TenantId() != "" { - c.TenantId = cloudConfig.TenantId() + c.TenantID = cloudConfig.TenantId() } } diff --git a/pkg/controller/logstorage/kubecontrollers/esgateway.go b/pkg/controller/logstorage/kubecontrollers/esgateway.go index feca28bbba..7410a548db 100644 --- a/pkg/controller/logstorage/kubecontrollers/esgateway.go +++ b/pkg/controller/logstorage/kubecontrollers/esgateway.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved. +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/controller/options/options.go b/pkg/controller/options/options.go index 51959effb3..b1f1c50b57 100644 --- a/pkg/controller/options/options.go +++ b/pkg/controller/options/options.go @@ -48,9 +48,9 @@ type ControllerOptions struct { // and instead will configure the cluster to use an external Elasticsearch. ElasticExternal bool - // Cloud indicates the operator is running as a Calico Cloud install. When set, controllers - // activate cloud-specific behavior (cloud render decorations, cloud config maps, etc.). When - // false the operator behaves as a regular Calico/Calico Enterprise install. + // Cloud indicates the operator is running in a Calico Cloud management cluster. When set, + // controllers activate cloud-specific behavior (cloud render decorations, cloud config maps, + // etc.). When false the operator behaves as a regular Calico/Calico Enterprise install. Cloud bool // ESMigration is enabled in the last phase of an ES migration, when we need to keep both an diff --git a/pkg/controller/tiers/tiers_controller_cloud.go b/pkg/controller/tiers/tiers_controller_cloud.go index 60766d4182..759c7dba80 100644 --- a/pkg/controller/tiers/tiers_controller_cloud.go +++ b/pkg/controller/tiers/tiers_controller_cloud.go @@ -23,8 +23,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -// cloudPatchTier removes the allow-tigera tier's app.kubernetes.io/instance label to fix Calico -// Cloud CD sync. Only invoked for cloud installs. +// cloudPatchTier removes the app.kubernetes.io/instance label from the calico-system tier to fix +// Calico Cloud CD sync (the label is applied out-of-band by Argo CD and confuses its diffing). +// Only invoked for cloud installs. +// TODO(cloud): consider folding this into the tier render instead of a post-create patch. func (r *ReconcileTiers) cloudPatchTier(ctx context.Context) error { tier := &v3.Tier{} err := r.client.Get(ctx, client.ObjectKey{Name: networkpolicy.CalicoTierName}, tier) diff --git a/pkg/render/common/cloudconfig/cloudconfig.go b/pkg/render/common/cloudconfig/cloudconfig.go index 33641551fc..f93dd3209e 100644 --- a/pkg/render/common/cloudconfig/cloudconfig.go +++ b/pkg/render/common/cloudconfig/cloudconfig.go @@ -1,3 +1,5 @@ +// Copyright (c) 2026 Tigera, Inc. All rights reserved. + // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at diff --git a/pkg/render/common/elasticsearch/multitenancy.go b/pkg/render/common/elasticsearch/multitenancy.go index a4a3b65333..b1a57e1200 100644 --- a/pkg/render/common/elasticsearch/multitenancy.go +++ b/pkg/render/common/elasticsearch/multitenancy.go @@ -1,4 +1,4 @@ -// Copyright (c) 2021 Tigera, Inc. All rights reserved. +// Copyright (c) 2021-2026 Tigera, Inc. All rights reserved. // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index b71f3ca19f..635671c5d8 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -116,9 +116,9 @@ type KubeControllersConfiguration struct { WASMCACert *corev1.ConfigMap TrustedBundle certificatemanagement.TrustedBundleRO - // Calico Cloud additions. TenantId is only set by the cloud-gated controller path; when empty + // Calico Cloud additions. TenantID is only set by the cloud-gated controller path; when empty // (regular Calico/Calico Enterprise) no cloud env is emitted. - TenantId string + TenantID string // Cloud indicates kube-controllers is being rendered for a Calico Cloud install. When false the // cloud-specific RBAC below is not granted and enterprise RBAC is unchanged. @@ -967,8 +967,8 @@ func (c *kubeControllersComponent) controllersDeployment() *appsv1.Deployment { {Name: "DISABLE_KUBE_CONTROLLERS_CONFIG_API", Value: strconv.FormatBool(c.cfg.Tenant.MultiTenant() && c.kubeControllerConfigName == "elasticsearch")}, } - if c.cfg.TenantId != "" { - env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.TenantId}) + if c.cfg.TenantID != "" { + env = append(env, corev1.EnvVar{Name: "TENANT_ID", Value: c.cfg.TenantID}) } env = append(env, c.cfg.K8sServiceEpPodNetwork.EnvVars()...) diff --git a/pkg/render/logstorage/esgateway/cloud.go b/pkg/render/logstorage/esgateway/cloud.go deleted file mode 100644 index a91fe0c12b..0000000000 --- a/pkg/render/logstorage/esgateway/cloud.go +++ /dev/null @@ -1,198 +0,0 @@ -// Copyright (c) 2022-2026 Tigera, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package esgateway - -import ( - "strconv" - - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" - - "github.com/tigera/operator/pkg/render" - rmeta "github.com/tigera/operator/pkg/render/common/meta" - "github.com/tigera/operator/pkg/render/common/networkpolicy" - "github.com/tigera/operator/pkg/render/common/secret" - "github.com/tigera/operator/pkg/render/logstorage" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - CloudPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "cloud-es-gateway-access" -) - -// CloudConfig holds Calico Cloud specific es-gateway configuration. Enabled gates all cloud -// behavior: when false (regular Calico/Calico Enterprise) the cloud decorators are no-ops. -type CloudConfig struct { - Enabled bool - EsAdminUserSecret *corev1.Secret - ExternalCertsSecret *corev1.Secret - TenantId string - EnableMTLS bool - ExternalElastic bool - ExternalESDomain string - ExternalKibanaDomain string -} - -func removeEnv(evs []corev1.EnvVar, name string) []corev1.EnvVar { - for i, ev := range evs { - if ev.Name == name { - evs = append(evs[:i], evs[i+1:]...) - } - } - return evs -} - -func (e *esGateway) modifyDeploymentForCloud(d *appsv1.Deployment) { - if !e.cfg.Cloud.Enabled { - return - } - - envs := d.Spec.Template.Spec.Containers[0].Env - // Enable prometheus metrics endpoint at :METRICS_PORT/metrics (Default is 9091). - envs = append(envs, corev1.EnvVar{Name: "ES_GATEWAY_METRICS_ENABLED", Value: "true"}) - - if e.cfg.Cloud.ExternalElastic { - // Find the following Envs and remove them - envs = removeEnv(envs, "ES_GATEWAY_ELASTIC_ENDPOINT") - envs = removeEnv(envs, "ES_GATEWAY_KIBANA_ENDPOINT") - envs = append(envs, corev1.EnvVar{ - Name: "ES_GATEWAY_ELASTIC_ENDPOINT", - Value: "https://" + e.cfg.Cloud.ExternalESDomain + ":443", - }) - envs = append(envs, corev1.EnvVar{ - Name: "ES_GATEWAY_KIBANA_ENDPOINT", - Value: "https://" + e.cfg.Cloud.ExternalKibanaDomain + ":443", - }) - // Enable this so that fluentd cannot modify ILM but the POSTs fluentd performs - // still succeed. - envs = append(envs, corev1.EnvVar{ - Name: "ES_GATEWAY_ILM_DUMMY_ROUTE_ENABLED", - Value: "true", - }) - } - - if e.cfg.Cloud.EnableMTLS { - // todo: delete these from the envVars - envs = removeEnv(envs, "ES_GATEWAY_KIBANA_CLIENT_CERT_PATH") - envs = removeEnv(envs, "ES_GATEWAY_ELASTIC_CLIENT_CERT_PATH") - - envs = append(envs, []corev1.EnvVar{ - {Name: "ES_GATEWAY_ELASTIC_CLIENT_CERT_PATH", Value: "/certs/elasticsearch/mtls/client.crt"}, - {Name: "ES_GATEWAY_ELASTIC_CLIENT_KEY_PATH", Value: "/certs/elasticsearch/mtls/client.key"}, - {Name: "ES_GATEWAY_ENABLE_ELASTIC_MUTUAL_TLS", Value: strconv.FormatBool(e.cfg.Cloud.EnableMTLS)}, - {Name: "ES_GATEWAY_KIBANA_CLIENT_CERT_PATH", Value: "/certs/kibana/mtls/client.crt"}, - {Name: "ES_GATEWAY_KIBANA_CLIENT_KEY_PATH", Value: "/certs/kibana/mtls/client.key"}, - {Name: "ES_GATEWAY_ENABLE_KIBANA_MUTUAL_TLS", Value: strconv.FormatBool(e.cfg.Cloud.EnableMTLS)}, - }...) - - d.Spec.Template.Spec.Volumes = append(d.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: logstorage.ExternalCertsVolumeName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: logstorage.ExternalCertsSecret, - }, - }, - }) - - d.Spec.Template.Spec.Containers[0].VolumeMounts = append(d.Spec.Template.Spec.Containers[0].VolumeMounts, - []corev1.VolumeMount{ - {Name: logstorage.ExternalCertsVolumeName, MountPath: "/certs/elasticsearch/mtls", ReadOnly: true}, - {Name: logstorage.ExternalCertsVolumeName, MountPath: "/certs/kibana/mtls", ReadOnly: true}, - }...) - } - - if e.cfg.Cloud.TenantId != "" { - envs = append(envs, corev1.EnvVar{Name: "ES_GATEWAY_TENANT_ID", Value: e.cfg.Cloud.TenantId}) - } - - d.Spec.Template.Spec.Containers[0].Env = envs - if e.cfg.Cloud.ExternalCertsSecret != nil { - d.Spec.Template.Annotations["hash.operator.tigera.io/cloud-external-es-secrets"] = rmeta.SecretsAnnotationHash(e.cfg.Cloud.ExternalCertsSecret) - } -} - -func (e *esGateway) getCloudObjects() (toCreate, toDelete []client.Object) { - if !e.cfg.Cloud.Enabled { - return nil, nil - } - - s := []client.Object{} - - if e.cfg.Cloud.ExternalCertsSecret != nil { - s = append(s, secret.ToRuntimeObjects(secret.CopyToNamespace(render.ElasticsearchNamespace, e.cfg.Cloud.ExternalCertsSecret)...)...) - } - - if e.cfg.Cloud.EsAdminUserSecret != nil { - s = append(s, secret.ToRuntimeObjects(secret.CopyToNamespace(render.ElasticsearchNamespace, e.cfg.Cloud.EsAdminUserSecret)...)...) - } - s = append(s, e.allowTigeraPolicyForCloud()) - - // allow-tigera Tier was renamed to calico-system - toDelete = append(toDelete, - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("cloud-es-gateway-access", render.ElasticsearchNamespace), - ) - return s, toDelete -} - -func (e *esGateway) allowTigeraPolicyForCloud() *v3.NetworkPolicy { - egressRules := []v3.Rule{} - if e.cfg.Cloud.ExternalElastic { - egressRules = append(egressRules, - v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: []numorstring.Port{{MinPort: 443, MaxPort: 443}}, - Domains: []string{e.cfg.Cloud.ExternalESDomain, e.cfg.Cloud.ExternalKibanaDomain}, - }, - }, - ) - } - - ingressRules := []v3.Rule{ - { - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: v3.EntityRule{ - NamespaceSelector: "projectcalico.org/name == 'monitoring'", - Selector: "app == 'prometheus'", - }, - // This matches the default. The metrics are enabled only on cloud (see - // ES_GATEWAY_METRICS_ENABLED which is added in this file). - Destination: v3.EntityRule{ - Ports: []numorstring.Port{{MinPort: 9091, MaxPort: 9091}}, - }, - }, - } - return &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: CloudPolicyName, - Namespace: render.ElasticsearchNamespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(DeploymentName), - Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - Ingress: ingressRules, - Egress: egressRules, - }, - } -} diff --git a/pkg/render/logstorage/esgateway/esgateway.go b/pkg/render/logstorage/esgateway/esgateway.go index 98e3423493..dfb7a675c7 100644 --- a/pkg/render/logstorage/esgateway/esgateway.go +++ b/pkg/render/logstorage/esgateway/esgateway.go @@ -27,6 +27,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/components" @@ -39,6 +40,7 @@ import ( "github.com/tigera/operator/pkg/render/common/secret" "github.com/tigera/operator/pkg/render/common/securitycontext" "github.com/tigera/operator/pkg/render/common/securitycontextconstraints" + "github.com/tigera/operator/pkg/render/logstorage" "github.com/tigera/operator/pkg/render/logstorage/esmetrics" "github.com/tigera/operator/pkg/render/logstorage/kibana" "github.com/tigera/operator/pkg/tls/certificatemanagement" @@ -57,6 +59,8 @@ const ( ElasticsearchHTTPSEndpoint = "https://tigera-secure-es-http.tigera-elasticsearch.svc:9200" KibanaHTTPSEndpoint = "https://tigera-secure-kb-http.tigera-kibana.svc:5601" + + CloudPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "cloud-es-gateway-access" ) func EsGateway(c *Config) render.Component { @@ -87,6 +91,19 @@ type Config struct { Cloud CloudConfig } +// CloudConfig holds Calico Cloud specific es-gateway configuration. Enabled gates all cloud +// behavior: when false (regular Calico/Calico Enterprise) the cloud decorators are no-ops. +type CloudConfig struct { + Enabled bool + EsAdminUserSecret *corev1.Secret + ExternalCertsSecret *corev1.Secret + TenantID string + EnableMTLS bool + ExternalElastic bool + ExternalESDomain string + ExternalKibanaDomain string +} + func (e *esGateway) ResolveImages(is *operatorv1.ImageSet) error { reg := e.cfg.Installation.Registry path := e.cfg.Installation.ImagePath @@ -118,9 +135,20 @@ func (e *esGateway) Objects() (toCreate, toDelete []client.Object) { toCreate = append(toCreate, e.esGatewayRoleBinding()) toCreate = append(toCreate, e.esGatewayServiceAccount()) - ccToCreate, ccToDelete := e.getCloudObjects() - toCreate = append(toCreate, ccToCreate...) - toDelete = append(toDelete, ccToDelete...) + if e.cfg.Cloud.Enabled { + // Copy the external ES certs and es-admin secret into the elasticsearch namespace so es-gateway can use them. + if e.cfg.Cloud.ExternalCertsSecret != nil { + toCreate = append(toCreate, secret.ToRuntimeObjects(secret.CopyToNamespace(render.ElasticsearchNamespace, e.cfg.Cloud.ExternalCertsSecret)...)...) + } + if e.cfg.Cloud.EsAdminUserSecret != nil { + toCreate = append(toCreate, secret.ToRuntimeObjects(secret.CopyToNamespace(render.ElasticsearchNamespace, e.cfg.Cloud.EsAdminUserSecret)...)...) + } + + toCreate = append(toCreate, e.cloudAccessNetworkPolicy()) + + // allow-tigera Tier was renamed to calico-system + toDelete = append(toDelete, networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("cloud-es-gateway-access", render.ElasticsearchNamespace)) + } // allow-tigera Tier was renamed to calico-system toDelete = append(toDelete, networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("es-gateway-access", e.cfg.Namespace)) @@ -197,15 +225,33 @@ func (e *esGateway) esGatewayRoleBinding() *rbacv1.RoleBinding { } func (e *esGateway) esGatewayDeployment() *appsv1.Deployment { + // The ES/Kibana endpoints and client cert paths default to the in-cluster values, but Calico + // Cloud points es-gateway at an external ES/Kibana and (optionally) mounts mTLS client certs. + // These are computed here rather than removed-and-re-added by the cloud path. + elasticEndpoint := ElasticsearchHTTPSEndpoint + kibanaEndpoint := KibanaHTTPSEndpoint + elasticClientCertPath := e.cfg.TrustedBundle.MountPath() + kibanaClientCertPath := e.cfg.TrustedBundle.MountPath() + if e.cfg.Cloud.Enabled { + if e.cfg.Cloud.ExternalElastic { + elasticEndpoint = "https://" + e.cfg.Cloud.ExternalESDomain + ":443" + kibanaEndpoint = "https://" + e.cfg.Cloud.ExternalKibanaDomain + ":443" + } + if e.cfg.Cloud.EnableMTLS { + elasticClientCertPath = "/certs/elasticsearch/mtls/client.crt" + kibanaClientCertPath = "/certs/kibana/mtls/client.crt" + } + } + envVars := []corev1.EnvVar{ {Name: "NAMESPACE", Value: e.cfg.Namespace}, {Name: "ES_GATEWAY_LOG_LEVEL", Value: "INFO"}, - {Name: "ES_GATEWAY_ELASTIC_ENDPOINT", Value: ElasticsearchHTTPSEndpoint}, - {Name: "ES_GATEWAY_KIBANA_ENDPOINT", Value: KibanaHTTPSEndpoint}, + {Name: "ES_GATEWAY_ELASTIC_ENDPOINT", Value: elasticEndpoint}, + {Name: "ES_GATEWAY_KIBANA_ENDPOINT", Value: kibanaEndpoint}, {Name: "ES_GATEWAY_HTTPS_CERT", Value: e.cfg.ESGatewayKeyPair.VolumeMountCertificateFilePath()}, {Name: "ES_GATEWAY_HTTPS_KEY", Value: e.cfg.ESGatewayKeyPair.VolumeMountKeyFilePath()}, - {Name: "ES_GATEWAY_KIBANA_CLIENT_CERT_PATH", Value: e.cfg.TrustedBundle.MountPath()}, - {Name: "ES_GATEWAY_ELASTIC_CLIENT_CERT_PATH", Value: e.cfg.TrustedBundle.MountPath()}, + {Name: "ES_GATEWAY_KIBANA_CLIENT_CERT_PATH", Value: kibanaClientCertPath}, + {Name: "ES_GATEWAY_ELASTIC_CLIENT_CERT_PATH", Value: elasticClientCertPath}, {Name: "ES_GATEWAY_ELASTIC_CA_BUNDLE_PATH", Value: e.cfg.TrustedBundle.MountPath()}, {Name: "ES_GATEWAY_KIBANA_CA_BUNDLE_PATH", Value: e.cfg.TrustedBundle.MountPath()}, {Name: "ES_GATEWAY_ELASTIC_USERNAME", Value: e.cfg.EsAdminUserName}, @@ -218,6 +264,29 @@ func (e *esGateway) esGatewayDeployment() *appsv1.Deployment { }, }}, } + + // Calico Cloud additions to the es-gateway env. + if e.cfg.Cloud.Enabled { + // Enable the prometheus metrics endpoint at :METRICS_PORT/metrics (default 9091). + envVars = append(envVars, corev1.EnvVar{Name: "ES_GATEWAY_METRICS_ENABLED", Value: "true"}) + if e.cfg.Cloud.ExternalElastic { + // Enable the ILM dummy route so fluentd cannot modify ILM but its POSTs still succeed. + envVars = append(envVars, corev1.EnvVar{Name: "ES_GATEWAY_ILM_DUMMY_ROUTE_ENABLED", Value: "true"}) + } + if e.cfg.Cloud.EnableMTLS { + // Cert paths are set above; here we add the client key paths and enable flags. + envVars = append(envVars, + corev1.EnvVar{Name: "ES_GATEWAY_ELASTIC_CLIENT_KEY_PATH", Value: "/certs/elasticsearch/mtls/client.key"}, + corev1.EnvVar{Name: "ES_GATEWAY_ENABLE_ELASTIC_MUTUAL_TLS", Value: "true"}, + corev1.EnvVar{Name: "ES_GATEWAY_KIBANA_CLIENT_KEY_PATH", Value: "/certs/kibana/mtls/client.key"}, + corev1.EnvVar{Name: "ES_GATEWAY_ENABLE_KIBANA_MUTUAL_TLS", Value: "true"}, + ) + } + if e.cfg.Cloud.TenantID != "" { + envVars = append(envVars, corev1.EnvVar{Name: "ES_GATEWAY_TENANT_ID", Value: e.cfg.Cloud.TenantID}) + } + } + sc := securitycontext.NewNonRootContext() var initContainers []corev1.Container if e.cfg.ESGatewayKeyPair.UseCertificateManagement() { @@ -237,6 +306,27 @@ func (e *esGateway) esGatewayDeployment() *appsv1.Deployment { annotations := e.cfg.TrustedBundle.HashAnnotations() annotations[e.cfg.ESGatewayKeyPair.HashAnnotationKey()] = e.cfg.ESGatewayKeyPair.HashAnnotationValue() + if e.cfg.Cloud.Enabled { + if e.cfg.Cloud.EnableMTLS { + // Mount the external certs secret so the mTLS client key/cert paths set above resolve. + volumes = append(volumes, corev1.Volume{ + Name: logstorage.ExternalCertsVolumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: logstorage.ExternalCertsSecret, + }, + }, + }) + volumeMounts = append(volumeMounts, + corev1.VolumeMount{Name: logstorage.ExternalCertsVolumeName, MountPath: "/certs/elasticsearch/mtls", ReadOnly: true}, + corev1.VolumeMount{Name: logstorage.ExternalCertsVolumeName, MountPath: "/certs/kibana/mtls", ReadOnly: true}, + ) + } + if e.cfg.Cloud.ExternalCertsSecret != nil { + annotations["hash.operator.tigera.io/cloud-external-es-secrets"] = rmeta.SecretsAnnotationHash(e.cfg.Cloud.ExternalCertsSecret) + } + } + tolerations := e.cfg.Installation.ControlPlaneTolerations if e.cfg.Installation.KubernetesProvider.IsGKE() { tolerations = append(tolerations, rmeta.TolerateGKEARM64NoSchedule) @@ -310,7 +400,6 @@ func (e *esGateway) esGatewayDeployment() *appsv1.Deployment { } } - e.modifyDeploymentForCloud(&d) return &d } @@ -448,3 +537,42 @@ func (e *esGateway) esGatewayCalicoSystemPolicy() *v3.NetworkPolicy { }, } } + +func (e *esGateway) cloudAccessNetworkPolicy() *v3.NetworkPolicy { + // When using external elastic, allow egress to the external ES/Kibana endpoints. + var egressRules []v3.Rule + if e.cfg.Cloud.ExternalElastic { + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: []numorstring.Port{{MinPort: 443, MaxPort: 443}}, + Domains: []string{e.cfg.Cloud.ExternalESDomain, e.cfg.Cloud.ExternalKibanaDomain}, + }, + }) + } + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{Name: CloudPolicyName, Namespace: render.ElasticsearchNamespace}, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(DeploymentName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: []v3.Rule{{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Source: v3.EntityRule{ + NamespaceSelector: "projectcalico.org/name == 'monitoring'", + Selector: "app == 'prometheus'", + }, + // Allow prometheus to scrape the metrics endpoint, which is enabled only on + // cloud (see ES_GATEWAY_METRICS_ENABLED added above). + Destination: v3.EntityRule{ + Ports: []numorstring.Port{{MinPort: 9091, MaxPort: 9091}}, + }, + }}, + Egress: egressRules, + }, + } +} diff --git a/pkg/render/logstorage/esgateway/esgateway_test.go b/pkg/render/logstorage/esgateway/esgateway_test.go index e108557663..13ae8d5c68 100644 --- a/pkg/render/logstorage/esgateway/esgateway_test.go +++ b/pkg/render/logstorage/esgateway/esgateway_test.go @@ -134,7 +134,7 @@ var _ = Describe("ES Gateway rendering tests", func() { Enabled: true, EsAdminUserSecret: &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: render.ElasticsearchAdminUserSecret, Namespace: common.OperatorNamespace()}}, ExternalCertsSecret: &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: logstorage.ExternalCertsSecret, Namespace: common.OperatorNamespace()}}, - TenantId: "tenantId", + TenantID: "tenantId", EnableMTLS: true, ExternalElastic: true, ExternalESDomain: "externalEs.com", diff --git a/pkg/render/logstorage/linseed/cloud.go b/pkg/render/logstorage/linseed/cloud.go deleted file mode 100644 index e081f3523e..0000000000 --- a/pkg/render/logstorage/linseed/cloud.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright (c) 2023-2026 Tigera, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package linseed - -import ( - "strconv" - - v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" - "github.com/tigera/api/pkg/lib/numorstring" - - "github.com/tigera/operator/pkg/render/common/networkpolicy" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" -) - -const ( - // Name of the network policy that adds CC specific rules to Linseed. - CloudPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "cloud-linseed-access" -) - -// modifyDeploymentForCloud applies Calico Cloud specific tweaks to the Linseed deployment. It is -// only called when rendering for cloud (gated by Config.Cloud), so it never affects enterprise. -func (c *linseed) modifyDeploymentForCloud(d *appsv1.Deployment) { - envs := d.Spec.Template.Spec.Containers[0].Env - - // Replica count for the policy activity index, kept consistent with the other index replicas. - envs = append(envs, corev1.EnvVar{Name: "ELASTIC_POLICY_ACTIVITY_INDEX_REPLICAS", Value: strconv.Itoa(c.cfg.ESClusterConfig.Replicas())}) - - // Enable prometheus metrics endpoint at :METRICS_PORT/metrics (Default is 9095). - // We use the same certificate for TLS on the metrics endpoint as we do for the main API. - envs = append(envs, corev1.EnvVar{Name: "LINSEED_ENABLE_METRICS", Value: "true"}) - envs = append(envs, corev1.EnvVar{Name: "LINSEED_METRICS_CERT", Value: c.cfg.KeyPair.VolumeMountCertificateFilePath()}) - envs = append(envs, corev1.EnvVar{Name: "LINSEED_METRICS_KEY", Value: c.cfg.KeyPair.VolumeMountKeyFilePath()}) - - if c.cfg.Tenant != nil { - if c.cfg.ExternalElastic { - // Overwrite policy activity index name until we create the tenant CR on all environments - envs = append(envs, corev1.EnvVar{Name: "ELASTIC_POLICY_ACTIVITY_BASE_INDEX_NAME", Value: "calico_policy_activity_standard"}) - } - } - d.Spec.Template.Spec.Containers[0].Env = envs -} - -func (c *linseed) getCloudObjects() (toCreate, toDelete []client.Object) { - s := []client.Object{} - s = append(s, c.allowTigeraPolicyForCloud()) - - // allow-tigera Tier was renamed to calico-system - toDelete = append(toDelete, - networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("cloud-linseed-access", c.namespace), - ) - - return s, toDelete -} - -func (c *linseed) allowTigeraPolicyForCloud() *v3.NetworkPolicy { - egressRules := []v3.Rule{} - if c.cfg.ElasticClientSecret != nil { - // TODO: At the moment, we only support mTLS for Elasticsearch when using an external ES cluster. - // That allows us to use the presence of the secret as a proxy for whether we should append this egress rule. - // In the future, we should support mTLS for internal ES clusters as well and switch this to a better check. - - // Allow egress traffic to the external Elasticsearch. - egressRules = append(egressRules, - v3.Rule{ - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Destination: v3.EntityRule{ - Ports: []numorstring.Port{{MinPort: 443, MaxPort: 443}}, - Domains: []string{c.cfg.ElasticHost}, - }, - }, - ) - } - - ingressRules := []v3.Rule{ - { - // Allow ingress traffic from Calico Cloud monitoring stack to the Linseed - // metrics port. - Action: v3.Allow, - Protocol: &networkpolicy.TCPProtocol, - Source: v3.EntityRule{ - NamespaceSelector: "projectcalico.org/name == 'monitoring'", - Selector: "app == 'prometheus'", - }, - Destination: v3.EntityRule{ - Ports: []numorstring.Port{{MinPort: 9095, MaxPort: 9095}}, - }, - }, - } - return &v3.NetworkPolicy{ - TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, - ObjectMeta: metav1.ObjectMeta{ - Name: CloudPolicyName, - Namespace: c.namespace, - }, - Spec: v3.NetworkPolicySpec{ - Order: &networkpolicy.HighPrecedenceOrder, - Tier: networkpolicy.CalicoTierName, - Selector: networkpolicy.KubernetesAppSelector(DeploymentName), - Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, - Ingress: ingressRules, - Egress: egressRules, - }, - } -} diff --git a/pkg/render/logstorage/linseed/linseed.go b/pkg/render/logstorage/linseed/linseed.go index 3b07b8f61f..86f98f456b 100644 --- a/pkg/render/logstorage/linseed/linseed.go +++ b/pkg/render/logstorage/linseed/linseed.go @@ -29,6 +29,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" v3 "github.com/tigera/api/pkg/apis/projectcalico/v3" + "github.com/tigera/api/pkg/lib/numorstring" operatorv1 "github.com/tigera/operator/api/v1" "github.com/tigera/operator/pkg/common" @@ -57,6 +58,9 @@ const ( ClusterRoleName = "tigera-linseed" MultiTenantManagedClustersAccessClusterRoleBindingName = "tigera-linseed-managed-cluster-access" ManagedClustersWatchRoleBindingName = "tigera-linseed-managed-cluster-watch" + + // CloudPolicyName Name of the network policy that adds CC specific rules to Linseed. + CloudPolicyName = networkpolicy.CalicoComponentPolicyPrefix + "cloud-linseed-access" ) func Linseed(c *Config) render.Component { @@ -177,10 +181,10 @@ func (l *linseed) Objects() (toCreate, toDelete []client.Object) { } if l.cfg.Cloud { - // Add in Calico Cloud resources. - ccToCreate, ccToDelete := l.getCloudObjects() - toCreate = append(toCreate, ccToCreate...) - toDelete = append(toDelete, ccToDelete...) + toCreate = append(toCreate, l.cloudAccessNetworkPolicy()) + + // allow-tigera Tier was renamed to calico-system + toDelete = append(toDelete, networkpolicy.DeprecatedAllowTigeraNetworkPolicyObject("cloud-linseed-access", l.namespace)) } return toCreate, toDelete @@ -454,6 +458,26 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { } annotations[l.cfg.TokenKeyPair.HashAnnotationKey()] = l.cfg.TokenKeyPair.HashAnnotationValue() } + + // Calico Cloud additions to the Linseed env. + if l.cfg.Cloud { + // Replica count for the policy activity index, kept consistent with the other index replicas. + envVars = append(envVars, corev1.EnvVar{Name: "ELASTIC_POLICY_ACTIVITY_INDEX_REPLICAS", Value: strconv.Itoa(l.cfg.ESClusterConfig.Replicas())}) + + // Enable the prometheus metrics endpoint at :METRICS_PORT/metrics (default 9095). We use the + // same certificate for TLS on the metrics endpoint as we do for the main API. + envVars = append(envVars, + corev1.EnvVar{Name: "LINSEED_ENABLE_METRICS", Value: "true"}, + corev1.EnvVar{Name: "LINSEED_METRICS_CERT", Value: l.cfg.KeyPair.VolumeMountCertificateFilePath()}, + corev1.EnvVar{Name: "LINSEED_METRICS_KEY", Value: l.cfg.KeyPair.VolumeMountKeyFilePath()}, + ) + + if l.cfg.Tenant != nil && l.cfg.ExternalElastic { + // Overwrite policy activity index name until we create the tenant CR on all environments. + envVars = append(envVars, corev1.EnvVar{Name: "ELASTIC_POLICY_ACTIVITY_BASE_INDEX_NAME", Value: "calico_policy_activity_standard"}) + } + } + tolerations := l.cfg.Installation.ControlPlaneTolerations if l.cfg.Installation.KubernetesProvider.IsGKE() { tolerations = append(tolerations, rmeta.TolerateGKEARM64NoSchedule) @@ -526,10 +550,6 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { }, } - if l.cfg.Cloud { - l.modifyDeploymentForCloud(&d) - } - if l.cfg.Tenant.MultiTenant() { if overrides := l.cfg.Tenant.Spec.LinseedDeployment; overrides != nil { rcomponents.ApplyDeploymentOverrides(&d, overrides) @@ -721,6 +741,50 @@ func (l *linseed) linseedCalicoSystemPolicy() *v3.NetworkPolicy { } } +func (l *linseed) cloudAccessNetworkPolicy() *v3.NetworkPolicy { + // Calico Cloud NetworkPolicy: allow the CC monitoring stack to scrape Linseed metrics, and + // (when using external ES) allow egress to the external Elasticsearch. + var egressRules []v3.Rule + if l.cfg.ElasticClientSecret != nil { + // TODO: At the moment, we only support mTLS for Elasticsearch when using an external ES cluster. + // That allows us to use the presence of the secret as a proxy for whether we should append this egress rule. + // In the future, we should support mTLS for internal ES clusters as well and switch this to a better check. + + // Allow egress traffic to the external Elasticsearch. + egressRules = append(egressRules, v3.Rule{ + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + Ports: []numorstring.Port{{MinPort: 443, MaxPort: 443}}, + Domains: []string{l.cfg.ElasticHost}, + }, + }) + } + return &v3.NetworkPolicy{ + TypeMeta: metav1.TypeMeta{Kind: "NetworkPolicy", APIVersion: "projectcalico.org/v3"}, + ObjectMeta: metav1.ObjectMeta{Name: CloudPolicyName, Namespace: l.namespace}, + Spec: v3.NetworkPolicySpec{ + Order: &networkpolicy.HighPrecedenceOrder, + Tier: networkpolicy.CalicoTierName, + Selector: networkpolicy.KubernetesAppSelector(DeploymentName), + Types: []v3.PolicyType{v3.PolicyTypeIngress, v3.PolicyTypeEgress}, + Ingress: []v3.Rule{{ + // Allow ingress traffic from the Calico Cloud monitoring stack to the Linseed metrics port. + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Source: v3.EntityRule{ + NamespaceSelector: "projectcalico.org/name == 'monitoring'", + Selector: "app == 'prometheus'", + }, + Destination: v3.EntityRule{ + Ports: []numorstring.Port{{MinPort: 9095, MaxPort: 9095}}, + }, + }}, + Egress: egressRules, + }, + } +} + // LinseedNamespace determine the namespace in which Linseed is running. // For management and standalone clusters, this is always the tigera-elasticsearch // namespace. For multi-tenant management clusters, this is the tenant namespace From e1efac6ce2d278a73bbb01c8168857eb9e784013 Mon Sep 17 00:00:00 2001 From: Brian McMahon Date: Mon, 13 Jul 2026 11:14:21 -0700 Subject: [PATCH 4/6] Bring over post-snapshot cloud differences from operator-cloud Cloud-only changes that landed on operator-cloud after the initial un-fork snapshot (or were missed by it), all behind the runtime Cloud gate so enterprise/OSS behavior is unchanged: - TSLA-11182: disable ui-apis Kibana login for all Calico Cloud (not just multi-tenant) via a cloud-gated decorateCloudUIAPIsContainer decorator. - TSLA-11580 (#1111): run cloud kube-controllers from the tesla-tagged calico image (components.CalicoCloudImage); gated on Config.Cloud, plumbed into both the calico- and es-kube-controllers render configs. - TSLA-11553 (#1103): pin the policy-activity index base name only for single-tenant external-ES (Tenant.SingleTenant()), not all tenants. - TSLA-11547: point enterprise images at the cloud registry in the build-hashrelease ArgoCI template so hashrelease images resolve. Co-Authored-By: Claude Opus 4.8 --- .../hashrelease/build-hashrelease.yaml | 4 ++++ pkg/components/combined.go | 10 ++++++++++ .../installation/core_controller.go | 6 ++++++ .../kubecontrollers/kube-controllers.go | 9 ++++++++- .../kubecontrollers/kube-controllers_test.go | 10 ++++++++++ pkg/render/logstorage/linseed/linseed.go | 10 ++++++++-- pkg/render/manager.go | 2 +- pkg/render/manager_cloud.go | 19 +++++++++++++++++++ pkg/render/manager_cloud_test.go | 6 ++++++ 9 files changed, 72 insertions(+), 4 deletions(-) diff --git a/.argoci/templates/hashrelease/build-hashrelease.yaml b/.argoci/templates/hashrelease/build-hashrelease.yaml index 1fc6850400..32ff165c2c 100644 --- a/.argoci/templates/hashrelease/build-hashrelease.yaml +++ b/.argoci/templates/hashrelease/build-hashrelease.yaml @@ -62,6 +62,10 @@ spec: value: "{{workflow.parameters.hashReleaseURL}}" - name: CLOUD_REGISTRY value: gcr.io/unique-caldron-775/cnx/ + # Point enterprise images at the cloud registry (not the quay.io default) so the + # hashrelease operator can pull them — otherwise ImagePullBackOff. + - name: ENTERPRISE_REGISTRY + value: gcr.io/unique-caldron-775/cnx/ - name: DEBUG value: "true" volumeMounts: diff --git a/pkg/components/combined.go b/pkg/components/combined.go index 312f9a081a..b9142802fe 100644 --- a/pkg/components/combined.go +++ b/pkg/components/combined.go @@ -30,3 +30,13 @@ func CombinedCalicoImage(installation *operatorv1.InstallationSpec) Component { } return ComponentCalico } + +// CalicoCloudImage returns the tesla-compiled variant of the combined calico image. It is the same +// image repository as ComponentTigeraCalico (so ImageSet digests still resolve by that name), +// published under a tesla- tag. It carries the Calico Cloud behavior for the components that need it +// — currently only kube-controllers, the sole component with tesla-gated code. See TSLA-11580. +func CalicoCloudImage() Component { + c := ComponentTigeraCalico + c.Version = "tesla-" + c.Version + return c +} diff --git a/pkg/controller/installation/core_controller.go b/pkg/controller/installation/core_controller.go index 6105249448..5cc14d0d03 100644 --- a/pkg/controller/installation/core_controller.go +++ b/pkg/controller/installation/core_controller.go @@ -363,6 +363,7 @@ func newReconciler(mgr manager.Manager, opts options.ControllerOptions) (*Reconc v3CRDs: opts.UseV3CRDs, kubernetesVersion: opts.KubernetesVersion, apiDiscovery: opts.APIDiscovery, + cloud: opts.Cloud, } r.status.Run(opts.ShutdownContext) r.typhaAutoscaler.start(opts.ShutdownContext) @@ -423,6 +424,10 @@ type ReconcileInstallation struct { kubernetesVersion *common.VersionInfo apiDiscovery *discovery.APIDiscovery + // cloud indicates the operator is running as a Calico Cloud install. When false the calico + // kube-controllers render config leaves cloud behavior (e.g. the tesla image) off. + cloud bool + // newComponentHandler returns a new component handler. Useful stub for unit testing. newComponentHandler func(log logr.Logger, client client.Client, scheme *runtime.Scheme, cr metav1.Object) utils.ComponentHandler } @@ -1722,6 +1727,7 @@ func (r *ReconcileInstallation) Reconcile(ctx context.Context, request reconcile // cert above. WAFWebhookCABundle: certificateManager.KeyPair().GetCertificatePEM(), RBACManagementEnabled: managerCR.RBACManagementEnabled(), + Cloud: r.cloud, } components = append(components, kubecontrollers.NewCalicoKubeControllers(&kubeControllersCfg)) diff --git a/pkg/render/kubecontrollers/kube-controllers.go b/pkg/render/kubecontrollers/kube-controllers.go index 635671c5d8..087329e664 100644 --- a/pkg/render/kubecontrollers/kube-controllers.go +++ b/pkg/render/kubecontrollers/kube-controllers.go @@ -319,7 +319,14 @@ func (c *kubeControllersComponent) ResolveImages(is *operatorv1.ImageSet) error path := c.cfg.Installation.ImagePath prefix := c.cfg.Installation.ImagePrefix var err error - c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) + if c.cfg.Cloud { + // Calico Cloud runs kube-controllers from the tesla-compiled variant of the combined image, + // which carries the Cloud behavior the enterprise mono image lacks. It is the same binary, + // so the container command and health probes below are unchanged. See TSLA-11580. + c.calicoImage, err = components.GetReference(components.CalicoCloudImage(), reg, path, prefix, is) + } else { + c.calicoImage, err = components.GetReference(components.CombinedCalicoImage(c.cfg.Installation), reg, path, prefix, is) + } if err != nil { return err } diff --git a/pkg/render/kubecontrollers/kube-controllers_test.go b/pkg/render/kubecontrollers/kube-controllers_test.go index 4ad441a65b..ffe4d68486 100644 --- a/pkg/render/kubecontrollers/kube-controllers_test.go +++ b/pkg/render/kubecontrollers/kube-controllers_test.go @@ -142,6 +142,16 @@ var _ = Describe("kube-controllers rendering tests", func() { })) }) + It("should use the tesla calico image for kube-controllers when Cloud is enabled (TSLA-11580)", func() { + instance.Variant = operatorv1.CalicoEnterprise + cfg.Cloud = true + component := kubecontrollers.NewCalicoKubeControllers(&cfg) + Expect(component.ResolveImages(nil)).To(BeNil()) + resources, _ := component.Objects() + dp := rtest.GetResource(resources, kubecontrollers.KubeController, common.CalicoNamespace, "apps", "v1", "Deployment").(*appsv1.Deployment) + Expect(dp.Spec.Template.Spec.Containers[0].Image).To(Equal("test-reg/tigera/calico:" + components.CalicoCloudImage().Version)) + }) + It("should include kubevirt.io RBAC rules in calico-kube-controllers ClusterRole", func() { component := kubecontrollers.NewCalicoKubeControllers(&cfg) Expect(component.ResolveImages(nil)).To(BeNil()) diff --git a/pkg/render/logstorage/linseed/linseed.go b/pkg/render/logstorage/linseed/linseed.go index 86f98f456b..c69e6a37f6 100644 --- a/pkg/render/logstorage/linseed/linseed.go +++ b/pkg/render/logstorage/linseed/linseed.go @@ -472,8 +472,14 @@ func (l *linseed) linseedDeployment() *appsv1.Deployment { corev1.EnvVar{Name: "LINSEED_METRICS_KEY", Value: l.cfg.KeyPair.VolumeMountKeyFilePath()}, ) - if l.cfg.Tenant != nil && l.cfg.ExternalElastic { - // Overwrite policy activity index name until we create the tenant CR on all environments. + if l.cfg.Tenant.SingleTenant() && l.cfg.ExternalElastic { + // Single-tenant external-ES clusters have an artificial Tenant CR, so index base names aren't + // carried via Tenant.Spec.Indices (that only happens for multi-tenant clusters). Policy + // activity is a brand new index, so these clusters have no existing policy activity data + // under the default calico_policy_activity name. Pin it to the multi-tenant "standard" + // name now so that when these clusters are consolidated into multi-tenant, the data + // already lives in the target index and needs no migration. Multi-tenant clusters skip + // this and rely on Tenant.Spec.Indices (which resolves free vs. standard from the data plan). envVars = append(envVars, corev1.EnvVar{Name: "ELASTIC_POLICY_ACTIVITY_BASE_INDEX_NAME", Value: "calico_policy_activity_standard"}) } } diff --git a/pkg/render/manager.go b/pkg/render/manager.go index a952084cd0..0f7d04148b 100644 --- a/pkg/render/manager.go +++ b/pkg/render/manager.go @@ -366,7 +366,7 @@ func (c *managerComponent) managerDeployment() *appsv1.Deployment { initContainers = append(initContainers, c.cfg.VoltronLinseedKeyPair.InitContainer(ManagerNamespace, securitycontext.NewNonRootContext())) } - managerPodContainers := []corev1.Container{c.managerUIAPIsContainer(), c.decorateCloudVoltronContainer(c.voltronContainer())} + managerPodContainers := []corev1.Container{c.decorateCloudUIAPIsContainer(c.managerUIAPIsContainer()), c.decorateCloudVoltronContainer(c.voltronContainer())} if c.cfg.Tenant == nil { managerPodContainers = append(managerPodContainers, c.dashboardContainer(), c.managerContainer()) } diff --git a/pkg/render/manager_cloud.go b/pkg/render/manager_cloud.go index 6071bba3ec..c296aec6dd 100644 --- a/pkg/render/manager_cloud.go +++ b/pkg/render/manager_cloud.go @@ -35,6 +35,25 @@ type ManagerCloudResources struct { ManagerExtraEnv map[string]string } +// decorateCloudUIAPIsContainer applies Calico Cloud-only overrides to the ui-apis container. +// TSLA-11182: Calico Cloud deprecates normal-user Kibana login, so the ui-apis Kibana login endpoint +// is disabled for all Calico Cloud deployments (upstream only disables it for multi-tenant). Kibana +// itself remains deployed for administrator use via the elastic superuser. Keeping this as a +// cloud-gated decorator rather than editing the shared managerUIAPIsContainer keeps the enterprise +// behavior (disable only for multi-tenant) unchanged. +func (c *managerComponent) decorateCloudUIAPIsContainer(container corev1.Container) corev1.Container { + if !c.cfg.Cloud { + return container + } + const kibanaDisabledEnv = "ELASTIC_KIBANA_DISABLED" + if i := slices.IndexFunc(container.Env, func(env corev1.EnvVar) bool { return env.Name == kibanaDisabledEnv }); i != -1 { + container.Env[i].Value = "true" + } else { + container.Env = append(container.Env, corev1.EnvVar{Name: kibanaDisabledEnv, Value: "true"}) + } + return container +} + func (c *managerComponent) decorateCloudVoltronContainer(container corev1.Container) corev1.Container { if !c.cfg.Cloud { return container diff --git a/pkg/render/manager_cloud_test.go b/pkg/render/manager_cloud_test.go index c59f0fe58b..05ddaff4e0 100644 --- a/pkg/render/manager_cloud_test.go +++ b/pkg/render/manager_cloud_test.go @@ -80,6 +80,12 @@ var _ = Describe("Tigera Secure Cloud Manager rendering tests", func() { )) }) + It("should disable Kibana login on the ui-apis container for all cloud (TSLA-11182)", func() { + // Even for a single-tenant cluster (where upstream leaves login enabled), cloud disables it. + uiAPIs := template.Containers[0] + Expect(uiAPIs.Env).Should(ContainElement(corev1.EnvVar{Name: "ELASTIC_KIBANA_DISABLED", Value: "true"})) + }) + It("should have default env vars overwritten by configmap override", func() { Expect(voltron.Env).ShouldNot(ContainElements( corev1.EnvVar{Name: "VOLTRON_K8S_CLIENT_QPS", Value: "20"}, From c990cd0dce673a1e72853843908b2e84f0269113 Mon Sep 17 00:00:00 2001 From: Brian McMahon Date: Mon, 13 Jul 2026 11:27:20 -0700 Subject: [PATCH 5/6] Regenerate enterprise ECK CRD bundle (eck 3.4.0 -> 3.4.1) make gen-versions picked up an ECK app.kubernetes.io/version label bump in the enterprise CRD source; regenerated 01-crd-eck-bundle.yaml to satisfy the CI dirty-check. Generated file only. Co-Authored-By: Claude Opus 4.8 --- .../crds/enterprise/01-crd-eck-bundle.yaml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pkg/imports/crds/enterprise/01-crd-eck-bundle.yaml b/pkg/imports/crds/enterprise/01-crd-eck-bundle.yaml index 86b772c282..903d3a5156 100644 --- a/pkg/imports/crds/enterprise/01-crd-eck-bundle.yaml +++ b/pkg/imports/crds/enterprise/01-crd-eck-bundle.yaml @@ -9,7 +9,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: agents.agent.k8s.elastic.co spec: group: agent.k8s.elastic.co @@ -502,7 +502,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: apmservers.apm.k8s.elastic.co spec: group: apm.k8s.elastic.co @@ -1024,7 +1024,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: autoopsagentpolicies.autoops.k8s.elastic.co spec: group: autoops.k8s.elastic.co @@ -1184,7 +1184,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: beats.beat.k8s.elastic.co spec: group: beat.k8s.elastic.co @@ -1423,7 +1423,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: elasticmapsservers.maps.k8s.elastic.co spec: group: maps.k8s.elastic.co @@ -1680,7 +1680,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: elasticsearchautoscalers.autoscaling.k8s.elastic.co spec: group: autoscaling.k8s.elastic.co @@ -1937,7 +1937,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: elasticsearches.elasticsearch.k8s.elastic.co spec: group: elasticsearch.k8s.elastic.co @@ -3322,7 +3322,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: enterprisesearches.enterprisesearch.k8s.elastic.co spec: group: enterprisesearch.k8s.elastic.co @@ -3800,7 +3800,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: kibanas.kibana.k8s.elastic.co spec: group: kibana.k8s.elastic.co @@ -4368,7 +4368,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: logstashes.logstash.k8s.elastic.co spec: group: logstash.k8s.elastic.co @@ -4911,7 +4911,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: packageregistries.packageregistry.k8s.elastic.co spec: group: packageregistry.k8s.elastic.co @@ -5153,7 +5153,7 @@ metadata: labels: app.kubernetes.io/instance: 'elastic-operator' app.kubernetes.io/name: 'eck-operator-crds' - app.kubernetes.io/version: '3.4.0' + app.kubernetes.io/version: '3.4.1' name: stackconfigpolicies.stackconfigpolicy.k8s.elastic.co spec: group: stackconfigpolicy.k8s.elastic.co From 02eed2c2e3b988dad3f402fa3271ab6e8b0355d2 Mon Sep 17 00:00:00 2001 From: Brian McMahon Date: Mon, 13 Jul 2026 15:00:17 -0700 Subject: [PATCH 6/6] Source external-ES from a single configmap knob (PR #4980 review) Casey flagged that ElasticExternal (cloud) and discovery.UseExternalElastic (enterprise bootstrap configmap) are the same knob read from two configmaps, OR'd together in main.go. Collapse to a single source: the operator now reads external-ES only from operator-bootstrap-config via discovery.UseExternalElastic. Calico Cloud provisions ELASTIC_EXTERNAL into operator-bootstrap-config too (in addition to cloud-operator-config, which cloud.Load still reads for its own startup verify), so cloud and enterprise share one downstream gate. Requires the cc-mgmt-config helm chart to populate operator-bootstrap-config (separate change). Co-Authored-By: Claude Opus 4.8 --- cmd/main.go | 15 ++++++++++----- pkg/cloud/cloud.go | 6 +++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index d8c0fa5009..d22547b1d1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -538,11 +538,16 @@ If a value other than 'all' is specified, the first CRD with a prefix of the spe ShutdownContext: ctx, K8sClientset: clientset, MultiTenant: multiTenant, - ElasticExternal: discovery.UseExternalElastic(bootConfig) || cloudOpts.ElasticExternal, - Cloud: cloudOpts.Cloud, - ESMigration: cloudOpts.ESMigration, - UseV3CRDs: v3CRDs, - APIDiscovery: apiDiscovery, + // External-ES is a single knob, sourced only from the operator bootstrap configmap + // (operator-bootstrap-config) via discovery.UseExternalElastic. Calico Cloud provisions + // ELASTIC_EXTERNAL there too (in addition to its own cloud-operator-config, which cloud.Load + // still reads for its startup verify), so cloud and enterprise share one downstream knob + // rather than the previous two-source OR. + ElasticExternal: discovery.UseExternalElastic(bootConfig), + Cloud: cloudOpts.Cloud, + ESMigration: cloudOpts.ESMigration, + UseV3CRDs: v3CRDs, + APIDiscovery: apiDiscovery, } // Before we start any controllers, make sure our options are valid. diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index b408f25c6d..c1c82666fa 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -61,7 +61,11 @@ func IsCloudBuild() bool { type Options struct { // Cloud indicates that this operator is running as a Calico Cloud install. It is true when the // cloud-operator-config ConfigMap is present or the relevant cloud env vars are set. - Cloud bool + Cloud bool + // ElasticExternal is parsed from cloud-operator-config for cloud's own startup verify (see verify). + // It is NOT the operator's external-ES gate: controllers read that from ControllerOptions.ElasticExternal, + // which main.go sources solely from operator-bootstrap-config via discovery.UseExternalElastic. Cloud + // provisions ELASTIC_EXTERNAL into both configmaps so the two stay consistent. ElasticExternal bool ESMigration bool }