From 35273f3584f3ded5e8546fc0acfc47bbb32e382f Mon Sep 17 00:00:00 2001 From: kgridou <32600911+kgridou@users.noreply.github.com> Date: Sat, 6 Jun 2026 02:29:24 +0200 Subject: [PATCH 1/2] Fix: give Jupyter deployment more rollout headroom in CI E2E Tests (Linux) started failing on main after #34 bumped the Spark images (3.5.0 -> 4.0.1). The larger images increase pull contention on the constrained minikube CI node, pushing the (unpinned, ~4GB) jupyter/pyspark-notebook:latest image past Terraform's default 10m rollout wait: Error: Waiting for rollout to finish: 1 replicas wanted; 0 replicas Ready with kubernetes_deployment_v1.jupyter The image was at 9m40s and still pulling when the wait expired, while Prometheus/Grafana each took ~5.5m on the same node, confirming it's slow image-pull, not a real deploy error. Add a 20m create/update timeout and bump progress_deadline_seconds to 1200 on the Jupyter deployment. Durable fix (pinning a smaller image) tracked in #36. Co-Authored-By: Claude Opus 4.8 --- terraform/main.tf | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/terraform/main.tf b/terraform/main.tf index 7434353..b55cef1 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -74,13 +74,23 @@ module "spark" { resource "kubernetes_deployment_v1" "jupyter" { depends_on = [module.spark, module.minio] + # The jupyter/pyspark-notebook image is large (~4GB); on a constrained + # node (e.g. minikube in CI) the initial pull can exceed the provider's + # default 10m rollout wait. Give it more headroom. See issue #36 for the + # durable fix (pinning a smaller image). + timeouts { + create = "20m" + update = "20m" + } + metadata { name = "jupyter" namespace = kubernetes_namespace_v1.ldp.metadata[0].name } spec { - replicas = 1 + replicas = 1 + progress_deadline_seconds = 1200 selector { match_labels = { From 09ed13236a5df167c4140ac9a023f8b12825005f Mon Sep 17 00:00:00 2001 From: kgridou <32600911+kgridou@users.noreply.github.com> Date: Sat, 6 Jun 2026 03:13:14 +0200 Subject: [PATCH 2/2] Harden CI deploy timeouts for heavier post-upgrade images The #34 version bumps (Spark 3.5->4.0.1, Airflow 3.0.2->3.1.7) increased total image bytes pulled onto the single minikube CI node. With more pull contention, whichever deployment loses the image-pull lottery exceeds its timeout - the failure moved between runs (jupyter, airflow, prometheus, grafana) rather than being one broken component: - platform-tests run: airflow still pulling when the 15m step timeout hit - ci run: airflow deployed fine (6m17s) but prometheus/grafana hit the provider's default 10m rollout wait Give the heavier deploys consistent headroom: - prometheus + grafana: timeouts.create/update=20m, progress_deadline=1200 (matching the jupyter fix in the previous commit) - airflow helm_release: timeout 900 -> 1500 (25m) - Deploy step timeout-minutes: platform-tests 15 -> 30, ci 20 -> 30 Durable fix (smaller/pinned images, pre-pull) tracked in #36. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 2 +- .github/workflows/platform-tests.yml | 2 +- .gitignore | 4 +++- terraform/modules/airflow/main.tf | 2 +- terraform/modules/monitoring/main.tf | 20 ++++++++++++++++++-- 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 38720ed..6c80c32 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -339,7 +339,7 @@ jobs: cd terraform terraform init -backend=false terraform apply -var-file=environments/local.tfvars -auto-approve - timeout-minutes: 20 + timeout-minutes: 30 - name: Wait for Airflow database migrations run: | diff --git a/.github/workflows/platform-tests.yml b/.github/workflows/platform-tests.yml index f5b25f1..cbdd2e9 100644 --- a/.github/workflows/platform-tests.yml +++ b/.github/workflows/platform-tests.yml @@ -369,7 +369,7 @@ jobs: run: | cd terraform terraform apply -var-file=environments/local.tfvars -auto-approve - timeout-minutes: 15 + timeout-minutes: 30 - name: Wait for Pods run: | diff --git a/.gitignore b/.gitignore index c41099a..e6ae2c9 100644 --- a/.gitignore +++ b/.gitignore @@ -161,4 +161,6 @@ build/ # Local development local/ -scratch/ \ No newline at end of file +scratch/ +# Claude Code harness state +.claude/ diff --git a/terraform/modules/airflow/main.tf b/terraform/modules/airflow/main.tf index a2c5c89..05fb090 100644 --- a/terraform/modules/airflow/main.tf +++ b/terraform/modules/airflow/main.tf @@ -76,7 +76,7 @@ resource "helm_release" "airflow" { EOT ] - timeout = 900 # Increased for CI environment (15 minutes) + timeout = 1500 # Increased for CI image-pull contention (25 minutes) wait = true wait_for_jobs = true } diff --git a/terraform/modules/monitoring/main.tf b/terraform/modules/monitoring/main.tf index 268c3fc..3a6f205 100644 --- a/terraform/modules/monitoring/main.tf +++ b/terraform/modules/monitoring/main.tf @@ -1,5 +1,12 @@ # Prometheus deployment for metrics collection resource "kubernetes_deployment_v1" "prometheus" { + # On a constrained node (e.g. minikube in CI) image-pull contention can + # push the rollout past the provider's default 10m wait. Give it headroom. + timeouts { + create = "20m" + update = "20m" + } + metadata { name = "prometheus" namespace = var.namespace @@ -9,7 +16,8 @@ resource "kubernetes_deployment_v1" "prometheus" { } spec { - replicas = 1 + replicas = 1 + progress_deadline_seconds = 1200 selector { match_labels = { @@ -194,6 +202,13 @@ resource "kubernetes_config_map_v1" "prometheus_config" { # Grafana deployment for visualization resource "kubernetes_deployment_v1" "grafana" { + # On a constrained node (e.g. minikube in CI) image-pull contention can + # push the rollout past the provider's default 10m wait. Give it headroom. + timeouts { + create = "20m" + update = "20m" + } + metadata { name = "grafana" namespace = var.namespace @@ -203,7 +218,8 @@ resource "kubernetes_deployment_v1" "grafana" { } spec { - replicas = 1 + replicas = 1 + progress_deadline_seconds = 1200 selector { match_labels = {