From f99ce76ada7b011430e9fd37bda0740391d85fcc Mon Sep 17 00:00:00 2001 From: kgridou <32600911+kgridou@users.noreply.github.com> Date: Sat, 6 Jun 2026 01:57:06 +0200 Subject: [PATCH] Fix version drift across Terraform/Docker and remove leftover Hive config The Phase 4-6 upgrade commits bumped the Docker image layer but left the deployed Terraform manifests on the old versions, so the running platform never actually moved to Spark 4.0 / Airflow 3.1. Align everything on one consistent latest set and drop the Hive catalog config left behind after the "Iceberg only" cleanup. Version drift: - Spark master/worker: apache/spark:3.5.0 -> 4.0.1 (matches Docker layer) - Airflow Helm chart: 1.18.0 -> 1.19.0; airflowVersion/defaultAirflowTag 3.0.2 -> 3.1.7 (chart 1.19.0's native appVersion) - Airflow Dockerfile: apache/airflow:3.1.5 -> 3.1.7-python3.13 - Iceberg operator runtime: iceberg-spark-runtime-3.5_2.12 -> 4.0_2.13 (Spark 4.0 uses Scala 2.13) and hadoop-aws 3.3.4 -> 3.4.1 Hive cleanup (Hive was removed in 1143622, Iceberg-only): - Drop spark_catalog=SparkSessionCatalog/type=hive from the Iceberg operator and spark-defaults.conf (needed a Hive metastore that no longer exists); the local Hadoop catalog is unaffected - Remove dead Hive metastore log4j logger filters Verified: terraform fmt -check and terraform validate both pass. Co-Authored-By: Claude Opus 4.8 --- airflow/plugins/custom_operators/iceberg_operator.py | 6 ++---- docker/airflow/Dockerfile | 2 +- docker/airflow/requirements.txt | 2 +- spark/config/log4j.properties | 2 -- spark/config/spark-defaults.conf | 2 -- terraform/modules/airflow/main.tf | 6 +++--- terraform/modules/spark/main.tf | 4 ++-- 7 files changed, 9 insertions(+), 15 deletions(-) diff --git a/airflow/plugins/custom_operators/iceberg_operator.py b/airflow/plugins/custom_operators/iceberg_operator.py index 7310a4a..3dd0d45 100644 --- a/airflow/plugins/custom_operators/iceberg_operator.py +++ b/airflow/plugins/custom_operators/iceberg_operator.py @@ -58,11 +58,9 @@ def __init__( def _get_spark_conf(self) -> Dict[str, str]: """Get Spark configuration for Iceberg.""" return { - 'spark.jars.packages': 'org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.10.0,' - 'org.apache.hadoop:hadoop-aws:3.3.4', + 'spark.jars.packages': 'org.apache.iceberg:iceberg-spark-runtime-4.0_2.13:1.10.0,' + 'org.apache.hadoop:hadoop-aws:3.4.1', 'spark.sql.extensions': 'org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions', - 'spark.sql.catalog.spark_catalog': 'org.apache.iceberg.spark.SparkSessionCatalog', - 'spark.sql.catalog.spark_catalog.type': 'hive', 'spark.sql.catalog.local': 'org.apache.iceberg.spark.SparkCatalog', 'spark.sql.catalog.local.type': 'hadoop', 'spark.sql.catalog.local.warehouse': self.warehouse_path, diff --git a/docker/airflow/Dockerfile b/docker/airflow/Dockerfile index 9d0f3a0..ec6d1eb 100644 --- a/docker/airflow/Dockerfile +++ b/docker/airflow/Dockerfile @@ -1,4 +1,4 @@ -FROM apache/airflow:3.1.5-python3.13 +FROM apache/airflow:3.1.7-python3.13 USER root diff --git a/docker/airflow/requirements.txt b/docker/airflow/requirements.txt index 6e7b1aa..e650be3 100644 --- a/docker/airflow/requirements.txt +++ b/docker/airflow/requirements.txt @@ -1,4 +1,4 @@ -# Airflow providers (Airflow 3.0 compatible) +# Airflow providers (Airflow 3.1 compatible) apache-airflow-providers-apache-spark>=5.0.0 apache-airflow-providers-amazon>=9.0.0 diff --git a/spark/config/log4j.properties b/spark/config/log4j.properties index a47aa88..e09ada9 100644 --- a/spark/config/log4j.properties +++ b/spark/config/log4j.properties @@ -15,5 +15,3 @@ log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO log4j.logger.org.apache.parquet=ERROR log4j.logger.parquet=ERROR -log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL -log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR diff --git a/spark/config/spark-defaults.conf b/spark/config/spark-defaults.conf index 47790b1..7b50065 100644 --- a/spark/config/spark-defaults.conf +++ b/spark/config/spark-defaults.conf @@ -23,8 +23,6 @@ spark.hadoop.fs.s3a.connection.ssl.enabled false # Iceberg Configuration spark.sql.extensions org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions -spark.sql.catalog.spark_catalog org.apache.iceberg.spark.SparkSessionCatalog -spark.sql.catalog.spark_catalog.type hive spark.sql.catalog.local org.apache.iceberg.spark.SparkCatalog spark.sql.catalog.local.type hadoop spark.sql.catalog.local.warehouse s3a://warehouse/ diff --git a/terraform/modules/airflow/main.tf b/terraform/modules/airflow/main.tf index e326a1d..a2c5c89 100644 --- a/terraform/modules/airflow/main.tf +++ b/terraform/modules/airflow/main.tf @@ -3,16 +3,16 @@ resource "helm_release" "airflow" { repository = "https://airflow.apache.org" chart = "airflow" namespace = var.namespace - version = "1.18.0" + version = "1.19.0" values = [ <<-EOT executor: "KubernetesExecutor" - airflowVersion: "3.0.2" + airflowVersion: "3.1.7" defaultAirflowRepository: apache/airflow - defaultAirflowTag: "3.0.2" + defaultAirflowTag: "3.1.7" webserver: service: diff --git a/terraform/modules/spark/main.tf b/terraform/modules/spark/main.tf index 960db49..ffdc9c7 100644 --- a/terraform/modules/spark/main.tf +++ b/terraform/modules/spark/main.tf @@ -33,7 +33,7 @@ resource "kubernetes_stateful_set_v1" "spark_master" { container { name = "spark-master" - image = "apache/spark:3.5.0" + image = "apache/spark:4.0.1" command = ["/opt/spark/bin/spark-class"] args = ["org.apache.spark.deploy.master.Master"] @@ -156,7 +156,7 @@ resource "kubernetes_deployment_v1" "spark_worker" { spec { container { name = "spark-worker" - image = "apache/spark:3.5.0" + image = "apache/spark:4.0.1" command = ["/opt/spark/bin/spark-class"] args = ["org.apache.spark.deploy.worker.Worker", "spark://spark-master:7077"]