From 28e8755bae6fab1dd5b5bf5121ca837b6caf548c Mon Sep 17 00:00:00 2001 From: Khalil LAGRIDA Date: Fri, 12 Jun 2026 12:22:35 +0200 Subject: [PATCH] feat(warehouse): add Hive as a LEGACY learning lab (Hive -> Iceberg migration) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trino only exposed an Iceberg catalog. Add an opt-in, clearly-legacy Hive lab so you can see the old table format and practise the Hive -> Iceberg migration (a real CDP interview topic) — without diluting the Iceberg-first story: it's off by default and flagged legacy in every file. - quickstart/docker-compose.hive.yml: overlay adding a standalone Hive Metastore (apache/hive:4.0.0, Thrift :9083, embedded Derby) and a `hive` catalog mounted into Trino. Lean by default (make hive = minio + trino + hive-metastore only), so it fits an 11GB laptop. - platform/warehouse/hive/catalog/: the lab's own Trino catalog dir, mounted AS /etc/trino/catalog (Compose dedups by target), carrying hive.properties + a copy of iceberg.properties (the migration target). Swapping the whole dir avoids nesting a file mount inside the base one, which would leave an empty stub in quickstart/trino/etc/catalog and break `make core`. - platform/warehouse/hive/core-site.xml + HADOOP_CLASSPATH: the bundled Hadoop S3A jars + MinIO config the metastore needs to resolve the s3a:// locations it stores (otherwise CREATE SCHEMA fails with ClassNotFound S3AFileSystem). - platform/warehouse/hive/hive-legacy.sh: runnable proof — create a Hive table on S3, query it, migrate it into Iceberg via CTAS, assert row counts and totals match. Verified locally end-to-end. - .github/workflows/hive-ci.yml: CI proof with the same timeout + Docker Hub retry hardening as the other compose workflows. - README + Makefile (make hive) wired in, with legacy warnings throughout. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/hive-ci.yml | 51 +++++++++++ Makefile | 7 +- platform/warehouse/README.md | 7 ++ platform/warehouse/hive/README.md | 57 +++++++++++++ .../warehouse/hive/catalog/hive.properties | 33 ++++++++ .../warehouse/hive/catalog/iceberg.properties | 24 ++++++ platform/warehouse/hive/core-site.xml | 41 +++++++++ platform/warehouse/hive/hive-legacy.sh | 84 +++++++++++++++++++ quickstart/docker-compose.hive.yml | 61 ++++++++++++++ 9 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/hive-ci.yml create mode 100644 platform/warehouse/hive/README.md create mode 100644 platform/warehouse/hive/catalog/hive.properties create mode 100644 platform/warehouse/hive/catalog/iceberg.properties create mode 100644 platform/warehouse/hive/core-site.xml create mode 100755 platform/warehouse/hive/hive-legacy.sh create mode 100644 quickstart/docker-compose.hive.yml diff --git a/.github/workflows/hive-ci.yml b/.github/workflows/hive-ci.yml new file mode 100644 index 0000000..cfedd7c --- /dev/null +++ b/.github/workflows/hive-ci.yml @@ -0,0 +1,51 @@ +name: hive-ci + +# LEGACY lab proof: bring up the Hive Metastore + Trino's `hive` catalog over +# MinIO, create a Hive table, query it, then migrate it into Iceberg and assert +# the row counts + totals match. Hive is kept for learning only — see +# platform/warehouse/hive/README.md. + +on: + push: + branches: [main] + paths: + - "quickstart/docker-compose.yml" + - "quickstart/docker-compose.hive.yml" + - "platform/warehouse/hive/**" + - ".github/workflows/hive-ci.yml" + pull_request: + paths: + - "quickstart/docker-compose.yml" + - "quickstart/docker-compose.hive.yml" + - "platform/warehouse/hive/**" + - ".github/workflows/hive-ci.yml" + workflow_dispatch: + +jobs: + hive-smoke: + runs-on: ubuntu-latest + # Cap the job so a Docker Hub pull stall fails fast, never the 6h default. + timeout-minutes: 20 + defaults: + run: + working-directory: quickstart + steps: + - uses: actions/checkout@v4 + + - name: Create .env from the example + run: test -f .env || cp .env.example .env + + - name: Start the lean Hive slice (MinIO + Trino + Hive Metastore) + run: | + for i in 1 2 3; do if docker compose -f docker-compose.yml -f docker-compose.hive.yml up -d minio minio-init trino hive-metastore; then break; fi; if [ "$i" = 3 ]; then echo "compose up still failing after $i attempts"; exit 1; fi; echo "compose up failed (attempt $i; likely a Docker Hub pull flake); retrying in 15s ..."; sleep 15; done + + - name: Hive legacy proof — query a Hive table, then migrate it to Iceberg + run: bash ../platform/warehouse/hive/hive-legacy.sh + + - name: Dump logs on failure + if: failure() + run: docker compose -f docker-compose.yml -f docker-compose.hive.yml logs --no-color + + - name: Tear down + if: always() + run: docker compose -f docker-compose.yml -f docker-compose.hive.yml down -v diff --git a/Makefile b/Makefile index e321191..132cd29 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ COMPOSE := docker compose --project-directory quickstart \ CORE := docker compose --project-directory quickstart -f quickstart/docker-compose.yml .DEFAULT_GOAL := help -.PHONY: help env core all governance services console opdb jupyterhub knox hue seed down ps logs +.PHONY: help env core all governance services console opdb jupyterhub knox hue hive seed down ps logs help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) \ @@ -63,6 +63,11 @@ hue: env ## Lakehouse + Hue (the web SQL editor over Trino) -f quickstart/docker-compose.yml \ -f quickstart/docker-compose.hue.yml up -d +hive: env ## LEGACY lab — Trino + Hive Metastore over MinIO (the Hive->Iceberg lesson; lean) + docker compose --project-directory quickstart \ + -f quickstart/docker-compose.yml \ + -f quickstart/docker-compose.hive.yml up -d minio minio-init trino hive-metastore + seed: ## Run the Spark seed job into iceberg.smoke.events $(COMPOSE) exec spark spark-submit /home/iceberg/jobs/seed_lakehouse.py diff --git a/platform/warehouse/README.md b/platform/warehouse/README.md index cddd40d..14e693a 100644 --- a/platform/warehouse/README.md +++ b/platform/warehouse/README.md @@ -1,3 +1,10 @@ # warehouse — Trino + Iceberg (≈ CDW) Data Warehouse: Trino querying Iceberg tables (Impala optional). + +- **Modern (default):** Trino over the shared **Iceberg** REST catalog. See the + runnable analytics lab in `run-analytics-sql.sh` and the catalog config in + `quickstart/trino/etc/catalog/iceberg.properties`. +- **Legacy (learning only):** `hive/` — a Hive Metastore + Trino `hive` catalog, + kept *only* to show the old table format and the Hive → Iceberg migration. + Off by default; start with `make hive`. ⚠️ Don't use it for new work. diff --git a/platform/warehouse/hive/README.md b/platform/warehouse/hive/README.md new file mode 100644 index 0000000..b758add --- /dev/null +++ b/platform/warehouse/hive/README.md @@ -0,0 +1,57 @@ +# Hive — the LEGACY table format (learning only) ⚠️ + +> **This is the old world, kept on purpose.** +> The modern lakehouse in this repo is **Iceberg** (one shared REST catalog, +> read by both Trino and Spark). **Hive + the Hive Metastore (HMS) is legacy** — +> in CDP it's the format you *migrate away from*, not what you build new +> workloads on. This lab exists so you can *see* the old path and practise the +> **Hive → Iceberg migration**, which is a real interview topic. Don't reach for +> the `hive` catalog for anything new — use `iceberg`. + +## What you get + +| Piece | Role | +|-------|------| +| `hive-metastore` (`apache/hive:4.0.0`) | The classic **Hive Metastore** on Thrift `:9083`, with an embedded Derby DB. Metadata only — and ephemeral (no volume), fitting for a throwaway demo. | +| Trino catalog `hive` | `platform/warehouse/hive/catalog/hive.properties` — Trino's Hive connector, pointed at HMS for metadata and at **MinIO/S3** (the same bucket as Iceberg) for data. The overlay mounts `catalog/` as Trino's whole catalog dir, so it also carries a copy of `iceberg.properties` (the migration target). | + +Why HMS *and* Iceberg's REST catalog exist side by side: HMS is the pre-Iceberg +metadata service. Iceberg replaced it here with the REST catalog +(`iceberg-rest`). Running both lets you compare them and move data across. + +## Run it + +It's an optional overlay on the base lakehouse — **off by default**, and lean +(only MinIO + Trino + HMS come up, so it fits the laptop): + +```bash +make hive # starts minio, minio-init, trino, hive-metastore +``` + +Then run the proof — it creates a legacy Hive table, queries it from Trino, and +**migrates it into Iceberg**, asserting the row counts and totals match: + +```bash +./platform/warehouse/hive/hive-legacy.sh +``` + +Tear down with `make down` (wipes volumes). + +## The migration, by hand + +```sql +-- legacy: a Hive table, metadata in HMS, data on S3 +CREATE SCHEMA hive.legacy WITH (location = 's3a://warehouse/hive/legacy'); +CREATE TABLE hive.legacy.sales (id bigint, region varchar, amount double) + WITH (format = 'PARQUET', external_location = 's3a://warehouse/hive/legacy/sales'); + +-- modern: lift it into Iceberg (snapshots, time travel, REST catalog) +CREATE TABLE iceberg.migrated.sales AS SELECT * FROM hive.legacy.sales; +``` + +In a real CDP cluster you'd often do an in-place `ALTER TABLE ... CONVERT TO +ICEBERG` / snapshot migration instead of a copy, but the CTAS above is the +clearest way to see the two formats next to each other. + +See the modern path in `quickstart/trino/etc/catalog/iceberg.properties` and the +analytics lab in `platform/warehouse/run-analytics-sql.sh`. diff --git a/platform/warehouse/hive/catalog/hive.properties b/platform/warehouse/hive/catalog/hive.properties new file mode 100644 index 0000000..9c42cff --- /dev/null +++ b/platform/warehouse/hive/catalog/hive.properties @@ -0,0 +1,33 @@ +# ─────────────────────────────────────────────────────────────── +# LEGACY — Trino `hive` catalog, kept ONLY for learning. +# +# The modern lakehouse here is Iceberg (see quickstart/trino/etc/catalog/ +# iceberg.properties). Hive + the Hive Metastore (HMS) is the OLD world: the +# table format you MIGRATE AWAY FROM in CDP. This catalog exists so you can see +# that old path and practise the Hive → Iceberg migration story — not because +# you'd run new workloads on it. +# +# Only loaded when you start the Hive overlay (`make hive`), which mounts this +# file into Trino at /etc/trino/catalog/hive.properties and starts an HMS. +# ─────────────────────────────────────────────────────────────── +connector.name=hive + +# Classic Hive Metastore over Thrift — the metadata service Hive predates +# Iceberg's REST catalog with. Provided by the hive-metastore container. +hive.metastore=thrift +hive.metastore.uri=thrift://hive-metastore:9083 + +# Native S3 filesystem pointed at MinIO — the SAME object store Iceberg uses, so +# Hive data lands next to the lakehouse. HMS only stores metadata; Trino itself +# does all the file IO through this client. +fs.native-s3.enabled=true +s3.endpoint=http://minio:9000 +s3.region=us-east-1 +s3.path-style-access=true +# dev-only creds, kept in sync with quickstart/.env.example. +s3.aws-access-key=admin +s3.aws-secret-key=password + +# Let the lab CREATE/INSERT into tables whose location we pin to the lakehouse +# bucket (external, non-managed writes). +hive.non-managed-table-writes-enabled=true diff --git a/platform/warehouse/hive/catalog/iceberg.properties b/platform/warehouse/hive/catalog/iceberg.properties new file mode 100644 index 0000000..ad81d9f --- /dev/null +++ b/platform/warehouse/hive/catalog/iceberg.properties @@ -0,0 +1,24 @@ +# Modern Iceberg catalog — the migration TARGET for the Hive legacy lab. +# +# A copy of quickstart/trino/etc/catalog/iceberg.properties: the Hive overlay +# mounts THIS directory as Trino's whole /etc/trino/catalog (replacing the base +# mount), so it must carry the iceberg catalog too — that's what the proof's +# `CREATE TABLE iceberg.migrated.sales AS SELECT ...` migration writes into. +# Keep it in sync with the base file; hive-ci queries `iceberg.*`, so drift here +# fails CI. +connector.name=iceberg + +# Use the REST catalog so Trino and Spark see the SAME tables. +iceberg.catalog.type=rest +iceberg.rest-catalog.uri=http://iceberg-rest:8181 +iceberg.rest-catalog.warehouse=s3://warehouse/ + +# Native S3 filesystem pointed at MinIO. +fs.native-s3.enabled=true +s3.endpoint=http://minio:9000 +s3.region=us-east-1 +s3.path-style-access=true +# NOTE: dev-only creds, kept in sync with quickstart/.env.example. +# For anything real, inject via env/secret instead of hardcoding here. +s3.aws-access-key=admin +s3.aws-secret-key=password diff --git a/platform/warehouse/hive/core-site.xml b/platform/warehouse/hive/core-site.xml new file mode 100644 index 0000000..cb9a43b --- /dev/null +++ b/platform/warehouse/hive/core-site.xml @@ -0,0 +1,41 @@ + + + + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3a.endpoint + http://minio:9000 + + + fs.s3a.path.style.access + true + + + fs.s3a.connection.ssl.enabled + false + + + fs.s3a.aws.credentials.provider + org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider + + + fs.s3a.access.key + admin + + + fs.s3a.secret.key + password + + diff --git a/platform/warehouse/hive/hive-legacy.sh b/platform/warehouse/hive/hive-legacy.sh new file mode 100755 index 0000000..0f4f836 --- /dev/null +++ b/platform/warehouse/hive/hive-legacy.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# ─────────────────────────────────────────────────────────────── +# LEGACY lab — prove the old Hive path works, then migrate it to Iceberg. +# +# This is deliberately the OLD world: a Hive table in the Hive Metastore (HMS), +# data on MinIO/S3. We create it, query it from Trino's `hive` catalog, then +# do the one move that matters in CDP — copy it into an Iceberg table and show +# the row counts match. That CTAS is the essence of the Hive → Iceberg +# migration story. +# +# Prereqs: start the overlay first — +# make hive +# +# TRINO how to invoke the Trino CLI +# (default: docker compose -p odp-quickstart exec -T trino trino) +# +# Usage: ./platform/warehouse/hive/hive-legacy.sh +# ─────────────────────────────────────────────────────────────── +set -euo pipefail + +TRINO=${TRINO:-docker compose -p odp-quickstart exec -T trino trino} + +# Run a single query as `admin` and strip Trino's quoting/whitespace. +q() { $TRINO --user admin --execute "$1" | tr -d '"[:space:]'; } + +# HMS can take a moment to open its Thrift port after the container starts. +echo "[hive] waiting for the Hive Metastore (catalog \`hive\`) to answer ..." +for i in $(seq 1 30); do + if $TRINO --user admin --execute "SHOW SCHEMAS FROM hive" >/dev/null 2>&1; then + break + fi + if [ "$i" = 30 ]; then + echo "[hive] metastore never came up — is \`make hive\` running?" >&2 + exit 1 + fi + sleep 5 +done + +echo "[hive] step 1 — create a LEGACY Hive table (HMS metadata, data on MinIO/S3)" +$TRINO --user admin <<'SQL' +CREATE SCHEMA IF NOT EXISTS hive.legacy + WITH (location = 's3a://warehouse/hive/legacy'); +DROP TABLE IF EXISTS hive.legacy.sales; +CREATE TABLE hive.legacy.sales ( + id bigint, region varchar, product varchar, amount double +) +WITH (format = 'PARQUET', external_location = 's3a://warehouse/hive/legacy/sales'); +INSERT INTO hive.legacy.sales VALUES + (1,'EU','widget', 19.99), + (2,'EU','gadget', 49.50), + (3,'US','widget', 21.00), + (4,'US','gizmo', 120.00), + (5,'APAC','gadget', 46.00); +SQL + +echo "[hive] step 2 — query the Hive table from Trino" +ROWS=$(q "SELECT count(*) FROM hive.legacy.sales") +echo " hive rows = ${ROWS}"; test "${ROWS}" = "5" + +echo "[hive] step 3 — confirm it's really a HIVE table, not Iceberg" +KIND=$(q "SELECT table_type FROM hive.information_schema.tables + WHERE table_schema = 'legacy' AND table_name = 'sales'") +echo " table_type = ${KIND}" # informational; HMS reports a Hive table type + +echo "[hive] step 4 — MIGRATE: copy the Hive table into Iceberg (CTAS)" +# This is the move you'd make in CDP: lift legacy Hive data into the modern +# Iceberg format, where you get snapshots, time travel, and the REST catalog. +$TRINO --user admin <<'SQL' +CREATE SCHEMA IF NOT EXISTS iceberg.migrated; +DROP TABLE IF EXISTS iceberg.migrated.sales; +CREATE TABLE iceberg.migrated.sales AS + SELECT * FROM hive.legacy.sales; +SQL + +echo "[hive] step 5 — verify the migrated Iceberg table matches row-for-row" +ICE=$(q "SELECT count(*) FROM iceberg.migrated.sales") +SUM_HIVE=$(q "SELECT cast(round(sum(amount),2) as varchar) FROM hive.legacy.sales") +SUM_ICE=$(q "SELECT cast(round(sum(amount),2) as varchar) FROM iceberg.migrated.sales") +echo " iceberg rows = ${ICE} (hive ${ROWS}) amount: hive=${SUM_HIVE} iceberg=${SUM_ICE}" +test "${ICE}" = "${ROWS}" +test "${SUM_ICE}" = "${SUM_HIVE}" + +echo "[hive] OK — legacy Hive table queried, then migrated to Iceberg (counts + totals match)." +echo "[hive] takeaway: Hive still works behind Trino, but Iceberg is where new work goes." diff --git a/quickstart/docker-compose.hive.yml b/quickstart/docker-compose.hive.yml new file mode 100644 index 0000000..c4a8a0e --- /dev/null +++ b/quickstart/docker-compose.hive.yml @@ -0,0 +1,61 @@ +# ─────────────────────────────────────────────────────────────── +# LEGACY lab — Apache Hive Metastore + Trino's `hive` connector. +# +# This is the OLD world on purpose. CDP's (and this repo's) modern table format +# is Iceberg; Hive tables are what you migrate away from. We keep a tiny Hive +# Metastore (HMS) around only so you can create a Hive table, query it from +# Trino, and then practise the Hive → Iceberg migration. See +# platform/warehouse/hive/README.md. +# +# Overlay it on the base lakehouse (it reuses MinIO + Trino): +# +# docker compose --project-directory quickstart \ +# -f quickstart/docker-compose.yml \ +# -f quickstart/docker-compose.hive.yml \ +# up -d minio minio-init trino hive-metastore +# # or, lean by default: make hive +# +# HMS (Thrift) : thrift://localhost:9083 +# Trino : http://localhost:8080 → catalog `hive` +# +# Run the proof: ./platform/warehouse/hive/hive-legacy.sh +# ─────────────────────────────────────────────────────────────── + +name: odp-quickstart + +services: + # The classic Hive Metastore, standalone, with the bundled embedded Derby DB. + # Single container, metadata is ephemeral (no volume) — fitting for a + # throwaway legacy demo. The official Apache image runs HMS with + # SERVICE_NAME=metastore and self-initialises the Derby schema. + hive-metastore: + image: apache/hive:4.0.0 + container_name: odp-hive-metastore + environment: + SERVICE_NAME: metastore + # Put the (already-bundled) Hadoop S3A jars on the metastore classpath so + # it can resolve the s3a:// locations it stores for Hive tables on MinIO. + HADOOP_CLASSPATH: /opt/hadoop/share/hadoop/tools/lib/hadoop-aws-3.3.6.jar:/opt/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-1.12.367.jar + volumes: + # S3A → MinIO config for the metastore (see the file's header). + - ../platform/warehouse/hive/core-site.xml:/opt/hive/conf/core-site.xml:ro + ports: + - "9083:9083" + networks: + - odp + + # Give Trino both catalogs by REPLACING its catalog dir with the lab's own + # (Compose dedups the /etc/trino/catalog mount by target, so this overrides the + # base). The lab dir carries `hive.properties` + a copy of `iceberg.properties` + # (the migration target). We swap the whole dir rather than nest a single file + # mount inside the base one — nesting needs a writable parent and leaves an + # empty stub file behind in quickstart/trino/etc/catalog on the host. + trino: + depends_on: + - hive-metastore + volumes: + - ../platform/warehouse/hive/catalog:/etc/trino/catalog:ro + +networks: + odp: + name: odp