From 7f2abe6269c1ec302643a0ef184cf7dc9d404391 Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 23 Aug 2026 21:42:31 -0500 Subject: [PATCH] Migrate production container to Python 3.13 --- .github/copilot-instructions.md | 11 +- .github/workflows/ci.yml | 41 +++++++ constraints/container-python313.txt | 5 + dockerfile | 68 ++++++++---- pytest-local.ini | 1 + scripts/container_smoke.py | 161 ++++++++++++++++++++++++++++ scripts/test-container-local.sh | 26 +++++ tests/test_container_smoke.py | 90 ++++++++++++++++ 8 files changed, 377 insertions(+), 26 deletions(-) create mode 100644 constraints/container-python313.txt create mode 100644 scripts/container_smoke.py create mode 100644 scripts/test-container-local.sh create mode 100644 tests/test_container_smoke.py diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 4cef966e..54fe567a 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -14,13 +14,13 @@ ## Tech Stack - **Python:** 3.11, 3.12, 3.13 (multi-version support) -- **Core Dependencies:** pandas (>=2.0), numpy, polars (1.33.0), pyyaml +- **Core Dependencies:** pandas (>=2.0,<3.0), numpy, polars (1.33.0), pyyaml - **Database Connectors:** sqlalchemy, pymssql, psycopg2-binary, pymysql, pymongo - **Cloud/External:** boto3 (AWS S3), simple-salesforce, fabric (SFTP) - **Data Formats:** openpyxl (Excel), xlsxwriter - **AI/ML:** OpenAI integration, Hugging Face models - **Testing:** pytest (9.0.2), pytest-mock, lorem (test data generation) -- **Containerization:** Production Docker image uses Python 3.11-slim-bookworm; development container uses Python 3.13-bookworm +- **Containerization:** Production Docker image uses Python 3.13-slim-bookworm with pandas 2.3.3 and NumPy 2.4.6; development container uses Python 3.13-bookworm ## Project Structure @@ -127,7 +127,7 @@ wrangles.recipe tests/samples/generate-data.wrgl.yml ### Docker Build The Dockerfile uses multi-stage builds for optimization: -1. Compile stage: Installs build dependencies and packages +1. Dependency stage: Installs binary NumPy and pandas wheels under the tracked production constraint; no compiler toolchain is installed 2. Build stage: Copies only necessary files (~400MB final image) 3. Special optimizations: Removes unused botocore AWS service definitions, pandas test data @@ -280,7 +280,8 @@ to define. - Pytest on Ubuntu + Windows across Python 3.11 + 3.13 for `main` PRs - Test pip installation - Generate and test JSON schema - - Build the Docker image, pushed on merges to `main` under the new policy + - Build the Docker image and, on PRs, run smoke checks, local recipes, and the credential-safe test suite against that exact image + - Push the image on merges to `main` under the new policy - Run container tests, then promote the mutable tag - **deploy-dev.yml** (*Deploy Dev*)**:** manually dispatch from `main`. The workflow still accepts `dev` temporarily; do not use that path for new work. @@ -325,6 +326,8 @@ Performance warnings from pandas are suppressed in `recipe.py` as they appear du ### Docker Image Size Optimization - Botocore data reduced to S3-only (removes ~300MB) - Pandas test data removed from final image +- Production data stack constrained to pandas 2.3.3 and NumPy 2.4.6 while the reusable package continues to allow pandas 2.x +- NumPy and pandas install from binary wheels; compiler tools are absent from the runtime image - Uses slim Debian base image for minimal footprint ## Common Commands diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 68bed975..b3149903 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -195,10 +195,45 @@ jobs: uses: docker/build-push-action@v7 with: context: . + load: ${{ needs.config.outputs.push_image != 'true' }} push: ${{ needs.config.outputs.push_image == 'true' }} tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.config.outputs.build_tag }} labels: ${{ steps.meta.outputs.labels }} + - name: Validate pull request image + if: needs.config.outputs.push_image != 'true' + env: + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.config.outputs.build_tag }} + run: | + docker run --rm \ + --volume "${GITHUB_WORKSPACE}:/workspace:ro" \ + "$IMAGE" \ + python /workspace/scripts/container_smoke.py + docker run --rm "$IMAGE" python -m pip check + docker run --rm "$IMAGE" python -m pip freeze + docker run --rm \ + --volume "${GITHUB_WORKSPACE}:/workspace:ro" \ + --workdir /workspace/tests/samples \ + "$IMAGE" \ + sh -c 'wrangles.recipe recipe-basic.wrgl.yml && wrangles.recipe recipe_custom_function.wrgl.yml -f custom_functions.py' + docker run --rm \ + --volume "${GITHUB_WORKSPACE}:/workspace:ro" \ + "$IMAGE" \ + sh /workspace/scripts/test-container-local.sh /workspace + + - name: Record pull request image size + if: needs.config.outputs.push_image != 'true' + env: + IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ needs.config.outputs.build_tag }} + PREVIOUS_IMAGE: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + run: | + docker image inspect "$IMAGE" --format 'Python 3.13 candidate: {{.Size}} bytes' + if docker pull "$PREVIOUS_IMAGE"; then + docker image inspect "$PREVIOUS_IMAGE" --format 'Current latest: {{.Size}} bytes' + else + echo 'Current latest image is not anonymously readable; record the comparison from an authenticated runner.' + fi + test-container: runs-on: ubuntu-latest if: needs.config.outputs.push_image == 'true' @@ -221,6 +256,12 @@ jobs: - name: Remove wrangles folder run: rm -r wrangles + - name: Validate production runtime + run: | + python scripts/container_smoke.py + python -m pip check + python -m pip freeze + - name: Install Test Dependencies run: | python -m pip install --upgrade pip diff --git a/constraints/container-python313.txt b/constraints/container-python313.txt new file mode 100644 index 00000000..d441ec2c --- /dev/null +++ b/constraints/container-python313.txt @@ -0,0 +1,5 @@ +# Reproducible data-stack baseline for the Python 3.13 production container. +# These exact application/runtime pins do not replace the reusable package +# compatibility ranges declared in requirements.txt. +numpy==2.4.6 +pandas==2.3.3 diff --git a/dockerfile b/dockerfile index 9bf6d006..6fef24b1 100644 --- a/dockerfile +++ b/dockerfile @@ -1,36 +1,60 @@ -FROM python:3.11-slim-bookworm AS compile-image +# syntax=docker/dockerfile:1 + +FROM python:3.13-slim-bookworm AS dependency-image # Copy package COPY . /pkg -# Install compile requirements -RUN apt-get update \ - && apt-get install -y build-essential gcc \ - gfortran python3-dev \ - --no-install-recommends - # Create a virtual env RUN python -m venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" -# Install package + dependencies -RUN pip install --no-cache-dir wheel -# Special install for numpy to reduce size -RUN CFLAGS="-g0 -Wl,--strip-all" pip install --no-cache-dir --compile --global-option=build_ext numpy==1.24.3 -# Regular install (without cache) for everything else -RUN pip install --no-cache-dir /pkg +# Install the package with the production data-stack constraint. NumPy and +# Pandas must resolve to binary wheels; the image no longer carries a compiler. +RUN python -m pip install \ + --no-cache-dir \ + --only-binary=numpy,pandas \ + --constraint /pkg/constraints/container-python313.txt \ + /pkg + +# Retain only the Botocore data used by the S3 connector and remove Pandas test +# data. Resolve installed-package locations instead of embedding a Python minor. +RUN python - <<'PY' +from pathlib import Path +import shutil + +import botocore +import pandas + +botocore_data = Path(botocore.__file__).resolve().parent / "data" +keep = { + "s3", + "_retry.json", + "endpoints.json", + "partitions.json", + "sdk-default-configuration.json", +} +for path in botocore_data.iterdir(): + if path.name in keep: + continue + if path.is_dir(): + shutil.rmtree(path) + else: + path.unlink() -# Botocore contains lots of definitions for all AWS services. We are only using S3. Remove all other files to save space -RUN cd /opt/venv/lib/python3.11/site-packages/botocore/data && cp -r s3 _retry.json endpoints.json partitions.json sdk-default-configuration.json /tmp/ -RUN rm -r /opt/venv/lib/python3.11/site-packages/botocore/data/* -RUN cp -r /tmp/s3 /tmp/_retry.json /tmp/endpoints.json /tmp/partitions.json /tmp/sdk-default-configuration.json /opt/venv/lib/python3.11/site-packages/botocore/data +pandas_tests = Path(pandas.__file__).resolve().parent / "tests" +if pandas_tests.exists(): + shutil.rmtree(pandas_tests) +PY -# Pandas contains a lot of unnecessary test data that we won't use -RUN rm -r /opt/venv/lib/python3.11/site-packages/pandas/tests/* +# Fail the build before the runtime stage if versions, metadata, package-data +# trimming, imports, or the credential-free data/S3 checks are incorrect. +RUN python -m pip check \ + && python /pkg/scripts/container_smoke.py # Create build image -FROM python:3.11-slim-bookworm AS build-image -COPY --from=compile-image /opt/venv /opt/venv +FROM python:3.13-slim-bookworm AS build-image +COPY --from=dependency-image /opt/venv /opt/venv LABEL maintainer="WrangleWorks" ENV PATH="/opt/venv/bin:$PATH" @@ -39,4 +63,4 @@ RUN mkdir /app COPY main.py /app/ WORKDIR /app/ -CMD python main.py \ No newline at end of file +CMD python main.py diff --git a/pytest-local.ini b/pytest-local.ini index 02f160ad..11c64814 100644 --- a/pytest-local.ini +++ b/pytest-local.ini @@ -2,6 +2,7 @@ testpaths = tests/test_ai_cache.py tests/test_ai_definition.py + tests/test_container_smoke.py tests/test_data.py tests/test_dataframe.py tests/test_openai_extract_ai.py diff --git a/scripts/container_smoke.py b/scripts/container_smoke.py new file mode 100644 index 00000000..a9cc9921 --- /dev/null +++ b/scripts/container_smoke.py @@ -0,0 +1,161 @@ +"""Credential-free validation for the production Wrangles container.""" + +from importlib import metadata +from pathlib import Path +import os +import shutil +import sys + + +EXPECTED_PYTHON = (3, 13) +EXPECTED_NUMPY = "2.4.6" +EXPECTED_PANDAS = "2.3.3" +RETAINED_BOTOCORE_DATA = frozenset( + { + "s3", + "_retry.json", + "endpoints.json", + "partitions.json", + "sdk-default-configuration.json", + } +) + + +def _require(condition, message): + if not condition: + raise RuntimeError(message) + + +def validate_runtime_versions(python_version, numpy_version, pandas_version): + """Validate the exact interpreter and constrained data-stack versions.""" + _require( + tuple(python_version[:2]) == EXPECTED_PYTHON, + f"Expected Python 3.13, found {python_version[0]}.{python_version[1]}", + ) + _require( + numpy_version == EXPECTED_NUMPY, + f"Expected NumPy {EXPECTED_NUMPY}, found {numpy_version}", + ) + _require( + pandas_version == EXPECTED_PANDAS, + f"Expected Pandas {EXPECTED_PANDAS}, found {pandas_version}", + ) + _require( + int(pandas_version.split(".", maxsplit=1)[0]) < 3, + f"Pandas 3.x is not supported by this image: {pandas_version}", + ) + + +def validate_wrangles_pandas_requirement(requirements): + """Prove the installed package metadata still excludes Pandas 3.""" + normalized = [requirement.replace(" ", "").lower() for requirement in requirements] + pandas_requirements = [ + requirement + for requirement in normalized + if requirement == "pandas" or requirement.startswith(("pandas<", "pandas>")) + ] + _require( + len(pandas_requirements) == 1, + f"Expected one Pandas package requirement, found {pandas_requirements}", + ) + _require( + "<3.0" in pandas_requirements[0], + f"Wrangles metadata must exclude Pandas 3: {pandas_requirements[0]}", + ) + + +def validate_trimmed_package_data(botocore_data, pandas_package): + """Validate the two package-data reductions used to control image size.""" + actual_botocore_data = {path.name for path in Path(botocore_data).iterdir()} + _require( + actual_botocore_data == RETAINED_BOTOCORE_DATA, + "Unexpected Botocore data after trimming: " + f"expected {sorted(RETAINED_BOTOCORE_DATA)}, " + f"found {sorted(actual_botocore_data)}", + ) + pandas_tests = Path(pandas_package) / "tests" + _require(not pandas_tests.exists(), f"Pandas tests were not removed: {pandas_tests}") + + +def validate_runtime_toolchain(which=shutil.which): + """Ensure compiler tools from the former source build are absent.""" + present = [tool for tool in ("gcc", "gfortran") if which(tool)] + _require(not present, f"Build-only compiler tools found in runtime image: {present}") + + +def validate_data_round_trip(): + """Exercise NumPy, Pandas, and PyArrow together without external data.""" + import numpy + import pandas + import pyarrow + + original = pandas.DataFrame( + { + "id": numpy.array([1, 2], dtype=numpy.int64), + "description": ["alpha", None], + } + ) + table = pyarrow.Table.from_pandas(original, preserve_index=False) + restored = table.to_pandas() + pandas.testing.assert_frame_equal(restored, original) + + +def validate_s3_model(): + """Load the retained S3 model without making a network request.""" + import boto3 + + os.environ.setdefault("AWS_EC2_METADATA_DISABLED", "true") + client = boto3.client( + "s3", + region_name="us-east-1", + aws_access_key_id="container-smoke-test", + aws_secret_access_key="container-smoke-test", + endpoint_url="https://example.invalid", + ) + try: + _require( + client.meta.service_model.service_name == "s3", + "The retained Botocore S3 service model did not load", + ) + finally: + client.close() + + +def main(): + import boto3 + import botocore + import numexpr + import numpy + import pandas + import polars + import pyarrow + import wrangles # noqa: F401 - import itself is part of the smoke check + + validate_runtime_versions(sys.version_info, numpy.__version__, pandas.__version__) + validate_wrangles_pandas_requirement(metadata.requires("wrangles") or ()) + validate_trimmed_package_data( + Path(botocore.__file__).resolve().parent / "data", + Path(pandas.__file__).resolve().parent, + ) + validate_runtime_toolchain() + validate_data_round_trip() + validate_s3_model() + + versions = { + "Python": sys.version.split()[0], + "Wrangles": metadata.version("wrangles"), + "NumPy": numpy.__version__, + "Pandas": pandas.__version__, + "PyArrow": pyarrow.__version__, + "Polars": polars.__version__, + "NumExpr": numexpr.__version__, + "Boto3": boto3.__version__, + "Botocore": botocore.__version__, + } + print("Production container smoke checks passed:") + for name, version in versions.items(): + print(f" {name}={version}") + + +if __name__ == "__main__": + main() diff --git a/scripts/test-container-local.sh b/scripts/test-container-local.sh new file mode 100644 index 00000000..d8d54a79 --- /dev/null +++ b/scripts/test-container-local.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu + +# Run the credential-safe local suite against the package already installed in +# the image. Work from a disposable copy so the checked-out source package +# cannot shadow that installation. +workspace=${1:-/workspace} +test_root=$(mktemp -d /tmp/wrangles-container-tests.XXXXXX) +trap 'rm -rf "$test_root"' EXIT + +cp -R "$workspace"/. "$test_root"/ +rm -rf "$test_root/.git" "$test_root/.test-local" "$test_root/wrangles" + +unset AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN +unset GEMINI_API_KEY HUGGINGFACE_TOKEN OPENAI_API_KEY SERPAPI_API_KEY +unset WRANGLES_PASSWORD WRANGLES_USER + +cd "$test_root" +python -m pip install \ + --no-cache-dir \ + --constraint constraints/container-python313.txt \ + --requirement requirements-full.txt \ + pytest==9.0.2 pytest-mock==3.15.1 +python -m pytest \ + -c pytest-local.ini \ + --basetemp=/tmp/wrangles-container-pytest diff --git a/tests/test_container_smoke.py b/tests/test_container_smoke.py new file mode 100644 index 00000000..19ec9d4b --- /dev/null +++ b/tests/test_container_smoke.py @@ -0,0 +1,90 @@ +from pathlib import Path + +import pytest + +from scripts import container_smoke + + +def test_runtime_versions_accept_migration_baseline(): + container_smoke.validate_runtime_versions((3, 13, 9), "2.4.6", "2.3.3") + + +@pytest.mark.parametrize( + ("python_version", "numpy_version", "pandas_version"), + [ + ((3, 12, 9), "2.4.6", "2.3.3"), + ((3, 13, 9), "2.5.2", "2.3.3"), + ((3, 13, 9), "2.4.6", "3.0.0"), + ], +) +def test_runtime_versions_reject_unapproved_versions( + python_version, + numpy_version, + pandas_version, +): + with pytest.raises(RuntimeError): + container_smoke.validate_runtime_versions( + python_version, + numpy_version, + pandas_version, + ) + + +def test_wrangles_metadata_requires_pandas_below_three(): + container_smoke.validate_wrangles_pandas_requirement( + ["numpy", "pandas<3.0,>=2.0", "requests"] + ) + + +@pytest.mark.parametrize( + "requirement", + ["pandas>=2.0", "pandas>=3.0,<4.0"], +) +def test_wrangles_metadata_rejects_pandas_three(requirement): + with pytest.raises(RuntimeError): + container_smoke.validate_wrangles_pandas_requirement([requirement]) + + +def test_trimmed_package_data_accepts_only_s3(tmp_path): + botocore_data = tmp_path / "botocore-data" + botocore_data.mkdir() + for name in container_smoke.RETAINED_BOTOCORE_DATA: + path = botocore_data / name + if name == "s3": + path.mkdir() + else: + path.touch() + + pandas_package = tmp_path / "pandas" + pandas_package.mkdir() + + container_smoke.validate_trimmed_package_data(botocore_data, pandas_package) + + +def test_trimmed_package_data_rejects_other_services(tmp_path): + botocore_data = tmp_path / "botocore-data" + botocore_data.mkdir() + for name in container_smoke.RETAINED_BOTOCORE_DATA | {"ec2"}: + path = botocore_data / name + if "." in name: + path.touch() + else: + path.mkdir() + + with pytest.raises(RuntimeError): + container_smoke.validate_trimmed_package_data(botocore_data, tmp_path / "pandas") + + +def test_runtime_toolchain_rejects_compiler(): + with pytest.raises(RuntimeError): + container_smoke.validate_runtime_toolchain( + lambda tool: Path("/usr/bin") / tool if tool == "gcc" else None + ) + + +def test_data_round_trip(): + container_smoke.validate_data_round_trip() + + +def test_s3_model_loads_without_network(): + container_smoke.validate_s3_model()