From ea2fd34df3991fe7563b6dc95221ffca85753945 Mon Sep 17 00:00:00 2001 From: xraymemory Date: Fri, 14 Aug 2026 12:36:04 -0400 Subject: [PATCH 1/3] fix(docker): pin the build's pixi and diagnose empty environments Every input to this image is pinned except the tool that builds it: the base image and the checkpoints are digests, CI's setup-pixi is v0.73.0, and the Dockerfile piped an unpinned install.sh. So two builds of the same commit could use different pixi versions, which is a bad property to have at the best of times and a worse one while the build is failing because pixi reports a successful install and leaves an empty prefix. The guard from #368 now prints what pixi actually left behind before it exits: the prefix listing, its conda-meta bookkeeping, and where pixi thinks it is installing. The empty install does not reproduce off this builder. Same manifest, same lock, same pixi, linux/amd64, cold and warm rattler cache: all five environments install correctly. The build log is the only place the cause can come from. --- Dockerfile | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index d19ca2d2..08350e7a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean -RUN curl -fsSL https://pixi.sh/install.sh | bash +# Pinned, because everything else about this build is: the base image is a +# digest, the checkpoints are a digest, and CI's setup-pixi is v0.73.0. An +# unpinned `curl | bash` made the build tool the one component that could change +# under us between two builds of the same commit, which is a bad property to +# have while chasing a build that installs nothing (see the guard below). Keep +# in step with `pixi-version` in .github/workflows/ci.yml. +ARG PIXI_VERSION=v0.73.0 +RUN curl -fsSL https://pixi.sh/install.sh | PIXI_VERSION="${PIXI_VERSION}" bash WORKDIR /app @@ -148,8 +155,17 @@ RUN --mount=type=bind,from=checkpoints,target=/ck \ # disagree with a prefix restored from a different build. The rattler and uv # caches stay: they hold downloaded packages and wheels, are what actually make # rebuilds fast, and were verified not to affect what lands in the layer. +# +# The guard prints diagnostics before it exits. The empty-environment install +# has not been reproducible outside this builder: the same manifest, lock and +# pixi version install all five environments correctly on linux/amd64 with both +# a cold and a warm rattler cache. So when it happens here, the build log is the +# only place the cause can come from, and "pixi said installed, nothing is +# there" is not enough to act on. RUN --mount=type=cache,target=/root/.cache/rattler \ --mount=type=cache,target=/root/.cache/uv \ + pixi --version && \ + pixi info && \ rm -rf /app/.pixi/envs && \ pixi install -e boltz --frozen && \ pixi install -e protenix --frozen && \ @@ -160,6 +176,15 @@ RUN --mount=type=cache,target=/root/.cache/rattler \ test -x "/app/.pixi/envs/${env}/bin/python" || { \ echo "FATAL: pixi environment '${env}' has no interpreter at /app/.pixi/envs/${env}/bin/python."; \ echo " pixi reported success but installed nothing — refusing to ship an empty environment."; \ + echo "--- what pixi left behind ---"; \ + ls -la "/app/.pixi/envs/${env}" 2>&1 | head -20; \ + echo "--- prefix bookkeeping (conda-meta) ---"; \ + ls -A "/app/.pixi/envs/${env}/conda-meta" 2>&1 | head -10; \ + head -c 400 "/app/.pixi/envs/${env}/conda-meta/pixi" 2>&1; echo; \ + echo "--- environments pixi thinks exist ---"; \ + ls -A /app/.pixi/envs 2>&1 | head; \ + echo "--- where pixi is installing to ---"; \ + pixi info 2>&1 | grep -iE "cache dir|environments|manifest|version" | head; \ exit 1; \ }; \ done From 4d9059720da3218bdb974853f59b468eb5f6e02a Mon Sep 17 00:00:00 2001 From: xraymemory Date: Fri, 14 Aug 2026 13:07:50 -0400 Subject: [PATCH 2/3] fix(docker): move the build off the builder's inherited pixi caches The empty-environment install is specific to diffuse-sh-builder. The same step, from the same base image with the same cache mounts and pixi 0.76.2, installs all five environments correctly on astera-sh-builder with both a cold and a warm cache (Astera-org/docker-images#25), and does the same on a laptop. Cache mounts live in the builder's own state. They survive --no-cache, a fresh checkout and a new builder container, so nothing in this repository could get the build off them. Giving them an explicit id can: BuildKit treats a new id as a new directory, so the next build starts from empty caches without anyone having to prune that host. --- Dockerfile | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 08350e7a..7ca47687 100644 --- a/Dockerfile +++ b/Dockerfile @@ -156,14 +156,23 @@ RUN --mount=type=bind,from=checkpoints,target=/ck \ # caches stay: they hold downloaded packages and wheels, are what actually make # rebuilds fast, and were verified not to affect what lands in the layer. # +# The cache mounts carry an explicit id so this build can be moved off the +# contents it inherited. Cache mounts live in the builder's own state and +# survive `--no-cache`, a fresh checkout and a new BuildKit builder container, +# so a poisoned one is invisible from the repository and unfixable from it. The +# same step, from the same base image with the same mounts, installs all five +# environments correctly on astera-sh-builder with pixi 0.76.2, cold and warm +# (Astera-org/docker-images#25); it is only diffuse-sh-builder that produces +# empty prefixes. Bump the suffix if that ever recurs. +# # The guard prints diagnostics before it exits. The empty-environment install # has not been reproducible outside this builder: the same manifest, lock and # pixi version install all five environments correctly on linux/amd64 with both # a cold and a warm rattler cache. So when it happens here, the build log is the # only place the cause can come from, and "pixi said installed, nothing is # there" is not enough to act on. -RUN --mount=type=cache,target=/root/.cache/rattler \ - --mount=type=cache,target=/root/.cache/uv \ +RUN --mount=type=cache,id=sampleworks-rattler-2,target=/root/.cache/rattler \ + --mount=type=cache,id=sampleworks-uv-2,target=/root/.cache/uv \ pixi --version && \ pixi info && \ rm -rf /app/.pixi/envs && \ From cf483a0e46853f5c7990f7247bc6176c80d770f2 Mon Sep 17 00:00:00 2001 From: xraymemory Date: Fri, 14 Aug 2026 14:37:19 -0400 Subject: [PATCH 3/3] fix(docker): force pixi environment materialization --- Dockerfile | 60 ++++++++++++++++++++++++------------------------------ 1 file changed, 27 insertions(+), 33 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7ca47687..d96a29f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -135,20 +135,13 @@ RUN --mount=type=bind,from=checkpoints,target=/ck \ # Docker layers duplicates shared conda packages (numpy, CUDA libs, etc.) and can # add tens of GB to the image. # -# The `rm -rf` and the assertion below are both load-bearing. Published images -# have shipped with all five environments as empty shells: the directories and -# pixi's own bookkeeping in conda-meta present, but no package records, no bin/, -# no lib/. `pixi install --frozen` then reports "The environment has been -# installed" against them and exits 0, so the breakage is invisible at build -# time and only surfaces when a scientist finds /app/.pixi/envs/protenix has no -# python. `SAMPLEWORKS_REQUIRE_PREBUILT_PIXI=1` makes the runner refuse to fall -# back, so the image promises environments it does not carry. -# -# Clearing the prefix first forces a genuine install even when a stale or stub -# prefix arrives from a cached layer, the base image, or the registry -# buildcache. The per-environment check then makes an empty env fail the build -# instead of shipping: a directory that exists but has no interpreter is exactly -# the state that got published, and `pixi install` alone does not catch it. +# Published images have shipped all five environments as empty shells: +# conda-meta/pixi exists, but there are no package records, bin/, or lib/. +# `pixi install --frozen` reports success against that state without running a +# package transaction. `pixi reinstall` is the supported command that bypasses +# that up-to-date decision and materializes every package again. It also creates +# a missing prefix, so clearing prefixes first covers both stale layers and the +# empty-prefix failure mode seen on diffuse-sh-builder. # # `/root/.cache/pixi` is deliberately not cached across builds — it carries # pixi's own "is this environment current" state, which is the thing that can @@ -156,14 +149,10 @@ RUN --mount=type=bind,from=checkpoints,target=/ck \ # caches stay: they hold downloaded packages and wheels, are what actually make # rebuilds fast, and were verified not to affect what lands in the layer. # -# The cache mounts carry an explicit id so this build can be moved off the -# contents it inherited. Cache mounts live in the builder's own state and -# survive `--no-cache`, a fresh checkout and a new BuildKit builder container, -# so a poisoned one is invisible from the repository and unfixable from it. The -# same step, from the same base image with the same mounts, installs all five -# environments correctly on astera-sh-builder with pixi 0.76.2, cold and warm -# (Astera-org/docker-images#25); it is only diffuse-sh-builder that produces -# empty prefixes. Bump the suffix if that ever recurs. +# The cache mounts have project-specific ids so they cannot alias mounts from +# unrelated Dockerfiles when a builder retains state. `sharing=locked` prevents +# concurrent steps on the same builder from mutating a package cache together. +# Bump the suffix only to recover from a demonstrably corrupt persistent cache. # # The guard prints diagnostics before it exits. The empty-environment install # has not been reproducible outside this builder: the same manifest, lock and @@ -171,20 +160,21 @@ RUN --mount=type=bind,from=checkpoints,target=/ck \ # a cold and a warm rattler cache. So when it happens here, the build log is the # only place the cause can come from, and "pixi said installed, nothing is # there" is not enough to act on. -RUN --mount=type=cache,id=sampleworks-rattler-2,target=/root/.cache/rattler \ - --mount=type=cache,id=sampleworks-uv-2,target=/root/.cache/uv \ +RUN --mount=type=cache,id=sampleworks-rattler-2,target=/root/.cache/rattler,sharing=locked \ + --mount=type=cache,id=sampleworks-uv-2,target=/root/.cache/uv,sharing=locked \ pixi --version && \ - pixi info && \ + pixi info --no-config && \ rm -rf /app/.pixi/envs && \ - pixi install -e boltz --frozen && \ - pixi install -e protenix --frozen && \ - pixi install -e rf3 --frozen && \ - pixi install -e protpardelle --frozen && \ - pixi install -e analysis --frozen && \ for env in boltz protenix rf3 protpardelle analysis; do \ - test -x "/app/.pixi/envs/${env}/bin/python" || { \ + echo "=== reinstalling ${env} ==="; \ + pixi reinstall -e "${env}" --frozen --no-config || { \ + echo "FATAL: pixi failed while reinstalling environment '${env}'."; \ + exit 1; \ + }; \ + python="/app/.pixi/envs/${env}/bin/python"; \ + test -x "${python}" || { \ echo "FATAL: pixi environment '${env}' has no interpreter at /app/.pixi/envs/${env}/bin/python."; \ - echo " pixi reported success but installed nothing — refusing to ship an empty environment."; \ + echo " pixi reinstall reported success but materialized no packages."; \ echo "--- what pixi left behind ---"; \ ls -la "/app/.pixi/envs/${env}" 2>&1 | head -20; \ echo "--- prefix bookkeeping (conda-meta) ---"; \ @@ -193,7 +183,11 @@ RUN --mount=type=cache,id=sampleworks-rattler-2,target=/root/.cache/rattler \ echo "--- environments pixi thinks exist ---"; \ ls -A /app/.pixi/envs 2>&1 | head; \ echo "--- where pixi is installing to ---"; \ - pixi info 2>&1 | grep -iE "cache dir|environments|manifest|version" | head; \ + pixi info --no-config 2>&1 | grep -iE "cache dir|environments|manifest|version" | head; \ + exit 1; \ + }; \ + "${python}" --version || { \ + echo "FATAL: environment '${env}' has an unusable Python interpreter."; \ exit 1; \ }; \ done