From f33fc76482b53c2f278da1a4e16bb8a1861bab7a Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Sun, 2 Aug 2026 18:46:14 -0500 Subject: [PATCH 1/2] feat: publish signed image provenance manifests (#896) Feedback from James is that the purpose of this PR was not clear, and that the summary was pretty heavy on jargon. The jargon did not phase me, presumably because I've been swimming in this stuff for longer/deeper. The purpose of this PR is to surface image contents in better ways. As part of our release process, the images that this repo produces get scanned with various tools to find CVEs and malware. These tools are prone to misidentifying some components, and that has been especially true for conda packages historically. We have frequently needed to override the detected package lists/SBOM via JIRA tickets with our security tools team. Part of filing those tickets is providing them with the actual contents of the images, which we often have to discover by looking into build logs. That's where this PR comes in. Instead of looking at build logs, we should export our known environment information to a more readily usable location. In theory, this could replace the scans, but I think it's better to keep doing the scans and use our manifests as validation/correction where necessary. That keeps us in line with how security tools team manages other projects. **Posted by Codex (GPT-5.6) on behalf of Michael Sarahan. Treat this description as LLM-generated.** ## Summary Add a signed, OCI-referrer RAPIDS image provenance manifest for published `base` and `notebooks` image digests. - Export exact `conda-meta` package facts through minimal BuildKit targets rather than downloading image layers. - Attach a platform-specific manifest to every pushed architecture image, then attach a multiarch index manifest that links those platform records. - Record the image digest, platform, source SHA, workflow URL, build arguments, and conda package name/version/build/channel/source URL. - Keylessly sign each attached OCI artifact with GitHub Actions OIDC and Cosign. - Keep pURLs explicitly unmapped until a verified upstream mapping is available; this intentionally does not invent `pkg:conda/...` identifiers. ## Relationship to #805 https://github.com/rapidsai/docker/pull/805 is complementary. It generates and embeds a CycloneDX SBOM in the final image. This change makes digest-bound build facts discoverable as registry metadata and supplies the platform-to-multiarch relationship needed by scan triage without pulling image layers. If #805 lands, the preferred follow-up is to add its SBOM digest/reference to this manifest rather than maintain a separate provenance model. Its Dockerfile stage renaming will require a small conflict resolution in this branch's `provenance-base` and `provenance-notebooks` export targets. --- .github/workflows/build-rapids-image.yml | 60 +++- .../workflows/build-test-publish-images.yml | 77 ++++- .github/workflows/pr.yml | 4 +- .../trigger-breaking-change-alert.yaml | 2 +- .gitignore | 2 + Dockerfile | 35 ++- README.md | 23 ++ ci/compute-build-args.sh | 3 +- ci/compute-matrix.sh | 14 +- ci/export-image-provenance.sh | 27 ++ ci/image_provenance_manifest.py | 264 ++++++++++++++++++ ci/publish-image-provenance-index.sh | 67 +++++ ci/publish-image-provenance.sh | 63 +++++ ci/release/update-version.sh | 12 +- context/.dockerignore | 1 + context/notebooks.sh | 6 +- context/scripts/export-pip-package-inventory | 42 +++ cuvs-bench/cpu/Dockerfile | 6 +- cuvs-bench/gpu/Dockerfile | 6 +- requirements-test.txt | 1 + tests/test_image_provenance_manifest.py | 174 ++++++++++++ 21 files changed, 859 insertions(+), 30 deletions(-) create mode 100755 ci/export-image-provenance.sh create mode 100755 ci/image_provenance_manifest.py create mode 100755 ci/publish-image-provenance-index.sh create mode 100755 ci/publish-image-provenance.sh create mode 100644 context/scripts/export-pip-package-inventory create mode 100644 requirements-test.txt create mode 100644 tests/test_image_provenance_manifest.py diff --git a/.github/workflows/build-rapids-image.yml b/.github/workflows/build-rapids-image.yml index 02947e37..af03aad9 100644 --- a/.github/workflows/build-rapids-image.yml +++ b/.github/workflows/build-rapids-image.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2025-2026, NVIDIA CORPORATION. +# Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. name: build-rapids on: @@ -25,6 +25,9 @@ on: RAPIDS_VER: required: true type: string + RAPIDS_NOTEBOOKS_REF: + required: true + type: string BASE_TAG: required: true type: string @@ -32,6 +35,10 @@ on: required: true type: string +permissions: + contents: read + id-token: write + env: # prevent buildx from creating arch-specific manifest lists... # we want single images, that are later referenced together in a multiarch manifest @@ -75,6 +82,18 @@ jobs: with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version: '1.25.x' + cache: false + - name: Install ORAS + run: | + GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0 + echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}" + "${RUNNER_TEMP}/oras-bin/oras" version + - name: Set up Cosign + uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3 - name: Set up Docker Context for Buildx id: buildx-context run: | @@ -99,8 +118,10 @@ jobs: LINUX_DISTRO_VER: ${{ inputs.LINUX_DISTRO_VER }} LINUX_VER: ${{ inputs.LINUX_VER }} PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_NOTEBOOKS_REF: ${{ inputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ inputs.RAPIDS_VER }} - name: Build base image + id: build-base uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: context @@ -113,7 +134,26 @@ jobs: tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true + - name: Export base image package provenance + env: + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }} + PROVENANCE_TARGET: provenance-base + run: ci/export-image-provenance.sh + - name: Publish base image provenance + env: + CUDA_VER: ${{ inputs.CUDA_VER }} + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + IMAGE_DIGEST: ${{ steps.build-base.outputs.digest }} + IMAGE_KIND: base + IMAGE_PLATFORM: linux/${{ matrix.ARCH }} + IMAGE_REFERENCE: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }} + PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_VER: ${{ inputs.RAPIDS_VER }} + run: ci/publish-image-provenance.sh - name: Build notebooks image + id: build-notebooks uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: context @@ -126,3 +166,21 @@ jobs: tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true + - name: Export notebooks image package provenance + env: + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }} + PROVENANCE_TARGET: provenance-notebooks + run: ci/export-image-provenance.sh + - name: Publish notebooks image provenance + env: + CUDA_VER: ${{ inputs.CUDA_VER }} + DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} + IMAGE_DIGEST: ${{ steps.build-notebooks.outputs.digest }} + IMAGE_KIND: notebooks + IMAGE_PLATFORM: linux/${{ matrix.ARCH }} + IMAGE_REFERENCE: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }} + PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }} + PYTHON_VER: ${{ inputs.PYTHON_VER }} + RAPIDS_VER: ${{ inputs.RAPIDS_VER }} + run: ci/publish-image-provenance.sh diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml index 270837de..990a4035 100644 --- a/.github/workflows/build-test-publish-images.yml +++ b/.github/workflows/build-test-publish-images.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2026, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. name: build and publish imgs workflow on: @@ -43,7 +43,7 @@ jobs: - build-cuvs-multiarch-manifest - test secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.08 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main # zizmor: ignore[unpinned-uses] checks: runs-on: ubuntu-latest steps: @@ -55,6 +55,10 @@ jobs: - name: Run hadolint run: | ci/lint-dockerfiles.sh + - name: Test image provenance manifest generator + run: | + python3 -m pip install --disable-pip-version-check -r requirements-test.txt + python3 -m pytest -q tests compute-matrix: runs-on: ubuntu-latest outputs: @@ -68,6 +72,7 @@ jobs: NOTEBOOKS_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.NOTEBOOKS_TAG_PREFIX }} CUVS_BENCH_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_TAG_PREFIX }} CUVS_BENCH_CPU_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_CPU_TAG_PREFIX }} + RAPIDS_NOTEBOOKS_REF: ${{ steps.compute-rapids-ver.outputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ steps.compute-rapids-ver.outputs.RAPIDS_VER }} ALPHA_TAG: ${{ steps.compute-rapids-ver.outputs.ALPHA_TAG }} steps: @@ -78,7 +83,15 @@ jobs: persist-credentials: false - name: Compute matrix id: compute-matrix + env: + BUILD_TYPE: ${{ inputs.build_type }} run: | + # PR builds temporarily use published 26.08 packages. Their + # aarch64 RAPIDS metapackage requires Python >=3.12, but still + # supports both CUDA 12 and CUDA 13 package lines. + if [[ $BUILD_TYPE == "pull-request" ]]; then + export MATRIX_PYTHON_VERSIONS='["3.12", "3.13", "3.14"]' + fi MATRIX=$(ci/compute-matrix.sh) echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT} - name: Compute tag prefix @@ -119,6 +132,8 @@ jobs: echo "CUVS_BENCH_CPU_IMAGE_REPO=${cuvs_bench_cpu_repo}" | tee -a ${GITHUB_OUTPUT} - name: Compute RAPIDS_VER id: compute-rapids-ver + env: + BUILD_TYPE: ${{ inputs.build_type }} run: | GIT_DESCRIBE_TAG="$(git describe --tags --first-parent --abbrev=0)" GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v' @@ -128,7 +143,20 @@ jobs: ALPHA_TAG="a" fi RAPIDS_VER="$(echo $GIT_DESCRIBE_TAG | awk 'BEGIN{FS=OFS="."} NF--')" # Convert full tag to YY.MM + RAPIDS_NOTEBOOKS_REF="main" + if [[ ! $GIT_DESCRIBE_TAG =~ [a-z] ]]; then + RAPIDS_NOTEBOOKS_REF="release/${RAPIDS_VER}" + fi + + # The source branch advances before its packages are published. Keep + # pull-request images on a matching published package and source line. + if [[ $BUILD_TYPE == "pull-request" ]]; then + RAPIDS_VER="26.08" + RAPIDS_NOTEBOOKS_REF="release/26.08" + ALPHA_TAG="a" + fi + echo "RAPIDS_NOTEBOOKS_REF=${RAPIDS_NOTEBOOKS_REF}" | tee -a ${GITHUB_OUTPUT} echo "RAPIDS_VER=${RAPIDS_VER}" | tee -a ${GITHUB_OUTPUT} echo "ALPHA_TAG=${ALPHA_TAG}" | tee -a ${GITHUB_OUTPUT} - name: Compute test matrix @@ -159,6 +187,7 @@ jobs: LINUX_DISTRO_VER: ${{ matrix.LINUX_DISTRO_VER }} LINUX_VER: ${{ matrix.LINUX_VER }} PYTHON_VER: ${{ matrix.PYTHON_VER }} + RAPIDS_NOTEBOOKS_REF: ${{ needs.compute-matrix.outputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }} BASE_TAG: "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\ @@ -224,6 +253,18 @@ jobs: with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} + - name: Install Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 + with: + go-version: '1.25.x' + cache: false + - name: Install ORAS + run: | + GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0 + echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}" + "${RUNNER_TEMP}/oras-bin/oras" version + - name: Set up Cosign + uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3 - name: Create multiarch manifest shell: bash env: @@ -239,6 +280,38 @@ jobs: GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} ARCHES: ${{ toJSON(matrix.ARCHES) }} run: ci/create-rapids-multiarch-manifest.sh + - name: Publish multiarch image provenance indexes + shell: bash + env: + BASE_IMAGE_REPO: ${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }} + BASE_TAG_PREFIX: ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }} + RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }} + ALPHA_TAG: ${{ needs.compute-matrix.outputs.ALPHA_TAG }} + CUDA_TAG: ${{ matrix.CUDA_TAG }} + PYTHON_VER: ${{ matrix.PYTHON_VER }} + NOTEBOOKS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }} + NOTEBOOKS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }} + ARCHES: ${{ toJSON(matrix.ARCHES) }} + run: | + base_reference="rapidsai/${BASE_IMAGE_REPO}:${BASE_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}" + notebooks_reference="rapidsai/${NOTEBOOKS_IMAGE_REPO}:${NOTEBOOKS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}" + for image_kind in base notebooks; do + if [[ "$image_kind" == base ]]; then + image_reference="$base_reference" + else + image_reference="$notebooks_reference" + fi + platform_references="$({ + while IFS= read -r arch; do + printf 'linux/%s=%s-%s\n' "$arch" "$image_reference" "$arch" + done < <(jq -r '.[]' <<< "$ARCHES") + })" + IMAGE_KIND="$image_kind" \ + IMAGE_REFERENCE="$image_reference" \ + PLATFORM_REFERENCES="$platform_references" \ + CUDA_VER="$CUDA_TAG" \ + ci/publish-image-provenance-index.sh + done build-cuvs-multiarch-manifest: needs: [build-cuvs, compute-matrix] strategy: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 97ce5de4..128e37b3 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2026, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. name: ci on: @@ -19,7 +19,7 @@ jobs: permissions: checks: write pull-requests: write - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.08 + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main # zizmor: ignore[unpinned-uses] if: always() with: needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/trigger-breaking-change-alert.yaml b/.github/workflows/trigger-breaking-change-alert.yaml index 96caa74d..d7752b9e 100644 --- a/.github/workflows/trigger-breaking-change-alert.yaml +++ b/.github/workflows/trigger-breaking-change-alert.yaml @@ -20,7 +20,7 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'breaking') permissions: contents: read - uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@release/26.08 + uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@main secrets: slack-webhook-url: ${{ secrets.NV_SLACK_BREAKING_CHANGE_NOTIFIER_APP }} with: diff --git a/.gitignore b/.gitignore index f99d452a..2d5c5c06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .vscode +__pycache__/ +*.py[cod] *.a *.bz2 diff --git a/Dockerfile b/Dockerfile index 808a7b26..5a7338d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -# Copyright (c) 2024-2026, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. ARG CUDA_VER=notset ARG LINUX_DISTRO=ubuntu @@ -7,15 +7,15 @@ ARG LINUX_DISTRO_VER=22.04 ARG LINUX_VER=${LINUX_DISTRO}${LINUX_DISTRO_VER} ARG MINIFORGE_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.08 +ARG RAPIDS_VER=26.10 # Gather dependency information FROM python:${PYTHON_VER} AS dependencies ARG CPU_ARCH=notset ARG CUDA_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_BRANCH="release/26.08" -ARG RAPIDS_VER=26.08 +ARG RAPIDS_NOTEBOOKS_REF="main" +ARG RAPIDS_VER=26.10 ARG YQ_VER=notset SHELL ["/bin/bash", "-euo", "pipefail", "-c"] @@ -122,7 +122,7 @@ EOF FROM miniforge-cuda AS base ARG CUDA_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.08 +ARG RAPIDS_VER=26.10 SHELL ["/bin/bash", "-euo", "pipefail", "-c"] @@ -249,7 +249,7 @@ LABEL com.nvidia.workbench.application.jupyterlab.webapp.url-cmd="jupyter lab li LABEL com.nvidia.workbench.cuda-version="$CUDA_VER" LABEL com.nvidia.workbench.description="RAPIDS with CUDA ${CUDA_VER}" LABEL com.nvidia.workbench.entrypoint-script="/home/rapids/entrypoint.sh" -LABEL com.nvidia.workbench.image-version="26.08.00" +LABEL com.nvidia.workbench.image-version="26.10.00" LABEL com.nvidia.workbench.labels="cuda${CUDA_VER}" LABEL com.nvidia.workbench.name="RAPIDS with CUDA ${CUDA_VER}" LABEL com.nvidia.workbench.os-distro-release="$LINUX_DISTRO_VER" @@ -258,6 +258,7 @@ LABEL com.nvidia.workbench.os="linux" LABEL com.nvidia.workbench.package-manager-environment.target="/opt/conda" LABEL com.nvidia.workbench.package-manager-environment.type="conda" LABEL com.nvidia.workbench.package-manager.apt.binary="/usr/bin/apt" + LABEL com.nvidia.workbench.package-manager.apt.installed-packages="" LABEL com.nvidia.workbench.package-manager.conda3.binary="/opt/conda/bin/conda" LABEL com.nvidia.workbench.package-manager.conda3.installed-packages="rapids cudf cuml cugraph rmm pylibraft cucim xgboost jupyterlab" @@ -268,3 +269,25 @@ LABEL com.nvidia.workbench.schema-version="v2" LABEL com.nvidia.workbench.user.gid="1000" LABEL com.nvidia.workbench.user.uid="1001" LABEL com.nvidia.workbench.user.username="rapids" + +# Minimal BuildKit export targets used to publish image provenance without +# exporting or pulling the image filesystem. Each contains only conda package +# metadata copied from the corresponding final image target. +FROM base AS provenance-base-inventory +COPY scripts/export-pip-package-inventory /usr/local/bin/export-pip-package-inventory +RUN python /usr/local/bin/export-pip-package-inventory /tmp/pip-packages.json + +FROM scratch AS provenance-base +COPY --from=base /opt/conda/conda-meta /conda-meta +COPY --from=provenance-base-inventory /tmp/pip-packages.json /pip-packages.json + +FROM notebooks AS provenance-notebooks-inventory +COPY scripts/export-pip-package-inventory /usr/local/bin/export-pip-package-inventory +RUN python /usr/local/bin/export-pip-package-inventory /tmp/pip-packages.json + +FROM scratch AS provenance-notebooks +COPY --from=notebooks /opt/conda/conda-meta /conda-meta +COPY --from=provenance-notebooks-inventory /tmp/pip-packages.json /pip-packages.json + +# Keep the default Docker build target runnable for local users. +FROM notebooks diff --git a/README.md b/README.md index d074ef1e..1f99d314 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,29 @@ The following environment variables can be passed to the `docker run` commands f - `PIP_TIMEOUT` - how long (in seconds) the `pip` install should wait before exiting - `UNQUOTE` - Whether the command line args to `docker run` should be [executed with or without being quoted](./context/entrypoint.sh). Default to false and it is unlikely that you need to change this. +## Image Provenance Manifests + +Published `base` and `notebooks` images have an OCI referrer containing a +RAPIDS image provenance manifest. It is attached to the immutable image digest, +not to a mutable tag, and can be retrieved from the registry without downloading +image layers. The platform-specific manifest records the source commit and +workflow run plus the exact conda package name, version, build, channel, and +source URL taken from `conda-meta`. The multiarch image-index manifest links the +corresponding platform manifests. + +Conda package records intentionally leave `purls` empty until an upstream pURL +mapping has been verified. Conda is a distribution format, not an upstream pURL +type, so consumers must not derive `pkg:conda/...` identifiers from these +records. The manifest uses the OCI artifact type +`application/vnd.rapids.image.provenance.v1+json` for platform images and +`application/vnd.rapids.image.provenance.index.v1+json` for multiarch indexes. + +The build publishes these records with [ORAS](https://oras.land/), using an OCI +referrer to the image digest, and keylessly signs the attached artifact with +GitHub Actions OIDC. Scanner and triage tooling can discover the referrer, +retrieve its JSON, and compare the scanner-reported component version with the +build facts without pulling the image. + ## Bind Mounts Mounting files/folders to the locations specified below provide additional functionality for the images. diff --git a/ci/compute-build-args.sh b/ci/compute-build-args.sh index 38ffb60c..fd52fb55 100755 --- a/ci/compute-build-args.sh +++ b/ci/compute-build-args.sh @@ -12,6 +12,7 @@ LINUX_DISTRO: ${LINUX_DISTRO:-notset} LINUX_DISTRO_VER: ${LINUX_DISTRO_VER:-notset} LINUX_VER: ${LINUX_VER} PYTHON_VER: ${PYTHON_VER} +RAPIDS_NOTEBOOKS_REF: ${RAPIDS_NOTEBOOKS_REF:-main} RAPIDS_VER: ${RAPIDS_VER} " export ARGS @@ -30,7 +31,7 @@ if [ -n "${GITHUB_ACTIONS:-}" ]; then # cat < "${GITHUB_OUTPUT:-/dev/stdout}" DOCKER_BUILD_ARGS< str: + return f"sha256:{hashlib.sha256(payload).hexdigest()}" + + +def parse_build_args(values: list[str]) -> dict[str, str]: + """Convert repeated ``NAME=VALUE`` CLI options to a stable mapping.""" + build_args: dict[str, str] = {} + for value in values: + name, separator, argument = value.partition("=") + if not separator or not name: + raise ValueError(f"build argument must use NAME=VALUE syntax: {value!r}") + build_args[name] = argument + return dict(sorted(build_args.items())) + + +def conda_packages(metadata_dir: Path | None) -> list[dict[str, Any]]: + """Read exact installed-package facts from conda-meta records.""" + if metadata_dir is None: + return [] + packages: list[dict[str, Any]] = [] + for path in sorted(metadata_dir.glob("*.json")): + try: + record = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid conda metadata JSON: {path}") from exc + if not isinstance(record, dict): + raise ValueError(f"conda metadata must be an object: {path}") + name = str(record.get("name") or "").strip() + version = str(record.get("version") or "").strip() + build = str(record.get("build") or "").strip() + if not name or not version: + raise ValueError(f"conda metadata lacks name or version: {path}") + packages.append( + { + "name": name, + "version": version, + "build": build, + "build_number": record.get("build_number"), + "channel": str(record.get("channel") or record.get("url") or ""), + "url": str(record.get("url") or ""), + # Conda is a distribution format, not an upstream pURL type. + # A pURL is populated only once a vetted package mapping exists. + "purls": [], + "purl_source": "unmapped", + } + ) + return sorted( + packages, + key=lambda package: ( + package["name"], + package["version"], + package["build"], + package["channel"], + ), + ) + + +def installed_packages( + conda_package_records: list[dict[str, Any]], + pip_packages_path: Path | None, +) -> list[dict[str, Any]]: + """Combine package-manager inventories without inventing upstream identities.""" + packages = [ + { + **package, + "ecosystem": "conda", + "installer": "conda", + "evidence": "conda-meta", + } + for package in conda_package_records + ] + if pip_packages_path is not None: + try: + pip_records = json.loads(pip_packages_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + raise ValueError(f"invalid pip package inventory: {pip_packages_path}") from exc + if not isinstance(pip_records, list): + raise ValueError("pip package inventory must be a JSON list") + for record in pip_records: + if not isinstance(record, dict): + raise ValueError("pip package inventory entries must be JSON objects") + name = str(record.get("name") or "").strip() + version = str(record.get("version") or "").strip() + purls = record.get("purls") + if not name or not version or not isinstance(purls, list): + raise ValueError("pip package inventory entries require name, version, and purls") + packages.append( + { + "name": name, + "version": version, + "purls": [str(purl) for purl in purls if str(purl).startswith("pkg:")], + "ecosystem": "pypi", + "installer": str(record.get("installer") or "pip"), + "evidence": str(record.get("evidence") or "python-distribution-metadata"), + } + ) + return sorted( + packages, + key=lambda package: ( + str(package["ecosystem"]), + str(package["name"]), + str(package["version"]), + ), + ) + + +def parse_platform_manifest(value: str) -> dict[str, str]: + """Parse ``os/architecture|reference|digest`` into a manifest entry.""" + platform, separator, remainder = value.partition("|") + reference, separator2, digest = remainder.partition("|") + _os_name, separator3, _architecture = platform.partition("/") + if not separator or not separator2 or not separator3: + raise ValueError( + "platform manifests must use os/architecture|reference|sha256:digest" + ) + if not digest.startswith("sha256:"): + raise ValueError(f"platform manifest digest must be sha256: {digest!r}") + return { + "platform": platform, + "reference": reference, + "digest": digest, + } + + +def build_manifest( + context: ManifestContext, + *, + metadata_dir: Path | None = None, + pip_packages_path: Path | None = None, + platform_manifests: list[str] | None = None, +) -> dict[str, Any]: + """Construct the schema payload for a platform image or multiarch index.""" + if not context.image_digest.startswith("sha256:"): + raise ValueError(f"image digest must be sha256: {context.image_digest!r}") + packages = conda_packages(metadata_dir) + package_payload = json.dumps(packages, sort_keys=True, separators=(",", ":")).encode() + all_packages = installed_packages(packages, pip_packages_path) + all_package_payload = json.dumps( + all_packages, sort_keys=True, separators=(",", ":") + ).encode() + subject: dict[str, Any] = { + "reference": context.image_reference, + "digest": context.image_digest, + } + if context.platform != "multiarch": + os_name, separator, architecture = context.platform.partition("/") + if not separator: + raise ValueError("platform must be os/architecture or multiarch") + subject["platform"] = {"os": os_name, "architecture": architecture} + manifest: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "generated_at": datetime.now(timezone.utc).isoformat(), + "subject": subject, + "image": { + "kind": context.image_kind, + "rapids_version": context.rapids_version, + "cuda_version": context.cuda_version, + "python_version": context.python_version, + }, + "build": { + "source_repository": context.source_repository, + "source_commit": context.source_commit, + "workflow_ref": context.workflow_ref, + "workflow_run_url": context.workflow_run_url, + "build_args": parse_build_args(context.build_args), + }, + "conda_packages": packages, + "conda_packages_sha256": _sha256(package_payload), + "installed_packages": all_packages, + "installed_packages_sha256": _sha256(all_package_payload), + "platform_manifests": sorted( + (parse_platform_manifest(value) for value in platform_manifests or []), + key=lambda item: item["platform"], + ), + } + return manifest + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--image-reference", required=True) + parser.add_argument("--image-digest", required=True) + parser.add_argument("--image-kind", required=True) + parser.add_argument("--rapids-version", required=True) + parser.add_argument("--cuda-version", required=True) + parser.add_argument("--python-version", required=True) + parser.add_argument("--platform", required=True) + parser.add_argument("--source-repository", required=True) + parser.add_argument("--source-commit", required=True) + parser.add_argument("--workflow-ref", required=True) + parser.add_argument("--workflow-run-url", required=True) + parser.add_argument("--conda-meta-dir", type=Path) + parser.add_argument("--pip-packages", type=Path) + parser.add_argument("--build-arg", action="append", default=[]) + parser.add_argument("--platform-manifest", action="append", default=[]) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + manifest = build_manifest( + ManifestContext( + image_reference=args.image_reference, + image_digest=args.image_digest, + image_kind=args.image_kind, + rapids_version=args.rapids_version, + cuda_version=args.cuda_version, + python_version=args.python_version, + platform=args.platform, + source_repository=args.source_repository, + source_commit=args.source_commit, + workflow_ref=args.workflow_ref, + workflow_run_url=args.workflow_run_url, + build_args=args.build_arg, + ), + metadata_dir=args.conda_meta_dir, + pip_packages_path=args.pip_packages, + platform_manifests=args.platform_manifest, + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + + +if __name__ == "__main__": + main() diff --git a/ci/publish-image-provenance-index.sh b/ci/publish-image-provenance-index.sh new file mode 100755 index 00000000..64032802 --- /dev/null +++ b/ci/publish-image-provenance-index.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eEuo pipefail + +: "${IMAGE_REFERENCE:?Set IMAGE_REFERENCE to the multiarch image tag}" +: "${IMAGE_KIND:?Set IMAGE_KIND to base or notebooks}" +: "${RAPIDS_VER:?Set RAPIDS_VER}" +: "${CUDA_VER:?Set CUDA_VER}" +: "${PYTHON_VER:?Set PYTHON_VER}" +: "${PLATFORM_REFERENCES:?Set PLATFORM_REFERENCES as platform=tag lines}" + +normalize_registry_reference() { + local reference="$1" + local first_component="${reference%%/*}" + if [[ $first_component != *.* && $first_component != *:* && $first_component != "localhost" ]]; then + reference="docker.io/${reference}" + fi + printf '%s\n' "$reference" +} + +output_dir="${RUNNER_TEMP:-/tmp}/image-provenance-index-${IMAGE_KIND}" +manifest_path="$output_dir/image-provenance-index.json" +workflow_run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" +registry_reference="$(normalize_registry_reference "$IMAGE_REFERENCE")" +mkdir -p "$output_dir" + +image_digest="$(oras manifest fetch --descriptor "$registry_reference" | jq -r '.digest')" +platform_args=() +while IFS='=' read -r platform reference; do + [[ -z "$platform" ]] && continue + platform_reference="$(normalize_registry_reference "$reference")" + digest="$(oras manifest fetch --descriptor "$platform_reference" | jq -r '.digest')" + platform_args+=(--platform-manifest "${platform}|${platform_reference}|${digest}") +done <<<"$PLATFORM_REFERENCES" + +python3 ci/image_provenance_manifest.py \ + --output "$manifest_path" \ + --image-reference "$IMAGE_REFERENCE" \ + --image-digest "$image_digest" \ + --image-kind "$IMAGE_KIND" \ + --rapids-version "$RAPIDS_VER" \ + --cuda-version "$CUDA_VER" \ + --python-version "$PYTHON_VER" \ + --platform multiarch \ + --source-repository "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --source-commit "$GITHUB_SHA" \ + --workflow-ref "${GITHUB_WORKFLOW_REF:-$GITHUB_WORKFLOW}" \ + --workflow-run-url "$workflow_run_url" \ + --build-arg "RAPIDS_VER=$RAPIDS_VER" \ + --build-arg "CUDA_VER=$CUDA_VER" \ + --build-arg "PYTHON_VER=$PYTHON_VER" \ + "${platform_args[@]}" + +attached_manifest="$output_dir/attached-image-provenance-index.json" +( + cd "$output_dir" + oras attach \ + --artifact-type application/vnd.rapids.image.provenance.index.v1+json \ + --export-manifest "$attached_manifest" \ + "${registry_reference%@*}@${image_digest}" \ + "image-provenance-index.json:application/vnd.rapids.image.provenance.index.v1+json" +) + +artifact_digest="sha256:$(sha256sum "$attached_manifest" | awk '{print $1}')" +cosign sign --yes "${registry_reference%@*}@${artifact_digest}" diff --git a/ci/publish-image-provenance.sh b/ci/publish-image-provenance.sh new file mode 100755 index 00000000..685d8017 --- /dev/null +++ b/ci/publish-image-provenance.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -eEuo pipefail + +: "${IMAGE_REFERENCE:?Set IMAGE_REFERENCE to the pushed image tag}" +: "${IMAGE_DIGEST:?Set IMAGE_DIGEST from docker/build-push-action output}" +: "${IMAGE_KIND:?Set IMAGE_KIND to base or notebooks}" +: "${IMAGE_PLATFORM:?Set IMAGE_PLATFORM to linux/architecture}" +: "${PROVENANCE_OUTPUT_DIR:?Set PROVENANCE_OUTPUT_DIR from export-image-provenance.sh}" +: "${RAPIDS_VER:?Set RAPIDS_VER}" +: "${CUDA_VER:?Set CUDA_VER}" +: "${PYTHON_VER:?Set PYTHON_VER}" + +normalize_registry_reference() { + local reference="$1" + local first_component="${reference%%/*}" + if [[ $first_component != *.* && $first_component != *:* && $first_component != "localhost" ]]; then + reference="docker.io/${reference}" + fi + printf '%s\n' "$reference" +} + +manifest_path="$PROVENANCE_OUTPUT_DIR/image-provenance.json" +workflow_run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" +registry_reference="$(normalize_registry_reference "$IMAGE_REFERENCE")" + +generator_args=() +while IFS= read -r argument; do + [[ -z "$argument" ]] && continue + generator_args+=(--build-arg "$argument") +done <<<"${DOCKER_BUILD_ARGS:-}" + +python3 ci/image_provenance_manifest.py \ + --output "$manifest_path" \ + --image-reference "$IMAGE_REFERENCE" \ + --image-digest "$IMAGE_DIGEST" \ + --image-kind "$IMAGE_KIND" \ + --rapids-version "$RAPIDS_VER" \ + --cuda-version "$CUDA_VER" \ + --python-version "$PYTHON_VER" \ + --platform "$IMAGE_PLATFORM" \ + --source-repository "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ + --source-commit "$GITHUB_SHA" \ + --workflow-ref "${GITHUB_WORKFLOW_REF:-$GITHUB_WORKFLOW}" \ + --workflow-run-url "$workflow_run_url" \ + --conda-meta-dir "$PROVENANCE_OUTPUT_DIR/conda-meta" \ + --pip-packages "$PROVENANCE_OUTPUT_DIR/pip-packages.json" \ + "${generator_args[@]}" + +attached_manifest="$PROVENANCE_OUTPUT_DIR/attached-image-provenance.json" +( + cd "$PROVENANCE_OUTPUT_DIR" + oras attach \ + --artifact-type application/vnd.rapids.image.provenance.v1+json \ + --export-manifest "$attached_manifest" \ + "${registry_reference%@*}@${IMAGE_DIGEST}" \ + "image-provenance.json:application/vnd.rapids.image.provenance.v1+json" +) + +artifact_digest="sha256:$(sha256sum "$attached_manifest" | awk '{print $1}')" +cosign sign --yes "${registry_reference%@*}@${artifact_digest}" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index d480f942..c5392dc2 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2026, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. ## Usage # Primary interface: bash update-version.sh [--run-context=main|release] @@ -76,11 +76,11 @@ NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} # Set branch references based on RUN_CONTEXT if [[ "${RUN_CONTEXT}" == "main" ]]; then - RAPIDS_BRANCH_NAME="main" + RAPIDS_NOTEBOOKS_REF="main" WORKFLOW_BRANCH_REF="main" echo "Preparing development branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting main branch)" elif [[ "${RUN_CONTEXT}" == "release" ]]; then - RAPIDS_BRANCH_NAME="release/${NEXT_SHORT_TAG}" + RAPIDS_NOTEBOOKS_REF="release/${NEXT_SHORT_TAG}" WORKFLOW_BRANCH_REF="release/${NEXT_SHORT_TAG}" echo "Preparing release branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting release/${NEXT_SHORT_TAG} branch)" fi @@ -96,9 +96,9 @@ done sed_runner "s/com\.nvidia\.workbench\.image-version=.*/com.nvidia.workbench.image-version=\"${NEXT_FULL_TAG}\"/g" Dockerfile -# Dockerfile RAPIDS_BRANCH -sed_runner "s|ARG RAPIDS_BRANCH=\"release/[0-9]\+\.[0-9]\+\"|ARG RAPIDS_BRANCH=\"${RAPIDS_BRANCH_NAME}\"|g" Dockerfile -sed_runner "s|ARG RAPIDS_BRANCH=\"main\"|ARG RAPIDS_BRANCH=\"${RAPIDS_BRANCH_NAME}\"|g" Dockerfile +# Dockerfile RAPIDS_NOTEBOOKS_REF +sed_runner "s|ARG RAPIDS_NOTEBOOKS_REF=\"release/[0-9]\+\.[0-9]\+\"|ARG RAPIDS_NOTEBOOKS_REF=\"${RAPIDS_NOTEBOOKS_REF}\"|g" Dockerfile +sed_runner "s|ARG RAPIDS_NOTEBOOKS_REF=\"main\"|ARG RAPIDS_NOTEBOOKS_REF=\"${RAPIDS_NOTEBOOKS_REF}\"|g" Dockerfile # docs sed_runner "s|RAPIDS_VER=[[:digit:]]\+\.[[:digit:]]|RAPIDS_VER=${NEXT_SHORT_TAG}|g" CONTRIBUTING.md diff --git a/context/.dockerignore b/context/.dockerignore index 90ffe897..1acef0b8 100644 --- a/context/.dockerignore +++ b/context/.dockerignore @@ -7,6 +7,7 @@ !pinned !scripts/configure-apt !scripts/configure-conda-base-environment +!scripts/export-pip-package-inventory !scripts/install-gha-tools !scripts/install-tzdata-packages !scripts/update-base-conda-environment diff --git a/context/notebooks.sh b/context/notebooks.sh index 5677cc29..f442a331 100755 --- a/context/notebooks.sh +++ b/context/notebooks.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -# Copyright (c) 2023-2026, NVIDIA CORPORATION. +# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # Clones repos with notebooks & compiles notebook test dependencies # Requires environment variables: -# RAPIDS_BRANCH +# RAPIDS_NOTEBOOKS_REF # CUDA_VER # PYTHON_VER @@ -14,7 +14,7 @@ NOTEBOOK_REPOS=(cudf cuml cugraph) mkdir -p /notebooks /dependencies for REPO in "${NOTEBOOK_REPOS[@]}"; do echo "Cloning $REPO..." - git clone -b "${RAPIDS_BRANCH}" --depth 1 --single-branch "https://github.com/rapidsai/$REPO" "$REPO" + git clone -b "${RAPIDS_NOTEBOOKS_REF}" --depth 1 --single-branch "https://github.com/rapidsai/$REPO" "$REPO" SOURCE="$REPO/notebooks" DESTINATION="/notebooks/$REPO" diff --git a/context/scripts/export-pip-package-inventory b/context/scripts/export-pip-package-inventory new file mode 100644 index 00000000..3202835e --- /dev/null +++ b/context/scripts/export-pip-package-inventory @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Export pip-installed distributions with their canonical PyPI pURLs.""" + +from __future__ import annotations + +import json +import re +import sys +from importlib import metadata +from pathlib import Path + + +def main() -> None: + output = Path(sys.argv[1]) + packages = [] + for distribution in metadata.distributions(): + name = str(distribution.metadata.get("Name") or "").strip() + version = str(distribution.version or "").strip() + installer = (distribution.read_text("INSTALLER") or "").strip().lower() + if not name or not version or installer != "pip": + continue + normalized_name = re.sub(r"[-_.]+", "-", name).lower() + packages.append( + { + "name": name, + "version": version, + "purls": [f"pkg:pypi/{normalized_name}"], + "installer": "pip", + "evidence": "python-distribution-metadata; INSTALLER=pip", + } + ) + output.write_text( + json.dumps(sorted(packages, key=lambda item: item["name"].lower())), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/cuvs-bench/cpu/Dockerfile b/cuvs-bench/cpu/Dockerfile index 6ad96841..5573d3ac 100644 --- a/cuvs-bench/cpu/Dockerfile +++ b/cuvs-bench/cpu/Dockerfile @@ -1,13 +1,13 @@ # syntax=docker/dockerfile:1 -# Copyright (c) 2024-2026, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. ARG MINIFORGE_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.08 +ARG RAPIDS_VER=26.10 FROM condaforge/miniforge3:${MINIFORGE_VER} AS cuvs-bench-cpu ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.08 +ARG RAPIDS_VER=26.10 COPY condarc /opt/conda/.condarc diff --git a/cuvs-bench/gpu/Dockerfile b/cuvs-bench/gpu/Dockerfile index 1a1a48f0..7a3f1846 100644 --- a/cuvs-bench/gpu/Dockerfile +++ b/cuvs-bench/gpu/Dockerfile @@ -1,11 +1,11 @@ # syntax=docker/dockerfile:1 -# Copyright (c) 2024-2026, NVIDIA CORPORATION. +# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. ARG CUDA_VER=notset ARG LINUX_VER=notset ARG MINIFORGE_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.08 +ARG RAPIDS_VER=26.10 # --- begin 'rapidsai/miniforge-cuda' --- # FROM condaforge/miniforge3:${MINIFORGE_VER} AS miniforge-upstream @@ -59,7 +59,7 @@ EOF FROM miniforge-cuda AS cuvs-bench ARG CUDA_VER=notset -ARG RAPIDS_VER=26.08 +ARG RAPIDS_VER=26.10 COPY condarc /opt/conda/.condarc diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..2c78728c --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1 @@ +pytest==8.3.5 diff --git a/tests/test_image_provenance_manifest.py b/tests/test_image_provenance_manifest.py new file mode 100644 index 00000000..948471b9 --- /dev/null +++ b/tests/test_image_provenance_manifest.py @@ -0,0 +1,174 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# ruff: noqa: S101 + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +MODULE_PATH = Path(__file__).parents[1] / "ci" / "image_provenance_manifest.py" +REPOSITORY_ROOT = MODULE_PATH.parents[1] +SPEC = importlib.util.spec_from_file_location("image_provenance_manifest", MODULE_PATH) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def manifest_context(**overrides: object) -> object: + """Create valid provenance inputs, with targeted test overrides.""" + defaults = { + "image_reference": "rapidsai/base:26.08-cuda12-py312-amd64", + "image_digest": "sha256:image", + "image_kind": "base", + "rapids_version": "26.08", + "cuda_version": "12", + "python_version": "3.12", + "platform": "linux/amd64", + "source_repository": "https://github.com/rapidsai/docker", + "source_commit": "abc123", + "workflow_ref": ".github/workflows/build-rapids-image.yml@abc123", + "workflow_run_url": "https://github.com/rapidsai/docker/actions/runs/1", + "build_args": ["PYTHON_VER=3.12", "CUDA_VER=12"], + } + defaults.update(overrides) + return MODULE.ManifestContext(**defaults) + + +def test_build_manifest_records_exact_conda_package_facts(tmp_path: Path) -> None: + metadata_dir = tmp_path / "conda-meta" + metadata_dir.mkdir() + (metadata_dir / "c-ares-1.34.8-h1.json").write_text( + json.dumps( + { + "name": "c-ares", + "version": "1.34.8", + "build": "h1", + "build_number": 2, + "channel": "https://conda.anaconda.org/conda-forge", + "url": "https://conda.anaconda.org/conda-forge/linux-64/c-ares.conda", + } + ) + ) + + manifest = MODULE.build_manifest( + manifest_context(), + metadata_dir=metadata_dir, + ) + + assert manifest["subject"]["platform"] == { + "os": "linux", + "architecture": "amd64", + } + assert manifest["conda_packages"] == [ + { + "build": "h1", + "build_number": 2, + "channel": "https://conda.anaconda.org/conda-forge", + "name": "c-ares", + "purl_source": "unmapped", + "purls": [], + "url": "https://conda.anaconda.org/conda-forge/linux-64/c-ares.conda", + "version": "1.34.8", + } + ] + assert manifest["build"]["build_args"] == {"CUDA_VER": "12", "PYTHON_VER": "3.12"} + assert manifest["conda_packages_sha256"].startswith("sha256:") + + +def test_build_manifest_records_direct_pip_purl_evidence(tmp_path: Path) -> None: + pip_packages = tmp_path / "pip-packages.json" + pip_packages.write_text( + json.dumps( + [ + { + "name": "torch", + "version": "2.8.0", + "purls": ["pkg:pypi/torch"], + "installer": "pip", + "evidence": "python-distribution-metadata; INSTALLER=pip", + } + ] + ) + ) + + manifest = MODULE.build_manifest( + manifest_context(), + pip_packages_path=pip_packages, + ) + + assert manifest["installed_packages"] == [ + { + "ecosystem": "pypi", + "evidence": "python-distribution-metadata; INSTALLER=pip", + "installer": "pip", + "name": "torch", + "purls": ["pkg:pypi/torch"], + "version": "2.8.0", + } + ] + assert manifest["installed_packages_sha256"].startswith("sha256:") + + +def test_build_manifest_links_platform_manifests_for_multiarch_images() -> None: + manifest = MODULE.build_manifest( + manifest_context( + image_reference="rapidsai/base:26.08-cuda12-py312", + image_digest="sha256:index", + platform="multiarch", + workflow_ref="workflow", + workflow_run_url="run", + build_args=[], + ), + platform_manifests=[ + "linux/arm64|rapidsai/base:tag-arm64|sha256:arm", + "linux/amd64|rapidsai/base:tag-amd64|sha256:amd", + ], + ) + + assert "platform" not in manifest["subject"] + assert [item["platform"] for item in manifest["platform_manifests"]] == [ + "linux/amd64", + "linux/arm64", + ] + + +def test_invalid_build_argument_is_rejected() -> None: + with pytest.raises(ValueError, match="NAME=VALUE"): + MODULE.parse_build_args(["CUDA_VER"]) + + +def test_pip_inventory_collector_is_in_docker_build_context() -> None: + dockerignore = (REPOSITORY_ROOT / "context" / ".dockerignore").read_text() + + assert "!scripts/export-pip-package-inventory" in dockerignore + + +def test_provenance_collector_writes_outside_the_nonroot_image_root() -> None: + dockerfile = (REPOSITORY_ROOT / "Dockerfile").read_text() + + assert "export-pip-package-inventory /tmp/pip-packages.json" in dockerfile + assert "COPY --from=provenance-base-inventory /tmp/pip-packages.json" in dockerfile + + +@pytest.mark.parametrize( + ("script_name", "safe_title"), + [ + ("publish-image-provenance.sh", "image-provenance.json"), + ("publish-image-provenance-index.sh", "image-provenance-index.json"), + ], +) +def test_published_provenance_uses_safe_oci_layer_titles( + script_name: str, + safe_title: str, +) -> None: + script = (REPOSITORY_ROOT / "ci" / script_name).read_text() + + assert "--disable-path-validation" not in script + assert f'"{safe_title}:application/vnd.rapids.image.provenance' in script From b5908b2028c3cbea9cd8e3cf07aba048d987a767 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Mon, 3 Aug 2026 10:45:20 -0500 Subject: [PATCH 2/2] Revert "feat: publish signed image provenance manifests (#896)" (#903) This reverts commit f33fc76482b53c2f278da1a4e16bb8a1861bab7a. This broke and should not have been merged into the release branch. Revert and re-do on main. See #901 and #902 for more info. Authors: - Mike Sarahan (https://github.com/msarahan) Approvers: - James Lamb (https://github.com/jameslamb) URL: https://github.com/rapidsai/docker/pull/903 --- .github/workflows/build-rapids-image.yml | 60 +--- .../workflows/build-test-publish-images.yml | 77 +---- .github/workflows/pr.yml | 4 +- .../trigger-breaking-change-alert.yaml | 2 +- .gitignore | 2 - Dockerfile | 35 +-- README.md | 23 -- ci/compute-build-args.sh | 3 +- ci/compute-matrix.sh | 14 +- ci/export-image-provenance.sh | 27 -- ci/image_provenance_manifest.py | 264 ------------------ ci/publish-image-provenance-index.sh | 67 ----- ci/publish-image-provenance.sh | 63 ----- ci/release/update-version.sh | 12 +- context/.dockerignore | 1 - context/notebooks.sh | 6 +- context/scripts/export-pip-package-inventory | 42 --- cuvs-bench/cpu/Dockerfile | 6 +- cuvs-bench/gpu/Dockerfile | 6 +- requirements-test.txt | 1 - tests/test_image_provenance_manifest.py | 174 ------------ 21 files changed, 30 insertions(+), 859 deletions(-) delete mode 100755 ci/export-image-provenance.sh delete mode 100755 ci/image_provenance_manifest.py delete mode 100755 ci/publish-image-provenance-index.sh delete mode 100755 ci/publish-image-provenance.sh delete mode 100644 context/scripts/export-pip-package-inventory delete mode 100644 requirements-test.txt delete mode 100644 tests/test_image_provenance_manifest.py diff --git a/.github/workflows/build-rapids-image.yml b/.github/workflows/build-rapids-image.yml index af03aad9..02947e37 100644 --- a/.github/workflows/build-rapids-image.yml +++ b/.github/workflows/build-rapids-image.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2025-2026, NVIDIA CORPORATION. name: build-rapids on: @@ -25,9 +25,6 @@ on: RAPIDS_VER: required: true type: string - RAPIDS_NOTEBOOKS_REF: - required: true - type: string BASE_TAG: required: true type: string @@ -35,10 +32,6 @@ on: required: true type: string -permissions: - contents: read - id-token: write - env: # prevent buildx from creating arch-specific manifest lists... # we want single images, that are later referenced together in a multiarch manifest @@ -82,18 +75,6 @@ jobs: with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} - - name: Install Go - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 - with: - go-version: '1.25.x' - cache: false - - name: Install ORAS - run: | - GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0 - echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}" - "${RUNNER_TEMP}/oras-bin/oras" version - - name: Set up Cosign - uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3 - name: Set up Docker Context for Buildx id: buildx-context run: | @@ -118,10 +99,8 @@ jobs: LINUX_DISTRO_VER: ${{ inputs.LINUX_DISTRO_VER }} LINUX_VER: ${{ inputs.LINUX_VER }} PYTHON_VER: ${{ inputs.PYTHON_VER }} - RAPIDS_NOTEBOOKS_REF: ${{ inputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ inputs.RAPIDS_VER }} - name: Build base image - id: build-base uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: context @@ -134,26 +113,7 @@ jobs: tags: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true - - name: Export base image package provenance - env: - DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} - PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }} - PROVENANCE_TARGET: provenance-base - run: ci/export-image-provenance.sh - - name: Publish base image provenance - env: - CUDA_VER: ${{ inputs.CUDA_VER }} - DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} - IMAGE_DIGEST: ${{ steps.build-base.outputs.digest }} - IMAGE_KIND: base - IMAGE_PLATFORM: linux/${{ matrix.ARCH }} - IMAGE_REFERENCE: ${{ inputs.BASE_TAG }}-${{ matrix.ARCH }} - PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-base-${{ matrix.ARCH }} - PYTHON_VER: ${{ inputs.PYTHON_VER }} - RAPIDS_VER: ${{ inputs.RAPIDS_VER }} - run: ci/publish-image-provenance.sh - name: Build notebooks image - id: build-notebooks uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6 with: context: context @@ -166,21 +126,3 @@ jobs: tags: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }} # ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types outputs: type=registry,oci-mediatypes=true - - name: Export notebooks image package provenance - env: - DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} - PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }} - PROVENANCE_TARGET: provenance-notebooks - run: ci/export-image-provenance.sh - - name: Publish notebooks image provenance - env: - CUDA_VER: ${{ inputs.CUDA_VER }} - DOCKER_BUILD_ARGS: ${{ steps.generate-build-args.outputs.DOCKER_BUILD_ARGS }} - IMAGE_DIGEST: ${{ steps.build-notebooks.outputs.digest }} - IMAGE_KIND: notebooks - IMAGE_PLATFORM: linux/${{ matrix.ARCH }} - IMAGE_REFERENCE: ${{ inputs.NOTEBOOKS_TAG }}-${{ matrix.ARCH }} - PROVENANCE_OUTPUT_DIR: ${{ runner.temp }}/provenance-notebooks-${{ matrix.ARCH }} - PYTHON_VER: ${{ inputs.PYTHON_VER }} - RAPIDS_VER: ${{ inputs.RAPIDS_VER }} - run: ci/publish-image-provenance.sh diff --git a/.github/workflows/build-test-publish-images.yml b/.github/workflows/build-test-publish-images.yml index 990a4035..270837de 100644 --- a/.github/workflows/build-test-publish-images.yml +++ b/.github/workflows/build-test-publish-images.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023-2026, NVIDIA CORPORATION. name: build and publish imgs workflow on: @@ -43,7 +43,7 @@ jobs: - build-cuvs-multiarch-manifest - test secrets: inherit # zizmor: ignore[secrets-inherit] - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main # zizmor: ignore[unpinned-uses] + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.08 checks: runs-on: ubuntu-latest steps: @@ -55,10 +55,6 @@ jobs: - name: Run hadolint run: | ci/lint-dockerfiles.sh - - name: Test image provenance manifest generator - run: | - python3 -m pip install --disable-pip-version-check -r requirements-test.txt - python3 -m pytest -q tests compute-matrix: runs-on: ubuntu-latest outputs: @@ -72,7 +68,6 @@ jobs: NOTEBOOKS_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.NOTEBOOKS_TAG_PREFIX }} CUVS_BENCH_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_TAG_PREFIX }} CUVS_BENCH_CPU_TAG_PREFIX: ${{ steps.compute-tag-prefix.outputs.CUVS_BENCH_CPU_TAG_PREFIX }} - RAPIDS_NOTEBOOKS_REF: ${{ steps.compute-rapids-ver.outputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ steps.compute-rapids-ver.outputs.RAPIDS_VER }} ALPHA_TAG: ${{ steps.compute-rapids-ver.outputs.ALPHA_TAG }} steps: @@ -83,15 +78,7 @@ jobs: persist-credentials: false - name: Compute matrix id: compute-matrix - env: - BUILD_TYPE: ${{ inputs.build_type }} run: | - # PR builds temporarily use published 26.08 packages. Their - # aarch64 RAPIDS metapackage requires Python >=3.12, but still - # supports both CUDA 12 and CUDA 13 package lines. - if [[ $BUILD_TYPE == "pull-request" ]]; then - export MATRIX_PYTHON_VERSIONS='["3.12", "3.13", "3.14"]' - fi MATRIX=$(ci/compute-matrix.sh) echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT} - name: Compute tag prefix @@ -132,8 +119,6 @@ jobs: echo "CUVS_BENCH_CPU_IMAGE_REPO=${cuvs_bench_cpu_repo}" | tee -a ${GITHUB_OUTPUT} - name: Compute RAPIDS_VER id: compute-rapids-ver - env: - BUILD_TYPE: ${{ inputs.build_type }} run: | GIT_DESCRIBE_TAG="$(git describe --tags --first-parent --abbrev=0)" GIT_DESCRIBE_TAG="${GIT_DESCRIBE_TAG:1}" # remove leading 'v' @@ -143,20 +128,7 @@ jobs: ALPHA_TAG="a" fi RAPIDS_VER="$(echo $GIT_DESCRIBE_TAG | awk 'BEGIN{FS=OFS="."} NF--')" # Convert full tag to YY.MM - RAPIDS_NOTEBOOKS_REF="main" - if [[ ! $GIT_DESCRIBE_TAG =~ [a-z] ]]; then - RAPIDS_NOTEBOOKS_REF="release/${RAPIDS_VER}" - fi - - # The source branch advances before its packages are published. Keep - # pull-request images on a matching published package and source line. - if [[ $BUILD_TYPE == "pull-request" ]]; then - RAPIDS_VER="26.08" - RAPIDS_NOTEBOOKS_REF="release/26.08" - ALPHA_TAG="a" - fi - echo "RAPIDS_NOTEBOOKS_REF=${RAPIDS_NOTEBOOKS_REF}" | tee -a ${GITHUB_OUTPUT} echo "RAPIDS_VER=${RAPIDS_VER}" | tee -a ${GITHUB_OUTPUT} echo "ALPHA_TAG=${ALPHA_TAG}" | tee -a ${GITHUB_OUTPUT} - name: Compute test matrix @@ -187,7 +159,6 @@ jobs: LINUX_DISTRO_VER: ${{ matrix.LINUX_DISTRO_VER }} LINUX_VER: ${{ matrix.LINUX_VER }} PYTHON_VER: ${{ matrix.PYTHON_VER }} - RAPIDS_NOTEBOOKS_REF: ${{ needs.compute-matrix.outputs.RAPIDS_NOTEBOOKS_REF }} RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }} BASE_TAG: "rapidsai/${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }}:\ @@ -253,18 +224,6 @@ jobs: with: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} - - name: Install Go - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 - with: - go-version: '1.25.x' - cache: false - - name: Install ORAS - run: | - GOBIN="${RUNNER_TEMP}/oras-bin" go install oras.land/oras/cmd/oras@v1.3.0 - echo "${RUNNER_TEMP}/oras-bin" >> "${GITHUB_PATH}" - "${RUNNER_TEMP}/oras-bin/oras" version - - name: Set up Cosign - uses: sigstore/cosign-installer@f713795cb21599bc4e5c4b58cbad1da852d7eeb9 # v3 - name: Create multiarch manifest shell: bash env: @@ -280,38 +239,6 @@ jobs: GPUCIBOT_DOCKERHUB_TOKEN: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} ARCHES: ${{ toJSON(matrix.ARCHES) }} run: ci/create-rapids-multiarch-manifest.sh - - name: Publish multiarch image provenance indexes - shell: bash - env: - BASE_IMAGE_REPO: ${{ needs.compute-matrix.outputs.BASE_IMAGE_REPO }} - BASE_TAG_PREFIX: ${{ needs.compute-matrix.outputs.BASE_TAG_PREFIX }} - RAPIDS_VER: ${{ needs.compute-matrix.outputs.RAPIDS_VER }} - ALPHA_TAG: ${{ needs.compute-matrix.outputs.ALPHA_TAG }} - CUDA_TAG: ${{ matrix.CUDA_TAG }} - PYTHON_VER: ${{ matrix.PYTHON_VER }} - NOTEBOOKS_IMAGE_REPO: ${{ needs.compute-matrix.outputs.NOTEBOOKS_IMAGE_REPO }} - NOTEBOOKS_TAG_PREFIX: ${{ needs.compute-matrix.outputs.NOTEBOOKS_TAG_PREFIX }} - ARCHES: ${{ toJSON(matrix.ARCHES) }} - run: | - base_reference="rapidsai/${BASE_IMAGE_REPO}:${BASE_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}" - notebooks_reference="rapidsai/${NOTEBOOKS_IMAGE_REPO}:${NOTEBOOKS_TAG_PREFIX}${RAPIDS_VER}${ALPHA_TAG}-cuda${CUDA_TAG}-py${PYTHON_VER}" - for image_kind in base notebooks; do - if [[ "$image_kind" == base ]]; then - image_reference="$base_reference" - else - image_reference="$notebooks_reference" - fi - platform_references="$({ - while IFS= read -r arch; do - printf 'linux/%s=%s-%s\n' "$arch" "$image_reference" "$arch" - done < <(jq -r '.[]' <<< "$ARCHES") - })" - IMAGE_KIND="$image_kind" \ - IMAGE_REFERENCE="$image_reference" \ - PLATFORM_REFERENCES="$platform_references" \ - CUDA_VER="$CUDA_TAG" \ - ci/publish-image-provenance-index.sh - done build-cuvs-multiarch-manifest: needs: [build-cuvs, compute-matrix] strategy: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 128e37b3..97ce5de4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023-2026, NVIDIA CORPORATION. name: ci on: @@ -19,7 +19,7 @@ jobs: permissions: checks: write pull-requests: write - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main # zizmor: ignore[unpinned-uses] + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@release/26.08 if: always() with: needs: ${{ toJSON(needs) }} diff --git a/.github/workflows/trigger-breaking-change-alert.yaml b/.github/workflows/trigger-breaking-change-alert.yaml index d7752b9e..96caa74d 100644 --- a/.github/workflows/trigger-breaking-change-alert.yaml +++ b/.github/workflows/trigger-breaking-change-alert.yaml @@ -20,7 +20,7 @@ jobs: if: contains(github.event.pull_request.labels.*.name, 'breaking') permissions: contents: read - uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@main + uses: rapidsai/shared-workflows/.github/workflows/breaking-change-alert.yaml@release/26.08 secrets: slack-webhook-url: ${{ secrets.NV_SLACK_BREAKING_CHANGE_NOTIFIER_APP }} with: diff --git a/.gitignore b/.gitignore index 2d5c5c06..f99d452a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ .vscode -__pycache__/ -*.py[cod] *.a *.bz2 diff --git a/Dockerfile b/Dockerfile index 5a7338d3..808a7b26 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024-2026, NVIDIA CORPORATION. ARG CUDA_VER=notset ARG LINUX_DISTRO=ubuntu @@ -7,15 +7,15 @@ ARG LINUX_DISTRO_VER=22.04 ARG LINUX_VER=${LINUX_DISTRO}${LINUX_DISTRO_VER} ARG MINIFORGE_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.10 +ARG RAPIDS_VER=26.08 # Gather dependency information FROM python:${PYTHON_VER} AS dependencies ARG CPU_ARCH=notset ARG CUDA_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_NOTEBOOKS_REF="main" -ARG RAPIDS_VER=26.10 +ARG RAPIDS_BRANCH="release/26.08" +ARG RAPIDS_VER=26.08 ARG YQ_VER=notset SHELL ["/bin/bash", "-euo", "pipefail", "-c"] @@ -122,7 +122,7 @@ EOF FROM miniforge-cuda AS base ARG CUDA_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.10 +ARG RAPIDS_VER=26.08 SHELL ["/bin/bash", "-euo", "pipefail", "-c"] @@ -249,7 +249,7 @@ LABEL com.nvidia.workbench.application.jupyterlab.webapp.url-cmd="jupyter lab li LABEL com.nvidia.workbench.cuda-version="$CUDA_VER" LABEL com.nvidia.workbench.description="RAPIDS with CUDA ${CUDA_VER}" LABEL com.nvidia.workbench.entrypoint-script="/home/rapids/entrypoint.sh" -LABEL com.nvidia.workbench.image-version="26.10.00" +LABEL com.nvidia.workbench.image-version="26.08.00" LABEL com.nvidia.workbench.labels="cuda${CUDA_VER}" LABEL com.nvidia.workbench.name="RAPIDS with CUDA ${CUDA_VER}" LABEL com.nvidia.workbench.os-distro-release="$LINUX_DISTRO_VER" @@ -258,7 +258,6 @@ LABEL com.nvidia.workbench.os="linux" LABEL com.nvidia.workbench.package-manager-environment.target="/opt/conda" LABEL com.nvidia.workbench.package-manager-environment.type="conda" LABEL com.nvidia.workbench.package-manager.apt.binary="/usr/bin/apt" - LABEL com.nvidia.workbench.package-manager.apt.installed-packages="" LABEL com.nvidia.workbench.package-manager.conda3.binary="/opt/conda/bin/conda" LABEL com.nvidia.workbench.package-manager.conda3.installed-packages="rapids cudf cuml cugraph rmm pylibraft cucim xgboost jupyterlab" @@ -269,25 +268,3 @@ LABEL com.nvidia.workbench.schema-version="v2" LABEL com.nvidia.workbench.user.gid="1000" LABEL com.nvidia.workbench.user.uid="1001" LABEL com.nvidia.workbench.user.username="rapids" - -# Minimal BuildKit export targets used to publish image provenance without -# exporting or pulling the image filesystem. Each contains only conda package -# metadata copied from the corresponding final image target. -FROM base AS provenance-base-inventory -COPY scripts/export-pip-package-inventory /usr/local/bin/export-pip-package-inventory -RUN python /usr/local/bin/export-pip-package-inventory /tmp/pip-packages.json - -FROM scratch AS provenance-base -COPY --from=base /opt/conda/conda-meta /conda-meta -COPY --from=provenance-base-inventory /tmp/pip-packages.json /pip-packages.json - -FROM notebooks AS provenance-notebooks-inventory -COPY scripts/export-pip-package-inventory /usr/local/bin/export-pip-package-inventory -RUN python /usr/local/bin/export-pip-package-inventory /tmp/pip-packages.json - -FROM scratch AS provenance-notebooks -COPY --from=notebooks /opt/conda/conda-meta /conda-meta -COPY --from=provenance-notebooks-inventory /tmp/pip-packages.json /pip-packages.json - -# Keep the default Docker build target runnable for local users. -FROM notebooks diff --git a/README.md b/README.md index 1f99d314..d074ef1e 100644 --- a/README.md +++ b/README.md @@ -39,29 +39,6 @@ The following environment variables can be passed to the `docker run` commands f - `PIP_TIMEOUT` - how long (in seconds) the `pip` install should wait before exiting - `UNQUOTE` - Whether the command line args to `docker run` should be [executed with or without being quoted](./context/entrypoint.sh). Default to false and it is unlikely that you need to change this. -## Image Provenance Manifests - -Published `base` and `notebooks` images have an OCI referrer containing a -RAPIDS image provenance manifest. It is attached to the immutable image digest, -not to a mutable tag, and can be retrieved from the registry without downloading -image layers. The platform-specific manifest records the source commit and -workflow run plus the exact conda package name, version, build, channel, and -source URL taken from `conda-meta`. The multiarch image-index manifest links the -corresponding platform manifests. - -Conda package records intentionally leave `purls` empty until an upstream pURL -mapping has been verified. Conda is a distribution format, not an upstream pURL -type, so consumers must not derive `pkg:conda/...` identifiers from these -records. The manifest uses the OCI artifact type -`application/vnd.rapids.image.provenance.v1+json` for platform images and -`application/vnd.rapids.image.provenance.index.v1+json` for multiarch indexes. - -The build publishes these records with [ORAS](https://oras.land/), using an OCI -referrer to the image digest, and keylessly signs the attached artifact with -GitHub Actions OIDC. Scanner and triage tooling can discover the referrer, -retrieve its JSON, and compare the scanner-reported component version with the -build facts without pulling the image. - ## Bind Mounts Mounting files/folders to the locations specified below provide additional functionality for the images. diff --git a/ci/compute-build-args.sh b/ci/compute-build-args.sh index fd52fb55..38ffb60c 100755 --- a/ci/compute-build-args.sh +++ b/ci/compute-build-args.sh @@ -12,7 +12,6 @@ LINUX_DISTRO: ${LINUX_DISTRO:-notset} LINUX_DISTRO_VER: ${LINUX_DISTRO_VER:-notset} LINUX_VER: ${LINUX_VER} PYTHON_VER: ${PYTHON_VER} -RAPIDS_NOTEBOOKS_REF: ${RAPIDS_NOTEBOOKS_REF:-main} RAPIDS_VER: ${RAPIDS_VER} " export ARGS @@ -31,7 +30,7 @@ if [ -n "${GITHUB_ACTIONS:-}" ]; then # cat < "${GITHUB_OUTPUT:-/dev/stdout}" DOCKER_BUILD_ARGS< str: - return f"sha256:{hashlib.sha256(payload).hexdigest()}" - - -def parse_build_args(values: list[str]) -> dict[str, str]: - """Convert repeated ``NAME=VALUE`` CLI options to a stable mapping.""" - build_args: dict[str, str] = {} - for value in values: - name, separator, argument = value.partition("=") - if not separator or not name: - raise ValueError(f"build argument must use NAME=VALUE syntax: {value!r}") - build_args[name] = argument - return dict(sorted(build_args.items())) - - -def conda_packages(metadata_dir: Path | None) -> list[dict[str, Any]]: - """Read exact installed-package facts from conda-meta records.""" - if metadata_dir is None: - return [] - packages: list[dict[str, Any]] = [] - for path in sorted(metadata_dir.glob("*.json")): - try: - record = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise ValueError(f"invalid conda metadata JSON: {path}") from exc - if not isinstance(record, dict): - raise ValueError(f"conda metadata must be an object: {path}") - name = str(record.get("name") or "").strip() - version = str(record.get("version") or "").strip() - build = str(record.get("build") or "").strip() - if not name or not version: - raise ValueError(f"conda metadata lacks name or version: {path}") - packages.append( - { - "name": name, - "version": version, - "build": build, - "build_number": record.get("build_number"), - "channel": str(record.get("channel") or record.get("url") or ""), - "url": str(record.get("url") or ""), - # Conda is a distribution format, not an upstream pURL type. - # A pURL is populated only once a vetted package mapping exists. - "purls": [], - "purl_source": "unmapped", - } - ) - return sorted( - packages, - key=lambda package: ( - package["name"], - package["version"], - package["build"], - package["channel"], - ), - ) - - -def installed_packages( - conda_package_records: list[dict[str, Any]], - pip_packages_path: Path | None, -) -> list[dict[str, Any]]: - """Combine package-manager inventories without inventing upstream identities.""" - packages = [ - { - **package, - "ecosystem": "conda", - "installer": "conda", - "evidence": "conda-meta", - } - for package in conda_package_records - ] - if pip_packages_path is not None: - try: - pip_records = json.loads(pip_packages_path.read_text(encoding="utf-8")) - except json.JSONDecodeError as exc: - raise ValueError(f"invalid pip package inventory: {pip_packages_path}") from exc - if not isinstance(pip_records, list): - raise ValueError("pip package inventory must be a JSON list") - for record in pip_records: - if not isinstance(record, dict): - raise ValueError("pip package inventory entries must be JSON objects") - name = str(record.get("name") or "").strip() - version = str(record.get("version") or "").strip() - purls = record.get("purls") - if not name or not version or not isinstance(purls, list): - raise ValueError("pip package inventory entries require name, version, and purls") - packages.append( - { - "name": name, - "version": version, - "purls": [str(purl) for purl in purls if str(purl).startswith("pkg:")], - "ecosystem": "pypi", - "installer": str(record.get("installer") or "pip"), - "evidence": str(record.get("evidence") or "python-distribution-metadata"), - } - ) - return sorted( - packages, - key=lambda package: ( - str(package["ecosystem"]), - str(package["name"]), - str(package["version"]), - ), - ) - - -def parse_platform_manifest(value: str) -> dict[str, str]: - """Parse ``os/architecture|reference|digest`` into a manifest entry.""" - platform, separator, remainder = value.partition("|") - reference, separator2, digest = remainder.partition("|") - _os_name, separator3, _architecture = platform.partition("/") - if not separator or not separator2 or not separator3: - raise ValueError( - "platform manifests must use os/architecture|reference|sha256:digest" - ) - if not digest.startswith("sha256:"): - raise ValueError(f"platform manifest digest must be sha256: {digest!r}") - return { - "platform": platform, - "reference": reference, - "digest": digest, - } - - -def build_manifest( - context: ManifestContext, - *, - metadata_dir: Path | None = None, - pip_packages_path: Path | None = None, - platform_manifests: list[str] | None = None, -) -> dict[str, Any]: - """Construct the schema payload for a platform image or multiarch index.""" - if not context.image_digest.startswith("sha256:"): - raise ValueError(f"image digest must be sha256: {context.image_digest!r}") - packages = conda_packages(metadata_dir) - package_payload = json.dumps(packages, sort_keys=True, separators=(",", ":")).encode() - all_packages = installed_packages(packages, pip_packages_path) - all_package_payload = json.dumps( - all_packages, sort_keys=True, separators=(",", ":") - ).encode() - subject: dict[str, Any] = { - "reference": context.image_reference, - "digest": context.image_digest, - } - if context.platform != "multiarch": - os_name, separator, architecture = context.platform.partition("/") - if not separator: - raise ValueError("platform must be os/architecture or multiarch") - subject["platform"] = {"os": os_name, "architecture": architecture} - manifest: dict[str, Any] = { - "schema_version": SCHEMA_VERSION, - "generated_at": datetime.now(timezone.utc).isoformat(), - "subject": subject, - "image": { - "kind": context.image_kind, - "rapids_version": context.rapids_version, - "cuda_version": context.cuda_version, - "python_version": context.python_version, - }, - "build": { - "source_repository": context.source_repository, - "source_commit": context.source_commit, - "workflow_ref": context.workflow_ref, - "workflow_run_url": context.workflow_run_url, - "build_args": parse_build_args(context.build_args), - }, - "conda_packages": packages, - "conda_packages_sha256": _sha256(package_payload), - "installed_packages": all_packages, - "installed_packages_sha256": _sha256(all_package_payload), - "platform_manifests": sorted( - (parse_platform_manifest(value) for value in platform_manifests or []), - key=lambda item: item["platform"], - ), - } - return manifest - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output", type=Path, required=True) - parser.add_argument("--image-reference", required=True) - parser.add_argument("--image-digest", required=True) - parser.add_argument("--image-kind", required=True) - parser.add_argument("--rapids-version", required=True) - parser.add_argument("--cuda-version", required=True) - parser.add_argument("--python-version", required=True) - parser.add_argument("--platform", required=True) - parser.add_argument("--source-repository", required=True) - parser.add_argument("--source-commit", required=True) - parser.add_argument("--workflow-ref", required=True) - parser.add_argument("--workflow-run-url", required=True) - parser.add_argument("--conda-meta-dir", type=Path) - parser.add_argument("--pip-packages", type=Path) - parser.add_argument("--build-arg", action="append", default=[]) - parser.add_argument("--platform-manifest", action="append", default=[]) - return parser.parse_args() - - -def main() -> None: - args = parse_args() - manifest = build_manifest( - ManifestContext( - image_reference=args.image_reference, - image_digest=args.image_digest, - image_kind=args.image_kind, - rapids_version=args.rapids_version, - cuda_version=args.cuda_version, - python_version=args.python_version, - platform=args.platform, - source_repository=args.source_repository, - source_commit=args.source_commit, - workflow_ref=args.workflow_ref, - workflow_run_url=args.workflow_run_url, - build_args=args.build_arg, - ), - metadata_dir=args.conda_meta_dir, - pip_packages_path=args.pip_packages, - platform_manifests=args.platform_manifest, - ) - args.output.parent.mkdir(parents=True, exist_ok=True) - args.output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") - - -if __name__ == "__main__": - main() diff --git a/ci/publish-image-provenance-index.sh b/ci/publish-image-provenance-index.sh deleted file mode 100755 index 64032802..00000000 --- a/ci/publish-image-provenance-index.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -eEuo pipefail - -: "${IMAGE_REFERENCE:?Set IMAGE_REFERENCE to the multiarch image tag}" -: "${IMAGE_KIND:?Set IMAGE_KIND to base or notebooks}" -: "${RAPIDS_VER:?Set RAPIDS_VER}" -: "${CUDA_VER:?Set CUDA_VER}" -: "${PYTHON_VER:?Set PYTHON_VER}" -: "${PLATFORM_REFERENCES:?Set PLATFORM_REFERENCES as platform=tag lines}" - -normalize_registry_reference() { - local reference="$1" - local first_component="${reference%%/*}" - if [[ $first_component != *.* && $first_component != *:* && $first_component != "localhost" ]]; then - reference="docker.io/${reference}" - fi - printf '%s\n' "$reference" -} - -output_dir="${RUNNER_TEMP:-/tmp}/image-provenance-index-${IMAGE_KIND}" -manifest_path="$output_dir/image-provenance-index.json" -workflow_run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" -registry_reference="$(normalize_registry_reference "$IMAGE_REFERENCE")" -mkdir -p "$output_dir" - -image_digest="$(oras manifest fetch --descriptor "$registry_reference" | jq -r '.digest')" -platform_args=() -while IFS='=' read -r platform reference; do - [[ -z "$platform" ]] && continue - platform_reference="$(normalize_registry_reference "$reference")" - digest="$(oras manifest fetch --descriptor "$platform_reference" | jq -r '.digest')" - platform_args+=(--platform-manifest "${platform}|${platform_reference}|${digest}") -done <<<"$PLATFORM_REFERENCES" - -python3 ci/image_provenance_manifest.py \ - --output "$manifest_path" \ - --image-reference "$IMAGE_REFERENCE" \ - --image-digest "$image_digest" \ - --image-kind "$IMAGE_KIND" \ - --rapids-version "$RAPIDS_VER" \ - --cuda-version "$CUDA_VER" \ - --python-version "$PYTHON_VER" \ - --platform multiarch \ - --source-repository "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ - --source-commit "$GITHUB_SHA" \ - --workflow-ref "${GITHUB_WORKFLOW_REF:-$GITHUB_WORKFLOW}" \ - --workflow-run-url "$workflow_run_url" \ - --build-arg "RAPIDS_VER=$RAPIDS_VER" \ - --build-arg "CUDA_VER=$CUDA_VER" \ - --build-arg "PYTHON_VER=$PYTHON_VER" \ - "${platform_args[@]}" - -attached_manifest="$output_dir/attached-image-provenance-index.json" -( - cd "$output_dir" - oras attach \ - --artifact-type application/vnd.rapids.image.provenance.index.v1+json \ - --export-manifest "$attached_manifest" \ - "${registry_reference%@*}@${image_digest}" \ - "image-provenance-index.json:application/vnd.rapids.image.provenance.index.v1+json" -) - -artifact_digest="sha256:$(sha256sum "$attached_manifest" | awk '{print $1}')" -cosign sign --yes "${registry_reference%@*}@${artifact_digest}" diff --git a/ci/publish-image-provenance.sh b/ci/publish-image-provenance.sh deleted file mode 100755 index 685d8017..00000000 --- a/ci/publish-image-provenance.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -eEuo pipefail - -: "${IMAGE_REFERENCE:?Set IMAGE_REFERENCE to the pushed image tag}" -: "${IMAGE_DIGEST:?Set IMAGE_DIGEST from docker/build-push-action output}" -: "${IMAGE_KIND:?Set IMAGE_KIND to base or notebooks}" -: "${IMAGE_PLATFORM:?Set IMAGE_PLATFORM to linux/architecture}" -: "${PROVENANCE_OUTPUT_DIR:?Set PROVENANCE_OUTPUT_DIR from export-image-provenance.sh}" -: "${RAPIDS_VER:?Set RAPIDS_VER}" -: "${CUDA_VER:?Set CUDA_VER}" -: "${PYTHON_VER:?Set PYTHON_VER}" - -normalize_registry_reference() { - local reference="$1" - local first_component="${reference%%/*}" - if [[ $first_component != *.* && $first_component != *:* && $first_component != "localhost" ]]; then - reference="docker.io/${reference}" - fi - printf '%s\n' "$reference" -} - -manifest_path="$PROVENANCE_OUTPUT_DIR/image-provenance.json" -workflow_run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" -registry_reference="$(normalize_registry_reference "$IMAGE_REFERENCE")" - -generator_args=() -while IFS= read -r argument; do - [[ -z "$argument" ]] && continue - generator_args+=(--build-arg "$argument") -done <<<"${DOCKER_BUILD_ARGS:-}" - -python3 ci/image_provenance_manifest.py \ - --output "$manifest_path" \ - --image-reference "$IMAGE_REFERENCE" \ - --image-digest "$IMAGE_DIGEST" \ - --image-kind "$IMAGE_KIND" \ - --rapids-version "$RAPIDS_VER" \ - --cuda-version "$CUDA_VER" \ - --python-version "$PYTHON_VER" \ - --platform "$IMAGE_PLATFORM" \ - --source-repository "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" \ - --source-commit "$GITHUB_SHA" \ - --workflow-ref "${GITHUB_WORKFLOW_REF:-$GITHUB_WORKFLOW}" \ - --workflow-run-url "$workflow_run_url" \ - --conda-meta-dir "$PROVENANCE_OUTPUT_DIR/conda-meta" \ - --pip-packages "$PROVENANCE_OUTPUT_DIR/pip-packages.json" \ - "${generator_args[@]}" - -attached_manifest="$PROVENANCE_OUTPUT_DIR/attached-image-provenance.json" -( - cd "$PROVENANCE_OUTPUT_DIR" - oras attach \ - --artifact-type application/vnd.rapids.image.provenance.v1+json \ - --export-manifest "$attached_manifest" \ - "${registry_reference%@*}@${IMAGE_DIGEST}" \ - "image-provenance.json:application/vnd.rapids.image.provenance.v1+json" -) - -artifact_digest="sha256:$(sha256sum "$attached_manifest" | awk '{print $1}')" -cosign sign --yes "${registry_reference%@*}@${artifact_digest}" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index c5392dc2..d480f942 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023-2026, NVIDIA CORPORATION. ## Usage # Primary interface: bash update-version.sh [--run-context=main|release] @@ -76,11 +76,11 @@ NEXT_SHORT_TAG=${NEXT_MAJOR}.${NEXT_MINOR} # Set branch references based on RUN_CONTEXT if [[ "${RUN_CONTEXT}" == "main" ]]; then - RAPIDS_NOTEBOOKS_REF="main" + RAPIDS_BRANCH_NAME="main" WORKFLOW_BRANCH_REF="main" echo "Preparing development branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting main branch)" elif [[ "${RUN_CONTEXT}" == "release" ]]; then - RAPIDS_NOTEBOOKS_REF="release/${NEXT_SHORT_TAG}" + RAPIDS_BRANCH_NAME="release/${NEXT_SHORT_TAG}" WORKFLOW_BRANCH_REF="release/${NEXT_SHORT_TAG}" echo "Preparing release branch update $CURRENT_TAG => $NEXT_FULL_TAG (targeting release/${NEXT_SHORT_TAG} branch)" fi @@ -96,9 +96,9 @@ done sed_runner "s/com\.nvidia\.workbench\.image-version=.*/com.nvidia.workbench.image-version=\"${NEXT_FULL_TAG}\"/g" Dockerfile -# Dockerfile RAPIDS_NOTEBOOKS_REF -sed_runner "s|ARG RAPIDS_NOTEBOOKS_REF=\"release/[0-9]\+\.[0-9]\+\"|ARG RAPIDS_NOTEBOOKS_REF=\"${RAPIDS_NOTEBOOKS_REF}\"|g" Dockerfile -sed_runner "s|ARG RAPIDS_NOTEBOOKS_REF=\"main\"|ARG RAPIDS_NOTEBOOKS_REF=\"${RAPIDS_NOTEBOOKS_REF}\"|g" Dockerfile +# Dockerfile RAPIDS_BRANCH +sed_runner "s|ARG RAPIDS_BRANCH=\"release/[0-9]\+\.[0-9]\+\"|ARG RAPIDS_BRANCH=\"${RAPIDS_BRANCH_NAME}\"|g" Dockerfile +sed_runner "s|ARG RAPIDS_BRANCH=\"main\"|ARG RAPIDS_BRANCH=\"${RAPIDS_BRANCH_NAME}\"|g" Dockerfile # docs sed_runner "s|RAPIDS_VER=[[:digit:]]\+\.[[:digit:]]|RAPIDS_VER=${NEXT_SHORT_TAG}|g" CONTRIBUTING.md diff --git a/context/.dockerignore b/context/.dockerignore index 1acef0b8..90ffe897 100644 --- a/context/.dockerignore +++ b/context/.dockerignore @@ -7,7 +7,6 @@ !pinned !scripts/configure-apt !scripts/configure-conda-base-environment -!scripts/export-pip-package-inventory !scripts/install-gha-tools !scripts/install-tzdata-packages !scripts/update-base-conda-environment diff --git a/context/notebooks.sh b/context/notebooks.sh index f442a331..5677cc29 100755 --- a/context/notebooks.sh +++ b/context/notebooks.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -# Copyright (c) 2023-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2023-2026, NVIDIA CORPORATION. # Clones repos with notebooks & compiles notebook test dependencies # Requires environment variables: -# RAPIDS_NOTEBOOKS_REF +# RAPIDS_BRANCH # CUDA_VER # PYTHON_VER @@ -14,7 +14,7 @@ NOTEBOOK_REPOS=(cudf cuml cugraph) mkdir -p /notebooks /dependencies for REPO in "${NOTEBOOK_REPOS[@]}"; do echo "Cloning $REPO..." - git clone -b "${RAPIDS_NOTEBOOKS_REF}" --depth 1 --single-branch "https://github.com/rapidsai/$REPO" "$REPO" + git clone -b "${RAPIDS_BRANCH}" --depth 1 --single-branch "https://github.com/rapidsai/$REPO" "$REPO" SOURCE="$REPO/notebooks" DESTINATION="/notebooks/$REPO" diff --git a/context/scripts/export-pip-package-inventory b/context/scripts/export-pip-package-inventory deleted file mode 100644 index 3202835e..00000000 --- a/context/scripts/export-pip-package-inventory +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Export pip-installed distributions with their canonical PyPI pURLs.""" - -from __future__ import annotations - -import json -import re -import sys -from importlib import metadata -from pathlib import Path - - -def main() -> None: - output = Path(sys.argv[1]) - packages = [] - for distribution in metadata.distributions(): - name = str(distribution.metadata.get("Name") or "").strip() - version = str(distribution.version or "").strip() - installer = (distribution.read_text("INSTALLER") or "").strip().lower() - if not name or not version or installer != "pip": - continue - normalized_name = re.sub(r"[-_.]+", "-", name).lower() - packages.append( - { - "name": name, - "version": version, - "purls": [f"pkg:pypi/{normalized_name}"], - "installer": "pip", - "evidence": "python-distribution-metadata; INSTALLER=pip", - } - ) - output.write_text( - json.dumps(sorted(packages, key=lambda item: item["name"].lower())), - encoding="utf-8", - ) - - -if __name__ == "__main__": - main() diff --git a/cuvs-bench/cpu/Dockerfile b/cuvs-bench/cpu/Dockerfile index 5573d3ac..6ad96841 100644 --- a/cuvs-bench/cpu/Dockerfile +++ b/cuvs-bench/cpu/Dockerfile @@ -1,13 +1,13 @@ # syntax=docker/dockerfile:1 -# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024-2026, NVIDIA CORPORATION. ARG MINIFORGE_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.10 +ARG RAPIDS_VER=26.08 FROM condaforge/miniforge3:${MINIFORGE_VER} AS cuvs-bench-cpu ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.10 +ARG RAPIDS_VER=26.08 COPY condarc /opt/conda/.condarc diff --git a/cuvs-bench/gpu/Dockerfile b/cuvs-bench/gpu/Dockerfile index 7a3f1846..1a1a48f0 100644 --- a/cuvs-bench/gpu/Dockerfile +++ b/cuvs-bench/gpu/Dockerfile @@ -1,11 +1,11 @@ # syntax=docker/dockerfile:1 -# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright (c) 2024-2026, NVIDIA CORPORATION. ARG CUDA_VER=notset ARG LINUX_VER=notset ARG MINIFORGE_VER=notset ARG PYTHON_VER=notset -ARG RAPIDS_VER=26.10 +ARG RAPIDS_VER=26.08 # --- begin 'rapidsai/miniforge-cuda' --- # FROM condaforge/miniforge3:${MINIFORGE_VER} AS miniforge-upstream @@ -59,7 +59,7 @@ EOF FROM miniforge-cuda AS cuvs-bench ARG CUDA_VER=notset -ARG RAPIDS_VER=26.10 +ARG RAPIDS_VER=26.08 COPY condarc /opt/conda/.condarc diff --git a/requirements-test.txt b/requirements-test.txt deleted file mode 100644 index 2c78728c..00000000 --- a/requirements-test.txt +++ /dev/null @@ -1 +0,0 @@ -pytest==8.3.5 diff --git a/tests/test_image_provenance_manifest.py b/tests/test_image_provenance_manifest.py deleted file mode 100644 index 948471b9..00000000 --- a/tests/test_image_provenance_manifest.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# ruff: noqa: S101 - -from __future__ import annotations - -import importlib.util -import json -import sys -from pathlib import Path - -import pytest - - -MODULE_PATH = Path(__file__).parents[1] / "ci" / "image_provenance_manifest.py" -REPOSITORY_ROOT = MODULE_PATH.parents[1] -SPEC = importlib.util.spec_from_file_location("image_provenance_manifest", MODULE_PATH) -assert SPEC is not None and SPEC.loader is not None -MODULE = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = MODULE -SPEC.loader.exec_module(MODULE) - - -def manifest_context(**overrides: object) -> object: - """Create valid provenance inputs, with targeted test overrides.""" - defaults = { - "image_reference": "rapidsai/base:26.08-cuda12-py312-amd64", - "image_digest": "sha256:image", - "image_kind": "base", - "rapids_version": "26.08", - "cuda_version": "12", - "python_version": "3.12", - "platform": "linux/amd64", - "source_repository": "https://github.com/rapidsai/docker", - "source_commit": "abc123", - "workflow_ref": ".github/workflows/build-rapids-image.yml@abc123", - "workflow_run_url": "https://github.com/rapidsai/docker/actions/runs/1", - "build_args": ["PYTHON_VER=3.12", "CUDA_VER=12"], - } - defaults.update(overrides) - return MODULE.ManifestContext(**defaults) - - -def test_build_manifest_records_exact_conda_package_facts(tmp_path: Path) -> None: - metadata_dir = tmp_path / "conda-meta" - metadata_dir.mkdir() - (metadata_dir / "c-ares-1.34.8-h1.json").write_text( - json.dumps( - { - "name": "c-ares", - "version": "1.34.8", - "build": "h1", - "build_number": 2, - "channel": "https://conda.anaconda.org/conda-forge", - "url": "https://conda.anaconda.org/conda-forge/linux-64/c-ares.conda", - } - ) - ) - - manifest = MODULE.build_manifest( - manifest_context(), - metadata_dir=metadata_dir, - ) - - assert manifest["subject"]["platform"] == { - "os": "linux", - "architecture": "amd64", - } - assert manifest["conda_packages"] == [ - { - "build": "h1", - "build_number": 2, - "channel": "https://conda.anaconda.org/conda-forge", - "name": "c-ares", - "purl_source": "unmapped", - "purls": [], - "url": "https://conda.anaconda.org/conda-forge/linux-64/c-ares.conda", - "version": "1.34.8", - } - ] - assert manifest["build"]["build_args"] == {"CUDA_VER": "12", "PYTHON_VER": "3.12"} - assert manifest["conda_packages_sha256"].startswith("sha256:") - - -def test_build_manifest_records_direct_pip_purl_evidence(tmp_path: Path) -> None: - pip_packages = tmp_path / "pip-packages.json" - pip_packages.write_text( - json.dumps( - [ - { - "name": "torch", - "version": "2.8.0", - "purls": ["pkg:pypi/torch"], - "installer": "pip", - "evidence": "python-distribution-metadata; INSTALLER=pip", - } - ] - ) - ) - - manifest = MODULE.build_manifest( - manifest_context(), - pip_packages_path=pip_packages, - ) - - assert manifest["installed_packages"] == [ - { - "ecosystem": "pypi", - "evidence": "python-distribution-metadata; INSTALLER=pip", - "installer": "pip", - "name": "torch", - "purls": ["pkg:pypi/torch"], - "version": "2.8.0", - } - ] - assert manifest["installed_packages_sha256"].startswith("sha256:") - - -def test_build_manifest_links_platform_manifests_for_multiarch_images() -> None: - manifest = MODULE.build_manifest( - manifest_context( - image_reference="rapidsai/base:26.08-cuda12-py312", - image_digest="sha256:index", - platform="multiarch", - workflow_ref="workflow", - workflow_run_url="run", - build_args=[], - ), - platform_manifests=[ - "linux/arm64|rapidsai/base:tag-arm64|sha256:arm", - "linux/amd64|rapidsai/base:tag-amd64|sha256:amd", - ], - ) - - assert "platform" not in manifest["subject"] - assert [item["platform"] for item in manifest["platform_manifests"]] == [ - "linux/amd64", - "linux/arm64", - ] - - -def test_invalid_build_argument_is_rejected() -> None: - with pytest.raises(ValueError, match="NAME=VALUE"): - MODULE.parse_build_args(["CUDA_VER"]) - - -def test_pip_inventory_collector_is_in_docker_build_context() -> None: - dockerignore = (REPOSITORY_ROOT / "context" / ".dockerignore").read_text() - - assert "!scripts/export-pip-package-inventory" in dockerignore - - -def test_provenance_collector_writes_outside_the_nonroot_image_root() -> None: - dockerfile = (REPOSITORY_ROOT / "Dockerfile").read_text() - - assert "export-pip-package-inventory /tmp/pip-packages.json" in dockerfile - assert "COPY --from=provenance-base-inventory /tmp/pip-packages.json" in dockerfile - - -@pytest.mark.parametrize( - ("script_name", "safe_title"), - [ - ("publish-image-provenance.sh", "image-provenance.json"), - ("publish-image-provenance-index.sh", "image-provenance-index.json"), - ], -) -def test_published_provenance_uses_safe_oci_layer_titles( - script_name: str, - safe_title: str, -) -> None: - script = (REPOSITORY_ROOT / "ci" / script_name).read_text() - - assert "--disable-path-validation" not in script - assert f'"{safe_title}:application/vnd.rapids.image.provenance' in script