diff --git a/.github/workflows/build-eckitlib.yml b/.github/workflows/build-eckitlib.yml new file mode 100644 index 00000000000..02b762726a2 --- /dev/null +++ b/.github/workflows/build-eckitlib.yml @@ -0,0 +1,335 @@ +# SPDX-FileCopyrightText: 2026 The RISE Project +# SPDX-License-Identifier: MIT +--- +# Upstream builds eckitlib via a private tool (ecmwf/reusable-workflows' +# python-wrapper-wheel.yml + the "wheelmaker" image), not a public workflow, so this +# replays ecmwf/eckit's own python/eckitlib/buildconfig CMAKE_PARAMS and +# post-build.sh directly. The built libraries use no Python C API, so this builds +# one py3-none wheel instead of upstream's byte-identical cp310..cp314 matrix. +# ENABLE_PYTHON is off: the Cython extension it adds is installed into the separate +# `eckit` distribution's source tree, never into this wheel. +name: Build eckitlib wheels (riscv64) + +on: + workflow_dispatch: + inputs: + version: + description: 'Version glob to (re)build; empty builds every version of docs/packages/eckitlib.yaml not released yet' + required: false + default: '' + pull_request: + branches: [main] + paths: + - '.github/workflows/build-eckitlib.yml' + - 'docs/packages/eckitlib.yaml' + push: + branches: [main] + paths: + - '.github/workflows/build-eckitlib.yml' + - 'docs/packages/eckitlib.yaml' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: read # to fetch code (actions/checkout) + +env: + MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 + ECBUILD_VERSION: '3.14.2' + +jobs: + setup: + uses: $/.github/workflows/_setup.yml + with: + package: eckitlib + version: ${{ inputs.version }} + + build_wheels: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Build eckitlib ${{ matrix.version }} py3-none-manylinux_riscv64 + runs-on: ubuntu-24.04-riscv + timeout-minutes: 720 + env: + PACKAGE_VERSION: ${{ matrix.version }} + + steps: + - name: Derive the eckit git tag from the package version + run: echo "ECKIT_VERSION=$(echo "$PACKAGE_VERSION" | sed -E 's/\.[0-9]+$//')" >> "$GITHUB_ENV" + + - name: Checkout eckit ${{ env.ECKIT_VERSION }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ecmwf/eckit + ref: ${{ env.ECKIT_VERSION }} + persist-credentials: false + + - name: Record the resolved commit + run: echo "ECKIT_COMMIT=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + + - name: Write the wheel packaging scripts + run: | + cat > collect_licences.py <<'PY' + import glob + import json + import os + import re + import shutil + import subprocess + import sys + + WHEEL_DIR = sys.argv[1] + dist_info = glob.glob(f"{WHEEL_DIR}/*.dist-info")[0] + sbom = json.load(open(f"{dist_info}/sboms/auditwheel.cdx.json")) + + + def rpm(*args): + return subprocess.run(["rpm", *args], capture_output=True, text=True).stdout + + + def licences_of(package): + return [ + path + for path in rpm("-ql", package).splitlines() + if path.startswith("/usr/share/licenses/") and os.path.isfile(path) + ] + + + by_source = [line.split() for line in rpm("-qa", "--qf", "%{SOURCERPM} %{NAME}\n").splitlines()] + + for name in sorted({c["name"] for c in sbom["components"] if c["purl"].startswith("pkg:rpm/")}): + source = rpm("-q", "--qf", "%{SOURCERPM}", name) + files = [ + path + for sibling_source, sibling in by_source + if sibling_source == source + for path in licences_of(sibling) + ] + if not files: + # lz4-libs is a runtime subpackage with no %license file and no installed + # sibling that has one, so its source package has to be pulled in. + base = re.sub(r"-[^-]+-[^-]+\.src\.rpm$", "", source) + subprocess.run(["dnf", "-y", "-q", "install", base], check=True) + files = licences_of(base) + if not files: + raise SystemExit(f"no licence file found for {name}") + dest = f"{dist_info}/licenses/{name}" + os.makedirs(dest, exist_ok=True) + for path in files: + shutil.copy(path, dest) + PY + + cat > pack_wheel.py <<'PY' + import hashlib + import os + import sys + import zipfile + from base64 import urlsafe_b64encode + + STAGE, OUT_DIR = sys.argv[1:3] + + dist_info = next(n for n in sorted(os.listdir(STAGE)) if n.endswith(".dist-info")) + name, version = dist_info[: -len(".dist-info")].split("-") + tags = [ + line.split(":", 1)[1].strip() + for line in open(os.path.join(STAGE, dist_info, "WHEEL")) + if line.startswith("Tag:") + ] + interpreter, abi = tags[0].split("-")[:2] + tag = f"{interpreter}-{abi}-{'.'.join(t.split('-', 2)[2] for t in tags)}" + + os.makedirs(OUT_DIR, exist_ok=True) + wheel_path = os.path.join(OUT_DIR, f"{name}-{version}-{tag}.whl") + record_name = f"{dist_info}/RECORD" + records = [] + + with zipfile.ZipFile(wheel_path, "w", zipfile.ZIP_DEFLATED) as zf: + for root, _dirs, files in os.walk(STAGE): + for filename in sorted(files): + path = os.path.join(root, filename) + arcname = os.path.relpath(path, STAGE) + if arcname == record_name: + continue + with open(path, "rb") as f: + data = f.read() + info = zipfile.ZipInfo(arcname) + info.external_attr = (os.stat(path).st_mode & 0xFFFF) << 16 + zf.writestr(info, data, zipfile.ZIP_DEFLATED) + digest = urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=").decode() + records.append(f"{arcname},sha256={digest},{len(data)}") + + zf.writestr(record_name, "\n".join(records + [f"{record_name},,"]) + "\n") + + print(wheel_path) + PY + + - name: Build eckit, then package and repair the wheel + run: | + docker run --rm \ + -v "$(pwd)":/workspace \ + --workdir /workspace \ + -e PACKAGE_VERSION="$PACKAGE_VERSION" \ + -e ECKIT_COMMIT="$ECKIT_COMMIT" \ + -e ECBUILD_VERSION="$ECBUILD_VERSION" \ + "$MANYLINUX_RISCV64_IMAGE" \ + bash -c ' + set -euo pipefail + dnf -y -q install libcurl-devel lz4-devel flex + DIST_INFO=/tmp/stage/eckitlib-$PACKAGE_VERSION.dist-info + mkdir -p /tmp/deps /tmp/build $DIST_INFO/licenses + + curl -fsSL "https://github.com/ecmwf/ecbuild/archive/refs/tags/$ECBUILD_VERSION.tar.gz" | tar xz -C /tmp/build + cmake -S "/tmp/build/ecbuild-$ECBUILD_VERSION" -B /tmp/build/ecbuild-build -D CMAKE_INSTALL_PREFIX=/tmp/deps + cmake --build /tmp/build/ecbuild-build -j "$(nproc)" + cmake --install /tmp/build/ecbuild-build + + cmake -S . -B /tmp/build/eckit-build \ + -D CMAKE_BUILD_TYPE=MinSizeRel \ + -D CMAKE_PREFIX_PATH=/tmp/deps \ + -D CMAKE_INSTALL_PREFIX=/tmp/stage/eckitlib \ + -D CMAKE_INSTALL_LIBDIR=lib64 \ + -D ENABLE_MPI=0 \ + -D ENABLE_ECKIT_GEO=1 \ + -D ENABLE_BUILD_TOOLS=OFF \ + -D ENABLE_AEC=0 \ + -D ENABLE_EIGEN=0 \ + -D ENABLE_LZ4=1 \ + -D ENABLE_PYTHON=0 + cmake --build /tmp/build/eckit-build -j "$(nproc)" + cmake --install /tmp/build/eckit-build + + printf "__version__ = \"%s\"\n__commit_hash__ = \"%s\"\nfindlibs_dependencies = []\n" \ + "$PACKAGE_VERSION" "$ECKIT_COMMIT" > /tmp/stage/eckitlib/__init__.py + + cp LICENSE AUTHORS $DIST_INFO/licenses/ + printf "eckitlib\n" > $DIST_INFO/top_level.txt + printf "Wheel-Version: 1.0\nGenerator: python-wheels-riscv64-port\nRoot-Is-Purelib: false\nTag: py3-none-linux_riscv64\n" > $DIST_INFO/WHEEL + printf "Metadata-Version: 2.4\nName: eckitlib\nVersion: %s\nSummary: Compiled eckit C++ toolkit libraries (ECMWF), no Python bindings\nHome-page: https://github.com/ecmwf/eckit\nLicense-Expression: Apache-2.0\n" \ + "$PACKAGE_VERSION" > $DIST_INFO/METADATA + + python3 pack_wheel.py /tmp/stage /tmp/unrepaired + auditwheel repair --plat manylinux_2_39_riscv64 -w /tmp/repaired /tmp/unrepaired/*.whl + + mkdir -p /tmp/final + cd /tmp/final && unzip -q /tmp/repaired/*.whl && cd /workspace + python3 collect_licences.py /tmp/final + # Rocky ships only the GPLv2 text covering the lz4 CLI tools, while the + # bundled liblz4 is BSD-2-Clause and no RPM carries that text; upstream + # pre-compile.sh fetches it from the lz4 repository for the same reason. + LZ4_LICENSES=/tmp/final/eckitlib-$PACKAGE_VERSION.dist-info/licenses/lz4-libs + LZ4_VERSION=$(rpm -q --qf "%{VERSION}" lz4-libs) + curl -fsSL "https://raw.githubusercontent.com/lz4/lz4/v$LZ4_VERSION/LICENSE" -o "$LZ4_LICENSES/LICENSE" + curl -fsSL "https://raw.githubusercontent.com/lz4/lz4/v$LZ4_VERSION/lib/LICENSE" -o "$LZ4_LICENSES/lib-LICENSE" + + python3 pack_wheel.py /tmp/final wheelhouse + ' + + - name: Check the wheel ships the libraries and every bundled licence + run: | + python3 - wheelhouse/*.whl <<'EOF' + import json, sys, zipfile + + zf = zipfile.ZipFile(sys.argv[1]) + names = zf.namelist() + for lib in ("libeckit.so", "libeckit_geo.so", "libeckit_sql.so", "libeckit_codec.so"): + assert any(n.endswith(f"lib64/{lib}") for n in names), (lib, names) + for licence in ("LICENSE", "AUTHORS"): + assert any(n.endswith(f".dist-info/licenses/{licence}") for n in names), licence + + sbom = json.loads(zf.read(next(n for n in names if n.endswith("auditwheel.cdx.json")))) + for component in sbom["components"]: + if component["purl"].startswith("pkg:rpm/"): + prefix = f".dist-info/licenses/{component['name']}/" + assert any(prefix in n for n in names), component["name"] + EOF + + - name: Smoke-test the built libraries + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq --no-install-recommends python3-venv + python3 -m venv .venv + . .venv/bin/activate + # gotcha 234: stock Ubuntu 24.04 apt pip (24.0) has zero manylinux_*_riscv64 + # tags, so it rejects the riscv64 wheel outright without upgrading first. + pip install -q --upgrade pip + pip install -q findlibs wheelhouse/*.whl + python3 -c " + import ctypes, glob, os + import eckitlib, findlibs + + for lib in sorted(glob.glob(os.path.join(os.path.dirname(eckitlib.__file__), 'lib64', '*.so'))): + ctypes.CDLL(lib) + print('loaded', os.path.basename(lib)) + + eckit = findlibs.load('eckit') + for name in ('eckit_version', 'eckit_version_str', 'eckit_git_sha1'): + getattr(eckit, name).restype = ctypes.c_char_p + eckit.eckit_version_int.restype = ctypes.c_uint + print(eckit.eckit_version().decode(), eckit.eckit_version_int(), eckit.eckit_git_sha1().decode()) + assert eckit.eckit_version_str().decode() == eckitlib.__version__.rsplit('.', 1)[0] + assert eckit.eckit_git_sha1().decode() == eckitlib.__commit_hash__ + " + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eckitlib-${{ matrix.version }}-py3-none-manylinux_riscv64 + path: wheelhouse/*.whl + if-no-files-found: error + + gpl_sources: + needs: [setup] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + name: Collect GPL sources for eckitlib ${{ matrix.version }} + runs-on: ubuntu-24.04-riscv + + env: + PACKAGE_VERSION: ${{ matrix.version }} + + steps: + - name: Checkout python-wheels + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + # The copyleft (GPL/LGPL) libraries auditwheel vendors out of the build image + # along libcurl's dependency closure. + - uses: ./actions/collect-gpl-sources + with: + image: ${{ env.MANYLINUX_RISCV64_IMAGE }} + packages: gcc keyutils-libs libssh libidn2 libunistring libxcrypt systemd-libs libcap pcre2 + output: gpl-sources.tar + + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: eckitlib-${{ env.PACKAGE_VERSION }}-gpl-sources + path: gpl-sources.tar + if-no-files-found: error + + publish: + name: Publish eckitlib ${{ matrix.version }} + needs: [setup, build_wheels, gpl_sources] + if: needs.setup.outputs.versions != '[]' + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.setup.outputs.versions) }} + permissions: + contents: write + pull-requests: write + uses: $/.github/workflows/_publish-wheel.yml + secrets: + app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} + with: + artifact-pattern: eckitlib-${{ matrix.version }}-*-manylinux_riscv64 + gpl-sources-artifact: eckitlib-${{ matrix.version }}-gpl-sources + gpl-sources-description: gcc and the copyleft libraries bundled in the wheel diff --git a/docs/packages/eckitlib.yaml b/docs/packages/eckitlib.yaml new file mode 100644 index 00000000000..bf25596f5dc --- /dev/null +++ b/docs/packages/eckitlib.yaml @@ -0,0 +1,5 @@ +package-name: eckitlib +source-code: https://github.com/ecmwf/eckit +license: Apache-2.0 +versions: +- version: 2.1.1.26