docs: Update projects #47
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: 2026 The RISE Project | |
| # SPDX-License-Identifier: MIT | |
| --- | |
| # This workflow is based on the CPU wheel job of: | |
| # https://github.com/johnnynunez/decord2/blob/v3.4.0/.github/workflows/pypi.yml | |
| name: Build decord2 wheels (riscv64) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| version: | |
| description: 'Version glob to (re)build; empty builds every version of docs/packages/decord2.yaml not released yet' | |
| required: false | |
| default: '' | |
| pull_request: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-decord2.yml' | |
| - 'docs/packages/decord2.yaml' | |
| - 'patches/decord2/**' | |
| push: | |
| branches: [main] | |
| paths: | |
| - '.github/workflows/build-decord2.yml' | |
| - 'docs/packages/decord2.yaml' | |
| - 'patches/decord2/**' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read # to fetch code (actions/checkout) | |
| env: | |
| MANYLINUX_RISCV64_IMAGE: quay.io/pypa/manylinux_2_39_riscv64 | |
| jobs: | |
| setup: | |
| uses: $/.github/workflows/_setup.yml | |
| with: | |
| package: decord2 | |
| version: ${{ inputs.version }} | |
| # The wheels bundle prebuilt FFmpeg libraries from a pyav-ffmpeg release, | |
| # including GPL x264/x265 and LGPL FFmpeg, GnuTLS, Nettle, GMP, libunistring, | |
| # alsa-lib and LAME. Publishing them obliges us to ship their licence texts | |
| # and to make the corresponding sources permanently available. | |
| vendor_sources: | |
| name: Collect decord2 ${{ matrix.version }} vendored FFmpeg sources | |
| needs: [setup] | |
| if: needs.setup.outputs.versions != '[]' | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| env: | |
| DECORD2_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout decord2 v${{ env.DECORD2_VERSION }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| repository: johnnynunez/decord2 | |
| ref: v${{ env.DECORD2_VERSION }} | |
| persist-credentials: false | |
| path: decord2 | |
| - name: Download the vendored sources and extract their licences | |
| run: | | |
| cat > "$RUNNER_TEMP/collect-vendor-sources.py" <<'PY' | |
| # SPDX-FileCopyrightText: 2026 The RISE Project | |
| # SPDX-License-Identifier: MIT | |
| """Collect the sources and licence texts of the FFmpeg stack decord2 vendors. | |
| decord2's wheels bundle prebuilt shared libraries fetched from a pyav-ffmpeg | |
| release. Several of them are GPL (x264, x265) or LGPL (FFmpeg, GnuTLS, | |
| Nettle, GMP, libunistring, alsa-lib, LAME), so redistributing the wheels | |
| carries a source-distribution obligation. pyav-ffmpeg pins every dependency | |
| by URL and SHA-256 in scripts/pkg.py, which is what this reads. | |
| """ | |
| import argparse | |
| import hashlib | |
| import json | |
| import re | |
| import subprocess | |
| import sys | |
| import tarfile | |
| import time | |
| from pathlib import Path | |
| LICENCE_RE = re.compile(r"^(COPYING|COPYRIGHT|LICEN[CS]E|NOTICE)", re.IGNORECASE) | |
| def load_packages(pkg_py: str): | |
| namespace: dict = {} | |
| exec(compile(pkg_py, "pkg.py", "exec"), namespace) | |
| # Linux riscv64 enables gnutls, alsa and libvpl; CUDA/AMF/nasm are x86-only. | |
| packages = ( | |
| namespace["gnutls_group"] | |
| + namespace["codec_group"] | |
| + [ | |
| namespace["alsa_package"], | |
| namespace["libvpl_package"], | |
| namespace["ffmpeg_package"], | |
| ] | |
| ) | |
| return sorted(packages, key=lambda p: p.name) | |
| def download(package, dest_dir: Path) -> Path: | |
| name = package.source_filename or package.source_url.rsplit("/", 1)[-1] | |
| # A few upstreams name their tarball after the tag alone ("v2.16.0.tar.gz"). | |
| if package.name.replace("-", "").lower() not in name.replace("-", "").lower(): | |
| name = f"{package.name}-{name}" | |
| path = dest_dir / name | |
| # GitLab generates an "-/archive/" tarball on demand and the bytes differ | |
| # from the cached copy the pin was taken against, so dav1d and x264 hash | |
| # wrong until a retry is served the cached one. | |
| for attempt in range(5): | |
| subprocess.run( | |
| ["curl", "--location", "--fail", "--silent", "--show-error", | |
| "--output", str(path), package.source_url], | |
| check=True, | |
| ) | |
| digest = hashlib.sha256(path.read_bytes()).hexdigest() | |
| if digest == package.sha256: | |
| break | |
| print(f"{package.name}: sha256 {digest}, retrying {package.source_url}") | |
| time.sleep(15) | |
| else: | |
| raise SystemExit( | |
| f"{package.name}: sha256 mismatch for {package.source_url}\n" | |
| f" expected {package.sha256}\n got {digest}" | |
| ) | |
| print(f"{package.name}: {name} ({path.stat().st_size} bytes, sha256 ok)") | |
| return path | |
| def extract_licences(package, tarball: Path, dest_dir: Path) -> None: | |
| chunks = [] | |
| with tarfile.open(tarball) as tar: | |
| for member in tar.getmembers(): | |
| parts = Path(member.name).parts | |
| # Top level of the archive, plus one nested directory (x265 keeps | |
| # its sources under source/, gnutls its licences under doc/). | |
| if not member.isfile() or len(parts) > 3: | |
| continue | |
| if not LICENCE_RE.match(parts[-1]): | |
| continue | |
| handle = tar.extractfile(member) | |
| if handle is None: | |
| continue | |
| text = handle.read().decode("utf-8", "replace") | |
| chunks.append(f"===== {'/'.join(parts[1:])} =====\n\n{text}") | |
| if not chunks: | |
| raise SystemExit(f"{package.name}: no licence file found in {tarball.name}") | |
| header = ( | |
| f"Licence texts for {package.name}, bundled in this wheel as a prebuilt\n" | |
| f"shared library. Source: {package.source_url}\n\n" | |
| ) | |
| (dest_dir / f"LICENSE.{package.name}").write_text(header + "\n\n".join(chunks)) | |
| print(f"{package.name}: {len(chunks)} licence file(s)") | |
| def main() -> None: | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--config-file", type=Path, required=True) | |
| parser.add_argument("--sources-dir", type=Path, required=True) | |
| parser.add_argument("--licenses-dir", type=Path, required=True) | |
| args = parser.parse_args() | |
| config = json.loads(args.config_file.read_text()) | |
| tag = config["url"].split("/download/")[1].split("/")[0] | |
| print(f"pyav-ffmpeg release: {tag}") | |
| args.sources_dir.mkdir(parents=True, exist_ok=True) | |
| args.licenses_dir.mkdir(parents=True, exist_ok=True) | |
| # pyav-ffmpeg carries the build recipe and the patches it applies to FFmpeg, | |
| # GMP, LAME and libvpx, so it is part of the corresponding source. | |
| recipe = args.sources_dir / f"pyav-ffmpeg-{tag}.tar.gz" | |
| subprocess.run( | |
| ["curl", "--location", "--fail", "--silent", "--show-error", "--output", str(recipe), | |
| f"https://github.com/PyAV-Org/pyav-ffmpeg/archive/refs/tags/{tag}.tar.gz"], | |
| check=True, | |
| ) | |
| with tarfile.open(recipe) as tar: | |
| member = next(m for m in tar.getmembers() if m.name.endswith("/scripts/pkg.py")) | |
| pkg_py = tar.extractfile(member).read().decode() | |
| for package in load_packages(pkg_py): | |
| tarball = download(package, args.sources_dir) | |
| extract_licences(package, tarball, args.licenses_dir) | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
| PY | |
| python3 "$RUNNER_TEMP/collect-vendor-sources.py" \ | |
| --config-file decord2/scripts/ffmpeg-8.1.json --sources-dir sources --licenses-dir licenses | |
| tar -cf gpl-sources.tar -C sources . | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: decord2-${{ env.DECORD2_VERSION }}-gpl-sources | |
| path: gpl-sources.tar | |
| if-no-files-found: error | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: decord2-${{ env.DECORD2_VERSION }}-vendor-licenses | |
| path: licenses/ | |
| if-no-files-found: error | |
| build_wheels: | |
| name: Build decord2 ${{ matrix.version }} ${{ matrix.python }}-manylinux_riscv64 | |
| needs: [setup, vendor_sources] | |
| if: needs.setup.outputs.versions != '[]' | |
| runs-on: ubuntu-24.04-riscv | |
| timeout-minutes: 360 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| python: ["cp312", "cp313", "cp314", "cp314t"] | |
| env: | |
| DECORD2_VERSION: ${{ matrix.version }} | |
| steps: | |
| - name: Checkout decord2 v${{ env.DECORD2_VERSION }} | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| repository: johnnynunez/decord2 | |
| ref: v${{ env.DECORD2_VERSION }} | |
| submodules: recursive | |
| persist-credentials: false | |
| - name: Checkout python-wheels | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| path: python-wheels | |
| persist-credentials: false | |
| - name: Patch decord2 source | |
| run: git apply python-wheels/patches/decord2/${{ env.DECORD2_VERSION }}/00*.patch | |
| - name: Fetch the vendored libraries' licence texts | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: decord2-${{ env.DECORD2_VERSION }}-vendor-licenses | |
| path: vendor-licenses | |
| # setuptools' default license-files glob is rooted at setup.py's own directory | |
| # (python/), not the checkout root where decord2's LICENSE lives. | |
| - name: Stage the licence texts for packaging | |
| run: cp LICENSE vendor-licenses/LICENSE.* python/ | |
| - name: Build wheel | |
| uses: pypa/cibuildwheel@1828c10ab37f080699c7b81cea34097c684a7074 # v4.2.0 | |
| with: | |
| package-dir: python | |
| output-dir: wheelhouse/ | |
| only: ${{ matrix.python }}-manylinux_riscv64 | |
| env: | |
| CIBW_MANYLINUX_RISCV64_IMAGE: ${{ env.MANYLINUX_RISCV64_IMAGE }} | |
| CIBW_BEFORE_BUILD: >- | |
| python {project}/scripts/fetch-vendor.py --config-file {project}/scripts/ffmpeg-8.1.json /tmp/vendor && | |
| cmake -S {project} -B {project}/build -DUSE_CUDA=OFF -DCMAKE_BUILD_TYPE=Release && | |
| cmake --build {project}/build --parallel $(nproc) | |
| CIBW_ENVIRONMENT_LINUX: >- | |
| LD_LIBRARY_PATH=/tmp/vendor/lib:$LD_LIBRARY_PATH | |
| PKG_CONFIG_PATH=/tmp/vendor/lib/pkgconfig | |
| PIP_EXTRA_INDEX_URL=https://pypi.riseproject.dev/simple/ | |
| CIBW_TEST_REQUIRES: pytest numpy | |
| CIBW_TEST_SOURCES: tests examples | |
| CIBW_TEST_COMMAND: pytest -v tests/python/unittests | |
| - name: Check the wheel carries libdecord, the vendored FFmpeg and its licences | |
| run: | | |
| python3 - wheelhouse/*.whl <<'EOF' | |
| import pathlib, sys, zipfile | |
| expected = {p.name for p in pathlib.Path("vendor-licenses").iterdir()} | {"LICENSE"} | |
| for whl in sys.argv[1:]: | |
| names = zipfile.ZipFile(whl).namelist() | |
| libs = [n for n in names if n.startswith("decord2.libs/")] | |
| shipped = {n.rsplit("/", 1)[1] for n in names if ".dist-info/licenses/" in n} - {""} | |
| assert "decord/libdecord.so" in names, f"no decord/libdecord.so in {whl}" | |
| assert any("libavcodec" in n for n in libs), f"no vendored FFmpeg in {whl}" | |
| assert expected <= shipped, f"{whl} is missing {sorted(expected - shipped)}" | |
| print(f"{whl}: {len(libs)} bundled libraries, {len(shipped)} licence files") | |
| EOF | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: decord2-${{ env.DECORD2_VERSION }}-${{ matrix.python }}-manylinux_riscv64 | |
| path: wheelhouse/*.whl | |
| if-no-files-found: error | |
| publish: | |
| name: Publish decord2 ${{ matrix.version }} | |
| needs: [setup, vendor_sources, build_wheels] | |
| if: needs.setup.outputs.versions != '[]' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: ${{ fromJSON(needs.setup.outputs.versions) }} | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| uses: $/.github/workflows/_publish-wheel.yml | |
| secrets: | |
| app-private-key: ${{ secrets.RISEPROJECT_APP_PRIVATE_KEY }} | |
| with: | |
| artifact-pattern: decord2-${{ matrix.version }}-*-manylinux_riscv64 | |
| gpl-sources-artifact: decord2-${{ matrix.version }}-gpl-sources | |
| gpl-sources-description: FFmpeg, x264, x265 |