From aeb0993a8ef8db5ca1d9f9b31243af6b96d69340 Mon Sep 17 00:00:00 2001 From: Oron Port Date: Tue, 8 Sep 2026 22:59:13 +0300 Subject: [PATCH 1/4] images: add pnr-xilinx (openXC7 place & route for Xilinx 7-series) Adds the toolchain DFHDL needs for `build -t foss` on Xilinx/AMD 7-series parts: nextpnr-xilinx, bbasm, prjxray (xc7frames2bit, xc7patch, bitread, fasm2frames) and the prjxray bit database. A separate image, not an extension of `pnr`: openXC7 is a fork of nextpnr (openXC7/nextpnr-xilinx, -DARCH=xilinx) rather than an arch of the YosysHQ tree, and it carries a ~500 MB bit database that ECP5/Gowin users should not pay for. Keeping them apart also means a Trellis or Apicula pin bump never rebuilds this image, and vice versa. The database is pruned to the families nextpnr-xilinx-meta also ships (artix7, spartan7, zynq7), since without site metadata the bba exporter cannot build a chipdb for the rest anyway. Built size: 100 MB sif from an 894 MB prefix. NEXTPNR_XILINX_REV must stay at or after 6680df93 (openXC7 PR #156). Below it create_clock does not propagate past the input BUFG, so every build silently reports "PASS at 12.00 MHz" whatever the real constraint. Pinned at 3ad30f57, 24 commits after the fix. The per-part chip databases nextpnr places against cannot live in the image (90-670 MB each, 7.8 GB for the full set) and should not be built on a user's machine, so scripts/chipdb.sc generates them from inside a freshly built sif and publishes each as its own immutable, content-addressed asset in a new `chipdbs` section of dftools.lock.json. Generating from the image is what makes them valid: nextpnr asserts a chipdb's interned IdString table matches its own constids.inc. The release workflow runs that job exactly when pnr-xilinx is rebuilt, since carrying a previous lockfile's chipdbs past a nextpnr-xilinx or prjxray-db bump would publish databases the new binary refuses. Validated locally: the image builds clean, all probes pass, and the full flow (yosys JSON -> nextpnr-xilinx -> fasm2frames -> xc7frames2bit) produces a valid 3,825,899 byte Nexys A7 bitstream from DFHDL's generated output at the design's real 100 MHz constraint. Both published chipdbs were generated from the image and verified against real place-and-route; xc7a100tcsg324 comes out byte-identical to the FPGAwars prebuilt. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/release.yml | 62 +++++++++++++-- README.md | 23 +++++- images/pnr-xilinx.def | 137 ++++++++++++++++++++++++++++++++++ pins.env | 12 +++ scripts/affected-images.sh | 3 +- scripts/build.sc | 6 +- scripts/chipdb.sc | 121 ++++++++++++++++++++++++++++++ scripts/lockfile.sc | 29 ++++++- scripts/test.sc | 5 ++ 9 files changed, 386 insertions(+), 12 deletions(-) create mode 100644 images/pnr-xilinx.def create mode 100644 scripts/chipdb.sc diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad1f93a..aa15275 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,6 +4,7 @@ name: build-test-release # # detect (which images changed) # -> build-test (only affected images x arch; per-image self-test) +# -> chipdb (only when pnr-xilinx was rebuilt: a chipdb is only valid for that build) # -> dfhdl-gate (DFHDL latest release + main, against the freshly built images) # -> publish (only on a version tag or manual dispatch, and only if the gate passed) # @@ -108,6 +109,49 @@ jobs: dftools-${{ matrix.image }}-${{ matrix.arch }}.MANIFEST.txt retention-days: 7 + # The nextpnr-xilinx chip databases are a pure function of NEXTPNR_XILINX_REV + PRJXRAY_DB_REV, + # i.e. of the pnr-xilinx image, so they are regenerated exactly when that image is rebuilt and + # never otherwise. Carrying the previous lockfile's chipdbs forward past a nextpnr-xilinx bump + # would be silently wrong: nextpnr asserts that a chipdb's interned IdString table matches its + # own constids.inc and refuses one built by another revision. + # A publish that does not rebuild pnr-xilinx skips this job and keeps the prior chipdb entries, + # which is correct precisely because the revisions did not move. + chipdb: + needs: [ detect, build-test ] + if: contains(needs.detect.outputs.images, 'pnr-xilinx') + runs-on: ubuntu-24.04 + # The parts published today take about a minute each (xc7a35t 46 s, xc7a100t 76 s, measured + # with the image's plain CPython), but upstream reports hours on the large zynq/virtex + # footprints, so the cap is set well under GitHub's 6 h hosted-runner limit rather than tight. + timeout-minutes: 350 + steps: + - uses: actions/checkout@v7 + # The .bba intermediate is ~0.5 GB for the parts published today, tens of GB on the largest. + - name: Free up disk space + uses: jlumbroso/free-disk-space@main + with: { tool-cache: true, large-packages: true, docker-images: true } + - name: Setup Apptainer + run: | + set -eux + sudo add-apt-repository -y ppa:apptainer/ppa + sudo apt-get update + sudo apt-get install -y apptainer + - name: Setup JDK + Scala CLI + uses: VirtusLab/scala-cli-setup@v1 + with: { jvm: 'temurin:21' } + # A chipdb is byte-identical on every platform, so one arch generates the set for both. + - uses: actions/download-artifact@v8 + with: { name: dftools-pnr-xilinx-linux-x64, path: dist } + - name: Generate chip databases + run: scala-cli run scripts/chipdb.sc -- dist/dftools-pnr-xilinx-linux-x64.sif chipdb + - uses: actions/upload-artifact@v8 + with: + name: dftools-chipdbs + path: | + chipdb/dftools-chipdb-*.bin.gz + chipdb/dftools-chipdb-*.bin.sha256 + retention-days: 7 + dfhdl-gate: needs: build-test strategy: @@ -159,8 +203,13 @@ jobs: # so freshly built images can be released and used to debug. Publish depends only on a # successful build-test; the dfhdl-gate still runs for visibility but does not block. # Restore `needs: dfhdl-gate` once the gate is green. - needs: build-test - if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.publish) + needs: [ build-test, chipdb ] + # `chipdb` is skipped whenever pnr-xilinx was not rebuilt, and a skipped dependency would + # otherwise skip this job too, so its result is checked explicitly. + if: | + !cancelled() && needs.build-test.result == 'success' + && (needs.chipdb.result == 'success' || needs.chipdb.result == 'skipped') + && (startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.publish)) runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 @@ -173,9 +222,10 @@ jobs: - name: Setup JDK + Scala CLI uses: VirtusLab/scala-cli-setup@v1 with: { jvm: 'temurin:21' } - # Rename the built sifs to immutable, content-addressed asset names and (re)generate the - # lockfile mapping each image+arch to its sha256 + asset. Seeded from the prior lockfile on - # the tag, so a partial re-publish carries unchanged images forward. + # Rename the built sifs and chip databases to immutable, content-addressed asset names and + # (re)generate the lockfile mapping each image+arch (and each chipdb part) to its sha256 + + # asset. Seeded from the prior lockfile on the tag, so a partial re-publish carries unchanged + # images (and the chipdb set, when this run did not regenerate it) forward. - name: Build lockfile + immutable asset names run: scala-cli run scripts/lockfile.sc -- ${{ steps.ver.outputs.tag }} dist # Attach the immutable per-image sifs + the lockfile to the release tag. Immutable names never @@ -190,4 +240,6 @@ jobs: dist/*.sif dist/*.sif.sha256 dist/*.MANIFEST.txt + dist/dftools-chipdb-*.bin.gz + dist/dftools-chipdb-*.bin.sha256 dist/dftools.lock.json diff --git a/README.md b/README.md index 7e9ca5e..3bda7c2 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ rebuilds the smallest possible image: | **synth-verilog** | yosys (in-tree slang SV frontend), eqy | no VHDL/LLVM — stays lean | | **synth-vhdl** | yosys + ghdl (frontend) + ghdl-yosys-plugin | `yosys -m ghdl`; carries LLVM | | **pnr** | nextpnr-ecp5, nextpnr-himbaechel, ecppack, gowin_pack | consumes yosys JSON | +| **pnr-xilinx** | nextpnr-xilinx, bbasm, fasm2frames, xc7frames2bit, prjxray-db | openXC7; Xilinx 7-series only⁴ | | **sim-llvm** | nvc + ghdl (simulator) | the two VHDL sims share one LLVM | | **sim-verilator** | verilator (+ g++/make/perl) | keeps a C++ build env at runtime¹ | | **sim-iverilog** | iverilog, vvp | small, self-contained | @@ -47,6 +48,15 @@ viewers alone. Revisit both (and the pin style) when the canonical [Ferrite-Engineering/wavecrux](https://github.com/Ferrite-Engineering/wavecrux) source repo opens post-beta. +⁴ openXC7 is a *fork* of nextpnr (`openXC7/nextpnr-xilinx`, built with `-DARCH=xilinx`), not an +arch of the YosysHQ tree in **pnr**, so it cannot share that image; it also carries a ~500 MB +prjxray bit database that ECP5/Gowin users should not pay for. Keeping them apart means a Trellis +or Apicula pin bump never rebuilds this image, and vice versa. The per-part chip databases nextpnr +places against are **not** in the image (90-670 MB each): they are generated by +[`scripts/chipdb.sc`](scripts/chipdb.sc) and published as their own assets, see below. +`NEXTPNR_XILINX_REV` must stay at or after `6680df93`: below it `create_clock` does not propagate +past the input BUFG and every build silently reports "PASS at 12.00 MHz". + **ghdl appears in two images on purpose**: the *synthesis frontend* ghdl (`synth-vhdl`, plugin-ABI-bound to yosys) and the *simulator* ghdl (`sim-llvm`) are pinned independently (`GHDL_SYNTH_REV` / `GHDL_SIM_REV`), so a simulator bump never @@ -65,7 +75,8 @@ Diamond, Gowin Designer, QuestaSim) are out of scope — used only via | [`scripts/build.sc`](scripts/build.sc) | `build.sc [dest.sif]` — Scalapptainer build driver | | [`scripts/test.sc`](scripts/test.sc) | `test.sc probe ` / `test.sc dfhdl ` | | [`scripts/affected-images.sh`](scripts/affected-images.sh) | maps a diff to the affected image(s) | -| [`scripts/lockfile.sc`](scripts/lockfile.sc) | rename sifs to immutable names + (re)generate `dftools.lock.json` | +| [`scripts/chipdb.sc`](scripts/chipdb.sc) | generate the nextpnr-xilinx chip databases from a built `pnr-xilinx` image | +| [`scripts/lockfile.sc`](scripts/lockfile.sc) | rename sifs and chipdbs to immutable names + (re)generate `dftools.lock.json` | | [`.github/workflows/release.yml`](.github/workflows/release.yml) | detect → build+test → DFHDL gate → publish | ## How each image stays minimal @@ -114,9 +125,17 @@ sha256 of its sif and the asset carrying those bytes: ```json { "tag": "v0.2.0", "images": { "sim-verilator": { "linux-x64": { "sha256": "ab12…", - "asset": "dftools-sim-verilator-linux-x64-ab12….sif" } } } } + "asset": "dftools-sim-verilator-linux-x64-ab12….sif" } } }, + "chipdbs": { "xc7a100tcsg324": { "sha256": "cd34…", + "asset": "dftools-chipdb-xc7a100tcsg324-cd34….bin.gz" } } } ``` +The `chipdbs` section carries the nextpnr-xilinx chip databases the same way, with two differences: +a chipdb is byte-identical on every platform, so it is not keyed by arch, and it is published +gzipped while the recorded digest is of the decompressed bytes, which is what DFHDL caches and +mmaps. A chipdb is only valid for the `nextpnr-xilinx` build that produced it, so the set is +regenerated exactly when the **pnr-xilinx** image is, and carried forward untouched otherwise. + DFHDL bundles this lockfile at build time and resolves/caches each image **by sha256**, not by tag — so bumping the tag re-downloads only the image(s) whose digest actually changed, and the rest of the set is a cache hit. Only rebuilt images are refreshed on a re-publish; the diff --git a/images/pnr-xilinx.def b/images/pnr-xilinx.def new file mode 100644 index 0000000..baab150 --- /dev/null +++ b/images/pnr-xilinx.def @@ -0,0 +1,137 @@ +# DFTools image: pnr-xilinx +# openXC7 place & route + bitstream generation for Xilinx/AMD 7-series. +# Consumes a Yosys netlist JSON (produced by the synth-* images) plus XDC constraints over the +# mounted workdir, and emits FASM -> frames -> .bit. +# +# Deliberately NOT part of the `pnr` image: nextpnr-xilinx is a separate fork of nextpnr +# (openXC7/nextpnr-xilinx, built with -DARCH=xilinx), not an arch of the YosysHQ tree, and it +# carries a ~500 MB prjxray bit database that ECP5/Gowin users should not pay for. Keeping the +# two apart also means a Trellis or Apicula pin bump never rebuilds this image. +# +# The chipdb (the memory-mapped device database nextpnr-xilinx actually places against) is NOT +# in this image: it is per-part, 90-670 MB each, and takes hours to generate. It is built by +# scripts/chipdb.sc into a separate set of release assets and resolved on demand by DFHDL. + +Bootstrap: docker +From: ubuntu:24.04 +Stage: build + +%files + pins.env /opt/pins.env + build/common.sh /opt/common.sh + build/strip-runtime.sh /opt/strip-runtime.sh + +%post + set -eux + export DEBIAN_FRONTEND=noninteractive + export PREFIX=/opt/dftools + . /opt/pins.env + apt-get update + apt-get install -y --no-install-recommends \ + ca-certificates git pkg-config \ + build-essential cmake ninja-build \ + libboost-all-dev libeigen3-dev zlib1g-dev uuid-dev \ + python3 python3-dev python3-venv python3-pip + rm -rf /var/lib/apt/lists/* + . /opt/common.sh + py_venv + + # -- nextpnr-xilinx (openXC7 fork): place & route + FASM writer, plus the bba assembler ---- + # `bbasm` is built but has no install rule upstream, so it is copied by hand. The python + # exporter, constids.inc and the site metadata are installed alongside because they are the + # chipdb generator's inputs (see scripts/chipdb.sc), which runs from this image. + clone https://github.com/openXC7/nextpnr-xilinx "$NEXTPNR_XILINX_REV" nextpnr-xilinx --recursive + ( cd /src/nextpnr-xilinx \ + && cmake -B build -G Ninja \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DARCH=xilinx \ + -DUSE_OPENMP=ON \ + -DBUILD_GUI=OFF -DCMAKE_BUILD_TYPE=Release \ + && ninja -C build && ninja -C build install \ + && install -m755 build/bbasm $PREFIX/bin/bbasm ) + mkdir -p $PREFIX/share/nextpnr-xilinx/external + cp /src/nextpnr-xilinx/xilinx/constids.inc $PREFIX/share/nextpnr-xilinx/constids.inc + cp -r /src/nextpnr-xilinx/xilinx/python $PREFIX/share/nextpnr-xilinx/python + cp -r /src/nextpnr-xilinx/xilinx/external/nextpnr-xilinx-meta \ + $PREFIX/share/nextpnr-xilinx/external/nextpnr-xilinx-meta + + # -- prjxray: FASM -> frames -> bitstream ------------------------------------------------ + # The C++ tools (xc7frames2bit, xc7patch, bitread, ...) link their third-party deps (absl, + # gflags, yaml-cpp) statically, so the final stage needs no extra runtime libraries for them. + clone https://github.com/openXC7/prjxray "$PRJXRAY_REV" prjxray --recursive + ( cd /src/prjxray \ + && cmake -B build -G Ninja \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DPRJXRAY_BUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release \ + && ninja -C build && ninja -C build install ) + # The python side: the `prjxray` package (database/FASM assembler) and `fasm` (the parser). + # `fasm2frames` is a script under utils/ that upstream's setup.py declares as a console entry + # point but never packages, so it is installed by hand with the venv interpreter's shebang. + pip install --no-cache-dir fasm + pip install --no-cache-dir /src/prjxray + install -m755 /src/prjxray/utils/fasm2frames.py $PREFIX/pyenv/bin/fasm2frames + sed -i "1s|.*|#!$PREFIX/pyenv/bin/python3|" $PREFIX/pyenv/bin/fasm2frames + + # -- prjxray-db: the bit database --------------------------------------------------------- + # Pinned independently of the nextpnr-xilinx submodule so the chipdb inputs are explicit. + # Only families that nextpnr-xilinx also has site metadata for are kept: without metadata the + # bba exporter cannot build a chipdb for them, so the rest is dead weight (hundreds of MB). + clone https://github.com/openXC7/prjxray-db "$PRJXRAY_DB_REV" prjxray-db + mkdir -p $PREFIX/share/prjxray-db + for fam in $(ls $PREFIX/share/nextpnr-xilinx/external/nextpnr-xilinx-meta); do + [ -d "/src/prjxray-db/$fam" ] || continue + cp -r "/src/prjxray-db/$fam" "$PREFIX/share/prjxray-db/$fam" + done + cp /src/prjxray-db/LICENSE $PREFIX/share/prjxray-db/LICENSE 2>/dev/null || true + # Let bbaexport.py's own `