From 802d42c1cebc89c0d9e7aa650d7082e4e663e586 Mon Sep 17 00:00:00 2001 From: PSi86 Date: Thu, 30 Jul 2026 18:59:05 +0200 Subject: [PATCH 1/2] Publish flashable release artifacts Releases published the application image only; the second-stage bootloader and the partition table stayed in the build directory. Commissioning a blank node therefore meant flashing stock WLED from install.wled.me and then pushing the RaceLink image through /update with "Ignore firmware validation" ticked -- a detour that routes the least experienced users through the most dangerous control in WLED's UI, because that checkbox disables all payload validation. Stage, per environment: the application image, the three pre-application images, and a merged factory image covering all four, plus a SHA-256 manifest and an assets.json sidecar giving environment, chip, device type, kind, flash offset, size and digest for every file. release_artifacts.py and release_staging.py are byte-identical to their RaceLink_Gateway counterparts. Both repositories publish the same asset shapes, and this is where a mistake produces an image that flashes cleanly and never boots, so there is one implementation of it rather than two. Offsets come from PlatformIO's metadata, the chip is read out of the image being merged, and the application offset comes from the generated partition table -- with a per-chip cross-check that aborts if the two sources disagree. That guard matters more here than in the gateway: the shipping set spans C3, S2 and S3, and the S2 bootloader sits at 0x1000 rather than 0x0. Metadata has to be collected inside the per-profile loop, while that profile's platformio_override.ini is still staged -- the next iteration overwrites it. Naming now states both versions, the RaceLink release and the WLED release it wraps, and the device type the master looks up: RaceLink_WLED-0.1.8-RaceLink_Node_v4_s3_llcc68-TYPE12-wled_v0.15.3-app.bin This also retires the old suffix handling, which joined Path.suffixes to preserve ".bin" but folded the whole WLED release name into the extension, because the upstream version contains dots. Nothing consumes these names: the host uploads whatever file the operator picks and WLED's /update ignores filenames. The build-and-stage loop moved into scripts/build_and_stage_profiles.sh so the new build.yml can rehearse the entire release on every pull request -- six profiles, staged, merged and verified, publishing nothing. Also records why RaceLink_Node_v7_classic_esp32_emac stays out of the shipping set: the omission was indistinguishable from an oversight. --- .github/workflows/build.yml | 181 +++++++++++++++++++ .github/workflows/release.yml | 113 ++++-------- .gitignore | 7 + scripts/build_and_stage_profiles.sh | 84 +++++++++ scripts/release_artifacts.py | 248 ++++++++++++++++++++++++++ scripts/release_profiles.py | 70 +++++--- scripts/release_staging.py | 190 ++++++++++++++++++++ scripts/stage_wled_profile.py | 83 ++++++++- tests/test_release_artifacts.py | 265 ++++++++++++++++++++++++++++ tests/test_release_workflow.py | 75 ++++++++ 10 files changed, 1210 insertions(+), 106 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 scripts/build_and_stage_profiles.sh create mode 100644 scripts/release_artifacts.py create mode 100644 scripts/release_staging.py create mode 100644 tests/test_release_artifacts.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..5a7805c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,181 @@ +name: Build Firmware + +# Compile-only counterpart to release.yml. That workflow is the only thing that +# ever built this repo, and it tags and publishes in the same run — so a broken +# build or a broken staging step surfaced in the middle of a real release. +# +# This job runs the identical build-and-stage script against the latest +# published WLED release, verifies the resulting factory images, and ships +# nothing. + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + inputs: + wled_ref: + description: "Optional WLED tag/ref override. Leave empty to use the latest published WLED release." + required: false + default: "" + type: string + +permissions: + contents: read + +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + + steps: + - name: Check out RaceLink_WLED repository + uses: actions/checkout@v6.0.2 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Run static tests + run: python -m unittest discover -s tests -p "test_*.py" + + - name: Resolve WLED source ref + id: wled_source + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ref="$(python scripts/resolve_wled_release.py --wled-ref "${{ inputs.wled_ref }}" --print ref)" + repository="$(python scripts/resolve_wled_release.py --wled-ref "${{ inputs.wled_ref }}" --print repository)" + echo "ref=${ref}" >> "$GITHUB_OUTPUT" + echo "repository=${repository}" >> "$GITHUB_OUTPUT" + + - name: Check out WLED repository + uses: actions/checkout@v6.0.2 + with: + repository: ${{ steps.wled_source.outputs.repository }} + ref: ${{ steps.wled_source.outputs.ref }} + path: external/WLED + fetch-depth: 1 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + cache-dependency-path: external/WLED/package-lock.json + + - name: Cache PlatformIO installation + uses: actions/cache@v6 + with: + path: | + ~/.platformio/.cache + ~/.platformio/packages + ~/.platformio/platforms + key: pio-${{ runner.os }}-${{ hashFiles('external/WLED/platformio.ini') }} + restore-keys: pio-${{ runner.os }}- + + - name: Install build prerequisites + run: | + python -m pip install --upgrade pip + python -m pip install -r external/WLED/requirements.txt + # Same pin as release.yml — this job rehearses the staging step. + python -m pip install "esptool>=5,<6" + npm --prefix external/WLED ci + + # The same script release.yml runs, so release day differs only in the + # version string. + - name: Build and stage every shipping profile + shell: bash + run: | + bash scripts/build_and_stage_profiles.sh \ + "$GITHUB_WORKSPACE" \ + "$GITHUB_WORKSPACE/external/WLED" \ + "0.0.0-ci" \ + "${{ steps.wled_source.outputs.ref }}" \ + "$GITHUB_WORKSPACE/dist" + + - name: Verify the staged factory images + run: | + python - <<'PY' + import json + from pathlib import Path + + manifest = json.loads( + next(Path("dist").glob("*-assets.json")).read_text(encoding="utf-8") + ) + for environment in manifest["environments"]: + assets = {asset["kind"]: asset for asset in environment["assets"]} + image = (Path("dist") / assets["factory"]["file"]).read_bytes() + app_offset = environment["app_offset"] + + expectations = [ + (assets["bootloader"]["offset"], b"\xe9", "bootloader"), + (assets["partitions"]["offset"], b"\xaa\x50", "partition table"), + (app_offset, b"\xe9", "application"), + ] + for offset, magic, label in expectations: + found = image[offset : offset + len(magic)] + if found != magic: + raise SystemExit( + f"{environment['env']}: no {label} at 0x{offset:x} " + f"(found {found.hex()}, expected {magic.hex()})" + ) + + published = (Path("dist") / assets["app"]["file"]).read_bytes() + if image[app_offset : app_offset + len(published)] != published: + raise SystemExit( + f"{environment['env']}: the application embedded in the factory " + "image differs from the published application image" + ) + print( + f"{environment['env']}: {environment['chip']}, DEV_TYPE " + f"{environment['dev_type']}, factory image verified ({len(image)} bytes)" + ) + PY + + - name: Write build summary + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + manifest = json.loads( + next(Path("dist").glob("*-assets.json")).read_text(encoding="utf-8") + ) + lines = [ + "## Build Summary", + "", + f"WLED ref: `{manifest.get('wled_ref')}`", + "", + "| Environment | Chip | DEV_TYPE | Application | Factory image |", + "|---|---|---|---|---|", + ] + for environment in manifest["environments"]: + assets = {asset["kind"]: asset for asset in environment["assets"]} + lines.append( + f"| `{environment['env']}` | {environment['chip']} | " + f"{environment['dev_type']} | {assets['app']['size'] / 1024:.1f} KiB | " + f"{assets['factory']['size'] / 1024:.1f} KiB |" + ) + Path(os.environ["GITHUB_STEP_SUMMARY"]).write_text( + "\n".join(lines) + "\n", encoding="utf-8" + ) + PY + + # The rehearsed release assets, so a pull request can be flashed on + # hardware — including the factory images — without cutting a release. + - name: Upload build output + uses: actions/upload-artifact@v7 + with: + name: racelink_wled_build + if-no-files-found: error + path: dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e832194..983fcae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,6 +120,10 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -r external/WLED/requirements.txt + # esptool 5 renamed merge_bin to merge-bin; pin the major so the + # rename cannot break a release, and so the defaults the staging + # code relies on (flash mode/freq/size = keep) stay put. + python -m pip install "esptool>=5,<6" npm --prefix external/WLED ci - name: Build shipping RaceLink firmware profiles @@ -129,88 +133,19 @@ jobs: RELEASE_VERSION: ${{ steps.release_version.outputs.version }} WLED_REF: ${{ steps.wled_source.outputs.ref }} run: | - set -euo pipefail - mkdir -p dist - : > dist/built_envs.txt - # Single source of truth: SHIPPING_PROFILE_FILENAMES in - # scripts/release_profiles.py (via iter_shipping_profiles, which also - # validates each file exists). Keep this in sync by editing that tuple - # only -- do NOT hard-code the profile list here. - mapfile -t profiles < <( - python -c "from pathlib import Path; from scripts.release_profiles import iter_shipping_profiles; [print('build_profiles/' + p.name) for p in iter_shipping_profiles(Path('build_profiles'))]" - ) - if [ "${#profiles[@]}" -eq 0 ]; then - echo "No shipping profiles resolved from SHIPPING_PROFILE_FILENAMES" - exit 1 - fi - printf 'Shipping profiles to build:\n'; printf ' %s\n' "${profiles[@]}" - - for profile in "${profiles[@]}"; do - rm -rf external/WLED/build_output/release - mkdir -p external/WLED/build_output/release - - mapfile -t envs < <( - python scripts/stage_wled_profile.py stage-profile \ - --repo-root "$GITHUB_WORKSPACE" \ - --wled-dir "$GITHUB_WORKSPACE/external/WLED" \ - --profile "$GITHUB_WORKSPACE/$profile" - ) - - if [ "${#envs[@]}" -eq 0 ]; then - echo "Profile $profile did not expose any envs" - exit 1 - fi - - for env_name in "${envs[@]}"; do - echo "$env_name" >> dist/built_envs.txt - done - - args=() - for env_name in "${envs[@]}"; do - args+=("-e" "$env_name") - done - - python -m platformio run \ - --project-dir "$GITHUB_WORKSPACE/external/WLED" \ - "${args[@]}" - - python scripts/stage_wled_profile.py stage-assets \ - --profile "$GITHUB_WORKSPACE/$profile" \ - --release-dir "$GITHUB_WORKSPACE/external/WLED/build_output/release" \ - --dist-dir "$GITHUB_WORKSPACE/dist" \ - --release-version "$RELEASE_VERSION" \ - --wled-ref "$WLED_REF" - done + bash scripts/build_and_stage_profiles.sh \ + "$GITHUB_WORKSPACE" \ + "$GITHUB_WORKSPACE/external/WLED" \ + "$RELEASE_VERSION" \ + "$WLED_REF" \ + "$GITHUB_WORKSPACE/dist" { echo "built_envs<> "$GITHUB_OUTPUT" - - name: Generate SHA256 manifest - run: | - python - <<'PY' - import hashlib - import json - from pathlib import Path - - dist = Path("dist") - lines = [] - for path in sorted(dist.glob("RaceLink_WLED-*")): - if not path.is_file(): - continue - digest = hashlib.sha256(path.read_bytes()).hexdigest() - lines.append(f"{digest} {path.name}") - if not lines: - raise SystemExit("No staged release artifacts found in dist/") - version = json.loads(Path("version.json").read_text(encoding="utf-8"))["version"] - (dist / f"RaceLink_WLED-{version}-sha256.txt").write_text( - "\n".join(lines) + "\n", - encoding="utf-8", - ) - PY - - name: Commit release metadata run: | git add version.json @@ -229,6 +164,7 @@ jobs: uses: actions/upload-artifact@v7 with: name: racelink_wled_release + if-no-files-found: error path: dist/* - name: Publish GitHub release @@ -237,6 +173,29 @@ jobs: tag_name: ${{ steps.release_version.outputs.tag }} target_commitish: ${{ inputs.target_branch }} generate_release_notes: true + body: | + Built against WLED `${{ steps.wled_source.outputs.ref }}`. Each + asset names both versions: its own RaceLink release and the WLED + release it wraps. + + ### Flashing + + * `…-app.bin` — application image. This is the file for WLED's + `/update` page and for the host's firmware-update dialog. + * `…-factory-usb-serial-only.bin` — bootloader, partition table, + OTA selector and application in one image, written at offset `0x0`. + **USB serial only.** Sent over OTA it is rejected in the normal + case, but with *Ignore firmware validation* ticked it is written + into the inactive slot and bricks the node until someone + re-flashes over serial. It also rewrites the partition table, so + WLED reformats its filesystem on the next boot: presets, + `cfg.json`, the master binding and the group id are all gone and + the node returns as unconfigured. A commissioning tool, never an + update tool. + * `…-bootloader.bin` / `…-partitions.bin` / `…-boot_app0.bin` — the + individual pieces, for flashing a blank chip manually. + * `…-assets.json` — machine-readable index: environment, chip, + device type, kind, flash offset, size and SHA-256 for every file. files: dist/* - name: Write workflow summary @@ -251,6 +210,6 @@ jobs: echo "- Built environments:" while IFS= read -r env_name; do echo " - \`${env_name}\`" - done < dist/built_envs.txt + done < built_envs.txt echo "- Trigger: \`workflow_dispatch\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 40fb24f..95b42db 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,10 @@ __pycache__/ .claude/ CLAUDE.md CLAUDE.local.md + +# Staged release artifacts and the scaffolding around them +# (scripts/build_and_stage_profiles.sh). +dist/ +built_envs.txt +metadata.json +external/ diff --git a/scripts/build_and_stage_profiles.sh b/scripts/build_and_stage_profiles.sh new file mode 100644 index 0000000..3af46d9 --- /dev/null +++ b/scripts/build_and_stage_profiles.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# +# Build every shipping profile into a WLED checkout and stage its output as +# release assets. +# +# Used by both release.yml and build.yml: the rehearsal on a pull request and +# the real release run the same code, so the only thing that differs on release +# day is the version string. +# +# Profiles are staged one at a time -- each writes platformio_override.ini into +# the WLED checkout, so the next iteration overwrites it. Everything that reads +# a profile's build configuration therefore has to happen inside the loop. + +set -euo pipefail + +repo_root="${1:?repository root}" +wled_dir="${2:?WLED checkout}" +release_version="${3:?release version}" +wled_ref="${4:?WLED ref}" +dist_dir="${5:?dist directory}" + +cd "$repo_root" +mkdir -p "$dist_dir" +# Outside dist/ on purpose: everything in there is published. +: > built_envs.txt + +# Single source of truth: SHIPPING_PROFILE_FILENAMES in +# scripts/release_profiles.py (via iter_shipping_profiles, which also validates +# each file exists). Keep this in sync by editing that tuple only -- do NOT +# hard-code the profile list here. +mapfile -t profiles < <( + python -c "from pathlib import Path; from scripts.release_profiles import iter_shipping_profiles; [print('build_profiles/' + p.name) for p in iter_shipping_profiles(Path('build_profiles'))]" +) +if [ "${#profiles[@]}" -eq 0 ]; then + echo "No shipping profiles resolved from SHIPPING_PROFILE_FILENAMES" + exit 1 +fi +printf 'Shipping profiles to build:\n'; printf ' %s\n' "${profiles[@]}" + +for profile in "${profiles[@]}"; do + echo "::group::$profile" + + mapfile -t envs < <( + python scripts/stage_wled_profile.py stage-profile \ + --repo-root "$repo_root" \ + --wled-dir "$wled_dir" \ + --profile "$repo_root/$profile" + ) + + if [ "${#envs[@]}" -eq 0 ]; then + echo "Profile $profile did not expose any envs" + exit 1 + fi + + args=() + for env_name in "${envs[@]}"; do + echo "$env_name" >> built_envs.txt + args+=("-e" "$env_name") + done + + python -m platformio run --project-dir "$wled_dir" "${args[@]}" + + # Reports the pre-application images and the offsets the platform itself + # would flash them at, so no per-SoC offset table is needed downstream. + python -m platformio project metadata \ + --project-dir "$wled_dir" \ + "${args[@]}" \ + --json-output-path "$repo_root/metadata.json" + + python scripts/stage_wled_profile.py stage-assets \ + --profile "$repo_root/$profile" \ + --build-root "$wled_dir/.pio/build" \ + --dist-dir "$dist_dir" \ + --release-version "$release_version" \ + --wled-ref "$wled_ref" \ + --metadata "$repo_root/metadata.json" + + echo "::endgroup::" +done + +python scripts/stage_wled_profile.py finalize \ + --dist-dir "$dist_dir" \ + --release-version "$release_version" \ + --wled-ref "$wled_ref" diff --git a/scripts/release_artifacts.py b/scripts/release_artifacts.py new file mode 100644 index 0000000..02b05d8 --- /dev/null +++ b/scripts/release_artifacts.py @@ -0,0 +1,248 @@ +"""Pure helpers for staging RaceLink release artifacts. + +Kept byte-identical in RaceLink_Gateway and RaceLink_WLED: both publish the same +asset shapes, and a fix to the offset or chip logic has to reach both. The +module is I/O-free so the release workflows' naming, offset and chip-detection +logic can be exercised without a toolchain; each repository's CLI wrapper does +the copying and calls out to esptool. + +Design notes: + +* **Offsets come from the build, never from a table.** PlatformIO's project + metadata reports the flash images (bootloader, partition table, boot_app0) + together with the offsets the platform itself would flash them at, so a new + SoC — where the bootloader moves from ``0x0`` to ``0x1000`` — needs no change + here. :func:`bootloader_offset_for_chip` exists only as a cross-check. +* **The chip is read out of the image being merged.** The ESP image header + carries the chip id, so the value handed to ``esptool merge-bin`` cannot + disagree with the binary it merges. +* **The application offset is read from the generated partition table**, which + is the same file that ends up in the factory image. +""" + +from __future__ import annotations + +import struct +import sys +from dataclasses import dataclass +from typing import Iterable, Sequence + +# --- ESP application image ------------------------------------------------ + +ESP_IMAGE_MAGIC = 0xE9 +_CHIP_ID_STRUCT = struct.Struct(" bool: + return self.type == PARTITION_TYPE_APP + + +@dataclass(frozen=True) +class FlashImage: + """One pre-application image and the offset it is flashed at.""" + + offset: int + path: str + + +def read_chip_name(image: bytes) -> str: + """Return the esptool chip name for an ESP application or bootloader image.""" + if len(image) < _CHIP_ID_OFFSET + _CHIP_ID_STRUCT.size: + raise ValueError("Image is too short to contain an ESP image header") + if image[0] != ESP_IMAGE_MAGIC: + raise ValueError(f"Not an ESP image: magic is 0x{image[0]:02X}, expected 0xE9") + (chip_id,) = _CHIP_ID_STRUCT.unpack_from(image, _CHIP_ID_OFFSET) + try: + return CHIP_NAMES[chip_id] + except KeyError: + raise ValueError( + f"Unknown ESP chip id 0x{chip_id:04X}. Add it to CHIP_NAMES and to " + "BOOTLOADER_OFFSETS before releasing for this SoC." + ) from None + + +def bootloader_offset_for_chip(chip: str) -> int: + """Expected second-stage bootloader offset, for cross-checking the build.""" + try: + return BOOTLOADER_OFFSETS[chip] + except KeyError: + raise ValueError( + f"No bootloader offset mapped for chip {chip!r}. Add it before " + "releasing for this SoC — a wrong offset yields an unbootable image." + ) from None + + +def parse_partition_table(data: bytes) -> list[Partition]: + """Parse a generated ``partitions.bin`` into its entries.""" + partitions: list[Partition] = [] + for start in range(0, len(data), PARTITION_ENTRY_SIZE): + entry = data[start : start + PARTITION_ENTRY_SIZE] + if len(entry) < PARTITION_ENTRY_SIZE or not entry.startswith(PARTITION_ENTRY_MAGIC): + break + offset, size = struct.unpack_from(" int: + """Offset of the first application partition — where firmware.bin goes.""" + for partition in sorted(partitions, key=lambda p: p.offset): + if partition.is_app: + return partition.offset + raise ValueError("Partition table declares no application partition") + + +def flash_images_from_metadata(metadata: dict, env: str) -> list[FlashImage]: + """Extract the pre-application images PlatformIO would flash for ``env``. + + Accepts either the per-environment metadata mapping produced by + ``pio project metadata --json-output`` or a single environment's idedata. + """ + entry = metadata.get(env, metadata) + extra = entry.get("extra") or {} + images = extra.get("flash_images") or [] + if not images: + raise ValueError( + f"PlatformIO reported no flash images for {env!r}; without the " + "bootloader and partition table a factory image cannot be built." + ) + return [FlashImage(offset=int(str(i["offset"]), 0), path=str(i["path"])) for i in images] + + +def define_value(defines: Iterable[str], name: str) -> str | None: + """Return the value of a ``NAME=value`` build define, if present.""" + prefix = f"{name}=" + for define in defines: + text = str(define) + if text.startswith(prefix): + return text[len(prefix) :] + return None + + +def device_type(defines: Iterable[str]) -> int | None: + """Parse ``-D DEV_TYPE=`` — the feature-set handle the master looks up.""" + raw = define_value(defines, "DEV_TYPE") + if raw is None: + return None + return int(raw, 0) + + +# --- Artifact naming ------------------------------------------------------ +# +# Field order: product, release version, environment, device type, variant, +# kind. The kind goes last because it is the most visible token in a file +# picker, and the factory image must never be mistaken for something OTA-able. +# The three pre-application images carry neither device type nor variant — they +# are build by-products of the environment, not firmware identity. +# +# `variant` carries the upstream version a build was made against, so a +# RaceLink_WLED asset states both its own release and the WLED release it wraps +# ("wled_v0.15.3"). Fields never contain "-", so the name stays parseable, but +# consumers should read the assets.json sidecar rather than the filename. + +FACTORY_KIND = "factory-usb-serial-only" +APP_KIND = "app" + + +def artifact_name( + *, + product: str, + version: str, + env: str, + kind: str, + dev_type: int | None = None, + variant: str | None = None, + extension: str = ".bin", +) -> str: + parts = [product, version, env] + if dev_type is not None: + parts.append(f"TYPE{dev_type}") + if variant is not None: + parts.append(variant) + parts.append(kind) + return "-".join(parts) + extension + + +def checksum_name(product: str, version: str) -> str: + return f"{product}-{version}-sha256.txt" + + +def manifest_name(product: str, version: str) -> str: + return f"{product}-{version}-assets.json" + + +def merge_command( + *, + chip: str, + output: str, + images: Sequence[tuple[int, str]], + python_executable: str | None = None, +) -> list[str]: + """Build the ``esptool merge-bin`` argv. + + Flash mode, frequency and size are deliberately not passed: ``merge-bin`` + defaults all three to ``keep``, which preserves the header the bootloader + was compiled with. Overriding them is how a merged image ends up + mismatching the board it was built for. + + The interpreter defaults to the running one so esptool is taken from the + same environment that installed it, rather than from whatever ``python`` + happens to be first on PATH. + """ + if not images: + raise ValueError("Refusing to merge an empty image list") + interpreter = python_executable or sys.executable + argv = [interpreter, "-m", "esptool", "--chip", chip, "merge-bin", "-o", output] + for offset, path in sorted(images, key=lambda item: item[0]): + argv += [hex(offset), path] + return argv diff --git a/scripts/release_profiles.py b/scripts/release_profiles.py index 2f7a236..6df856d 100644 --- a/scripts/release_profiles.py +++ b/scripts/release_profiles.py @@ -7,6 +7,16 @@ from dataclasses import dataclass from pathlib import Path +from scripts.release_staging import stage_environment + +# The profiles a release actually builds and publishes. +# +# RaceLink_Node_v7_classic_esp32_emac is committed but deliberately absent: +# its internal-EMAC Ethernet support is still in bring-up and has never been +# released. The omission is intentional, not an oversight — add it here once +# Ethernet ships, and note that it is the first classic ESP32 in the set, so +# its bootloader sits at 0x1000 rather than 0x0 (release_staging.py cross- +# checks that and will refuse to stage it if the offsets disagree). SHIPPING_PROFILE_FILENAMES = ( "RaceLink_Node_v1_c3_ct62.platformio_override.ini", "RaceLink_Node_v3_s2_llcc68.platformio_override.ini", @@ -140,32 +150,48 @@ def sanitize_ref_for_filename(raw_ref: str) -> str: return value or "unknown-ref" +def wled_variant(wled_ref: str) -> str: + """Filename fragment naming the upstream release a build wraps. + + One dash-free field, so an asset states both its own RaceLink version and + the WLED version it was built against without making the name ambiguous. + """ + return f"wled_{sanitize_ref_for_filename(wled_ref)}" + + def stage_release_assets( *, profile_path: Path, - release_dir: Path, + build_root: Path, dist_dir: Path, release_version: str, wled_ref: str, -) -> list[Path]: - """Rename WLED output artifacts to stable RaceLink release filenames.""" + metadata: dict, + product: str = "RaceLink_WLED", +) -> list[dict]: + """Stage every asset for one profile's environments. + + Returns a manifest entry per environment; the caller accumulates them + across profiles and writes the release index once. + + Note the source: PlatformIO's build directory, not WLED's + ``build_output/release``. Both hold the same application image, but the + build directory also holds the bootloader and the partition table that the + factory image needs, and it is what the flash-image offsets in PlatformIO's + metadata point at. + """ dist_dir.mkdir(parents=True, exist_ok=True) - staged_paths: list[Path] = [] - ref_fragment = sanitize_ref_for_filename(wled_ref) - - for env in parse_profile_environments(profile_path): - candidates = sorted(release_dir.glob(f"WLED_*_{env.release_name}.bin*")) - if not candidates: - raise FileNotFoundError( - f"Could not find build_output/release artifact for {env.release_name}" - ) - - for candidate in candidates: - suffix = "".join(candidate.suffixes) - target = dist_dir / ( - f"RaceLink_WLED-{release_version}-{env.name}-{ref_fragment}{suffix}" - ) - shutil.copy2(candidate, target) - staged_paths.append(target) - - return staged_paths + variant = wled_variant(wled_ref) + + return [ + stage_environment( + env=env.name, + product=product, + version=release_version, + build_dir=build_root / env.name, + dist_dir=dist_dir, + metadata=metadata, + variant=variant, + ) + for env in parse_profile_environments(profile_path) + ] diff --git a/scripts/release_staging.py b/scripts/release_staging.py new file mode 100644 index 0000000..06b4058 --- /dev/null +++ b/scripts/release_staging.py @@ -0,0 +1,190 @@ +"""Stage one build environment's output as release assets. + +Kept byte-identical in RaceLink_Gateway and RaceLink_WLED, for the same reason +as :mod:`scripts.release_artifacts`: this is where a mistake produces an image +that flashes cleanly and never boots, so there should be exactly one +implementation of it. The repositories differ in how they drive this — the +gateway iterates a flat environment list, RaceLink_WLED stages one profile at a +time into an external WLED checkout — but not in what a staged environment +looks like. + +Produced per environment: + +* the application image, which is what OTA and the host's firmware dialog take, +* the three pre-application images (bootloader, partition table, boot_app0), so + a blank chip can be commissioned without a local toolchain, +* a merged factory image covering all four, written at offset 0. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +from pathlib import Path + +from scripts.release_artifacts import ( + APP_KIND, + FACTORY_KIND, + application_offset, + artifact_name, + bootloader_offset_for_chip, + checksum_name, + device_type, + flash_images_from_metadata, + manifest_name, + merge_command, + parse_partition_table, + read_chip_name, +) + +# PlatformIO names the pre-application images by file; map them to the asset +# kind so the sidecar reads as intent rather than as a filename. +PART_KINDS = { + "bootloader.bin": "bootloader", + "partitions.bin": "partitions", + "boot_app0.bin": "boot_app0", +} + + +def sha256_of(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _stage(source: Path, target: Path) -> Path: + if not source.is_file(): + raise SystemExit(f"Expected build artifact not found: {source}") + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + return target + + +def stage_environment( + *, + env: str, + product: str, + version: str, + build_dir: Path, + dist_dir: Path, + metadata: dict, + variant: str | None = None, +) -> dict: + """Stage every asset for one environment and return its manifest entry.""" + firmware = build_dir / "firmware.bin" + if not firmware.is_file(): + raise SystemExit(f"Expected firmware artifact not found: {firmware}") + + chip = read_chip_name(firmware.read_bytes()) + env_metadata = metadata.get(env, metadata) + flash_images = flash_images_from_metadata(metadata, env) + + by_name = {Path(image.path).name: image for image in flash_images} + for required in ("bootloader.bin", "partitions.bin"): + if required not in by_name: + raise SystemExit(f"{env}: PlatformIO reported no {required}") + + partitions = parse_partition_table(Path(by_name["partitions.bin"].path).read_bytes()) + app_offset = application_offset(partitions) + + # Cross-check the offset PlatformIO reported against the one this chip is + # known to need. Either source alone could be wrong silently; disagreement + # cannot be. This is the guard that catches an S2 or classic-ESP32 target + # whose bootloader sits at 0x1000 rather than 0x0. + expected_offset = bootloader_offset_for_chip(chip) + if by_name["bootloader.bin"].offset != expected_offset: + raise SystemExit( + f"{env}: PlatformIO flashes the bootloader at " + f"0x{by_name['bootloader.bin'].offset:x} but {chip} expects " + f"0x{expected_offset:x}. Refusing to publish a factory image." + ) + + dev_type = device_type(env_metadata.get("defines") or []) + assets: list[dict] = [] + + def record(path: Path, kind: str, offset: int) -> None: + assets.append( + { + "file": path.name, + "kind": kind, + "offset": offset, + "size": path.stat().st_size, + "sha256": sha256_of(path), + } + ) + + app_target = _stage( + firmware, + dist_dir + / artifact_name( + product=product, + version=version, + env=env, + kind=APP_KIND, + dev_type=dev_type, + variant=variant, + ), + ) + record(app_target, APP_KIND, app_offset) + + merge_inputs: list[tuple[int, str]] = [(app_offset, str(firmware))] + for image in flash_images: + source = Path(image.path) + kind = PART_KINDS.get(source.name) + if kind is None: + raise SystemExit(f"{env}: unexpected flash image {source.name}") + target = _stage( + source, + dist_dir / artifact_name(product=product, version=version, env=env, kind=kind), + ) + record(target, kind, image.offset) + merge_inputs.append((image.offset, str(source))) + + factory_target = dist_dir / artifact_name( + product=product, + version=version, + env=env, + kind=FACTORY_KIND, + dev_type=dev_type, + variant=variant, + ) + command = merge_command(chip=chip, output=str(factory_target), images=merge_inputs) + print(" ".join(command), flush=True) + subprocess.run(command, check=True) + record(factory_target, "factory", 0) + + return { + "env": env, + "chip": chip, + "dev_type": dev_type, + "app_offset": app_offset, + "assets": assets, + } + + +def write_release_index( + *, + dist_dir: Path, + product: str, + version: str, + environments: list[dict], + extra: dict | None = None, +) -> tuple[Path, Path]: + """Write the assets.json sidecar and the SHA-256 manifest.""" + if not environments: + raise SystemExit("No environments were staged") + + manifest = {"product": product, "version": version, **(extra or {})} + manifest["environments"] = environments + manifest_path = dist_dir / manifest_name(product, version) + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + checksum_lines = [ + f"{asset['sha256']} {asset['file']}" + for environment in environments + for asset in environment["assets"] + ] + checksum_path = dist_dir / checksum_name(product, version) + checksum_path.write_text("\n".join(checksum_lines) + "\n", encoding="utf-8") + + return manifest_path, checksum_path diff --git a/scripts/stage_wled_profile.py b/scripts/stage_wled_profile.py index 558e336..d2a2249 100644 --- a/scripts/stage_wled_profile.py +++ b/scripts/stage_wled_profile.py @@ -3,6 +3,8 @@ from __future__ import annotations import argparse +import json +import shutil import sys from pathlib import Path @@ -16,6 +18,12 @@ stage_profile_override, stage_release_assets, ) +from scripts.release_staging import write_release_index + +PRODUCT = "RaceLink_WLED" + +# Per-profile manifest fragments, assembled by `finalize` and removed again. +FRAGMENT_DIR = ".staging" def _build_parser() -> argparse.ArgumentParser: @@ -34,13 +42,32 @@ def _build_parser() -> argparse.ArgumentParser: stage_assets = subparsers.add_parser( "stage-assets", - help="Rename built WLED artifacts into RaceLink release assets.", + help="Stage one profile's build output as RaceLink release assets.", ) stage_assets.add_argument("--profile", required=True, type=Path) - stage_assets.add_argument("--release-dir", required=True, type=Path) + stage_assets.add_argument( + "--build-root", + required=True, + type=Path, + help="PlatformIO build directory of the WLED checkout (.pio/build).", + ) stage_assets.add_argument("--dist-dir", required=True, type=Path) stage_assets.add_argument("--release-version", required=True) stage_assets.add_argument("--wled-ref", required=True) + stage_assets.add_argument( + "--metadata", + required=True, + type=Path, + help="JSON from `pio project metadata`, collected while this profile is staged.", + ) + + finalize = subparsers.add_parser( + "finalize", + help="Write the assets.json sidecar and the SHA-256 manifest.", + ) + finalize.add_argument("--dist-dir", required=True, type=Path) + finalize.add_argument("--release-version", required=True) + finalize.add_argument("--wled-ref", required=True) return parser @@ -57,15 +84,55 @@ def _run_stage_profile(args: argparse.Namespace) -> int: def _run_stage_assets(args: argparse.Namespace) -> int: - staged = stage_release_assets( + dist_dir = args.dist_dir.resolve() + environments = stage_release_assets( profile_path=args.profile.resolve(), - release_dir=args.release_dir.resolve(), - dist_dir=args.dist_dir.resolve(), + build_root=args.build_root.resolve(), + dist_dir=dist_dir, release_version=args.release_version, wled_ref=args.wled_ref, + metadata=json.loads(args.metadata.resolve().read_text(encoding="utf-8")), + ) + + # Each profile is built and staged in turn, then its override file is + # overwritten by the next one -- so the manifest is accumulated on disk + # rather than held in memory, and assembled by `finalize` at the end. + fragments = dist_dir / FRAGMENT_DIR + fragments.mkdir(parents=True, exist_ok=True) + (fragments / f"{args.profile.stem}.json").write_text( + json.dumps(environments, indent=2) + "\n", encoding="utf-8" ) - for path in staged: - sys.stdout.write(f"{path}\n") + + for environment in environments: + for asset in environment["assets"]: + sys.stdout.write(f"{asset['file']}\n") + return 0 + + +def _run_finalize(args: argparse.Namespace) -> int: + dist_dir = args.dist_dir.resolve() + fragments = sorted((dist_dir / FRAGMENT_DIR).glob("*.json")) + if not fragments: + raise SystemExit(f"No staged profiles found in {dist_dir / FRAGMENT_DIR}") + + environments = [ + environment + for fragment in fragments + for environment in json.loads(fragment.read_text(encoding="utf-8")) + ] + + manifest_path, checksum_path = write_release_index( + dist_dir=dist_dir, + product=PRODUCT, + version=args.release_version, + environments=environments, + extra={"wled_ref": args.wled_ref}, + ) + + # The fragments are scaffolding, not release assets. + shutil.rmtree(dist_dir / FRAGMENT_DIR) + + sys.stdout.write(f"{manifest_path.name}\n{checksum_path.name}\n") return 0 @@ -76,6 +143,8 @@ def main() -> int: return _run_stage_profile(args) if args.command == "stage-assets": return _run_stage_assets(args) + if args.command == "finalize": + return _run_finalize(args) parser.error(f"Unsupported command: {args.command}") return 2 diff --git a/tests/test_release_artifacts.py b/tests/test_release_artifacts.py new file mode 100644 index 0000000..15935a9 --- /dev/null +++ b/tests/test_release_artifacts.py @@ -0,0 +1,265 @@ +"""Tests for the release-artifact helpers. + +The guardrails that matter here are the ones whose failure mode is a published +artifact that flashes cleanly and never boots: an unmapped SoC, a bootloader +offset taken from the wrong source, or an application offset that no longer +matches the partition table shipped inside the same image. +""" + +from __future__ import annotations + +import struct +import unittest + +from scripts.release_artifacts import ( + APP_KIND, + BOOTLOADER_OFFSETS, + CHIP_NAMES, + FACTORY_KIND, + application_offset, + artifact_name, + bootloader_offset_for_chip, + checksum_name, + device_type, + flash_images_from_metadata, + manifest_name, + merge_command, + parse_partition_table, + read_chip_name, +) + + +def _esp_image(chip_id: int, magic: int = 0xE9) -> bytes: + header = bytearray(24) + header[0] = magic + header[1] = 5 # segment count + struct.pack_into(" bytes: + entry = bytearray(32) + entry[0:2] = b"\xaa\x50" + entry[2] = ptype + entry[3] = subtype + struct.pack_into(" bytes: + return b"".join( + [ + _partition_entry("nvs", 0x01, 0x02, 0x9000, 0x5000), + _partition_entry("otadata", 0x01, 0x00, 0xE000, 0x2000), + _partition_entry("app0", 0x00, 0x10, 0x10000, 0x330000), + _partition_entry("app1", 0x00, 0x11, 0x340000, 0x330000), + _partition_entry("spiffs", 0x01, 0x82, 0x670000, 0x180000), + ] + ) + b"\xff" * 32 + + +class ChipDetectionTests(unittest.TestCase): + def test_reads_the_chip_from_the_image_header(self) -> None: + self.assertEqual(read_chip_name(_esp_image(0x0009)), "esp32s3") + self.assertEqual(read_chip_name(_esp_image(0x0002)), "esp32s2") + self.assertEqual(read_chip_name(_esp_image(0x0000)), "esp32") + + def test_rejects_a_non_esp_image(self) -> None: + with self.assertRaises(ValueError): + read_chip_name(_esp_image(0x0009, magic=0x00)) + + def test_rejects_an_unmapped_chip_id(self) -> None: + with self.assertRaises(ValueError): + read_chip_name(_esp_image(0x00FF)) + + def test_every_known_chip_has_a_bootloader_offset(self) -> None: + for chip in CHIP_NAMES.values(): + with self.subTest(chip=chip): + self.assertIn(chip, BOOTLOADER_OFFSETS) + + def test_bootloader_offsets_match_the_documented_layout(self) -> None: + # The two families that differ; getting these wrong is the classic + # unbootable-factory-image bug. + self.assertEqual(bootloader_offset_for_chip("esp32"), 0x1000) + self.assertEqual(bootloader_offset_for_chip("esp32s2"), 0x1000) + self.assertEqual(bootloader_offset_for_chip("esp32s3"), 0x0000) + self.assertEqual(bootloader_offset_for_chip("esp32c3"), 0x0000) + + def test_rejects_an_unmapped_chip_name(self) -> None: + with self.assertRaises(ValueError): + bootloader_offset_for_chip("esp32p4") + + +class PartitionTableTests(unittest.TestCase): + def test_parses_entries_until_the_table_ends(self) -> None: + partitions = parse_partition_table(_partition_table()) + + self.assertEqual([p.name for p in partitions], ["nvs", "otadata", "app0", "app1", "spiffs"]) + self.assertEqual(partitions[0].offset, 0x9000) + + def test_application_offset_is_the_first_app_partition(self) -> None: + self.assertEqual(application_offset(parse_partition_table(_partition_table())), 0x10000) + + def test_rejects_a_table_without_an_application(self) -> None: + data = _partition_entry("nvs", 0x01, 0x02, 0x9000, 0x5000) + with self.assertRaises(ValueError): + application_offset(parse_partition_table(data)) + + def test_rejects_an_empty_table(self) -> None: + with self.assertRaises(ValueError): + parse_partition_table(b"\xff" * 32) + + +class MetadataTests(unittest.TestCase): + METADATA = { + "WirelessStickV3-ESP32S3": { + "defines": ["PLATFORMIO=60119", "DEV_TYPE=1", 'DEV_TYPE_STR="RaceLink_Gateway_v4"'], + "extra": { + "flash_images": [ + {"offset": "0x0000", "path": "/build/bootloader.bin"}, + {"offset": "0x8000", "path": "/build/partitions.bin"}, + {"offset": "0xe000", "path": "/framework/boot_app0.bin"}, + ] + }, + } + } + + def test_reads_flash_images_for_an_environment(self) -> None: + images = flash_images_from_metadata(self.METADATA, "WirelessStickV3-ESP32S3") + + self.assertEqual([image.offset for image in images], [0x0, 0x8000, 0xE000]) + + def test_accepts_single_environment_idedata(self) -> None: + single = self.METADATA["WirelessStickV3-ESP32S3"] + + images = flash_images_from_metadata(single, "WirelessStickV3-ESP32S3") + + self.assertEqual(len(images), 3) + + def test_rejects_metadata_without_flash_images(self) -> None: + with self.assertRaises(ValueError): + flash_images_from_metadata({"env": {"extra": {}}}, "env") + + def test_reads_the_device_type_from_the_build_defines(self) -> None: + defines = self.METADATA["WirelessStickV3-ESP32S3"]["defines"] + + self.assertEqual(device_type(defines), 1) + + def test_device_type_is_optional(self) -> None: + self.assertIsNone(device_type(["PLATFORMIO=60119"])) + + +class NamingTests(unittest.TestCase): + def test_application_image_name(self) -> None: + self.assertEqual( + artifact_name( + product="RaceLink_Gateway", + version="0.1.6", + env="WirelessStickV3-ESP32S3", + kind=APP_KIND, + dev_type=1, + ), + "RaceLink_Gateway-0.1.6-WirelessStickV3-ESP32S3-TYPE1-app.bin", + ) + + def test_factory_image_name_reads_as_a_warning(self) -> None: + name = artifact_name( + product="RaceLink_Gateway", + version="0.1.6", + env="WirelessStickV3-ESP32S3", + kind=FACTORY_KIND, + dev_type=1, + ) + + self.assertEqual( + name, + "RaceLink_Gateway-0.1.6-WirelessStickV3-ESP32S3-TYPE1-factory-usb-serial-only.bin", + ) + self.assertIn("usb-serial-only", name) + + def test_variant_states_the_upstream_release_a_build_wraps(self) -> None: + name = artifact_name( + product="RaceLink_WLED", + version="0.1.8", + env="RaceLink_Node_v4_s3_llcc68", + kind=APP_KIND, + dev_type=12, + variant="wled_v0.15.3", + ) + + self.assertEqual( + name, + "RaceLink_WLED-0.1.8-RaceLink_Node_v4_s3_llcc68-TYPE12-wled_v0.15.3-app.bin", + ) + # Both versions have to stay readable: the RaceLink release and the + # upstream WLED release it was built against. + self.assertIn("-0.1.8-", name) + self.assertIn("wled_v0.15.3", name) + + def test_pre_application_images_carry_no_device_type(self) -> None: + self.assertEqual( + artifact_name( + product="RaceLink_Gateway", + version="0.1.6", + env="WirelessStickV3-ESP32S3", + kind="bootloader", + ), + "RaceLink_Gateway-0.1.6-WirelessStickV3-ESP32S3-bootloader.bin", + ) + + def test_sidecar_names(self) -> None: + self.assertEqual( + checksum_name("RaceLink_Gateway", "0.1.6"), "RaceLink_Gateway-0.1.6-sha256.txt" + ) + self.assertEqual( + manifest_name("RaceLink_Gateway", "0.1.6"), "RaceLink_Gateway-0.1.6-assets.json" + ) + + +class MergeCommandTests(unittest.TestCase): + def test_orders_images_by_offset(self) -> None: + argv = merge_command( + chip="esp32s3", + output="factory.bin", + images=[(0x10000, "firmware.bin"), (0x0, "bootloader.bin"), (0x8000, "partitions.bin")], + ) + + self.assertEqual( + argv[argv.index("-o") + 2 :], + ["0x0", "bootloader.bin", "0x8000", "partitions.bin", "0x10000", "firmware.bin"], + ) + + def test_names_the_chip_and_uses_the_dashed_subcommand(self) -> None: + argv = merge_command(chip="esp32s2", output="out.bin", images=[(0x1000, "bootloader.bin")]) + + self.assertIn("--chip", argv) + self.assertEqual(argv[argv.index("--chip") + 1], "esp32s2") + # esptool 5 renamed merge_bin to merge-bin; the workflow pins esptool 5. + self.assertIn("merge-bin", argv) + + def test_runs_esptool_from_the_current_interpreter(self) -> None: + argv = merge_command( + chip="esp32s3", + output="out.bin", + images=[(0x0, "bootloader.bin")], + python_executable="/venv/bin/python", + ) + + self.assertEqual(argv[:3], ["/venv/bin/python", "-m", "esptool"]) + + def test_does_not_override_the_compiled_flash_header(self) -> None: + argv = merge_command(chip="esp32s3", output="out.bin", images=[(0x0, "bootloader.bin")]) + + # merge-bin defaults these to "keep"; passing them is how a merged image + # ends up mismatching the board it was built for. + for flag in ("--flash-mode", "--flash-freq", "--flash-size", "--flash_mode"): + self.assertNotIn(flag, argv) + + def test_rejects_an_empty_image_list(self) -> None: + with self.assertRaises(ValueError): + merge_command(chip="esp32s3", output="out.bin", images=[]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index faf3f84..c8c4a74 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -2,6 +2,81 @@ import unittest ROOT = pathlib.Path(__file__).resolve().parents[1] +BUILD_WORKFLOW = ROOT / ".github" / "workflows" / "build.yml" +RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" +BUILD_SCRIPT = ROOT / "scripts" / "build_and_stage_profiles.sh" + + +class ArtifactStagingTests(unittest.TestCase): + """Guards for the flashable-artifact pipeline. + + These failure modes are all silent: an esptool major bump that renames the + merge subcommand, a rehearsal that drifts from the real release, or a + publish glob that quietly drops the new assets. + """ + + def test_both_workflows_run_the_same_build_and_stage_script(self): + # The rehearsal on a pull request is only worth anything if it runs the + # same code the release does. + for workflow in (BUILD_WORKFLOW, RELEASE_WORKFLOW): + with self.subTest(workflow=workflow.name): + source = workflow.read_text(encoding="utf-8") + self.assertIn("scripts/build_and_stage_profiles.sh", source) + + def test_esptool_major_is_pinned_in_both_workflows(self): + # esptool 5 renamed merge_bin to merge-bin; release_artifacts.py emits + # the dashed form, so an unpinned major would break a release. + for workflow in (BUILD_WORKFLOW, RELEASE_WORKFLOW): + with self.subTest(workflow=workflow.name): + source = workflow.read_text(encoding="utf-8") + self.assertIn('python -m pip install "esptool>=5,<6"', source) + + def test_metadata_is_collected_before_staging(self): + source = BUILD_SCRIPT.read_text(encoding="utf-8") + metadata = source.index("--json-output-path") + staging = source.index("stage-assets") + + self.assertLess(metadata, staging, "stage-assets reads metadata.json before it is written") + + def test_release_index_is_written_after_every_profile(self): + source = BUILD_SCRIPT.read_text(encoding="utf-8") + + self.assertLess(source.index("stage-assets"), source.index("finalize")) + + def test_built_envs_list_is_kept_out_of_the_published_directory(self): + # dist/ is published wholesale; scaffolding in there would ship. + source = BUILD_SCRIPT.read_text(encoding="utf-8") + + self.assertNotIn("dist/built_envs.txt", source) + self.assertIn(": > built_envs.txt", source) + + def test_every_staged_asset_is_published(self): + source = RELEASE_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn("files: dist/*", source) + self.assertIn("path: dist/*", source) + + def test_release_notes_warn_about_the_factory_image(self): + # Collapse whitespace: the notes are a wrapped YAML block, so asserting + # on exact line breaks would fail the next time someone rewraps them. + source = " ".join(RELEASE_WORKFLOW.read_text(encoding="utf-8").split()) + + self.assertIn("USB serial only", source) + self.assertIn("commissioning tool, never an update tool", source) + + def test_build_workflow_publishes_nothing(self): + source = BUILD_WORKFLOW.read_text(encoding="utf-8") + + self.assertNotIn("action-gh-release", source) + self.assertNotIn("git tag", source) + self.assertNotIn("git push", source) + self.assertIn("contents: read", source) + + def test_build_workflow_runs_on_pull_requests(self): + source = BUILD_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn("pull_request:", source) + self.assertIn("workflow_dispatch:", source) class ReleaseWorkflowTests(unittest.TestCase): From 8c2a4ed77fe90a80fb7f24d841bbd9b1ba3b9e74 Mon Sep 17 00:00:00 2001 From: PSi86 Date: Thu, 30 Jul 2026 19:05:58 +0200 Subject: [PATCH 2/2] Collect build metadata before the build, not after `pio project metadata` cleans the build directory, so collecting it after `pio run` deleted the firmware.bin the staging step was about to read. The trigger is visible in the failing run: the metadata pass resolved and installed a dependency the build had not, which changes PlatformIO's project checksum and makes it wipe the build directory. Everything the metadata reports is derived from the configuration -- the flash offsets, and paths like $BUILD_DIR/bootloader.bin -- so the files it names do not have to exist yet, and collecting it first also means any dependency resolution happens before the build rather than invalidating it afterwards. --- scripts/build_and_stage_profiles.sh | 11 +++++++---- tests/test_release_workflow.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/scripts/build_and_stage_profiles.sh b/scripts/build_and_stage_profiles.sh index 3af46d9..37027f4 100644 --- a/scripts/build_and_stage_profiles.sh +++ b/scripts/build_and_stage_profiles.sh @@ -58,15 +58,18 @@ for profile in "${profiles[@]}"; do args+=("-e" "$env_name") done - python -m platformio run --project-dir "$wled_dir" "${args[@]}" - - # Reports the pre-application images and the offsets the platform itself - # would flash them at, so no per-SoC offset table is needed downstream. + # Before the build, not after: `project metadata` cleans the build directory, + # so collecting it afterwards deletes the very firmware.bin about to be + # staged. Everything it reports is derived from the configuration -- the + # offsets, and paths like $BUILD_DIR/bootloader.bin -- so the files it names + # do not have to exist yet. python -m platformio project metadata \ --project-dir "$wled_dir" \ "${args[@]}" \ --json-output-path "$repo_root/metadata.json" + python -m platformio run --project-dir "$wled_dir" "${args[@]}" + python scripts/stage_wled_profile.py stage-assets \ --profile "$repo_root/$profile" \ --build-root "$wled_dir/.pio/build" \ diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index c8c4a74..da8b443 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -31,12 +31,18 @@ def test_esptool_major_is_pinned_in_both_workflows(self): source = workflow.read_text(encoding="utf-8") self.assertIn('python -m pip install "esptool>=5,<6"', source) - def test_metadata_is_collected_before_staging(self): + def test_metadata_is_collected_before_the_build(self): + # `pio project metadata` cleans the build directory, so collecting it + # after `pio run` deletes the firmware.bin that is about to be staged. + # Everything it reports is derived from the configuration, so the files + # it names do not have to exist yet. source = BUILD_SCRIPT.read_text(encoding="utf-8") - metadata = source.index("--json-output-path") + metadata = source.index("project metadata") + build = source.index("platformio run") staging = source.index("stage-assets") - self.assertLess(metadata, staging, "stage-assets reads metadata.json before it is written") + self.assertLess(metadata, build, "project metadata must run before the build") + self.assertLess(build, staging, "stage-assets needs the built firmware") def test_release_index_is_written_after_every_profile(self): source = BUILD_SCRIPT.read_text(encoding="utf-8")