diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..5a7805c --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,181 @@ +name: Build Firmware + +# Compile-only counterpart to release.yml. That workflow is the only thing that +# ever built this repo, and it tags and publishes in the same run — so a broken +# build or a broken staging step surfaced in the middle of a real release. +# +# This job runs the identical build-and-stage script against the latest +# published WLED release, verifies the resulting factory images, and ships +# nothing. + +on: + pull_request: + branches: [main] + push: + branches: [main] + workflow_dispatch: + inputs: + wled_ref: + description: "Optional WLED tag/ref override. Leave empty to use the latest published WLED release." + required: false + default: "" + type: string + +permissions: + contents: read + +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + + steps: + - name: Check out RaceLink_WLED repository + uses: actions/checkout@v6.0.2 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Run static tests + run: python -m unittest discover -s tests -p "test_*.py" + + - name: Resolve WLED source ref + id: wled_source + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + ref="$(python scripts/resolve_wled_release.py --wled-ref "${{ inputs.wled_ref }}" --print ref)" + repository="$(python scripts/resolve_wled_release.py --wled-ref "${{ inputs.wled_ref }}" --print repository)" + echo "ref=${ref}" >> "$GITHUB_OUTPUT" + echo "repository=${repository}" >> "$GITHUB_OUTPUT" + + - name: Check out WLED repository + uses: actions/checkout@v6.0.2 + with: + repository: ${{ steps.wled_source.outputs.repository }} + ref: ${{ steps.wled_source.outputs.ref }} + path: external/WLED + fetch-depth: 1 + + - name: Set up Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + cache: npm + cache-dependency-path: external/WLED/package-lock.json + + - name: Cache PlatformIO installation + uses: actions/cache@v6 + with: + path: | + ~/.platformio/.cache + ~/.platformio/packages + ~/.platformio/platforms + key: pio-${{ runner.os }}-${{ hashFiles('external/WLED/platformio.ini') }} + restore-keys: pio-${{ runner.os }}- + + - name: Install build prerequisites + run: | + python -m pip install --upgrade pip + python -m pip install -r external/WLED/requirements.txt + # Same pin as release.yml — this job rehearses the staging step. + python -m pip install "esptool>=5,<6" + npm --prefix external/WLED ci + + # The same script release.yml runs, so release day differs only in the + # version string. + - name: Build and stage every shipping profile + shell: bash + run: | + bash scripts/build_and_stage_profiles.sh \ + "$GITHUB_WORKSPACE" \ + "$GITHUB_WORKSPACE/external/WLED" \ + "0.0.0-ci" \ + "${{ steps.wled_source.outputs.ref }}" \ + "$GITHUB_WORKSPACE/dist" + + - name: Verify the staged factory images + run: | + python - <<'PY' + import json + from pathlib import Path + + manifest = json.loads( + next(Path("dist").glob("*-assets.json")).read_text(encoding="utf-8") + ) + for environment in manifest["environments"]: + assets = {asset["kind"]: asset for asset in environment["assets"]} + image = (Path("dist") / assets["factory"]["file"]).read_bytes() + app_offset = environment["app_offset"] + + expectations = [ + (assets["bootloader"]["offset"], b"\xe9", "bootloader"), + (assets["partitions"]["offset"], b"\xaa\x50", "partition table"), + (app_offset, b"\xe9", "application"), + ] + for offset, magic, label in expectations: + found = image[offset : offset + len(magic)] + if found != magic: + raise SystemExit( + f"{environment['env']}: no {label} at 0x{offset:x} " + f"(found {found.hex()}, expected {magic.hex()})" + ) + + published = (Path("dist") / assets["app"]["file"]).read_bytes() + if image[app_offset : app_offset + len(published)] != published: + raise SystemExit( + f"{environment['env']}: the application embedded in the factory " + "image differs from the published application image" + ) + print( + f"{environment['env']}: {environment['chip']}, DEV_TYPE " + f"{environment['dev_type']}, factory image verified ({len(image)} bytes)" + ) + PY + + - name: Write build summary + run: | + python - <<'PY' + import json + import os + from pathlib import Path + + manifest = json.loads( + next(Path("dist").glob("*-assets.json")).read_text(encoding="utf-8") + ) + lines = [ + "## Build Summary", + "", + f"WLED ref: `{manifest.get('wled_ref')}`", + "", + "| Environment | Chip | DEV_TYPE | Application | Factory image |", + "|---|---|---|---|---|", + ] + for environment in manifest["environments"]: + assets = {asset["kind"]: asset for asset in environment["assets"]} + lines.append( + f"| `{environment['env']}` | {environment['chip']} | " + f"{environment['dev_type']} | {assets['app']['size'] / 1024:.1f} KiB | " + f"{assets['factory']['size'] / 1024:.1f} KiB |" + ) + Path(os.environ["GITHUB_STEP_SUMMARY"]).write_text( + "\n".join(lines) + "\n", encoding="utf-8" + ) + PY + + # The rehearsed release assets, so a pull request can be flashed on + # hardware — including the factory images — without cutting a release. + - name: Upload build output + uses: actions/upload-artifact@v7 + with: + name: racelink_wled_build + if-no-files-found: error + path: dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e832194..983fcae 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -120,6 +120,10 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install -r external/WLED/requirements.txt + # esptool 5 renamed merge_bin to merge-bin; pin the major so the + # rename cannot break a release, and so the defaults the staging + # code relies on (flash mode/freq/size = keep) stay put. + python -m pip install "esptool>=5,<6" npm --prefix external/WLED ci - name: Build shipping RaceLink firmware profiles @@ -129,88 +133,19 @@ jobs: RELEASE_VERSION: ${{ steps.release_version.outputs.version }} WLED_REF: ${{ steps.wled_source.outputs.ref }} run: | - set -euo pipefail - mkdir -p dist - : > dist/built_envs.txt - # Single source of truth: SHIPPING_PROFILE_FILENAMES in - # scripts/release_profiles.py (via iter_shipping_profiles, which also - # validates each file exists). Keep this in sync by editing that tuple - # only -- do NOT hard-code the profile list here. - mapfile -t profiles < <( - python -c "from pathlib import Path; from scripts.release_profiles import iter_shipping_profiles; [print('build_profiles/' + p.name) for p in iter_shipping_profiles(Path('build_profiles'))]" - ) - if [ "${#profiles[@]}" -eq 0 ]; then - echo "No shipping profiles resolved from SHIPPING_PROFILE_FILENAMES" - exit 1 - fi - printf 'Shipping profiles to build:\n'; printf ' %s\n' "${profiles[@]}" - - for profile in "${profiles[@]}"; do - rm -rf external/WLED/build_output/release - mkdir -p external/WLED/build_output/release - - mapfile -t envs < <( - python scripts/stage_wled_profile.py stage-profile \ - --repo-root "$GITHUB_WORKSPACE" \ - --wled-dir "$GITHUB_WORKSPACE/external/WLED" \ - --profile "$GITHUB_WORKSPACE/$profile" - ) - - if [ "${#envs[@]}" -eq 0 ]; then - echo "Profile $profile did not expose any envs" - exit 1 - fi - - for env_name in "${envs[@]}"; do - echo "$env_name" >> dist/built_envs.txt - done - - args=() - for env_name in "${envs[@]}"; do - args+=("-e" "$env_name") - done - - python -m platformio run \ - --project-dir "$GITHUB_WORKSPACE/external/WLED" \ - "${args[@]}" - - python scripts/stage_wled_profile.py stage-assets \ - --profile "$GITHUB_WORKSPACE/$profile" \ - --release-dir "$GITHUB_WORKSPACE/external/WLED/build_output/release" \ - --dist-dir "$GITHUB_WORKSPACE/dist" \ - --release-version "$RELEASE_VERSION" \ - --wled-ref "$WLED_REF" - done + bash scripts/build_and_stage_profiles.sh \ + "$GITHUB_WORKSPACE" \ + "$GITHUB_WORKSPACE/external/WLED" \ + "$RELEASE_VERSION" \ + "$WLED_REF" \ + "$GITHUB_WORKSPACE/dist" { echo "built_envs<> "$GITHUB_OUTPUT" - - name: Generate SHA256 manifest - run: | - python - <<'PY' - import hashlib - import json - from pathlib import Path - - dist = Path("dist") - lines = [] - for path in sorted(dist.glob("RaceLink_WLED-*")): - if not path.is_file(): - continue - digest = hashlib.sha256(path.read_bytes()).hexdigest() - lines.append(f"{digest} {path.name}") - if not lines: - raise SystemExit("No staged release artifacts found in dist/") - version = json.loads(Path("version.json").read_text(encoding="utf-8"))["version"] - (dist / f"RaceLink_WLED-{version}-sha256.txt").write_text( - "\n".join(lines) + "\n", - encoding="utf-8", - ) - PY - - name: Commit release metadata run: | git add version.json @@ -229,6 +164,7 @@ jobs: uses: actions/upload-artifact@v7 with: name: racelink_wled_release + if-no-files-found: error path: dist/* - name: Publish GitHub release @@ -237,6 +173,29 @@ jobs: tag_name: ${{ steps.release_version.outputs.tag }} target_commitish: ${{ inputs.target_branch }} generate_release_notes: true + body: | + Built against WLED `${{ steps.wled_source.outputs.ref }}`. Each + asset names both versions: its own RaceLink release and the WLED + release it wraps. + + ### Flashing + + * `…-app.bin` — application image. This is the file for WLED's + `/update` page and for the host's firmware-update dialog. + * `…-factory-usb-serial-only.bin` — bootloader, partition table, + OTA selector and application in one image, written at offset `0x0`. + **USB serial only.** Sent over OTA it is rejected in the normal + case, but with *Ignore firmware validation* ticked it is written + into the inactive slot and bricks the node until someone + re-flashes over serial. It also rewrites the partition table, so + WLED reformats its filesystem on the next boot: presets, + `cfg.json`, the master binding and the group id are all gone and + the node returns as unconfigured. A commissioning tool, never an + update tool. + * `…-bootloader.bin` / `…-partitions.bin` / `…-boot_app0.bin` — the + individual pieces, for flashing a blank chip manually. + * `…-assets.json` — machine-readable index: environment, chip, + device type, kind, flash offset, size and SHA-256 for every file. files: dist/* - name: Write workflow summary @@ -251,6 +210,6 @@ jobs: echo "- Built environments:" while IFS= read -r env_name; do echo " - \`${env_name}\`" - done < dist/built_envs.txt + done < built_envs.txt echo "- Trigger: \`workflow_dispatch\`" } >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index 40fb24f..95b42db 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,10 @@ __pycache__/ .claude/ CLAUDE.md CLAUDE.local.md + +# Staged release artifacts and the scaffolding around them +# (scripts/build_and_stage_profiles.sh). +dist/ +built_envs.txt +metadata.json +external/ diff --git a/scripts/build_and_stage_profiles.sh b/scripts/build_and_stage_profiles.sh new file mode 100644 index 0000000..37027f4 --- /dev/null +++ b/scripts/build_and_stage_profiles.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# +# Build every shipping profile into a WLED checkout and stage its output as +# release assets. +# +# Used by both release.yml and build.yml: the rehearsal on a pull request and +# the real release run the same code, so the only thing that differs on release +# day is the version string. +# +# Profiles are staged one at a time -- each writes platformio_override.ini into +# the WLED checkout, so the next iteration overwrites it. Everything that reads +# a profile's build configuration therefore has to happen inside the loop. + +set -euo pipefail + +repo_root="${1:?repository root}" +wled_dir="${2:?WLED checkout}" +release_version="${3:?release version}" +wled_ref="${4:?WLED ref}" +dist_dir="${5:?dist directory}" + +cd "$repo_root" +mkdir -p "$dist_dir" +# Outside dist/ on purpose: everything in there is published. +: > built_envs.txt + +# Single source of truth: SHIPPING_PROFILE_FILENAMES in +# scripts/release_profiles.py (via iter_shipping_profiles, which also validates +# each file exists). Keep this in sync by editing that tuple only -- do NOT +# hard-code the profile list here. +mapfile -t profiles < <( + python -c "from pathlib import Path; from scripts.release_profiles import iter_shipping_profiles; [print('build_profiles/' + p.name) for p in iter_shipping_profiles(Path('build_profiles'))]" +) +if [ "${#profiles[@]}" -eq 0 ]; then + echo "No shipping profiles resolved from SHIPPING_PROFILE_FILENAMES" + exit 1 +fi +printf 'Shipping profiles to build:\n'; printf ' %s\n' "${profiles[@]}" + +for profile in "${profiles[@]}"; do + echo "::group::$profile" + + mapfile -t envs < <( + python scripts/stage_wled_profile.py stage-profile \ + --repo-root "$repo_root" \ + --wled-dir "$wled_dir" \ + --profile "$repo_root/$profile" + ) + + if [ "${#envs[@]}" -eq 0 ]; then + echo "Profile $profile did not expose any envs" + exit 1 + fi + + args=() + for env_name in "${envs[@]}"; do + echo "$env_name" >> built_envs.txt + args+=("-e" "$env_name") + done + + # Before the build, not after: `project metadata` cleans the build directory, + # so collecting it afterwards deletes the very firmware.bin about to be + # staged. Everything it reports is derived from the configuration -- the + # offsets, and paths like $BUILD_DIR/bootloader.bin -- so the files it names + # do not have to exist yet. + python -m platformio project metadata \ + --project-dir "$wled_dir" \ + "${args[@]}" \ + --json-output-path "$repo_root/metadata.json" + + python -m platformio run --project-dir "$wled_dir" "${args[@]}" + + python scripts/stage_wled_profile.py stage-assets \ + --profile "$repo_root/$profile" \ + --build-root "$wled_dir/.pio/build" \ + --dist-dir "$dist_dir" \ + --release-version "$release_version" \ + --wled-ref "$wled_ref" \ + --metadata "$repo_root/metadata.json" + + echo "::endgroup::" +done + +python scripts/stage_wled_profile.py finalize \ + --dist-dir "$dist_dir" \ + --release-version "$release_version" \ + --wled-ref "$wled_ref" diff --git a/scripts/release_artifacts.py b/scripts/release_artifacts.py new file mode 100644 index 0000000..02b05d8 --- /dev/null +++ b/scripts/release_artifacts.py @@ -0,0 +1,248 @@ +"""Pure helpers for staging RaceLink release artifacts. + +Kept byte-identical in RaceLink_Gateway and RaceLink_WLED: both publish the same +asset shapes, and a fix to the offset or chip logic has to reach both. The +module is I/O-free so the release workflows' naming, offset and chip-detection +logic can be exercised without a toolchain; each repository's CLI wrapper does +the copying and calls out to esptool. + +Design notes: + +* **Offsets come from the build, never from a table.** PlatformIO's project + metadata reports the flash images (bootloader, partition table, boot_app0) + together with the offsets the platform itself would flash them at, so a new + SoC — where the bootloader moves from ``0x0`` to ``0x1000`` — needs no change + here. :func:`bootloader_offset_for_chip` exists only as a cross-check. +* **The chip is read out of the image being merged.** The ESP image header + carries the chip id, so the value handed to ``esptool merge-bin`` cannot + disagree with the binary it merges. +* **The application offset is read from the generated partition table**, which + is the same file that ends up in the factory image. +""" + +from __future__ import annotations + +import struct +import sys +from dataclasses import dataclass +from typing import Iterable, Sequence + +# --- ESP application image ------------------------------------------------ + +ESP_IMAGE_MAGIC = 0xE9 +_CHIP_ID_STRUCT = struct.Struct(" bool: + return self.type == PARTITION_TYPE_APP + + +@dataclass(frozen=True) +class FlashImage: + """One pre-application image and the offset it is flashed at.""" + + offset: int + path: str + + +def read_chip_name(image: bytes) -> str: + """Return the esptool chip name for an ESP application or bootloader image.""" + if len(image) < _CHIP_ID_OFFSET + _CHIP_ID_STRUCT.size: + raise ValueError("Image is too short to contain an ESP image header") + if image[0] != ESP_IMAGE_MAGIC: + raise ValueError(f"Not an ESP image: magic is 0x{image[0]:02X}, expected 0xE9") + (chip_id,) = _CHIP_ID_STRUCT.unpack_from(image, _CHIP_ID_OFFSET) + try: + return CHIP_NAMES[chip_id] + except KeyError: + raise ValueError( + f"Unknown ESP chip id 0x{chip_id:04X}. Add it to CHIP_NAMES and to " + "BOOTLOADER_OFFSETS before releasing for this SoC." + ) from None + + +def bootloader_offset_for_chip(chip: str) -> int: + """Expected second-stage bootloader offset, for cross-checking the build.""" + try: + return BOOTLOADER_OFFSETS[chip] + except KeyError: + raise ValueError( + f"No bootloader offset mapped for chip {chip!r}. Add it before " + "releasing for this SoC — a wrong offset yields an unbootable image." + ) from None + + +def parse_partition_table(data: bytes) -> list[Partition]: + """Parse a generated ``partitions.bin`` into its entries.""" + partitions: list[Partition] = [] + for start in range(0, len(data), PARTITION_ENTRY_SIZE): + entry = data[start : start + PARTITION_ENTRY_SIZE] + if len(entry) < PARTITION_ENTRY_SIZE or not entry.startswith(PARTITION_ENTRY_MAGIC): + break + offset, size = struct.unpack_from(" int: + """Offset of the first application partition — where firmware.bin goes.""" + for partition in sorted(partitions, key=lambda p: p.offset): + if partition.is_app: + return partition.offset + raise ValueError("Partition table declares no application partition") + + +def flash_images_from_metadata(metadata: dict, env: str) -> list[FlashImage]: + """Extract the pre-application images PlatformIO would flash for ``env``. + + Accepts either the per-environment metadata mapping produced by + ``pio project metadata --json-output`` or a single environment's idedata. + """ + entry = metadata.get(env, metadata) + extra = entry.get("extra") or {} + images = extra.get("flash_images") or [] + if not images: + raise ValueError( + f"PlatformIO reported no flash images for {env!r}; without the " + "bootloader and partition table a factory image cannot be built." + ) + return [FlashImage(offset=int(str(i["offset"]), 0), path=str(i["path"])) for i in images] + + +def define_value(defines: Iterable[str], name: str) -> str | None: + """Return the value of a ``NAME=value`` build define, if present.""" + prefix = f"{name}=" + for define in defines: + text = str(define) + if text.startswith(prefix): + return text[len(prefix) :] + return None + + +def device_type(defines: Iterable[str]) -> int | None: + """Parse ``-D DEV_TYPE=`` — the feature-set handle the master looks up.""" + raw = define_value(defines, "DEV_TYPE") + if raw is None: + return None + return int(raw, 0) + + +# --- Artifact naming ------------------------------------------------------ +# +# Field order: product, release version, environment, device type, variant, +# kind. The kind goes last because it is the most visible token in a file +# picker, and the factory image must never be mistaken for something OTA-able. +# The three pre-application images carry neither device type nor variant — they +# are build by-products of the environment, not firmware identity. +# +# `variant` carries the upstream version a build was made against, so a +# RaceLink_WLED asset states both its own release and the WLED release it wraps +# ("wled_v0.15.3"). Fields never contain "-", so the name stays parseable, but +# consumers should read the assets.json sidecar rather than the filename. + +FACTORY_KIND = "factory-usb-serial-only" +APP_KIND = "app" + + +def artifact_name( + *, + product: str, + version: str, + env: str, + kind: str, + dev_type: int | None = None, + variant: str | None = None, + extension: str = ".bin", +) -> str: + parts = [product, version, env] + if dev_type is not None: + parts.append(f"TYPE{dev_type}") + if variant is not None: + parts.append(variant) + parts.append(kind) + return "-".join(parts) + extension + + +def checksum_name(product: str, version: str) -> str: + return f"{product}-{version}-sha256.txt" + + +def manifest_name(product: str, version: str) -> str: + return f"{product}-{version}-assets.json" + + +def merge_command( + *, + chip: str, + output: str, + images: Sequence[tuple[int, str]], + python_executable: str | None = None, +) -> list[str]: + """Build the ``esptool merge-bin`` argv. + + Flash mode, frequency and size are deliberately not passed: ``merge-bin`` + defaults all three to ``keep``, which preserves the header the bootloader + was compiled with. Overriding them is how a merged image ends up + mismatching the board it was built for. + + The interpreter defaults to the running one so esptool is taken from the + same environment that installed it, rather than from whatever ``python`` + happens to be first on PATH. + """ + if not images: + raise ValueError("Refusing to merge an empty image list") + interpreter = python_executable or sys.executable + argv = [interpreter, "-m", "esptool", "--chip", chip, "merge-bin", "-o", output] + for offset, path in sorted(images, key=lambda item: item[0]): + argv += [hex(offset), path] + return argv diff --git a/scripts/release_profiles.py b/scripts/release_profiles.py index 2f7a236..6df856d 100644 --- a/scripts/release_profiles.py +++ b/scripts/release_profiles.py @@ -7,6 +7,16 @@ from dataclasses import dataclass from pathlib import Path +from scripts.release_staging import stage_environment + +# The profiles a release actually builds and publishes. +# +# RaceLink_Node_v7_classic_esp32_emac is committed but deliberately absent: +# its internal-EMAC Ethernet support is still in bring-up and has never been +# released. The omission is intentional, not an oversight — add it here once +# Ethernet ships, and note that it is the first classic ESP32 in the set, so +# its bootloader sits at 0x1000 rather than 0x0 (release_staging.py cross- +# checks that and will refuse to stage it if the offsets disagree). SHIPPING_PROFILE_FILENAMES = ( "RaceLink_Node_v1_c3_ct62.platformio_override.ini", "RaceLink_Node_v3_s2_llcc68.platformio_override.ini", @@ -140,32 +150,48 @@ def sanitize_ref_for_filename(raw_ref: str) -> str: return value or "unknown-ref" +def wled_variant(wled_ref: str) -> str: + """Filename fragment naming the upstream release a build wraps. + + One dash-free field, so an asset states both its own RaceLink version and + the WLED version it was built against without making the name ambiguous. + """ + return f"wled_{sanitize_ref_for_filename(wled_ref)}" + + def stage_release_assets( *, profile_path: Path, - release_dir: Path, + build_root: Path, dist_dir: Path, release_version: str, wled_ref: str, -) -> list[Path]: - """Rename WLED output artifacts to stable RaceLink release filenames.""" + metadata: dict, + product: str = "RaceLink_WLED", +) -> list[dict]: + """Stage every asset for one profile's environments. + + Returns a manifest entry per environment; the caller accumulates them + across profiles and writes the release index once. + + Note the source: PlatformIO's build directory, not WLED's + ``build_output/release``. Both hold the same application image, but the + build directory also holds the bootloader and the partition table that the + factory image needs, and it is what the flash-image offsets in PlatformIO's + metadata point at. + """ dist_dir.mkdir(parents=True, exist_ok=True) - staged_paths: list[Path] = [] - ref_fragment = sanitize_ref_for_filename(wled_ref) - - for env in parse_profile_environments(profile_path): - candidates = sorted(release_dir.glob(f"WLED_*_{env.release_name}.bin*")) - if not candidates: - raise FileNotFoundError( - f"Could not find build_output/release artifact for {env.release_name}" - ) - - for candidate in candidates: - suffix = "".join(candidate.suffixes) - target = dist_dir / ( - f"RaceLink_WLED-{release_version}-{env.name}-{ref_fragment}{suffix}" - ) - shutil.copy2(candidate, target) - staged_paths.append(target) - - return staged_paths + variant = wled_variant(wled_ref) + + return [ + stage_environment( + env=env.name, + product=product, + version=release_version, + build_dir=build_root / env.name, + dist_dir=dist_dir, + metadata=metadata, + variant=variant, + ) + for env in parse_profile_environments(profile_path) + ] diff --git a/scripts/release_staging.py b/scripts/release_staging.py new file mode 100644 index 0000000..06b4058 --- /dev/null +++ b/scripts/release_staging.py @@ -0,0 +1,190 @@ +"""Stage one build environment's output as release assets. + +Kept byte-identical in RaceLink_Gateway and RaceLink_WLED, for the same reason +as :mod:`scripts.release_artifacts`: this is where a mistake produces an image +that flashes cleanly and never boots, so there should be exactly one +implementation of it. The repositories differ in how they drive this — the +gateway iterates a flat environment list, RaceLink_WLED stages one profile at a +time into an external WLED checkout — but not in what a staged environment +looks like. + +Produced per environment: + +* the application image, which is what OTA and the host's firmware dialog take, +* the three pre-application images (bootloader, partition table, boot_app0), so + a blank chip can be commissioned without a local toolchain, +* a merged factory image covering all four, written at offset 0. +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +from pathlib import Path + +from scripts.release_artifacts import ( + APP_KIND, + FACTORY_KIND, + application_offset, + artifact_name, + bootloader_offset_for_chip, + checksum_name, + device_type, + flash_images_from_metadata, + manifest_name, + merge_command, + parse_partition_table, + read_chip_name, +) + +# PlatformIO names the pre-application images by file; map them to the asset +# kind so the sidecar reads as intent rather than as a filename. +PART_KINDS = { + "bootloader.bin": "bootloader", + "partitions.bin": "partitions", + "boot_app0.bin": "boot_app0", +} + + +def sha256_of(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _stage(source: Path, target: Path) -> Path: + if not source.is_file(): + raise SystemExit(f"Expected build artifact not found: {source}") + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, target) + return target + + +def stage_environment( + *, + env: str, + product: str, + version: str, + build_dir: Path, + dist_dir: Path, + metadata: dict, + variant: str | None = None, +) -> dict: + """Stage every asset for one environment and return its manifest entry.""" + firmware = build_dir / "firmware.bin" + if not firmware.is_file(): + raise SystemExit(f"Expected firmware artifact not found: {firmware}") + + chip = read_chip_name(firmware.read_bytes()) + env_metadata = metadata.get(env, metadata) + flash_images = flash_images_from_metadata(metadata, env) + + by_name = {Path(image.path).name: image for image in flash_images} + for required in ("bootloader.bin", "partitions.bin"): + if required not in by_name: + raise SystemExit(f"{env}: PlatformIO reported no {required}") + + partitions = parse_partition_table(Path(by_name["partitions.bin"].path).read_bytes()) + app_offset = application_offset(partitions) + + # Cross-check the offset PlatformIO reported against the one this chip is + # known to need. Either source alone could be wrong silently; disagreement + # cannot be. This is the guard that catches an S2 or classic-ESP32 target + # whose bootloader sits at 0x1000 rather than 0x0. + expected_offset = bootloader_offset_for_chip(chip) + if by_name["bootloader.bin"].offset != expected_offset: + raise SystemExit( + f"{env}: PlatformIO flashes the bootloader at " + f"0x{by_name['bootloader.bin'].offset:x} but {chip} expects " + f"0x{expected_offset:x}. Refusing to publish a factory image." + ) + + dev_type = device_type(env_metadata.get("defines") or []) + assets: list[dict] = [] + + def record(path: Path, kind: str, offset: int) -> None: + assets.append( + { + "file": path.name, + "kind": kind, + "offset": offset, + "size": path.stat().st_size, + "sha256": sha256_of(path), + } + ) + + app_target = _stage( + firmware, + dist_dir + / artifact_name( + product=product, + version=version, + env=env, + kind=APP_KIND, + dev_type=dev_type, + variant=variant, + ), + ) + record(app_target, APP_KIND, app_offset) + + merge_inputs: list[tuple[int, str]] = [(app_offset, str(firmware))] + for image in flash_images: + source = Path(image.path) + kind = PART_KINDS.get(source.name) + if kind is None: + raise SystemExit(f"{env}: unexpected flash image {source.name}") + target = _stage( + source, + dist_dir / artifact_name(product=product, version=version, env=env, kind=kind), + ) + record(target, kind, image.offset) + merge_inputs.append((image.offset, str(source))) + + factory_target = dist_dir / artifact_name( + product=product, + version=version, + env=env, + kind=FACTORY_KIND, + dev_type=dev_type, + variant=variant, + ) + command = merge_command(chip=chip, output=str(factory_target), images=merge_inputs) + print(" ".join(command), flush=True) + subprocess.run(command, check=True) + record(factory_target, "factory", 0) + + return { + "env": env, + "chip": chip, + "dev_type": dev_type, + "app_offset": app_offset, + "assets": assets, + } + + +def write_release_index( + *, + dist_dir: Path, + product: str, + version: str, + environments: list[dict], + extra: dict | None = None, +) -> tuple[Path, Path]: + """Write the assets.json sidecar and the SHA-256 manifest.""" + if not environments: + raise SystemExit("No environments were staged") + + manifest = {"product": product, "version": version, **(extra or {})} + manifest["environments"] = environments + manifest_path = dist_dir / manifest_name(product, version) + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + checksum_lines = [ + f"{asset['sha256']} {asset['file']}" + for environment in environments + for asset in environment["assets"] + ] + checksum_path = dist_dir / checksum_name(product, version) + checksum_path.write_text("\n".join(checksum_lines) + "\n", encoding="utf-8") + + return manifest_path, checksum_path diff --git a/scripts/stage_wled_profile.py b/scripts/stage_wled_profile.py index 558e336..d2a2249 100644 --- a/scripts/stage_wled_profile.py +++ b/scripts/stage_wled_profile.py @@ -3,6 +3,8 @@ from __future__ import annotations import argparse +import json +import shutil import sys from pathlib import Path @@ -16,6 +18,12 @@ stage_profile_override, stage_release_assets, ) +from scripts.release_staging import write_release_index + +PRODUCT = "RaceLink_WLED" + +# Per-profile manifest fragments, assembled by `finalize` and removed again. +FRAGMENT_DIR = ".staging" def _build_parser() -> argparse.ArgumentParser: @@ -34,13 +42,32 @@ def _build_parser() -> argparse.ArgumentParser: stage_assets = subparsers.add_parser( "stage-assets", - help="Rename built WLED artifacts into RaceLink release assets.", + help="Stage one profile's build output as RaceLink release assets.", ) stage_assets.add_argument("--profile", required=True, type=Path) - stage_assets.add_argument("--release-dir", required=True, type=Path) + stage_assets.add_argument( + "--build-root", + required=True, + type=Path, + help="PlatformIO build directory of the WLED checkout (.pio/build).", + ) stage_assets.add_argument("--dist-dir", required=True, type=Path) stage_assets.add_argument("--release-version", required=True) stage_assets.add_argument("--wled-ref", required=True) + stage_assets.add_argument( + "--metadata", + required=True, + type=Path, + help="JSON from `pio project metadata`, collected while this profile is staged.", + ) + + finalize = subparsers.add_parser( + "finalize", + help="Write the assets.json sidecar and the SHA-256 manifest.", + ) + finalize.add_argument("--dist-dir", required=True, type=Path) + finalize.add_argument("--release-version", required=True) + finalize.add_argument("--wled-ref", required=True) return parser @@ -57,15 +84,55 @@ def _run_stage_profile(args: argparse.Namespace) -> int: def _run_stage_assets(args: argparse.Namespace) -> int: - staged = stage_release_assets( + dist_dir = args.dist_dir.resolve() + environments = stage_release_assets( profile_path=args.profile.resolve(), - release_dir=args.release_dir.resolve(), - dist_dir=args.dist_dir.resolve(), + build_root=args.build_root.resolve(), + dist_dir=dist_dir, release_version=args.release_version, wled_ref=args.wled_ref, + metadata=json.loads(args.metadata.resolve().read_text(encoding="utf-8")), + ) + + # Each profile is built and staged in turn, then its override file is + # overwritten by the next one -- so the manifest is accumulated on disk + # rather than held in memory, and assembled by `finalize` at the end. + fragments = dist_dir / FRAGMENT_DIR + fragments.mkdir(parents=True, exist_ok=True) + (fragments / f"{args.profile.stem}.json").write_text( + json.dumps(environments, indent=2) + "\n", encoding="utf-8" ) - for path in staged: - sys.stdout.write(f"{path}\n") + + for environment in environments: + for asset in environment["assets"]: + sys.stdout.write(f"{asset['file']}\n") + return 0 + + +def _run_finalize(args: argparse.Namespace) -> int: + dist_dir = args.dist_dir.resolve() + fragments = sorted((dist_dir / FRAGMENT_DIR).glob("*.json")) + if not fragments: + raise SystemExit(f"No staged profiles found in {dist_dir / FRAGMENT_DIR}") + + environments = [ + environment + for fragment in fragments + for environment in json.loads(fragment.read_text(encoding="utf-8")) + ] + + manifest_path, checksum_path = write_release_index( + dist_dir=dist_dir, + product=PRODUCT, + version=args.release_version, + environments=environments, + extra={"wled_ref": args.wled_ref}, + ) + + # The fragments are scaffolding, not release assets. + shutil.rmtree(dist_dir / FRAGMENT_DIR) + + sys.stdout.write(f"{manifest_path.name}\n{checksum_path.name}\n") return 0 @@ -76,6 +143,8 @@ def main() -> int: return _run_stage_profile(args) if args.command == "stage-assets": return _run_stage_assets(args) + if args.command == "finalize": + return _run_finalize(args) parser.error(f"Unsupported command: {args.command}") return 2 diff --git a/tests/test_release_artifacts.py b/tests/test_release_artifacts.py new file mode 100644 index 0000000..15935a9 --- /dev/null +++ b/tests/test_release_artifacts.py @@ -0,0 +1,265 @@ +"""Tests for the release-artifact helpers. + +The guardrails that matter here are the ones whose failure mode is a published +artifact that flashes cleanly and never boots: an unmapped SoC, a bootloader +offset taken from the wrong source, or an application offset that no longer +matches the partition table shipped inside the same image. +""" + +from __future__ import annotations + +import struct +import unittest + +from scripts.release_artifacts import ( + APP_KIND, + BOOTLOADER_OFFSETS, + CHIP_NAMES, + FACTORY_KIND, + application_offset, + artifact_name, + bootloader_offset_for_chip, + checksum_name, + device_type, + flash_images_from_metadata, + manifest_name, + merge_command, + parse_partition_table, + read_chip_name, +) + + +def _esp_image(chip_id: int, magic: int = 0xE9) -> bytes: + header = bytearray(24) + header[0] = magic + header[1] = 5 # segment count + struct.pack_into(" bytes: + entry = bytearray(32) + entry[0:2] = b"\xaa\x50" + entry[2] = ptype + entry[3] = subtype + struct.pack_into(" bytes: + return b"".join( + [ + _partition_entry("nvs", 0x01, 0x02, 0x9000, 0x5000), + _partition_entry("otadata", 0x01, 0x00, 0xE000, 0x2000), + _partition_entry("app0", 0x00, 0x10, 0x10000, 0x330000), + _partition_entry("app1", 0x00, 0x11, 0x340000, 0x330000), + _partition_entry("spiffs", 0x01, 0x82, 0x670000, 0x180000), + ] + ) + b"\xff" * 32 + + +class ChipDetectionTests(unittest.TestCase): + def test_reads_the_chip_from_the_image_header(self) -> None: + self.assertEqual(read_chip_name(_esp_image(0x0009)), "esp32s3") + self.assertEqual(read_chip_name(_esp_image(0x0002)), "esp32s2") + self.assertEqual(read_chip_name(_esp_image(0x0000)), "esp32") + + def test_rejects_a_non_esp_image(self) -> None: + with self.assertRaises(ValueError): + read_chip_name(_esp_image(0x0009, magic=0x00)) + + def test_rejects_an_unmapped_chip_id(self) -> None: + with self.assertRaises(ValueError): + read_chip_name(_esp_image(0x00FF)) + + def test_every_known_chip_has_a_bootloader_offset(self) -> None: + for chip in CHIP_NAMES.values(): + with self.subTest(chip=chip): + self.assertIn(chip, BOOTLOADER_OFFSETS) + + def test_bootloader_offsets_match_the_documented_layout(self) -> None: + # The two families that differ; getting these wrong is the classic + # unbootable-factory-image bug. + self.assertEqual(bootloader_offset_for_chip("esp32"), 0x1000) + self.assertEqual(bootloader_offset_for_chip("esp32s2"), 0x1000) + self.assertEqual(bootloader_offset_for_chip("esp32s3"), 0x0000) + self.assertEqual(bootloader_offset_for_chip("esp32c3"), 0x0000) + + def test_rejects_an_unmapped_chip_name(self) -> None: + with self.assertRaises(ValueError): + bootloader_offset_for_chip("esp32p4") + + +class PartitionTableTests(unittest.TestCase): + def test_parses_entries_until_the_table_ends(self) -> None: + partitions = parse_partition_table(_partition_table()) + + self.assertEqual([p.name for p in partitions], ["nvs", "otadata", "app0", "app1", "spiffs"]) + self.assertEqual(partitions[0].offset, 0x9000) + + def test_application_offset_is_the_first_app_partition(self) -> None: + self.assertEqual(application_offset(parse_partition_table(_partition_table())), 0x10000) + + def test_rejects_a_table_without_an_application(self) -> None: + data = _partition_entry("nvs", 0x01, 0x02, 0x9000, 0x5000) + with self.assertRaises(ValueError): + application_offset(parse_partition_table(data)) + + def test_rejects_an_empty_table(self) -> None: + with self.assertRaises(ValueError): + parse_partition_table(b"\xff" * 32) + + +class MetadataTests(unittest.TestCase): + METADATA = { + "WirelessStickV3-ESP32S3": { + "defines": ["PLATFORMIO=60119", "DEV_TYPE=1", 'DEV_TYPE_STR="RaceLink_Gateway_v4"'], + "extra": { + "flash_images": [ + {"offset": "0x0000", "path": "/build/bootloader.bin"}, + {"offset": "0x8000", "path": "/build/partitions.bin"}, + {"offset": "0xe000", "path": "/framework/boot_app0.bin"}, + ] + }, + } + } + + def test_reads_flash_images_for_an_environment(self) -> None: + images = flash_images_from_metadata(self.METADATA, "WirelessStickV3-ESP32S3") + + self.assertEqual([image.offset for image in images], [0x0, 0x8000, 0xE000]) + + def test_accepts_single_environment_idedata(self) -> None: + single = self.METADATA["WirelessStickV3-ESP32S3"] + + images = flash_images_from_metadata(single, "WirelessStickV3-ESP32S3") + + self.assertEqual(len(images), 3) + + def test_rejects_metadata_without_flash_images(self) -> None: + with self.assertRaises(ValueError): + flash_images_from_metadata({"env": {"extra": {}}}, "env") + + def test_reads_the_device_type_from_the_build_defines(self) -> None: + defines = self.METADATA["WirelessStickV3-ESP32S3"]["defines"] + + self.assertEqual(device_type(defines), 1) + + def test_device_type_is_optional(self) -> None: + self.assertIsNone(device_type(["PLATFORMIO=60119"])) + + +class NamingTests(unittest.TestCase): + def test_application_image_name(self) -> None: + self.assertEqual( + artifact_name( + product="RaceLink_Gateway", + version="0.1.6", + env="WirelessStickV3-ESP32S3", + kind=APP_KIND, + dev_type=1, + ), + "RaceLink_Gateway-0.1.6-WirelessStickV3-ESP32S3-TYPE1-app.bin", + ) + + def test_factory_image_name_reads_as_a_warning(self) -> None: + name = artifact_name( + product="RaceLink_Gateway", + version="0.1.6", + env="WirelessStickV3-ESP32S3", + kind=FACTORY_KIND, + dev_type=1, + ) + + self.assertEqual( + name, + "RaceLink_Gateway-0.1.6-WirelessStickV3-ESP32S3-TYPE1-factory-usb-serial-only.bin", + ) + self.assertIn("usb-serial-only", name) + + def test_variant_states_the_upstream_release_a_build_wraps(self) -> None: + name = artifact_name( + product="RaceLink_WLED", + version="0.1.8", + env="RaceLink_Node_v4_s3_llcc68", + kind=APP_KIND, + dev_type=12, + variant="wled_v0.15.3", + ) + + self.assertEqual( + name, + "RaceLink_WLED-0.1.8-RaceLink_Node_v4_s3_llcc68-TYPE12-wled_v0.15.3-app.bin", + ) + # Both versions have to stay readable: the RaceLink release and the + # upstream WLED release it was built against. + self.assertIn("-0.1.8-", name) + self.assertIn("wled_v0.15.3", name) + + def test_pre_application_images_carry_no_device_type(self) -> None: + self.assertEqual( + artifact_name( + product="RaceLink_Gateway", + version="0.1.6", + env="WirelessStickV3-ESP32S3", + kind="bootloader", + ), + "RaceLink_Gateway-0.1.6-WirelessStickV3-ESP32S3-bootloader.bin", + ) + + def test_sidecar_names(self) -> None: + self.assertEqual( + checksum_name("RaceLink_Gateway", "0.1.6"), "RaceLink_Gateway-0.1.6-sha256.txt" + ) + self.assertEqual( + manifest_name("RaceLink_Gateway", "0.1.6"), "RaceLink_Gateway-0.1.6-assets.json" + ) + + +class MergeCommandTests(unittest.TestCase): + def test_orders_images_by_offset(self) -> None: + argv = merge_command( + chip="esp32s3", + output="factory.bin", + images=[(0x10000, "firmware.bin"), (0x0, "bootloader.bin"), (0x8000, "partitions.bin")], + ) + + self.assertEqual( + argv[argv.index("-o") + 2 :], + ["0x0", "bootloader.bin", "0x8000", "partitions.bin", "0x10000", "firmware.bin"], + ) + + def test_names_the_chip_and_uses_the_dashed_subcommand(self) -> None: + argv = merge_command(chip="esp32s2", output="out.bin", images=[(0x1000, "bootloader.bin")]) + + self.assertIn("--chip", argv) + self.assertEqual(argv[argv.index("--chip") + 1], "esp32s2") + # esptool 5 renamed merge_bin to merge-bin; the workflow pins esptool 5. + self.assertIn("merge-bin", argv) + + def test_runs_esptool_from_the_current_interpreter(self) -> None: + argv = merge_command( + chip="esp32s3", + output="out.bin", + images=[(0x0, "bootloader.bin")], + python_executable="/venv/bin/python", + ) + + self.assertEqual(argv[:3], ["/venv/bin/python", "-m", "esptool"]) + + def test_does_not_override_the_compiled_flash_header(self) -> None: + argv = merge_command(chip="esp32s3", output="out.bin", images=[(0x0, "bootloader.bin")]) + + # merge-bin defaults these to "keep"; passing them is how a merged image + # ends up mismatching the board it was built for. + for flag in ("--flash-mode", "--flash-freq", "--flash-size", "--flash_mode"): + self.assertNotIn(flag, argv) + + def test_rejects_an_empty_image_list(self) -> None: + with self.assertRaises(ValueError): + merge_command(chip="esp32s3", output="out.bin", images=[]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index faf3f84..da8b443 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -2,6 +2,87 @@ import unittest ROOT = pathlib.Path(__file__).resolve().parents[1] +BUILD_WORKFLOW = ROOT / ".github" / "workflows" / "build.yml" +RELEASE_WORKFLOW = ROOT / ".github" / "workflows" / "release.yml" +BUILD_SCRIPT = ROOT / "scripts" / "build_and_stage_profiles.sh" + + +class ArtifactStagingTests(unittest.TestCase): + """Guards for the flashable-artifact pipeline. + + These failure modes are all silent: an esptool major bump that renames the + merge subcommand, a rehearsal that drifts from the real release, or a + publish glob that quietly drops the new assets. + """ + + def test_both_workflows_run_the_same_build_and_stage_script(self): + # The rehearsal on a pull request is only worth anything if it runs the + # same code the release does. + for workflow in (BUILD_WORKFLOW, RELEASE_WORKFLOW): + with self.subTest(workflow=workflow.name): + source = workflow.read_text(encoding="utf-8") + self.assertIn("scripts/build_and_stage_profiles.sh", source) + + def test_esptool_major_is_pinned_in_both_workflows(self): + # esptool 5 renamed merge_bin to merge-bin; release_artifacts.py emits + # the dashed form, so an unpinned major would break a release. + for workflow in (BUILD_WORKFLOW, RELEASE_WORKFLOW): + with self.subTest(workflow=workflow.name): + source = workflow.read_text(encoding="utf-8") + self.assertIn('python -m pip install "esptool>=5,<6"', source) + + def test_metadata_is_collected_before_the_build(self): + # `pio project metadata` cleans the build directory, so collecting it + # after `pio run` deletes the firmware.bin that is about to be staged. + # Everything it reports is derived from the configuration, so the files + # it names do not have to exist yet. + source = BUILD_SCRIPT.read_text(encoding="utf-8") + metadata = source.index("project metadata") + build = source.index("platformio run") + staging = source.index("stage-assets") + + self.assertLess(metadata, build, "project metadata must run before the build") + self.assertLess(build, staging, "stage-assets needs the built firmware") + + def test_release_index_is_written_after_every_profile(self): + source = BUILD_SCRIPT.read_text(encoding="utf-8") + + self.assertLess(source.index("stage-assets"), source.index("finalize")) + + def test_built_envs_list_is_kept_out_of_the_published_directory(self): + # dist/ is published wholesale; scaffolding in there would ship. + source = BUILD_SCRIPT.read_text(encoding="utf-8") + + self.assertNotIn("dist/built_envs.txt", source) + self.assertIn(": > built_envs.txt", source) + + def test_every_staged_asset_is_published(self): + source = RELEASE_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn("files: dist/*", source) + self.assertIn("path: dist/*", source) + + def test_release_notes_warn_about_the_factory_image(self): + # Collapse whitespace: the notes are a wrapped YAML block, so asserting + # on exact line breaks would fail the next time someone rewraps them. + source = " ".join(RELEASE_WORKFLOW.read_text(encoding="utf-8").split()) + + self.assertIn("USB serial only", source) + self.assertIn("commissioning tool, never an update tool", source) + + def test_build_workflow_publishes_nothing(self): + source = BUILD_WORKFLOW.read_text(encoding="utf-8") + + self.assertNotIn("action-gh-release", source) + self.assertNotIn("git tag", source) + self.assertNotIn("git push", source) + self.assertIn("contents: read", source) + + def test_build_workflow_runs_on_pull_requests(self): + source = BUILD_WORKFLOW.read_text(encoding="utf-8") + + self.assertIn("pull_request:", source) + self.assertIn("workflow_dispatch:", source) class ReleaseWorkflowTests(unittest.TestCase):