diff --git a/.github/workflows/matrix.yml b/.github/workflows/matrix.yml index 0c8c387..7c6ea4d 100644 --- a/.github/workflows/matrix.yml +++ b/.github/workflows/matrix.yml @@ -40,16 +40,24 @@ jobs: compression-level: 0 - name: capture matrix id: capture-matrix - run: > - echo "matrix=$(cat matrix.json)" >> "${GITHUB_OUTPUT}" + run: | + # Split matrix.json into two strategy-matrix-shaped outputs: one keyed + # on `builds` (per-arch build jobs) and one on `merges` (per + # (version, flavor) manifest-list assembly jobs). Each iteration of the + # build job sees matrix.builds.* and each merge sees matrix.merges.*. + BUILDS=$(jq -c '{builds: .builds}' matrix.json) + MERGES=$(jq -c '{merges: .merges}' matrix.json) + echo "builds=${BUILDS}" >> "${GITHUB_OUTPUT}" + echo "merges=${MERGES}" >> "${GITHUB_OUTPUT}" outputs: - matrix: "${{ steps.capture-matrix.outputs.matrix }}" + builds: "${{ steps.capture-matrix.outputs.builds }}" + merges: "${{ steps.capture-matrix.outputs.merges }}" build: - name: "build ${{ matrix.builds.version }} ${{ matrix.builds.flavor }}" + name: "build ${{ matrix.builds.version }} ${{ matrix.builds.flavor }} ${{ matrix.builds.arch }}" needs: matrix strategy: fail-fast: false - matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + matrix: ${{ fromJSON(needs.matrix.outputs.builds) }} runs-on: "${{ matrix.builds.runner }}" env: KERNEL_PUBLISH: "${{ inputs.publish }}" @@ -59,7 +67,7 @@ jobs: FIRMWARE_SIG_URL: "${{ matrix.builds.firmware_sig_url }}" KERNEL_FLAVOR: "${{ matrix.builds.flavor }}" KERNEL_TAGS: "${{ join(matrix.builds.tags, ',') }}" - KERNEL_ARCHITECTURES: "${{ join(matrix.builds.architectures, ',') }}" + KERNEL_ARCH: "${{ matrix.builds.arch }}" steps: - name: Harden the runner (Audit all outbound calls) uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 @@ -91,19 +99,29 @@ jobs: # restore-keys is important here - it lets us restore the most recent cache key, # *ignoring* the specific run ID, as a fuzzy match. So we can use previous build's # caches for this flavor/arch even if the runid is not the same - key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}" + key: "ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}-${{ github.run_id }}" restore-keys: | - ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}- + ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}- - name: generate docker script run: "./hack/build/generate-docker-script.sh" - name: upload docker script uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - name: "build-${{ matrix.builds.version }}-${{ matrix.builds.flavor }}.sh" + name: "build-${{ matrix.builds.version }}-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}.sh" path: "docker.sh" compression-level: 0 - name: run docker script run: sh -x docker.sh + - name: upload digests + # Only produced when publishing — push-by-digest path writes digests.json. + if: ${{ inputs.publish }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: "digests-${{ matrix.builds.version }}-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}" + path: "digests.json" + if-no-files-found: error + compression-level: 0 + retention-days: 1 - name: save ccache uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.2 with: @@ -111,4 +129,57 @@ jobs: # The run_id here is just for write-key uniqueness, as GH doesn't allow overwriting # existing cache keys - the `restore` action will fuzzy-match and ignore the run_id # for subsequent runs. - key: "ccache-${{ matrix.builds.flavor }}-${{ join(matrix.builds.architectures, '-') }}-${{ github.run_id }}" + key: "ccache-${{ matrix.builds.flavor }}-${{ matrix.builds.arch }}-${{ github.run_id }}" + merge: + # Stitch the per-arch single-platform pushes from `build` into multi-arch + # manifest lists. Only runs when publishing; no-op when nothing was pushed. + name: "merge ${{ matrix.merges.version }} ${{ matrix.merges.flavor }}" + needs: [matrix, build] + if: ${{ inputs.publish && needs.matrix.outputs.merges != '' }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.merges) }} + runs-on: ubuntu-latest + env: + KERNEL_PUBLISH: "${{ inputs.publish }}" + KERNEL_VERSION: "${{ matrix.merges.version }}" + KERNEL_FLAVOR: "${{ matrix.merges.flavor }}" + KERNEL_PRODUCES: "${{ join(matrix.merges.produces, ',') }}" + DIGESTS_DIR: digests + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@fa2e9d605c4eeb9fcad4c99c224cee0c6c7f3594 # v2.16.0 + with: + egress-policy: audit + - name: checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v4 + - name: install cosign + uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1 + - name: docker setup buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + - name: docker login ghcr.io + uses: Wandalen/wretry.action@e68c23e6309f2871ca8ae4763e7629b9c258e1ea # v3.8.0 + with: + action: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + with: | + registry: ghcr.io + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + - name: download digest artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + # Each per-arch build uploads its digests under a uniquely-named + # artifact; pattern + default merge-multiple=false drops each artifact + # into its own subdirectory under digests/. + pattern: "digests-${{ matrix.merges.version }}-${{ matrix.merges.flavor }}-*" + path: digests + - name: generate merge script + run: python3 ./hack/build/generate-merge-script.py + - name: upload merge script + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: "merge-${{ matrix.merges.version }}-${{ matrix.merges.flavor }}.sh" + path: "merge.sh" + compression-level: 0 + - name: run merge script + run: sh -x merge.sh diff --git a/config.yaml b/config.yaml index 4981856..54b493b 100644 --- a/config.yaml +++ b/config.yaml @@ -1,8 +1,15 @@ imageNameFormat: "ghcr.io/edera-dev/[image]:[tag]" +# Default set of architectures applied to any flavor that does not override it. architectures: - x86_64 flavors: - name: zone + # zone is the only flavor we currently publish for aarch64. Other flavors fall + # back to the global `architectures` list above; add an `architectures:` entry + # here to opt them in. + architectures: + - x86_64 + - aarch64 constraints: lower: '6.1' - name: zone-amdgpu @@ -103,9 +110,14 @@ images: name: "[flavor]-kernel-sdk" format: kernel.sdk runners: +# Match order matters: first runner whose constraints match wins. +- name: ubuntu-24.04-arm + arch: aarch64 - name: edera-large + arch: x86_64 flavors: - host - zone-amdgpu - zone-nvidiagpu - name: ubuntu-latest + arch: x86_64 diff --git a/hack/build/generate-docker-script.py b/hack/build/generate-docker-script.py index 25be81c..fed8687 100644 --- a/hack/build/generate-docker-script.py +++ b/hack/build/generate-docker-script.py @@ -18,6 +18,9 @@ # Targets skipped during the packaging phase (handled separately or not needed). SKIP_PACKAGING_TARGETS = {"kernelsrc", "buildenv"} +# Per-arch digests file consumed by the merge job to assemble manifest lists. +DIGESTS_FILE = "digests.json" + def is_publish_enabled() -> bool: root_publish = os.getenv("KERNEL_PUBLISH", "false") @@ -34,14 +37,6 @@ def dockerify_version(version_string: str) -> str: return version_string.replace('+', '-') -def docker_platforms(architectures: list[str]) -> list[str]: - platforms = [] - for arch in architectures: - platform = arch_to_platform(arch) - platforms.append(platform) - return platforms - - def arch_to_platform(arch: str) -> str: if arch == "aarch64": return "linux/aarch64" @@ -51,10 +46,21 @@ def arch_to_platform(arch: str) -> str: sys.exit(1) +def docker_platforms(architectures: list[str]) -> list[str]: + return [arch_to_platform(a) for a in architectures] + + +def metadata_path(image_root: str, target: str) -> str: + # buildx metadata file paths must be unique per build invocation; + # use a sanitized image-name + target as the key. + safe = image_root.replace("/", "_").replace(":", "_") + return "metadata-%s-%s.json" % (safe, target) + + def docker_build_staged( version: str, flavor: str, - architectures: list[str], + archs: list[str], src_url: str, firmware_url: str, firmware_sig_url: str, @@ -81,7 +87,7 @@ def docker_build_staged( "--target", quoted(target), "--iidfile", quoted(iidfile), ] - for platform in docker_platforms(architectures): + for platform in docker_platforms(archs): command += ["--platform", quoted(platform)] command += ["--build-arg", quoted("KERNEL_SRC_URL=%s" % src_url)] if has_firmware: @@ -92,13 +98,15 @@ def docker_build_staged( if has_nvidia: command += ["--build-arg", quoted("NV_MODULES_TARBALL_URL=%s" % nv_modules_url)] command += ["."] - return [""] + smart_script_split(command, "stage=stage flavor=%s version=%s" % (flavor, version)) + return [""] + smart_script_split( + command, "stage=stage flavor=%s version=%s arch=%s" % (flavor, version, ",".join(archs)) + ) def docker_compile( version: str, flavor: str, - architectures: list[str], + archs: list[str], firmware_url: str, firmware_sig_url: str, ) -> list[str]: @@ -118,7 +126,7 @@ def docker_compile( lines += ["", "rm -rf target && mkdir -p target && chmod a+rwX target"] lines += ['mkdir -p "${HOME}/.cache/kernel-ccache" && chmod -R a+rwX "${HOME}/.cache/kernel-ccache"'] - for arch in architectures: + for arch in archs: platform = arch_to_platform(arch) compile_command = [ "docker", @@ -162,7 +170,7 @@ def docker_build( version: str, version_info: Version, tags: list[str], - architectures: list[str], + archs: list[str], publish: bool, pass_build_args: bool, mark_format: Optional[str], @@ -180,15 +188,21 @@ def docker_build( # They differ when building from a non-main branch to avoid tag collisions. # Tags in the list are already branch-suffixed (applied during matrix generation). tag_version = "%s-%s" % (version, tag_suffix) if tag_suffix else version - oci_tags = list(tags) - root = format_image_name( + image_full = format_image_name( image_name_format=CONFIG["imageNameFormat"], flavor=flavor, version_info=version_info, name=name, tag=tag_version, ) + image_root = image_full.split(":")[0] + + # CI path: single-arch matrix entry + publish → push by digest only, let the + # merge job tag and assemble the manifest list once every arch is done. + # Everything else (multi-arch local, single-arch local) uses the original + # single-invocation buildx flow with --load or --push --tag. + use_push_by_digest = publish and len(archs) == 1 image_build_command = [ "docker", @@ -196,17 +210,13 @@ def docker_build( "build", "--builder", "edera", - "--load", "-f", quoted("Dockerfile"), "--target", quoted(actual_target), - "--iidfile", - quoted("image-id-%s-%s-%s" % (tag_version, flavor, actual_target)), ] - - for build_platform in docker_platforms(architectures): - image_build_command += ["--platform", quoted(build_platform)] + for platform in docker_platforms(archs): + image_build_command += ["--platform", quoted(platform)] if actual_target != target: image_build_command += ["--build-context", quoted("ccachebuild=target")] @@ -217,6 +227,8 @@ def docker_build( quoted("dev.edera.kernel.version=%s" % version), "--annotation", quoted("dev.edera.kernel.flavor=%s" % flavor), + "--annotation", + quoted("dev.edera.%s.format=1" % mark_format), ] if pass_build_args: @@ -227,67 +239,94 @@ def docker_build( quoted("KERNEL_FLAVOR=%s" % flavor), ] - if mark_format is not None: + if use_push_by_digest: + metadata_file = metadata_path(image_root, actual_target) image_build_command += [ - "--annotation", - quoted("dev.edera.%s.format=1" % mark_format), + "--metadata-file", quoted(metadata_file), + "--output", + quoted( + "type=image,name=%s,push-by-digest=true,name-canonical=true,push=true" + % image_root + ), + ".", ] + lines += [""] + lines += smart_script_split( + image_build_command, "stage=build image=%s arch=%s" % (image_root, archs[0]) + ) + record_command = [ + "python3", "hack/build/record-digest.py", + quoted(image_root), + quoted(metadata_file), + quoted(DIGESTS_FILE), + ] + lines += [""] + lines += smart_script_split( + record_command, "stage=record-digest image=%s arch=%s" % (image_root, archs[0]) + ) + return lines + # Tagged path: either local --load (no publish) or multi-arch --push (local + # multi-arch publish, e.g. someone running the env-driven script with + # KERNEL_ARCHITECTURES=x86_64,aarch64 and KERNEL_PUBLISH=true). Both produce + # a tagged image (or multi-arch manifest list) in one buildx invocation. + all_tags = sorted({tag_version, *tags}) + for tag in all_tags: + tagged = format_image_name( + image_name_format=CONFIG["imageNameFormat"], + flavor=flavor, + version_info=version_info, + name=name, + tag=tag, + ) + image_build_command += ["--tag", quoted(tagged)] + + iidfile = "image-id-%s-%s-%s" % (tag_version, flavor, actual_target) + image_build_command += ["--iidfile", quoted(iidfile)] if publish: image_build_command += ["--push"] + else: + image_build_command += ["--load"] + image_build_command += ["."] + lines += [""] + lines += smart_script_split( + image_build_command, "stage=build image=%s arch=%s" % (image_root, ",".join(archs)) + ) - all_tags = [root] - additional_tags = [] - - for tag in oci_tags: - if tag == tag_version: - continue - additional_tags.append( - format_image_name( + if publish: + for tag in all_tags: + tagged = format_image_name( image_name_format=CONFIG["imageNameFormat"], flavor=flavor, version_info=version_info, name=name, tag=tag, ) - ) - - all_tags += additional_tags - all_tags.sort() - for tag in all_tags: - image_build_command += [ - "--tag", - quoted(tag), - ] - - image_build_command += ["."] - lines += [""] - lines += smart_script_split(image_build_command, "stage=build image=%s" % root) - - if publish: - for tag in all_tags: image_signing_command = [ "cosign", "sign", "--yes", - quoted( - '%s@$(cat "image-id-%s-%s-%s")' % (tag, tag_version, flavor, actual_target) - ), + quoted('%s@$(cat "%s")' % (tagged, iidfile)), ] lines += [""] lines += smart_script_split( - image_signing_command, "stage=sign image=%s" % tag + image_signing_command, "stage=sign image=%s" % tagged ) return lines def generate_header() -> list[str]: - return [ + lines = [ "#!/bin/sh", "set -e", "docker buildx create --name edera --config hack/build/buildkitd.toml", 'trap "docker buildx rm edera" EXIT', ] + if is_publish_enabled(): + # Start fresh so a re-run within the same workspace doesn't pick up + # digests from a previous (failed) invocation. + lines += ['rm -f "%s"' % DIGESTS_FILE] + return lines def generate_builds( @@ -295,7 +334,7 @@ def generate_builds( kernel_flavor: str, kernel_src_url: str, kernel_tags: list[str], - kernel_architectures: list[str], + kernel_archs: list[str], firmware_url: str, firmware_sig_url: str, tag_suffix: Optional[str] = None, @@ -308,7 +347,7 @@ def generate_builds( lines += docker_build_staged( version=kernel_version, flavor=kernel_flavor, - architectures=kernel_architectures, + archs=kernel_archs, src_url=kernel_src_url, firmware_url=firmware_url, firmware_sig_url=firmware_sig_url, @@ -318,7 +357,7 @@ def generate_builds( lines += docker_compile( version=kernel_version, flavor=kernel_flavor, - architectures=kernel_architectures, + archs=kernel_archs, firmware_url=firmware_url, firmware_sig_url=firmware_sig_url, ) @@ -346,7 +385,7 @@ def generate_builds( pass_build_args=should_pass_build_args, mark_format=image_format, flavor=kernel_flavor, - architectures=kernel_architectures, + archs=kernel_archs, firmware_url=firmware_url, firmware_sig_url=firmware_sig_url, tag_suffix=tag_suffix, @@ -355,19 +394,36 @@ def generate_builds( def generate_build_from_env() -> list[str]: + """Env-driven (non-matrix) build, e.g. invoked directly from a developer's + shell. Multi-arch is preserved here: pass KERNEL_ARCHITECTURES=x86_64,aarch64 + to build a multi-platform image with one buildx invocation (relies on QEMU + + containerd-snapshotter locally). KERNEL_ARCH is also accepted as a + convenience alias for a single arch.""" root_kernel_version = os.getenv("KERNEL_VERSION") root_kernel_flavor = os.getenv("KERNEL_FLAVOR") root_kernel_src_url = os.getenv("KERNEL_SRC_URL") root_firmware_url = os.getenv("FIRMWARE_URL") root_firmware_sig_url = os.getenv("FIRMWARE_SIG_URL") root_kernel_tags = os.getenv("KERNEL_TAGS", "").split(",") - root_kernel_architectures = os.getenv("KERNEL_ARCHITECTURES").split(",") + + archs_env = os.getenv("KERNEL_ARCHITECTURES", "") + arch_env = os.getenv("KERNEL_ARCH", "") + if archs_env: + root_kernel_archs = [a.strip() for a in archs_env.split(",") if a.strip()] + elif arch_env: + root_kernel_archs = [arch_env.strip()] + else: + print( + "ERROR: KERNEL_ARCHITECTURES (or KERNEL_ARCH) must be set", + file=sys.stderr, + ) + sys.exit(1) return generate_builds( kernel_version=root_kernel_version, kernel_flavor=root_kernel_flavor, kernel_src_url=root_kernel_src_url, kernel_tags=root_kernel_tags, - kernel_architectures=root_kernel_architectures, + kernel_archs=root_kernel_archs, firmware_url=root_firmware_url, firmware_sig_url=root_firmware_sig_url, tag_suffix=get_branch_tag_suffix(), @@ -385,13 +441,15 @@ def generate_builds_from_matrix(matrix) -> list[str]: firmware_url = build["firmware_url"] firmware_sig_url = build["firmware_sig_url"] build_tags = build["tags"] - build_architectures = build["architectures"] + # Matrix entries are always single-arch; wrap into a list for the + # shared generate_builds helper. + build_archs = [build["arch"]] lines += generate_builds( kernel_version=build_version, kernel_flavor=build_flavor, kernel_src_url=build_source, kernel_tags=build_tags, - kernel_architectures=build_architectures, + kernel_archs=build_archs, firmware_url=firmware_url, firmware_sig_url=firmware_sig_url, tag_suffix=tag_suffix, diff --git a/hack/build/generate-matrix.py b/hack/build/generate-matrix.py index dca830f..38ad79e 100644 --- a/hack/build/generate-matrix.py +++ b/hack/build/generate-matrix.py @@ -113,9 +113,11 @@ def construct_manual_matrix(exact_versions): build["tags"] = ["%s-%s" % (t, branch_suffix) for t in build["tags"]] build["produces"] = ["%s-%s" % (p, branch_suffix) for p in build["produces"]] -print("generated %s builds" % len(final_matrix)) +merges = matrix.generate_merges(final_matrix) + +print("generated %s builds, %s merges" % (len(final_matrix), len(merges))) matrix.summarize_matrix(final_matrix) with open("matrix.json", "w") as mf: - json.dump({"builds": final_matrix}, mf) + json.dump({"builds": final_matrix, "merges": merges}, mf) mf.write("\n") diff --git a/hack/build/generate-merge-script.py b/hack/build/generate-merge-script.py new file mode 100644 index 0000000..15a19d2 --- /dev/null +++ b/hack/build/generate-merge-script.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Generate merge.sh for one (version, flavor) merge entry. + +The CI build matrix produces per-arch images pushed to the registry by digest +only (no tags). This script consumes the digest artifacts produced by those +per-arch jobs and emits a shell script that: + + 1. Runs `docker buildx imagetools create` once per produced image, attaching + all per-arch digests under the desired tags. This is what stitches the + single-platform pushes into the published multi-arch manifest list. + 2. Signs each published tag with cosign. + +Env vars consumed: + KERNEL_PUBLISH "true" to actually run; otherwise the script no-ops. + KERNEL_PRODUCES Comma-separated image:tag list (from the merge matrix entry). + DIGESTS_DIR Directory containing downloaded artifact subdirs, each with + a digests.json. Defaults to "digests". + +The merge matrix entry already contains the canonical produces list and tags, +so we don't need to re-derive them here. +""" +import json +import os +import stat +import sys +from collections import OrderedDict + +from util import parse_text_bool, smart_script_split + + +def quoted(text: str) -> str: + return '"%s"' % text + + +def collect_digests(digests_dir: str) -> dict[str, list[str]]: + """Walk digests_dir for all digests.json files and union them. + + Returns {image_name: [digest, digest, ...]} with one digest per arch. + """ + image_digests: dict[str, list[str]] = OrderedDict() + if not os.path.isdir(digests_dir): + return image_digests + for root, _, files in os.walk(digests_dir): + for fname in files: + if fname != "digests.json": + continue + with open(os.path.join(root, fname)) as f: + data = json.load(f) + for image_name, digest in data.items(): + bucket = image_digests.setdefault(image_name, []) + if digest not in bucket: + bucket.append(digest) + return image_digests + + +def main() -> None: + publish = parse_text_bool(os.getenv("KERNEL_PUBLISH", "false")) + digests_dir = os.getenv("DIGESTS_DIR", "digests") + produces_env = os.getenv("KERNEL_PRODUCES", "") + + lines = ["#!/bin/sh", "set -e"] + + if not publish: + lines += [ + 'echo "merge: KERNEL_PUBLISH is not true; nothing to merge."', + "exit 0", + ] + elif not produces_env: + print("ERROR: KERNEL_PRODUCES not set", file=sys.stderr) + sys.exit(1) + else: + produce_list = [p for p in produces_env.split(",") if p] + image_to_tags: dict[str, list[str]] = OrderedDict() + for produce in produce_list: + image_name, tag = produce.rsplit(":", 1) + image_to_tags.setdefault(image_name, []).append(tag) + + image_digests = collect_digests(digests_dir) + for image_name, tags in image_to_tags.items(): + digests = image_digests.get(image_name) + if not digests: + print( + "ERROR: no digests found for %s under %s" + % (image_name, digests_dir), + file=sys.stderr, + ) + sys.exit(1) + + create_command = ["docker", "buildx", "imagetools", "create"] + for tag in tags: + create_command += ["-t", quoted("%s:%s" % (image_name, tag))] + for digest in digests: + create_command += [quoted("%s@%s" % (image_name, digest))] + lines += [""] + lines += smart_script_split( + create_command, "stage=merge image=%s archs=%d" % (image_name, len(digests)) + ) + + for tag in tags: + ref = "%s:%s" % (image_name, tag) + sign_command = ["cosign", "sign", "--yes", quoted(ref)] + lines += [""] + lines += smart_script_split( + sign_command, "stage=sign image=%s" % ref + ) + + with open("merge.sh", "w") as out: + out.write("\n".join(lines)) + out.write("\n") + s = os.stat("merge.sh") + os.chmod("merge.sh", s.st_mode | stat.S_IEXEC) + + +if __name__ == "__main__": + main() diff --git a/hack/build/matrix.py b/hack/build/matrix.py index 136ff29..e0e18bd 100644 --- a/hack/build/matrix.py +++ b/hack/build/matrix.py @@ -23,7 +23,7 @@ @cache -def build_architectures(): +def default_architectures() -> list[str]: architecture_env = os.getenv("KERNEL_ARCHITECTURES", "") if len(architecture_env) > 0: return [arch.strip() for arch in architecture_env.split(",")] @@ -32,6 +32,13 @@ def build_architectures(): return architectures +def flavor_architectures(flavor_info: dict[str, any]) -> list[str]: + """Per-flavor architectures override; falls back to the global default.""" + if "architectures" in flavor_info: + return flavor_info["architectures"] + return default_architectures() + + @cache def get_current_kernel_releases() -> dict[str, any]: with urllib.request.urlopen("https://www.kernel.org/releases.json") as response: @@ -87,16 +94,13 @@ def merge_matrix(matrix_list: list[list[dict[str, any]]]) -> list[dict[str, any] all_builds = OrderedDict() # type: dict[str, dict[str, any]] for builds in matrix_list: for item in builds: - key = "%s::%s" % (item["version"], item["flavor"]) + key = "%s::%s::%s" % (item["version"], item["flavor"], item["arch"]) if key not in all_builds: all_builds[key] = item else: for tag in item["tags"]: if tag not in all_builds[key]["tags"]: all_builds[key]["tags"].append(tag) - for arch in item["architectures"]: - if arch not in all_builds[key]["architectures"]: - all_builds[key]["architectures"].append(arch) builds = list(all_builds.values()) builds.sort(key=lambda build: parse(build["version"])) @@ -131,14 +135,21 @@ def find_existing_tags(images: list[str]) -> dict[str, list[str]]: def validate_produce_conflicts(builds: list[dict[str, any]]): - produce_check = {} + # produces are intentionally shared across arches for the same (version, flavor); + # each arch build pushes its single-platform image by digest, and the merge step + # later tags the combined manifest list. Only flag when the *same* image:tag is + # claimed by two different (version, flavor) combinations. + produce_owner = {} for build in builds: + owner = "%s::%s" % (build["version"], build["flavor"]) for produce in build["produces"]: - if produce in produce_check: - raise Exception("ERROR: %s was produced more than once" % produce) - else: - produce_check[produce] = produce + if produce in produce_owner and produce_owner[produce] != owner: + raise Exception( + "ERROR: %s is produced by both %s and %s" + % (produce, produce_owner[produce], owner) + ) + produce_owner[produce] = owner def filter_new_builds(builds: list[dict[str, any]]) -> list[dict[str, any]]: @@ -194,7 +205,11 @@ def filter_matrix( flavor = build["flavor"] is_current_release = is_release_current(version_info.base_version) should_build = matches_constraints( - version_info, flavor, constraint, is_current_release=is_current_release + version_info, + flavor, + constraint, + is_current_release=is_current_release, + arch=build.get("arch"), ) if should_build: output_builds.append(build) @@ -269,6 +284,8 @@ def generate_matrix(tags: dict[str, str]) -> list[dict[str, any]]: ): continue + architectures = flavor_architectures(flavor_info) + if "local_tags" in flavor_info: for local_tag in flavor_info["local_tags"]: produces = [] @@ -284,18 +301,19 @@ def generate_matrix(tags: dict[str, str]) -> list[dict[str, any]]: ) produces.append(kernel_output) produces.append(kernel_sdk_output) - version_builds.append( - { - "version": version+"+"+local_tag, - "firmware_url": firmware_url, - "firmware_sig_url": firmware_sig_url, - "tags": local_version_tags, - "source": src_url, - "flavor": flavor, - "architectures": build_architectures(), - "produces": produces, - } - ) + for arch in architectures: + version_builds.append( + { + "version": version+"+"+local_tag, + "firmware_url": firmware_url, + "firmware_sig_url": firmware_sig_url, + "tags": local_version_tags, + "source": src_url, + "flavor": flavor, + "arch": arch, + "produces": produces, + } + ) else: produces = [] for tag in version_tags: @@ -307,18 +325,19 @@ def generate_matrix(tags: dict[str, str]) -> list[dict[str, any]]: ) produces.append(kernel_output) produces.append(kernel_sdk_output) - version_builds.append( - { - "version": version, - "firmware_url": firmware_url, - "firmware_sig_url": firmware_sig_url, - "tags": version_tags, - "source": src_url, - "flavor": flavor, - "architectures": build_architectures(), - "produces": produces, - } - ) + for arch in architectures: + version_builds.append( + { + "version": version, + "firmware_url": firmware_url, + "firmware_sig_url": firmware_sig_url, + "tags": version_tags, + "source": src_url, + "flavor": flavor, + "arch": arch, + "produces": produces, + } + ) return version_builds @@ -339,7 +358,7 @@ def summarize_matrix(builds: list[dict[str, any]]): % ( build["flavor"], build["version"], - ", ".join(build["architectures"]), + build["arch"], ", ".join(tags), ", ".join(image_names), build["runner"], @@ -429,9 +448,14 @@ def pick_runner(build: dict[str, any]) -> str: version: str = build["version"] version_info: Version = parse(version) flavor: str = build["flavor"] + arch: str = build["arch"] for runner in CONFIG["runners"]: if matches_constraints( - version_info, flavor, runner, is_current_release=is_release_current(version_info.base_version) + version_info, + flavor, + runner, + is_current_release=is_release_current(version_info.base_version), + arch=arch, ): return runner["name"] raise Exception("No runner found for build %s" % build) @@ -444,3 +468,27 @@ def fill_runners(builds: list[dict[str, any]]): def sort_matrix(builds: list[dict[str, any]]): builds.sort(key=lambda build: Version(build["version"])) + + +def generate_merges(builds: list[dict[str, any]]) -> list[dict[str, any]]: + """Group per-arch builds into one merge entry per (version, flavor). + + The merge job runs after all per-arch build jobs for that (version, flavor) + complete; it stitches the single-platform pushes into a manifest list per + produced image:tag. + """ + merges = OrderedDict() # type: dict[str, dict[str, any]] + for build in builds: + key = "%s::%s" % (build["version"], build["flavor"]) + if key not in merges: + merges[key] = { + "version": build["version"], + "flavor": build["flavor"], + "tags": list(build["tags"]), + "produces": list(build["produces"]), + "archs": [build["arch"]], + } + else: + if build["arch"] not in merges[key]["archs"]: + merges[key]["archs"].append(build["arch"]) + return list(merges.values()) diff --git a/hack/build/record-digest.py b/hack/build/record-digest.py new file mode 100644 index 0000000..ddcb5a6 --- /dev/null +++ b/hack/build/record-digest.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Record an image digest emitted by `docker buildx build --metadata-file`. + +Reads the buildx metadata JSON, extracts containerimage.digest, and appends an +entry {: } to the cumulative digests file for this build. +The merge job consumes this file (one per (version, flavor, arch)) to create the +final manifest list with `docker buildx imagetools create`. +""" +import json +import os +import sys + + +def main(): + if len(sys.argv) != 4: + print( + "Usage: record-digest.py ", + file=sys.stderr, + ) + sys.exit(1) + + image_name = sys.argv[1] + metadata_path = sys.argv[2] + digests_path = sys.argv[3] + + with open(metadata_path) as f: + metadata = json.load(f) + + digest = metadata.get("containerimage.digest") + if not digest: + print( + "ERROR: %s missing containerimage.digest key" % metadata_path, + file=sys.stderr, + ) + sys.exit(1) + + existing = {} + if os.path.exists(digests_path): + with open(digests_path) as f: + existing = json.load(f) + existing[image_name] = digest + with open(digests_path, "w") as f: + json.dump(existing, f, indent=2, sort_keys=True) + f.write("\n") + + +if __name__ == "__main__": + main() diff --git a/hack/build/util.py b/hack/build/util.py index d710f9c..15aff86 100644 --- a/hack/build/util.py +++ b/hack/build/util.py @@ -36,11 +36,21 @@ def maybe(m: dict[str, any], k: str, default_value: any = None) -> any: return default_value def matches_constraints( - version: Version, flavor: str, constraints: dict[str, any], is_current_release=None + version: Version, + flavor: str, + constraints: dict[str, any], + is_current_release=None, + arch: Optional[str] = None, ) -> bool: if "any" in constraints: for constraint in constraints["any"]: - if matches_constraints(version, flavor, constraint, is_current_release=is_current_release): + if matches_constraints( + version, + flavor, + constraint, + is_current_release=is_current_release, + arch=arch, + ): return True return False @@ -53,6 +63,7 @@ def matches_constraints( upper = maybe(constraints, "upper") exact = maybe(constraints, "exact") current = maybe(constraints, "current") + arch_constraint = maybe(constraints, "arch") if lower is not None: lower = Version(lower) @@ -94,6 +105,12 @@ def matches_constraints( if exact is not None and not version_string in exact: applies = False + if arch_constraint is not None and arch is not None: + if type(arch_constraint) is str: + arch_constraint = [arch_constraint] + if arch not in arch_constraint: + applies = False + return applies @@ -134,7 +151,7 @@ def parse_text_constraint(text: str) -> dict[str, any]: constraint[key] = parse_text_bool(value) elif key == "lower" or key == "upper": constraint[key] = value - elif key == "flavors" or key == "flavor" or key == "series" or key == "exact": + elif key == "flavors" or key == "flavor" or key == "series" or key == "exact" or key == "arch": if key == "flavor": key = "flavors" constraint[key] = value.split(",")